model-router v0.4.4
Claude Code plugin: routes subagents by how well-defined the brief is, verifies dev work on a second model in the agent's own clone, and estimates savings against Opus 5.5. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1 @@
|
||||
{ "modules": ["./register.ts"] }
|
||||
@@ -0,0 +1,185 @@
|
||||
// Cost of a run, and what the same work would have cost on Opus. Pure: no `$`.
|
||||
//
|
||||
// Prices: USD per million tokens, from platform.claude.com/docs/en/about-claude/pricing
|
||||
// (read 2026-10-08). A setting can replace them.
|
||||
|
||||
export type Usage = {
|
||||
input_tokens: number
|
||||
output_tokens: number
|
||||
cache_creation_input_tokens?: number | null
|
||||
cache_read_input_tokens?: number | null
|
||||
}
|
||||
|
||||
/** in, out, cache write (5 min), cache read: USD per MTok. */
|
||||
export type Price = { in: number; out: number; write: number; read: number }
|
||||
|
||||
export type PriceRow = { match: string; price: Price; label: string }
|
||||
|
||||
// First match wins: more specific ids come first.
|
||||
export const DEFAULT_PRICES: readonly PriceRow[] = [
|
||||
{ match: 'fable-5-1', label: 'Fable 5.1', price: { in: 10, out: 50, write: 12.5, read: 0.25 } },
|
||||
{ match: 'mythos-5-1', label: 'Mythos 5.1', price: { in: 10, out: 50, write: 12.5, read: 0.25 } },
|
||||
{ match: 'opus-5-5', label: 'Opus 5.5', price: { in: 4, out: 20, write: 5, read: 0.2 } },
|
||||
{ match: 'sonnet-5-5', label: 'Sonnet 5.5', price: { in: 2, out: 10, write: 2.5, read: 0.1 } },
|
||||
// Haiku 5.5 doubles past 100k-token prompts; a run's summed usage cannot tell, so the low tier is used.
|
||||
{ match: 'haiku-5-5', label: 'Haiku 5.5', price: { in: 0.1, out: 0.5, write: 0.125, read: 0.01 } },
|
||||
{ match: 'haiku-4-5', label: 'Haiku 4.5', price: { in: 1, out: 5, write: 1.25, read: 0.1 } },
|
||||
{ match: 'fable-5', label: 'Fable 5', price: { in: 10, out: 50, write: 12.5, read: 1 } },
|
||||
{ match: 'opus-5', label: 'Opus 5', price: { in: 5, out: 25, write: 6.25, read: 0.5 } },
|
||||
{ match: 'sonnet-5', label: 'Sonnet 5', price: { in: 2, out: 10, write: 2.5, read: 0.2 } },
|
||||
]
|
||||
|
||||
/** The model every saving is measured against. */
|
||||
export const BASELINE = 'opus-5-5'
|
||||
|
||||
/**
|
||||
* Models before Claude 4.7 use a tokenizer that yields ~30% fewer tokens for
|
||||
* the same text, so their counts are scaled up before pricing them as Opus 5.5.
|
||||
*/
|
||||
const OLD_TOKENIZER = /(haiku-4-5|sonnet-4|opus-4-[0-6]|opus-4-1|opus-4(?!-)|haiku-3)/
|
||||
export const tokenizerFactor = (modelId: string) => (OLD_TOKENIZER.test(modelId) ? 1.3 : 1)
|
||||
|
||||
export const priceRow = (modelId: string, table: readonly PriceRow[] = DEFAULT_PRICES) =>
|
||||
table.find(r => modelId.toLowerCase().includes(r.match))
|
||||
|
||||
/** USD for a usage at a price; tokens scaled by `factor`. */
|
||||
export const cost = (u: Usage, p: Price, factor = 1) =>
|
||||
(factor *
|
||||
(u.input_tokens * p.in +
|
||||
u.output_tokens * p.out +
|
||||
(u.cache_creation_input_tokens ?? 0) * p.write +
|
||||
(u.cache_read_input_tokens ?? 0) * p.read)) /
|
||||
1_000_000
|
||||
|
||||
export type Priced = { actual: number; opus: number; known: boolean; label: string }
|
||||
|
||||
/** What a run cost on its own model, and the same tokens on the baseline. */
|
||||
export const priceRun = (modelId: string, u: Usage, table: readonly PriceRow[] = DEFAULT_PRICES): Priced => {
|
||||
const row = priceRow(modelId, table)
|
||||
const base = priceRow(BASELINE, table)
|
||||
if (!row || !base) return { actual: 0, opus: 0, known: false, label: modelId }
|
||||
return {
|
||||
actual: cost(u, row.price),
|
||||
opus: cost(u, base.price, tokenizerFactor(modelId)),
|
||||
known: true,
|
||||
label: row.label,
|
||||
}
|
||||
}
|
||||
|
||||
/** A task's running totals, across every agent and round it took. */
|
||||
export type TaskCost = {
|
||||
task: string
|
||||
/** Writer runs on each model, USD. */
|
||||
writerActual: number
|
||||
/** Writer runs below Opus, priced at Opus: what Opus would have cost for the same tokens. */
|
||||
writerAsOpus: number
|
||||
/** Writer runs that were on Opus already, USD. */
|
||||
writerOnOpus: number
|
||||
/** Classifier and verifier calls the router added, USD. */
|
||||
overhead: number
|
||||
models: string[]
|
||||
unpriced: string[]
|
||||
}
|
||||
|
||||
export const emptyTask = (task: string): TaskCost => ({
|
||||
task,
|
||||
writerActual: 0,
|
||||
writerAsOpus: 0,
|
||||
writerOnOpus: 0,
|
||||
overhead: 0,
|
||||
models: [],
|
||||
unpriced: [],
|
||||
})
|
||||
|
||||
export const addWriterRun = (t: TaskCost, modelId: string, u: Usage, table?: readonly PriceRow[]): TaskCost => {
|
||||
const p = priceRun(modelId, u, table)
|
||||
const next = { ...t, models: [...t.models], unpriced: [...t.unpriced] }
|
||||
if (!p.known) {
|
||||
if (!next.unpriced.includes(modelId)) next.unpriced.push(modelId)
|
||||
return next
|
||||
}
|
||||
if (!next.models.includes(p.label)) next.models.push(p.label)
|
||||
next.writerActual += p.actual
|
||||
if (modelId.toLowerCase().includes(BASELINE)) next.writerOnOpus += p.actual
|
||||
else next.writerAsOpus += p.opus
|
||||
return next
|
||||
}
|
||||
|
||||
export const addOverhead = (t: TaskCost, modelId: string, u: Usage, table?: readonly PriceRow[]): TaskCost => {
|
||||
const p = priceRun(modelId, u, table)
|
||||
return p.known ? { ...t, overhead: t.overhead + p.actual } : t
|
||||
}
|
||||
|
||||
/**
|
||||
* What the task would have cost had it gone to Opus at once.
|
||||
*
|
||||
* When Opus wrote part of it (a rewrite after failed rounds, or an open
|
||||
* design), Opus's own runs are taken as what Opus alone would have needed:
|
||||
* the failed cheaper rounds then count as pure loss. Otherwise every writer
|
||||
* token is priced at Opus, an upper bound, since Opus may need fewer tokens.
|
||||
*/
|
||||
export const counterfactual = (t: TaskCost) => (t.writerOnOpus > 0 ? t.writerOnOpus : t.writerAsOpus)
|
||||
|
||||
export const actualTotal = (t: TaskCost) => t.writerActual + t.overhead
|
||||
|
||||
export const saved = (t: TaskCost) => counterfactual(t) - actualTotal(t)
|
||||
|
||||
export const savedPct = (t: TaskCost) => {
|
||||
const c = counterfactual(t)
|
||||
return c > 0 ? (saved(t) / c) * 100 : 0
|
||||
}
|
||||
|
||||
/** Reads a prices setting: a JSON list of rows, or empty for the defaults. */
|
||||
export const parsePrices = (json: string): readonly PriceRow[] => {
|
||||
if (!json.trim()) return DEFAULT_PRICES
|
||||
try {
|
||||
const rows = JSON.parse(json) as PriceRow[]
|
||||
const ok = Array.isArray(rows) && rows.every(r => typeof r.match === 'string' && typeof r.price?.in === 'number')
|
||||
return ok ? [...rows.map(r => ({ ...r, label: r.label ?? r.match })), ...DEFAULT_PRICES] : DEFAULT_PRICES
|
||||
} catch {
|
||||
return DEFAULT_PRICES
|
||||
}
|
||||
}
|
||||
|
||||
export const usd = (n: number) => (Math.abs(n) < 0.01 && n !== 0 ? `$${n.toFixed(4)}` : `$${n.toFixed(2)}`)
|
||||
|
||||
/**
|
||||
* The savings table: per task, what it cost, what Opus would have cost, and
|
||||
* the difference, newest first; then the total over every task.
|
||||
*/
|
||||
export const savingsReport = (costs: Readonly<Record<string, TaskCost>>, limit: number, only?: string): string => {
|
||||
const ids = Object.keys(costs)
|
||||
.filter(id => !only || id === only)
|
||||
.sort((a, b) => Number(b.slice(1)) - Number(a.slice(1)))
|
||||
if (ids.length === 0) return only ? `model-router: no costs recorded for ${only}.` : 'savings: no agent runs recorded yet.'
|
||||
|
||||
const row = (id: string, t: TaskCost) => {
|
||||
const pct = counterfactual(t) > 0 ? `${savedPct(t).toFixed(0)}%` : '-'
|
||||
const how = t.writerActual === 0 ? 'running' : t.writerOnOpus > 0 ? 'opus ran' : 'upper bound'
|
||||
return `${id.padEnd(5)} ${usd(actualTotal(t)).padStart(9)} ${usd(counterfactual(t)).padStart(9)} ${usd(saved(t)).padStart(9)} ${pct.padStart(5)} ${how.padEnd(11)} ${t.models.join('+').padEnd(20)} ${t.task.slice(0, 32)}${t.unpriced.length ? ` [no price: ${t.unpriced.join(', ')}]` : ''}`
|
||||
}
|
||||
|
||||
let actual = 0
|
||||
let opus = 0
|
||||
for (const id of Object.keys(costs)) {
|
||||
const t = costs[id]
|
||||
// a task whose writer has not finished has no comparison yet
|
||||
if (!t || t.writerActual === 0) continue
|
||||
actual += actualTotal(t)
|
||||
opus += counterfactual(t)
|
||||
}
|
||||
const shown = ids.slice(0, limit)
|
||||
const lines = [
|
||||
`savings vs Opus 5.5 (USD, API list prices)${only ? '' : ` - ${shown.length} of ${ids.length} tasks`}`,
|
||||
`task actual if opus saved % basis models task`,
|
||||
...shown.map(id => row(id, costs[id] as TaskCost)),
|
||||
]
|
||||
if (!only)
|
||||
lines.push(
|
||||
`total ${usd(actual).padStart(9)} ${usd(opus).padStart(9)} ${usd(opus - actual).padStart(9)} ${(opus > 0 ? `${(((opus - actual) / opus) * 100).toFixed(0)}%` : '-').padStart(5)}`,
|
||||
)
|
||||
lines.push(
|
||||
'actual = writer runs + classifier + verifier. "upper bound": all tokens priced at Opus; Opus may need fewer. "opus ran": the Opus run stands for Opus alone, cheaper failed rounds count as loss.',
|
||||
)
|
||||
return lines.join('\n')
|
||||
}
|
||||
@@ -0,0 +1,681 @@
|
||||
import type { EngineInterface, Register } from 'claude-code'
|
||||
|
||||
import {
|
||||
addOverhead,
|
||||
addWriterRun,
|
||||
emptyTask,
|
||||
parsePrices,
|
||||
savingsReport,
|
||||
type PriceRow,
|
||||
type TaskCost,
|
||||
type Usage,
|
||||
} from './pricing'
|
||||
import {
|
||||
REWRITE_MARK,
|
||||
asTier,
|
||||
clip,
|
||||
decide,
|
||||
onFail,
|
||||
parseBrief,
|
||||
parsePatterns,
|
||||
parseVerdict,
|
||||
safetyHit,
|
||||
skipReason,
|
||||
validTimeout,
|
||||
verifierFor,
|
||||
type Role,
|
||||
type Tier,
|
||||
} from './router'
|
||||
|
||||
// ---------- stored records ----------
|
||||
|
||||
/** One agent the router started, kept from spawn to its last verified round. */
|
||||
type AgentRec = {
|
||||
taskId: string
|
||||
tier: Tier
|
||||
role?: Role
|
||||
clone?: string
|
||||
baseBranch?: string
|
||||
critical: boolean
|
||||
prompt: string
|
||||
/** Failed verdicts in this task's chain, carried over to a rewrite agent. */
|
||||
fails: number
|
||||
rounds: number
|
||||
/** The tree after the last verified round: the next round diffs from here. */
|
||||
lastSnap?: string
|
||||
lastUntracked?: string[]
|
||||
lastIssues?: string[]
|
||||
}
|
||||
|
||||
type Pending = { prompt: string; tier: Tier; fails: number; critical: boolean }
|
||||
|
||||
type Outcome =
|
||||
| 'pass'
|
||||
| 'resume-requested'
|
||||
| 'rewrite-requested'
|
||||
| 'handed-back'
|
||||
| 'no-changes'
|
||||
| 'no-base'
|
||||
| 'tests-timeout'
|
||||
| 'tests-error'
|
||||
| 'verify-error'
|
||||
| 'test-agent'
|
||||
| 'review-agent'
|
||||
| 'no-clone'
|
||||
| 'opus-unverified'
|
||||
| 'agent-ended'
|
||||
|
||||
type LogEntry = {
|
||||
id: string
|
||||
agentId?: string
|
||||
at: number
|
||||
task: string
|
||||
tier: Tier
|
||||
role?: Role
|
||||
reason: string
|
||||
round?: number
|
||||
verifier?: Tier
|
||||
outcome: Outcome | 'spawned'
|
||||
opusRequested?: boolean
|
||||
}
|
||||
|
||||
const S = {
|
||||
counter: 'counter',
|
||||
pending: 'pending',
|
||||
agents: 'agents',
|
||||
log: 'log',
|
||||
costs: 'costs',
|
||||
agentTask: 'agentTask',
|
||||
aliases: 'aliases',
|
||||
} as const
|
||||
|
||||
/** Model ids assumed for an alias until an agent run shows what it resolves to. */
|
||||
const ALIAS_DEFAULT: Record<Tier, string> = {
|
||||
haiku: 'claude-haiku-4-5',
|
||||
sonnet: 'claude-sonnet-5-5',
|
||||
opus: 'claude-opus-5-5',
|
||||
fable: 'claude-fable-5-1',
|
||||
}
|
||||
const LOG_LIMIT = 400
|
||||
|
||||
// ---------- prompts ----------
|
||||
|
||||
const CLASSIFIER_SYSTEM = `You route coding tasks to the smallest model that will do them right.
|
||||
Judge how well-defined the task is, NOT how important or low-level the code is. In this codebase nearly everything touches control loops, asyncio and RS485; that alone never means opus. A stronger model verifies every change below Opus, and a review agent reviews every PR.
|
||||
Answer with exactly one word: haiku, sonnet or opus.
|
||||
|
||||
haiku: read-only lookups and summaries, mechanical changes fully spelled out.
|
||||
sonnet: the approach is decided: the brief says what to change, or gives Expected behaviour, acceptance criteria or a test to make pass. Size and domain do not matter.
|
||||
opus: the approach is still open: a design or trade-off to decide, a bug whose cause is unknown, or a brief so vague the writer must invent the plan.
|
||||
|
||||
When unsure between sonnet and opus, pick sonnet: a failed verification escalates.`
|
||||
|
||||
const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not.
|
||||
Judge only whether the change correctly and completely does the task, without breaking anything nearby.
|
||||
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
|
||||
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
|
||||
|
||||
Reply with JSON only:
|
||||
{"verdict": "PASS" | "FAIL", "issues": ["concrete issue with file:line where possible", ...], "rewrite_brief": "what must change, empty on PASS"}`
|
||||
|
||||
const ORCHESTRATOR_NOTE = `## Model router (model-router mod)
|
||||
The model-router mod picks each subagent's model and verifies dev work before it reaches a PR.
|
||||
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
|
||||
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
|
||||
- Review and test agents keep the model you name (Opus when you name none).
|
||||
- When a dev agent below Opus finishes a round, the router verifies its clone's diff on Opus and posts a "[model-router]" verdict. Wait for that verdict before opening or advancing the PR.
|
||||
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. Do not fix rejected work yourself.`
|
||||
|
||||
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
||||
|
||||
// Agents start and finish in parallel. Every read-modify-write of the store
|
||||
// goes through this queue, one at a time, or parallel hooks overwrite each
|
||||
// other's writes (and hand out the same task id).
|
||||
let queue: Promise<unknown> = Promise.resolve()
|
||||
|
||||
function serial<T>(work: () => Promise<T>): Promise<T> {
|
||||
const run = queue.then(work)
|
||||
queue = run.catch(() => undefined)
|
||||
return run
|
||||
}
|
||||
|
||||
async function getMap<T>($: EngineInterface, key: string) {
|
||||
return ((await $.store.get(key)) ?? {}) as Record<string, T>
|
||||
}
|
||||
|
||||
/** Replaces one entry of a stored map, atomically with respect to this plugin. */
|
||||
async function updateIn<T>($: EngineInterface, key: string, id: string, change: (old: T | undefined) => T | undefined) {
|
||||
return serial(async () => {
|
||||
const all = await getMap<T>($, key)
|
||||
const value = change(all[id])
|
||||
if (value === undefined) delete all[id]
|
||||
else all[id] = value
|
||||
await $.store.set(key, all)
|
||||
return value
|
||||
})
|
||||
}
|
||||
|
||||
async function putIn<T>($: EngineInterface, key: string, id: string, value: T | undefined) {
|
||||
await updateIn<T>($, key, id, () => value)
|
||||
}
|
||||
|
||||
async function nextId($: EngineInterface) {
|
||||
return serial(async () => {
|
||||
const n = Number((await $.store.get(S.counter)) ?? 0) + 1
|
||||
await $.store.set(S.counter, n)
|
||||
return `T${n}`
|
||||
})
|
||||
}
|
||||
|
||||
async function log($: EngineInterface, entry: LogEntry) {
|
||||
await serial(async () => {
|
||||
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
||||
list.push(entry)
|
||||
await $.store.set(S.log, list.slice(-LOG_LIMIT))
|
||||
})
|
||||
}
|
||||
|
||||
async function sh($: EngineInterface, argv: string[], cwd: string, timeoutMs = 30_000) {
|
||||
try {
|
||||
const r = await $.process.run(argv, { cwd, timeoutMs })
|
||||
return { ok: r.exitCode === 0, out: r.stdout, err: r.stderr, code: r.exitCode }
|
||||
} catch (err) {
|
||||
return { ok: false, out: '', err: String(err), code: -1 }
|
||||
}
|
||||
}
|
||||
|
||||
async function untracked($: EngineInterface, cwd: string) {
|
||||
const r = await sh($, ['git', 'ls-files', '--others', '--exclude-standard'], cwd)
|
||||
return r.ok ? r.out.split('\n').filter(Boolean) : []
|
||||
}
|
||||
|
||||
/** A commit holding the clone's tree as it is now (HEAD when clean). */
|
||||
async function snapshot($: EngineInterface, cwd: string) {
|
||||
const stash = await sh($, ['git', 'stash', 'create'], cwd)
|
||||
if (stash.ok && stash.out.trim()) return stash.out.trim()
|
||||
const head = await sh($, ['git', 'rev-parse', 'HEAD'], cwd)
|
||||
return head.ok ? head.out.trim() : undefined
|
||||
}
|
||||
|
||||
/** Where a first round diffs from: the merge base with the PR's base branch. */
|
||||
async function firstBase($: EngineInterface, rec: AgentRec, prBaseCommand: string, fallback: string) {
|
||||
const cwd = rec.clone as string
|
||||
let branch: string | undefined
|
||||
if (prBaseCommand) {
|
||||
const r = await sh($, ['sh', '-c', prBaseCommand], cwd)
|
||||
const b = r.out.trim().split('\n')[0]?.trim()
|
||||
if (r.ok && b) branch = b
|
||||
}
|
||||
branch = branch ?? rec.baseBranch ?? fallback
|
||||
for (const ref of [`origin/${branch}`, branch]) {
|
||||
const r = await sh($, ['git', 'merge-base', ref, 'HEAD'], cwd)
|
||||
if (r.ok && r.out.trim()) return { base: r.out.trim(), branch }
|
||||
}
|
||||
return { base: undefined, branch }
|
||||
}
|
||||
|
||||
async function classify($: EngineInterface, prompt: string): Promise<{ tier: Tier; reason: string; usage?: Usage }> {
|
||||
const r = await $.model.complete({
|
||||
model: 'haiku',
|
||||
system: CLASSIFIER_SYSTEM,
|
||||
prompt: `Task:\n${clip(prompt, 6000)}`,
|
||||
maxTokens: 5,
|
||||
timeoutMs: 15_000,
|
||||
})
|
||||
const tier = r.isAnswered ? asTier(r.text) : undefined
|
||||
if (tier && tier !== 'fable') return { tier, reason: 'classifier', usage: r.usage }
|
||||
return { tier: 'sonnet', reason: 'classifier unavailable, default', usage: r.usage }
|
||||
}
|
||||
|
||||
async function tell($: EngineInterface, text: string) {
|
||||
await $.prompt.submit({ text })
|
||||
}
|
||||
|
||||
// ---------- cost tracking ----------
|
||||
|
||||
async function modelIdFor($: EngineInterface, alias: Tier) {
|
||||
const seen = (await getMap<string>($, S.aliases))[alias]
|
||||
return seen ?? ALIAS_DEFAULT[alias]
|
||||
}
|
||||
|
||||
/** Adds a router call (classifier, verifier) to a task's overhead. */
|
||||
async function chargeOverhead($: EngineInterface, taskId: string, alias: Tier, u: Usage | undefined, prices: readonly PriceRow[]) {
|
||||
if (!u) return
|
||||
const modelId = await modelIdFor($, alias)
|
||||
await updateIn<TaskCost>($, S.costs, taskId, old => addOverhead(old ?? emptyTask(taskId), modelId, u, prices))
|
||||
}
|
||||
|
||||
type TestRun = { status: 'ran' | 'timeout' | 'error'; code: number | null; tail: string; seconds: number }
|
||||
|
||||
/**
|
||||
* Runs the test command in the clone, streamed: `$.process.run` stops at ten
|
||||
* minutes, a stream has no limit of its own. GNU `timeout` does the kill.
|
||||
* A timeout or a run that could not start is no verdict on the code.
|
||||
*/
|
||||
async function runTests($: EngineInterface, cmd: string, cwd: string, timeoutSec: number): Promise<TestRun> {
|
||||
const started = await $.clock.now()
|
||||
const seconds = async () => Math.round(((await $.clock.now()) - started) / 1000)
|
||||
let out = ''
|
||||
let errTail = ''
|
||||
try {
|
||||
const stream = $.process.spawn({
|
||||
argv: ['timeout', '--verbose', '--kill-after=30', String(Math.max(1, Math.round(timeoutSec))), 'sh', '-c', cmd],
|
||||
cwd,
|
||||
})
|
||||
for await (const piece of stream) {
|
||||
out = (out + piece.text).slice(-65_536)
|
||||
if (piece.stream === 'stderr') errTail = (errTail + piece.text).slice(-65_536)
|
||||
}
|
||||
// GNU timeout --verbose says so on stderr when, and only when, it fires.
|
||||
// Read from the joined stderr, so a line split across pieces still counts.
|
||||
const isFired = errTail.includes('timeout: sending signal')
|
||||
const end = await stream.result
|
||||
const tail = clip(out.trim().split('\n').slice(-80).join('\n'), 8000)
|
||||
const took = await seconds()
|
||||
// Only a timeout that fired is a timeout: a 137 without it is the code's own
|
||||
// (e.g. out of memory) and goes to the verifier as a failed run.
|
||||
if (isFired) return { status: 'timeout', code: end.code, tail, seconds: took }
|
||||
if (end.code === 126 || end.code === 127) return { status: 'error', code: end.code, tail, seconds: took }
|
||||
return { status: 'ran', code: end.code, tail, seconds: took }
|
||||
} catch (err) {
|
||||
return { status: 'error', code: null, tail: String(err).slice(0, 500), seconds: await seconds() }
|
||||
}
|
||||
}
|
||||
|
||||
type Settings = {
|
||||
testCommand: string
|
||||
testTimeoutSec: number
|
||||
prBaseCommand: string
|
||||
defaultBaseBranch: string
|
||||
maxFails: number
|
||||
allowFable: boolean
|
||||
prices: readonly PriceRow[]
|
||||
}
|
||||
|
||||
/** Verifies one finished round of a dev agent and tells the manager. */
|
||||
async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, cfg: Settings) {
|
||||
const cwd = rec.clone as string
|
||||
const round = rec.rounds + 1
|
||||
const entry: LogEntry = {
|
||||
id: rec.taskId,
|
||||
agentId,
|
||||
at: await $.clock.now(),
|
||||
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
||||
tier: rec.tier,
|
||||
role: rec.role,
|
||||
reason: `round ${round}`,
|
||||
round,
|
||||
outcome: 'verify-error',
|
||||
}
|
||||
const head = `[model-router] ${rec.taskId} (agent ${agentId}, ${rec.tier}, round ${round})`
|
||||
|
||||
// 1. What this round changed.
|
||||
let base = rec.lastSnap
|
||||
let baseLabel = 'the previous verified round'
|
||||
if (!base) {
|
||||
const fb = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch)
|
||||
base = fb.base
|
||||
baseLabel = `merge-base with ${fb.branch}`
|
||||
if (!base) {
|
||||
entry.outcome = 'no-base'
|
||||
await log($, entry)
|
||||
await tell($, `${head}: NOT VERIFIED - no merge base with ${fb.branch} in ${cwd}. Review the change yourself before the PR.`)
|
||||
return
|
||||
}
|
||||
}
|
||||
const diff = await sh($, ['git', 'diff', base], cwd, 60_000)
|
||||
const known = new Set(rec.lastUntracked ?? [])
|
||||
const fresh = (await untracked($, cwd)).filter(f => !known.has(f))
|
||||
let added = ''
|
||||
for (const f of fresh.slice(0, 20)) {
|
||||
try {
|
||||
added += `\n--- new file: ${f}\n${clip(await $.fs.read(`${cwd}/${f}`), 8000)}\n`
|
||||
} catch {
|
||||
added += `\n--- new file: ${f} (unreadable)\n`
|
||||
}
|
||||
}
|
||||
const changes = (diff.out + added).trim()
|
||||
if (!changes) {
|
||||
entry.outcome = 'no-changes'
|
||||
await log($, entry)
|
||||
await tell($, `${head}: no changes since ${baseLabel}; nothing to verify.`)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Tests, in the clone.
|
||||
let tests = ''
|
||||
if (cfg.testCommand) {
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} running tests`)
|
||||
const t = await runTests($, cfg.testCommand, cwd, cfg.testTimeoutSec)
|
||||
if (t.status !== 'ran') {
|
||||
// No verdict on the code: no verifier call, no fail counted, no rewrite.
|
||||
entry.outcome = t.status === 'timeout' ? 'tests-timeout' : 'tests-error'
|
||||
await log($, entry)
|
||||
const why =
|
||||
t.status === 'timeout'
|
||||
? `the tests did not finish within ${cfg.testTimeoutSec} s (testTimeoutSec)`
|
||||
: `the test command could not run (exit ${t.code ?? 'none'})`
|
||||
await tell(
|
||||
$,
|
||||
`${head}: NOT VERIFIED - ${why}. This says nothing about the code: do not resume or rewrite the agent for it. ` +
|
||||
`Fix the test setup or raise testTimeoutSec, or review the change yourself.\nLast output:\n${clip(t.tail, 2000)}`,
|
||||
)
|
||||
return
|
||||
}
|
||||
tests = `exit code ${t.code} after ${t.seconds} s\n${t.tail}`
|
||||
}
|
||||
|
||||
// 3. The verdict, from another model.
|
||||
const verifier = verifierFor(rec.tier)
|
||||
entry.verifier = verifier
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
|
||||
const v = await $.model.complete({
|
||||
model: verifier,
|
||||
system: VERIFIER_SYSTEM,
|
||||
prompt: [
|
||||
`## Task\n${clip(rec.prompt, 12_000)}`,
|
||||
rec.lastIssues?.length
|
||||
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
|
||||
: '',
|
||||
`## Changes (${baseLabel})\n${clip(changes, 60_000)}`,
|
||||
cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n\n'),
|
||||
maxTokens: 2000,
|
||||
timeoutMs: 180_000,
|
||||
})
|
||||
await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices)
|
||||
if (!v.isAnswered) {
|
||||
await log($, entry)
|
||||
await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`)
|
||||
return
|
||||
}
|
||||
const verdict = parseVerdict(v.text)
|
||||
const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)'
|
||||
|
||||
// 4. Remember where this round ended.
|
||||
const after: AgentRec = {
|
||||
...rec,
|
||||
rounds: round,
|
||||
lastSnap: (await snapshot($, cwd)) ?? rec.lastSnap,
|
||||
lastUntracked: await untracked($, cwd),
|
||||
lastIssues: verdict.pass ? [] : verdict.issues,
|
||||
}
|
||||
|
||||
if (verdict.pass) {
|
||||
entry.outcome = 'pass'
|
||||
await putIn($, S.agents, agentId, after)
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} ✓ ${rec.tier}, verified by ${verifier}`)
|
||||
await tell($, `${head}: PASS from verifier ${verifier}. Continue to the PR.`)
|
||||
return
|
||||
}
|
||||
|
||||
const fails = rec.fails + 1
|
||||
after.fails = fails
|
||||
const step = onFail(fails, rec.tier, cfg.maxFails, cfg.allowFable)
|
||||
await putIn($, S.agents, agentId, after)
|
||||
|
||||
if (step.kind === 'resume') {
|
||||
entry.outcome = 'resume-requested'
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} ✗ round ${round}, resume ${agentId}`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: FAIL from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
||||
(verdict.brief ? `What must change: ${verdict.brief}\n` : '') +
|
||||
`Resume agent ${agentId} with SendMessage, passing these issues. Do not fix it yourself.`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (step.kind === 'new-agent') {
|
||||
await putIn<Pending>($, S.pending, rec.taskId, {
|
||||
prompt:
|
||||
`${rec.prompt.split('\n\n## Previous attempts rejected')[0]}\n\n## Previous attempts rejected\n` +
|
||||
`Agents on ${rec.tier} were rejected ${fails} times by an independent verifier (${verifier}). Their work is in the clone ${cwd}: continue from it, fix or replace it, do not start over elsewhere.\n` +
|
||||
`Issues:\n${issues}\n` +
|
||||
(verdict.brief ? `What must change: ${verdict.brief}\n` : ''),
|
||||
tier: step.tier,
|
||||
fails,
|
||||
critical: rec.critical,
|
||||
})
|
||||
entry.outcome = 'rewrite-requested'
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} ✗ again, new agent on ${step.tier}`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: FAIL again from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
||||
`Stop agent ${agentId}, then start a new agent on ${step.tier} in the same clone: call Agent with prompt [router-rewrite ${rec.taskId}] (same subagent_type).`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
entry.outcome = 'handed-back'
|
||||
await log($, entry)
|
||||
$.ui.toast(`model-router: ${rec.taskId} failed verification ${fails}x - handed back`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: FAIL from verifier ${verifier}, ${fails} failed verdicts. Handed back: stop and show the person these issues.\n${issues}` +
|
||||
(verdict.brief ? `\nVerifier's brief: ${verdict.brief}` : ''),
|
||||
)
|
||||
}
|
||||
|
||||
/** Logs an agent run that is not verified; tells the manager when a dev agent waits on a verdict. */
|
||||
async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, outcome: Outcome) {
|
||||
await log($, {
|
||||
id: rec.taskId,
|
||||
agentId,
|
||||
at: await $.clock.now(),
|
||||
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
||||
tier: rec.tier,
|
||||
role: rec.role,
|
||||
reason: outcome,
|
||||
outcome,
|
||||
})
|
||||
if (rec.role === 'dev' && outcome === 'no-clone')
|
||||
await tell($, `[model-router] ${rec.taskId} (agent ${agentId}): NOT VERIFIED - the brief names no clone path.`)
|
||||
}
|
||||
|
||||
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, cfg: Settings) {
|
||||
const skip = skipReason(rec, ended)
|
||||
if (skip) await skipRound($, agentId, rec, skip)
|
||||
else await verifyRound($, agentId, rec, cfg)
|
||||
}
|
||||
|
||||
// ---------- the hooks ----------
|
||||
|
||||
export const register: Register = (on, options) => {
|
||||
const patterns = parsePatterns(String(options.safetyPatterns ?? ''))
|
||||
const respectExplicit = options.respectExplicitModel === true
|
||||
const cfg: Settings = {
|
||||
testCommand: String(options.testCommand ?? '').trim(),
|
||||
testTimeoutSec: validTimeout(options.testTimeoutSec),
|
||||
prBaseCommand: String(options.prBaseCommand ?? '').trim(),
|
||||
defaultBaseBranch: String(options.defaultBaseBranch ?? 'release/1.0').trim(),
|
||||
maxFails: Number(options.maxRewrites ?? 2),
|
||||
allowFable: options.allowFable === true,
|
||||
prices: parsePrices(String(options.prices ?? '')),
|
||||
}
|
||||
|
||||
on('session.start', async ($, e, next) => {
|
||||
await $.command.register({
|
||||
name: 'router',
|
||||
description: 'Model router: decisions, verdicts and savings vs Opus (`/router savings [Tn]`, `/router clear`)',
|
||||
})
|
||||
return next(e)
|
||||
})
|
||||
|
||||
on('prompt.compose', async ($, e, next) => {
|
||||
const out = await next(e)
|
||||
if (e.traits.includes('bare')) return out
|
||||
return { sections: [...out.sections, { id: 'model-router:note', text: ORCHESTRATOR_NOTE, scope: 'session' as const }] }
|
||||
})
|
||||
|
||||
// Routing: every agent the model starts, background ones included.
|
||||
on('agent.spawn', async ($, e, next) => {
|
||||
if (e.fork || e.isTeammate || e.workflow) return next(e)
|
||||
|
||||
let taskId: string
|
||||
let prompt = e.prompt
|
||||
let tier: Tier
|
||||
let reason: string
|
||||
let fails = 0
|
||||
let critical: boolean
|
||||
let opusRequested = false
|
||||
let classifierUsage: Usage | undefined
|
||||
// Agents whose model the router chose; kept models (review, test) are not a saving.
|
||||
let isRouted = true
|
||||
|
||||
const markId = e.prompt.match(REWRITE_MARK)?.[1]
|
||||
const pending = markId ? (await getMap<Pending>($, S.pending))[markId] : undefined
|
||||
|
||||
if (markId && pending) {
|
||||
taskId = markId
|
||||
const extra = e.prompt.replace(REWRITE_MARK, '').trim()
|
||||
prompt = pending.prompt + (extra ? `\n\nAdditional guidance from the orchestrator:\n${extra}` : '')
|
||||
tier = pending.tier
|
||||
fails = pending.fails
|
||||
critical = pending.critical
|
||||
reason = `new agent after ${fails} failed verdicts`
|
||||
await putIn<Pending>($, S.pending, markId, undefined)
|
||||
} else {
|
||||
taskId = await nextId($)
|
||||
const brief = parseBrief(e.prompt)
|
||||
critical = safetyHit(`${e.description}\n${e.prompt}`, patterns) !== undefined
|
||||
const d = decide({
|
||||
role: brief.role,
|
||||
named: asTier(e.model),
|
||||
subagentType: e.subagentType,
|
||||
prompt: e.prompt,
|
||||
respectExplicit,
|
||||
isAmbiguous: brief.isAmbiguous,
|
||||
})
|
||||
opusRequested = d.opusRequested === true
|
||||
isRouted =
|
||||
brief.role !== 'review' &&
|
||||
brief.role !== 'test' &&
|
||||
!(d.kind === 'fixed' && d.reason === 'model chosen by orchestrator')
|
||||
if (d.kind === 'fixed') {
|
||||
tier = d.tier
|
||||
reason = d.reason
|
||||
} else {
|
||||
const c = await classify($, e.prompt)
|
||||
tier = c.tier
|
||||
reason = c.reason
|
||||
classifierUsage = c.usage
|
||||
}
|
||||
}
|
||||
|
||||
const brief = parseBrief(prompt)
|
||||
const ran = await next({ ...e, prompt, model: tier })
|
||||
if (ran.deny !== undefined || !ran.agentId) return ran
|
||||
|
||||
// Learn what the alias resolves to, so router calls are priced on the real model.
|
||||
if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model)
|
||||
if (isRouted) {
|
||||
await putIn($, S.agentTask, ran.agentId, { taskId, task: e.description })
|
||||
await updateIn<TaskCost>($, S.costs, taskId, old => old ?? emptyTask(e.description))
|
||||
await chargeOverhead($, taskId, 'haiku', classifierUsage, cfg.prices)
|
||||
}
|
||||
|
||||
// Agents with a role (dev, review, test) get a record; only dev rounds are verified.
|
||||
if (brief.role)
|
||||
await putIn<AgentRec>($, S.agents, ran.agentId, {
|
||||
taskId,
|
||||
tier,
|
||||
role: brief.role,
|
||||
clone: brief.clone,
|
||||
baseBranch: brief.baseBranch,
|
||||
critical,
|
||||
prompt,
|
||||
fails,
|
||||
rounds: 0,
|
||||
})
|
||||
|
||||
await log($, {
|
||||
id: taskId,
|
||||
agentId: ran.agentId,
|
||||
at: await $.clock.now(),
|
||||
task: e.description,
|
||||
tier,
|
||||
role: brief.role,
|
||||
reason,
|
||||
outcome: 'spawned',
|
||||
opusRequested,
|
||||
})
|
||||
$.ui.status(`router: ${taskId} ${brief.role ?? 'agent'} → ${tier}`)
|
||||
return ran
|
||||
}).catch(($, e, next) => {
|
||||
// Fail open: a router bug must never block delegation.
|
||||
$.ui.status('router: error, agent ran unrouted')
|
||||
return next(e)
|
||||
})
|
||||
|
||||
// Verification: each finished run of a tracked agent, a resumed SendMessage round included.
|
||||
on('turn.complete', async ($, e, next) => {
|
||||
const out = await next(e)
|
||||
const agentId = e.agentId
|
||||
if (!agentId) return out
|
||||
|
||||
const owner = (await getMap<{ taskId: string; task: string }>($, S.agentTask))[agentId]
|
||||
if (owner && e.usage) {
|
||||
const usage = e.usage
|
||||
await updateIn<TaskCost>($, S.costs, owner.taskId, old =>
|
||||
addWriterRun(old ?? emptyTask(owner.task), usage.model, usage, cfg.prices),
|
||||
)
|
||||
}
|
||||
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
||||
if (!rec) return out
|
||||
// Verification takes minutes: run it after this event has gone on.
|
||||
$.clock.after(0, () => void runRound($, agentId, rec, e.reason, cfg))
|
||||
return out
|
||||
})
|
||||
|
||||
on('command.run', { command: 'router' }, async ($, e) => {
|
||||
if (e.args.trim() === 'clear') {
|
||||
await serial(async () => {
|
||||
await $.store.set(S.log, [])
|
||||
await $.store.set(S.pending, {})
|
||||
await $.store.set(S.agents, {})
|
||||
await $.store.set(S.costs, {})
|
||||
await $.store.set(S.agentTask, {})
|
||||
})
|
||||
return { text: 'model-router: log, agents, costs and pending rewrites cleared.' }
|
||||
}
|
||||
const [sub, arg] = e.args.trim().split(/\s+/)
|
||||
if (sub === 'savings') return { text: savingsReport(await getMap<TaskCost>($, S.costs), Infinity, arg) }
|
||||
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
||||
if (list.length === 0) return { text: 'model-router: nothing routed yet.' }
|
||||
|
||||
const count = (key: (l: LogEntry) => string | undefined) => {
|
||||
const r: Record<string, number> = {}
|
||||
for (const l of list) {
|
||||
const k = key(l)
|
||||
if (k) r[k] = (r[k] ?? 0) + 1
|
||||
}
|
||||
return Object.entries(r)
|
||||
.map(([k, n]) => `${k} ${n}`)
|
||||
.join(' · ')
|
||||
}
|
||||
const rows = list
|
||||
.slice(-15)
|
||||
.map(
|
||||
l =>
|
||||
`${l.id.padEnd(5)} ${(l.role ?? '-').padEnd(6)} ${l.tier.padEnd(6)} ${(l.verifier ? '→' + l.verifier : '').padEnd(8)} ${l.outcome.padEnd(17)} ${(l.task ?? '').slice(0, 40)}${l.opusRequested ? ' [opus requested]' : ''}`,
|
||||
)
|
||||
.join('\n')
|
||||
const pending = Object.keys(await getMap<Pending>($, S.pending))
|
||||
return {
|
||||
text:
|
||||
`model-router - last ${Math.min(15, list.length)} of ${list.length}\n` +
|
||||
`writers: ${count(l => (l.outcome === 'spawned' ? `${l.role ?? 'other'}:${l.tier}` : undefined))}\n` +
|
||||
`results: ${count(l => (l.outcome === 'spawned' ? undefined : l.outcome))}\n` +
|
||||
`opus requested on dev briefs: ${list.filter(l => l.opusRequested).length}\n` +
|
||||
(pending.length ? `pending rewrites: ${pending.join(', ')}\n` : '') +
|
||||
`\n${rows}\n\n` +
|
||||
savingsReport(await getMap<TaskCost>($, S.costs), 5),
|
||||
}
|
||||
})
|
||||
}
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
// Pure routing logic: no `$`, so every rule here is unit-tested.
|
||||
|
||||
export type Tier = 'haiku' | 'sonnet' | 'opus' | 'fable'
|
||||
export type Role = 'dev' | 'review' | 'test'
|
||||
|
||||
export const TIERS: readonly Tier[] = ['haiku', 'sonnet', 'opus', 'fable']
|
||||
|
||||
export const rank = (t: Tier) => TIERS.indexOf(t)
|
||||
|
||||
export const asTier = (s: string | undefined): Tier | undefined => {
|
||||
if (!s) return undefined
|
||||
const lower = s.toLowerCase()
|
||||
return TIERS.find(t => lower.includes(t))
|
||||
}
|
||||
|
||||
export const parsePatterns = (csv: string): string[] =>
|
||||
csv
|
||||
.split(',')
|
||||
.map(s => s.trim().toLowerCase())
|
||||
.filter(s => s.length > 0)
|
||||
|
||||
/** The first safety pattern the text mentions, or undefined. */
|
||||
export const safetyHit = (text: string, patterns: readonly string[]): string | undefined => {
|
||||
const lower = text.toLowerCase()
|
||||
return patterns.find(p => lower.includes(p))
|
||||
}
|
||||
|
||||
// ---------- the brief ----------
|
||||
|
||||
export type Brief = {
|
||||
clone?: string
|
||||
ticket?: string
|
||||
role?: Role
|
||||
baseBranch?: string
|
||||
/** Several clone paths and no `Clone:` line: the role cannot be told. */
|
||||
isAmbiguous?: boolean
|
||||
}
|
||||
|
||||
// A ticket is LETTERS-digits plus any extra parts (OPT-11-P4); the role is the last part.
|
||||
const TICKET = '[A-Za-z]+-\\d+(?:-[A-Za-z0-9]+)*?'
|
||||
const CLONE_PATH = new RegExp(`(\\S*?\\/clones\\/(${TICKET})-(dev|review|test))\\/?`)
|
||||
const CLONE_PATHS = new RegExp(CLONE_PATH.source, 'g')
|
||||
// The explicit line every brief carries: `Clone: <path>` with an optional `(Role: dev)`.
|
||||
const CLONE_LINE = /^[ \t>*-]*Clone:[ \t]*`?([^\s`]+?)\/?`?[ \t]*(?:\(Role:[ \t]*(dev|review|test)\))?[ \t]*$/im
|
||||
const BASE =
|
||||
/\b(?:base(?:\s+branch)?|target(?:\s+branch)?|merge\s+into|PR\s+(?:against|into))\s*[:=]?\s*`?((?:release|main|master|develop)[\w./-]*)/i
|
||||
|
||||
const trimPath = (p: string) => p.replace(/^[`'"(]+/, '').replace(/[`'")]+$/, '').replace(/\/+$/, '')
|
||||
|
||||
/**
|
||||
* Reads the clone path, ticket, role and base branch out of a brief.
|
||||
*
|
||||
* The `Clone:` line decides when present: its `(Role: ...)`, else the role in
|
||||
* its path. Without one, a brief that names one clone is read from that path;
|
||||
* one that names several is ambiguous and gets no role.
|
||||
*/
|
||||
export const parseBrief = (prompt: string): Brief => {
|
||||
const b = prompt.match(BASE)
|
||||
// a sentence's full stop or a trailing slash is no part of the branch
|
||||
const baseBranch = b?.[1]?.replace(/[./-]+$/, '')
|
||||
|
||||
const line = prompt.match(CLONE_LINE)
|
||||
if (line?.[1]) {
|
||||
const clone = trimPath(line[1])
|
||||
const fromPath = clone.match(CLONE_PATH)
|
||||
const role = (line[2] ?? fromPath?.[3]) as Role | undefined
|
||||
return { clone, ticket: fromPath?.[2], role, baseBranch }
|
||||
}
|
||||
|
||||
const paths = [...new Set([...prompt.matchAll(CLONE_PATHS)].map(m => trimPath(m[1] ?? '')))]
|
||||
if (paths.length > 1) return { baseBranch, isAmbiguous: true }
|
||||
const only = paths[0]?.match(CLONE_PATH)
|
||||
if (!only) return { baseBranch }
|
||||
return { clone: trimPath(only[1] ?? ''), ticket: only[2], role: only[3] as Role, baseBranch }
|
||||
}
|
||||
|
||||
// ---------- routing ----------
|
||||
|
||||
const READ_ONLY_TYPES = /^(explore|scout|runner|claude-code-guide|statusline-setup)$|:(scout|runner)$/i
|
||||
|
||||
// Light work, only for tasks outside the dev/review/test roles. No bare
|
||||
// "docs" or "run tests": both appear in nearly every brief.
|
||||
const LIGHT = [
|
||||
/\b(find|locate|grep|where is|which (files|modules)|summari[sz]e|read and report)\b/,
|
||||
/\b(fix (a |the )?typo|update (the )?(readme|changelog)|add docstrings?)\b/,
|
||||
/\b(run (the )?(unit )?tests? and report|report (the )?test results)\b/,
|
||||
]
|
||||
|
||||
// Opus writes only when the approach itself is still open. Domain words
|
||||
// (control loop, asyncio, RS485) are everywhere in this codebase and say nothing.
|
||||
const HEAVY = [
|
||||
/\b(design|architect|redesign|decide (how|whether)|propose an approach|trade-?offs?)\b/,
|
||||
/\b(root cause|unclear why|no idea why|investigate why|figure out why|intermittent|flaky)\b/,
|
||||
]
|
||||
|
||||
export type Route = { tier: Tier; reason: string }
|
||||
|
||||
/** A clear signal from the text, or undefined (ask the classifier). */
|
||||
export const heuristic = (subagentType: string | undefined, prompt: string, role?: Role): Route | undefined => {
|
||||
if (!role && subagentType && READ_ONLY_TYPES.test(subagentType))
|
||||
return { tier: 'haiku', reason: `read-only agent type ${subagentType}` }
|
||||
const lower = prompt.toLowerCase()
|
||||
const heavy = HEAVY.find(r => r.test(lower))
|
||||
if (heavy) return { tier: 'opus', reason: 'approach still open' }
|
||||
if (role) return undefined // dev briefs never take the light path
|
||||
const light = LIGHT.find(r => r.test(lower))
|
||||
if (light && prompt.length < 1500) return { tier: 'haiku', reason: 'light task' }
|
||||
return undefined
|
||||
}
|
||||
|
||||
export type Decision =
|
||||
| { kind: 'fixed'; tier: Tier; reason: string; opusRequested?: boolean }
|
||||
| { kind: 'classify'; opusRequested?: boolean }
|
||||
|
||||
/**
|
||||
* Who decides the writer's model.
|
||||
* - review and test agents: a named model is kept; unnamed, they run on Opus
|
||||
* (they are the gates, and a test agent's work has no verifier behind it).
|
||||
* - dev agents: always routed; a named Opus is only logged.
|
||||
* - anything else: routed, unless respectExplicit and a model is named.
|
||||
*/
|
||||
export const decide = (args: {
|
||||
role?: Role
|
||||
named?: Tier
|
||||
subagentType?: string
|
||||
prompt: string
|
||||
respectExplicit: boolean
|
||||
isAmbiguous?: boolean
|
||||
}): Decision => {
|
||||
const { role, named, subagentType, prompt, respectExplicit, isAmbiguous } = args
|
||||
// Could be a test or review agent: never risk a weaker model on it.
|
||||
if (isAmbiguous)
|
||||
return { kind: 'fixed', tier: named && named !== 'haiku' && named !== 'sonnet' ? named : 'opus', reason: 'several clone paths and no Clone: line' }
|
||||
if (role === 'review' || role === 'test')
|
||||
return named
|
||||
? { kind: 'fixed', tier: named, reason: `named model kept for ${role} agent` }
|
||||
: { kind: 'fixed', tier: 'opus', reason: `${role} agent default` }
|
||||
if (!role && respectExplicit && named) return { kind: 'fixed', tier: named, reason: 'model chosen by orchestrator' }
|
||||
const opusRequested = role === 'dev' && (named === 'opus' || named === 'fable')
|
||||
const h = heuristic(subagentType, prompt, role)
|
||||
return h ? { kind: 'fixed', ...h, opusRequested } : { kind: 'classify', opusRequested }
|
||||
}
|
||||
|
||||
// ---------- verification ----------
|
||||
|
||||
export type Skip = 'test-agent' | 'review-agent' | 'no-clone' | 'opus-unverified' | 'agent-ended'
|
||||
|
||||
/** Why an agent run is not verified, or undefined when it is. */
|
||||
export const skipReason = (rec: { role?: Role; clone?: string; tier: Tier }, ended: string): Skip | undefined => {
|
||||
if (ended !== 'answer') return 'agent-ended'
|
||||
if (rec.role === 'test') return 'test-agent'
|
||||
if (rec.role === 'review') return 'review-agent'
|
||||
if (!rec.clone) return 'no-clone'
|
||||
if (rank(rec.tier) >= rank('opus')) return 'opus-unverified'
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** The verifier never runs on the writer's model; below Opus it is Opus. */
|
||||
export const verifierFor = (writer: Tier): Tier => (rank(writer) >= rank('opus') ? 'sonnet' : 'opus')
|
||||
|
||||
/** The model a rewrite runs on: one tier up, never the same model. */
|
||||
export const rewriteTier = (writer: Tier, allowFable: boolean): Tier | undefined => {
|
||||
if (writer === 'haiku') return 'sonnet'
|
||||
if (writer === 'sonnet') return 'opus'
|
||||
if (writer === 'opus' && allowFable) return 'fable'
|
||||
return undefined
|
||||
}
|
||||
|
||||
export type FailStep = { kind: 'resume' } | { kind: 'new-agent'; tier: Tier } | { kind: 'hand-back' }
|
||||
|
||||
/**
|
||||
* After the n-th failed verdict in a task's chain: the first resumes the same
|
||||
* agent, the second starts a new agent one model up in the same clone, and
|
||||
* past maxFails (or with no model above) it goes back to the manager.
|
||||
*/
|
||||
export const onFail = (fails: number, writer: Tier, maxFails: number, allowFable: boolean): FailStep => {
|
||||
if (fails > maxFails) return { kind: 'hand-back' }
|
||||
if (fails === 1) return { kind: 'resume' }
|
||||
const up = rewriteTier(writer, allowFable)
|
||||
return up ? { kind: 'new-agent', tier: up } : { kind: 'hand-back' }
|
||||
}
|
||||
|
||||
export type Verdict = { pass: boolean; issues: string[]; brief: string }
|
||||
|
||||
/** Reads the verifier's reply; anything unreadable counts as a failure. */
|
||||
export const parseVerdict = (text: string): Verdict => {
|
||||
const match = text.match(/\{[\s\S]*\}/)
|
||||
if (match) {
|
||||
try {
|
||||
const v = JSON.parse(match[0]) as { verdict?: string; issues?: unknown; rewrite_brief?: unknown }
|
||||
const issues = Array.isArray(v.issues) ? v.issues.map(String) : []
|
||||
const pass = String(v.verdict ?? '').toUpperCase() === 'PASS'
|
||||
return { pass, issues, brief: typeof v.rewrite_brief === 'string' ? v.rewrite_brief : '' }
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
return { pass: false, issues: ['Verifier reply was not valid JSON: ' + text.slice(0, 300)], brief: '' }
|
||||
}
|
||||
|
||||
export const REWRITE_MARK = /^\s*\[router-rewrite (T\d+)\]/
|
||||
|
||||
export const clip = (s: string, n: number) =>
|
||||
s.length <= n ? s : s.slice(0, n) + `\n... [${s.length - n} more characters cut]`
|
||||
|
||||
/** A usable test time limit in seconds: anything not a positive number is the default. */
|
||||
export function validTimeout(raw: unknown): number {
|
||||
const n = Number(raw)
|
||||
return Number.isFinite(n) && n >= 1 ? Math.round(n) : 1800
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user