Files
model-router/hooks/register.ts
T
glenn schrooyenandClaude Opus 5.5 86bb9d79cc 0.5.0: pipeline support for ems-dev-tooling
- test-designer and release-reviewer keep a named model (Opus when none, never Haiku)
- verifier checks every T-id and N-id of a "## Test plan" section
- manager guidance: record each verdict with `ticket` before resume or rewrite

Already running on the homelab install (applied from ems-dev-tooling patches/model-router-0.5.0-pipeline.patch).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bdqr7PiaHcBnzPituu2yTC
2026-11-17 21:28:36 +01:00

943 lines
37 KiB
TypeScript

import type { EngineInterface, Register } from 'claude-code'
import {
addOverhead,
addWriterRun,
emptyTask,
parsePrices,
savingsReport,
type PriceRow,
type TaskCost,
type Usage,
} from './pricing'
import {
REWRITE_MARK,
asTier,
clip,
decide,
onFail,
parseBrief,
parsePatterns,
parseVerdict,
handBackLine,
ranTier,
turnEndKind,
WATCH_EVERY_MS,
WATCH_FOR_MS,
safetyHit,
skipReason,
validTimeout,
fileCommand,
isSafeFile,
issueRegex,
parseIssueLine,
positiveInt,
rerunTable,
type FileKind,
type FileRun,
type RerunRow,
verifierFor,
type Role,
type Tier,
} from './router'
// ---------- stored records ----------
/** One agent the router started, kept from spawn to its last verified round. */
type AgentRec = {
taskId: string
/** The model the last round actually ran on (the routing choice until a round reports one). */
tier: Tier
/** The model the engine resolved at spawn; tier can differ when the run used another. */
routed?: Tier
/** The name the Agent call gave it, which SendMessage may address it by. */
name?: string
/** Later instructions to this agent (resumes, SendMessage), oldest first: part of its task. */
followUps?: string[]
role?: Role
clone?: string
baseBranch?: string
critical: boolean
prompt: string
/** Failed verdicts in this task's chain, carried over to a rewrite agent. */
fails: number
rounds: number
/** The tree after the last verified round: the next round diffs from here. */
lastSnap?: string
lastUntracked?: string[]
lastIssues?: string[]
}
type Pending = { prompt: string; tier: Tier; fails: number; critical: boolean }
type Outcome =
| 'pass'
| 'resume-requested'
| 'rewrite-requested'
| 'handed-back'
| 'no-changes'
| 'no-base'
| 'tests-timeout'
| 'tests-error'
| 'verify-error'
| 'verdict-unreadable'
| 'test-agent'
| 'review-agent'
| 'no-clone'
| 'opus-unverified'
| 'agent-ended'
type LogEntry = {
id: string
agentId?: string
at: number
task: string
tier: Tier
role?: Role
reason: string
round?: number
verifier?: Tier
outcome: Outcome | 'spawned'
opusRequested?: boolean
}
const S = {
counter: 'counter',
pending: 'pending',
agents: 'agents',
log: 'log',
costs: 'costs',
agentTask: 'agentTask',
aliases: 'aliases',
} as const
/** Model ids assumed for an alias until an agent run shows what it resolves to. */
const ALIAS_DEFAULT: Record<Tier, string> = {
haiku: 'claude-haiku-4-5',
sonnet: 'claude-sonnet-5-5',
opus: 'claude-opus-5-5',
fable: 'claude-fable-5-1',
}
const LOG_LIMIT = 400
// ---------- prompts ----------
const CLASSIFIER_SYSTEM = `You route coding tasks to the smallest model that will do them right.
Judge how well-defined the task is, NOT how important or low-level the code is. In this codebase nearly everything touches control loops, asyncio and RS485; that alone never means opus. A stronger model verifies every change below Opus, and a review agent reviews every PR.
Answer with exactly one word: haiku, sonnet or opus.
haiku: read-only lookups and summaries, mechanical changes fully spelled out.
sonnet: the approach is decided: the brief says what to change, or gives Expected behaviour, acceptance criteria or a test to make pass. Size and domain do not matter.
opus: the approach is still open: a design or trade-off to decide, a bug whose cause is unknown, or a brief so vague the writer must invent the plan.
When unsure between sonnet and opus, pick sonnet: a failed verification escalates.`
const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not.
Judge only whether the change correctly and completely does the task, without breaking anything nearby.
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
Scope added in "Later instructions" was asked for after the brief: judge it as in scope, not as unrequested work.
When a table of files rerun one by one is given: a file that passes alone does not count against the change (the full run was overloaded or flaky); one that also fails or times out on base is not caused by this change: mention it, do not fail for it. Name every file you count against the change in issues.
When the task has a "## Test plan" section with T-ids and N-ids: every T-id must be implemented by a test that can fail, and every N-id asserted somewhere, unless the developer says why not. A missing or untestable T-id/N-id is a FAIL issue naming the id. A test that cannot fail on the old code where the plan says "red on base" is a FAIL issue.
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
Reply with JSON only:
{"verdict": "PASS" | "FAIL", "issues": ["concrete issue with file:line where possible", ...], "rewrite_brief": "what must change, empty on PASS"}`
const ORCHESTRATOR_NOTE = `## Model router (model-router mod)
The model-router mod picks each subagent's model and verifies dev work before it reaches a PR.
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
- Review and test agents keep the model you name (Opus when you name none).
- Every dev hand-back gets one "[model-router] Tn" line at once; an interim stop while the dev waits on its own background work gets none. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on.
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person.
- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself.
- With ems-dev-tooling: record every verdict with \`ticket\` first (PASS: \`ticket advance <T> --to review --sha <sha>\`; FAIL: \`ticket fail <T> --stage router\`), then resume or rewrite. pipeline-guard refuses a resume while the ticket is not back in development.`
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
// Agents start and finish in parallel. Every read-modify-write of the store
// goes through this queue, one at a time, or parallel hooks overwrite each
// other's writes (and hand out the same task id).
let queue: Promise<unknown> = Promise.resolve()
function serial<T>(work: () => Promise<T>): Promise<T> {
const run = queue.then(work)
queue = run.catch(() => undefined)
return run
}
async function getMap<T>($: EngineInterface, key: string) {
return ((await $.store.get(key)) ?? {}) as Record<string, T>
}
/** Replaces one entry of a stored map, atomically with respect to this plugin. */
async function updateIn<T>($: EngineInterface, key: string, id: string, change: (old: T | undefined) => T | undefined) {
return serial(async () => {
const all = await getMap<T>($, key)
const value = change(all[id])
if (value === undefined) delete all[id]
else all[id] = value
await $.store.set(key, all)
return value
})
}
async function putIn<T>($: EngineInterface, key: string, id: string, value: T | undefined) {
await updateIn<T>($, key, id, () => value)
}
async function nextId($: EngineInterface) {
return serial(async () => {
const n = Number((await $.store.get(S.counter)) ?? 0) + 1
await $.store.set(S.counter, n)
return `T${n}`
})
}
async function log($: EngineInterface, entry: LogEntry) {
await serial(async () => {
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
list.push(entry)
await $.store.set(S.log, list.slice(-LOG_LIMIT))
})
}
async function sh($: EngineInterface, argv: string[], cwd: string, timeoutMs = 30_000) {
try {
const r = await $.process.run(argv, { cwd, timeoutMs })
return { ok: r.exitCode === 0, out: r.stdout, err: r.stderr, code: r.exitCode }
} catch (err) {
return { ok: false, out: '', err: String(err), code: -1 }
}
}
async function untracked($: EngineInterface, cwd: string) {
const r = await sh($, ['git', 'ls-files', '--others', '--exclude-standard'], cwd)
return r.ok ? r.out.split('\n').filter(Boolean) : []
}
/** A commit holding the clone's tree as it is now (HEAD when clean). */
async function snapshot($: EngineInterface, cwd: string) {
const stash = await sh($, ['git', 'stash', 'create'], cwd)
if (stash.ok && stash.out.trim()) return stash.out.trim()
const head = await sh($, ['git', 'rev-parse', 'HEAD'], cwd)
return head.ok ? head.out.trim() : undefined
}
/** Where a first round diffs from: the merge base with the PR's base branch. */
async function firstBase($: EngineInterface, rec: AgentRec, prBaseCommand: string, fallback: string) {
const cwd = rec.clone as string
let branch: string | undefined
if (prBaseCommand) {
const r = await sh($, ['sh', '-c', prBaseCommand], cwd)
const b = r.out.trim().split('\n')[0]?.trim()
if (r.ok && b) branch = b
}
branch = branch ?? rec.baseBranch ?? fallback
for (const ref of [`origin/${branch}`, branch]) {
const r = await sh($, ['git', 'merge-base', ref, 'HEAD'], cwd)
if (r.ok && r.out.trim()) return { base: r.out.trim(), branch }
}
return { base: undefined, branch }
}
async function classify($: EngineInterface, prompt: string): Promise<{ tier: Tier; reason: string; usage?: Usage }> {
const r = await $.model.complete({
model: 'haiku',
system: CLASSIFIER_SYSTEM,
prompt: `Task:\n${clip(prompt, 6000)}`,
maxTokens: 5,
timeoutMs: 15_000,
})
const tier = r.isAnswered ? asTier(r.text) : undefined
if (tier && tier !== 'fable') return { tier, reason: 'classifier', usage: r.usage }
return { tier: 'sonnet', reason: 'classifier unavailable, default', usage: r.usage }
}
async function tell($: EngineInterface, text: string) {
await $.prompt.submit({ text })
}
// ---------- cost tracking ----------
async function modelIdFor($: EngineInterface, alias: Tier) {
const seen = (await getMap<string>($, S.aliases))[alias]
return seen ?? ALIAS_DEFAULT[alias]
}
/** Adds a router call (classifier, verifier) to a task's overhead. */
async function chargeOverhead($: EngineInterface, taskId: string, alias: Tier, u: Usage | undefined, prices: readonly PriceRow[]) {
if (!u) return
const modelId = await modelIdFor($, alias)
await updateIn<TaskCost>($, S.costs, taskId, old => addOverhead(old ?? emptyTask(taskId), modelId, u, prices))
}
type TestRun = {
status: 'ran' | 'timeout' | 'error'
code: number | null
tail: string
seconds: number
/** Files the run reported as TIMEOUT or FAIL, in order, each once. */
issues: { kind: FileKind; file: string }[]
}
/**
* Runs the test command in the clone, streamed: `$.process.run` stops at ten
* minutes, a stream has no limit of its own. GNU `timeout` does the kill.
* A timeout or a run that could not start is no verdict on the code.
*/
async function runTests($: EngineInterface, cmd: string, cwd: string, timeoutSec: number, issueRe?: RegExp): Promise<TestRun> {
const started = await $.clock.now()
const seconds = async () => Math.round(((await $.clock.now()) - started) / 1000)
let out = ''
let errTail = ''
// reported files are read line by line as the run streams, not from the kept tail
const issues: TestRun['issues'] = []
const partial: Record<string, string> = { stdout: '', stderr: '' }
const readLines = (stream: string, text: string, isLast = false) => {
if (!issueRe) return
const lines = ((partial[stream] ?? '') + text).split('\n')
partial[stream] = isLast ? '' : lines.pop() ?? ''
for (const line of lines) {
const hit = parseIssueLine(line, issueRe)
if (hit && !issues.some(i => i.file === hit.file)) issues.push(hit)
}
}
try {
const stream = $.process.spawn({
argv: ['timeout', '--verbose', '--kill-after=30', String(Math.max(1, Math.round(timeoutSec))), 'sh', '-c', cmd],
cwd,
})
for await (const piece of stream) {
out = (out + piece.text).slice(-65_536)
if (piece.stream === 'stderr') errTail = (errTail + piece.text).slice(-65_536)
readLines(piece.stream, piece.text)
}
readLines('stdout', '', true)
readLines('stderr', '', true)
// GNU timeout --verbose says so on stderr when, and only when, it fires.
// Read from the joined stderr, so a line split across pieces still counts.
const isFired = errTail.includes('timeout: sending signal')
const end = await stream.result
const tail = clip(out.trim().split('\n').slice(-80).join('\n'), 8000)
const took = await seconds()
// Only a timeout that fired is a timeout: a 137 without it is the code's own
// (e.g. out of memory) and goes to the verifier as a failed run.
if (isFired) return { status: 'timeout', code: end.code, tail, seconds: took, issues }
if (end.code === 126 || end.code === 127) return { status: 'error', code: end.code, tail, seconds: took, issues }
return { status: 'ran', code: end.code, tail, seconds: took, issues }
} catch (err) {
return { status: 'error', code: null, tail: String(err).slice(0, 500), seconds: await seconds(), issues }
}
}
/** The result of one file run alone. */
function fileRunOf(t: TestRun): FileRun {
if (t.status === 'timeout') return 'timeout'
if (t.status === 'error') return 'error'
return t.code === 0 ? 'pass' : 'fail'
}
/** Runs `work` over the items, at most `jobs` at a time, keeping their order. */
async function inPool<T, R>(items: readonly T[], jobs: number, work: (item: T) => Promise<R>): Promise<R[]> {
const out: R[] = new Array(items.length)
let next = 0
const lane = async () => {
while (next < items.length) {
const i = next++
out[i] = await work(items[i] as T)
}
}
await Promise.all(Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, lane))
return out
}
/**
* Reruns the files a full run reported, one by one in the clone; those that
* still fail or time out alone run again on the base, checked out next to the
* clone (`<clone>-verifier-base`, removed afterwards).
*/
async function rerunReported($: EngineInterface, rec: AgentRec, issues: TestRun['issues'], cfg: Settings): Promise<RerunRow[]> {
const cwd = rec.clone as string
const picked = issues.filter(i => isSafeFile(i.file)).slice(0, cfg.testRerunMax)
const rows: RerunRow[] = picked.map(i => ({ file: i.file, full: i.kind, head: 'not run', base: 'not run' }))
const runAll = (dir: string, which: RerunRow[]) =>
inPool(which, cfg.testRerunJobs, async r =>
fileRunOf(await runTests($, fileCommand(cfg.testFileCommand, r.file), dir, cfg.testFileTimeoutSec)),
)
$.ui.status(`router: ${rec.taskId} rerunning ${rows.length} reported file(s) alone`)
const head = await runAll(cwd, rows)
rows.forEach((r, i) => (r.head = head[i] ?? 'error'))
const onBase = rows.filter(r => r.head === 'fail' || r.head === 'timeout')
if (onBase.length === 0) return rows
const { base } = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch)
if (!base) return rows
const tree = `${cwd.replace(/\/+$/, '')}-verifier-base`
await sh($, ['git', '-C', cwd, 'worktree', 'remove', '--force', tree], cwd)
await sh($, ['git', '-C', cwd, 'worktree', 'prune'], cwd)
const added = await sh($, ['git', '-C', cwd, 'worktree', 'add', '--detach', tree, base], cwd, 120_000)
if (!added.ok) return rows
try {
$.ui.status(`router: ${rec.taskId} rerunning ${onBase.length} file(s) on base`)
const results = await runAll(tree, onBase)
onBase.forEach((r, i) => (r.base = results[i] ?? 'error'))
} finally {
await sh($, ['git', '-C', cwd, 'worktree', 'remove', '--force', tree], cwd, 120_000)
await sh($, ['git', '-C', cwd, 'worktree', 'prune'], cwd)
}
return rows
}
type Settings = {
testCommand: string
testTimeoutSec: number
testFileCommand: string
testFileTimeoutSec: number
testRerunJobs: number
testRerunMax: number
testIssueRe: RegExp
prBaseCommand: string
defaultBaseBranch: string
maxFails: number
allowFable: boolean
prices: readonly PriceRow[]
}
/** Verifies one finished round of a dev agent and tells the manager. */
async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, cfg: Settings) {
const cwd = rec.clone as string
const round = rec.rounds + 1
const entry: LogEntry = {
id: rec.taskId,
agentId,
at: await $.clock.now(),
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
tier: rec.tier,
role: rec.role,
reason: `round ${round}`,
round,
outcome: 'verify-error',
}
const head = `[model-router] ${rec.taskId} (agent ${agentId}, ${rec.tier}, round ${round})`
// 1. What this round changed.
let base = rec.lastSnap
let baseLabel = 'the previous verified round'
if (!base) {
const fb = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch)
base = fb.base
baseLabel = `merge-base with ${fb.branch}`
if (!base) {
entry.outcome = 'no-base'
await log($, entry)
await tell($, `${head}: NOT VERIFIED - no merge base with ${fb.branch} in ${cwd}. Review the change yourself before the PR.`)
return
}
}
const diff = await sh($, ['git', 'diff', base], cwd, 60_000)
const known = new Set(rec.lastUntracked ?? [])
const fresh = (await untracked($, cwd)).filter(f => !known.has(f))
let added = ''
for (const f of fresh.slice(0, 20)) {
try {
added += `\n--- new file: ${f}\n${clip(await $.fs.read(`${cwd}/${f}`), 8000)}\n`
} catch {
added += `\n--- new file: ${f} (unreadable)\n`
}
}
const changes = (diff.out + added).trim()
if (!changes) {
entry.outcome = 'no-changes'
await log($, entry)
await tell($, `${head}: no changes since ${baseLabel}; nothing to verify.`)
return
}
// 2. Tests, in the clone.
let tests = ''
if (cfg.testCommand) {
$.ui.status(`router: ${rec.taskId} round ${round} running tests`)
const t = await runTests($, cfg.testCommand, cwd, cfg.testTimeoutSec, cfg.testIssueRe)
if (t.status !== 'ran') {
// No verdict on the code: no verifier call, no fail counted, no rewrite.
entry.outcome = t.status === 'timeout' ? 'tests-timeout' : 'tests-error'
await log($, entry)
const why =
t.status === 'timeout'
? `the tests did not finish within ${cfg.testTimeoutSec} s (testTimeoutSec)`
: `the test command could not run (exit ${t.code ?? 'none'})`
await tell(
$,
`${head}: NOT VERIFIED - ${why}. This says nothing about the code: do not resume or rewrite the agent for it. ` +
`Fix the test setup or raise testTimeoutSec, or review the change yourself.\nLast output:\n${clip(t.tail, 2000)}`,
)
return
}
tests = `exit code ${t.code} after ${t.seconds} s\n${t.tail}`
if (cfg.testFileCommand && t.issues.length > 0) {
const rows = await rerunReported($, rec, t.issues, cfg)
const skipped = t.issues.length - rows.length
tests +=
`\n\n### Files the full run reported, rerun one by one\n${rerunTable(rows)}` +
(skipped > 0 ? `\n(${skipped} more reported file(s) not rerun: unsafe name or over testRerunMax.)` : '')
}
}
// 3. The verdict, from another model.
const verifier = verifierFor(rec.tier)
entry.verifier = verifier
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
const question = [
`## Task\n${clip(rec.prompt, 12_000)}`,
rec.followUps?.length
? `## Later instructions to this agent (after the brief; part of the task, in scope)\n${rec.followUps
.map((f, i) => `${i + 1}. ${f}`)
.join('\n\n')}`
: '',
rec.lastIssues?.length
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
: '',
`## Changes (${baseLabel})\n${clip(changes, 60_000)}`,
cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '',
]
.filter(Boolean)
.join('\n\n')
const v = await $.model.complete({ model: verifier, system: VERIFIER_SYSTEM, prompt: question, maxTokens: 2000, timeoutMs: 180_000 })
await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices)
if (!v.isAnswered) {
await log($, entry)
await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`)
return
}
let reading = parseVerdict(v.text)
// the first reply is where the verifier judged the code: quote that one
const firstPreview = v.text.slice(0, 300)
if (!reading.isReadable) {
// Ask once more, for the JSON alone. A reply that still cannot be read is
// no verdict on the code: hand it back, never count it as a FAIL.
$.ui.status(`router: ${rec.taskId} round ${round} verdict unreadable, asking ${verifier} again`)
const again = await $.model.complete({
model: verifier,
system: VERIFIER_SYSTEM,
prompt:
`${question}\n\n## Your previous reply could not be read\n${clip(v.text, 4000)}\n\n` +
'Reply again with the JSON object only: no code fence, no text before or after it.',
maxTokens: 2000,
timeoutMs: 180_000,
})
await chargeOverhead($, rec.taskId, verifier, again.usage, cfg.prices)
if (again.isAnswered) reading = parseVerdict(again.text)
}
if (!reading.isReadable) {
entry.outcome = 'verdict-unreadable'
await log($, entry)
$.ui.status(`router: ${rec.taskId} verdict unreadable, handed back`)
await tell(
$,
`${head}: NOT VERIFIED - verdict unreadable. The verifier (${verifier}) did not give a readable PASS or FAIL, also when asked again. ` +
`This says nothing about the code: do not resume or rewrite the agent for it. Review the change yourself.\n` +
`Start of its reply: ${firstPreview}`,
)
return
}
const verdict = reading
const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)'
// 4. Remember where this round ended.
const after: AgentRec = {
...rec,
rounds: round,
lastSnap: (await snapshot($, cwd)) ?? rec.lastSnap,
lastUntracked: await untracked($, cwd),
lastIssues: verdict.pass ? [] : verdict.issues,
}
if (verdict.pass) {
entry.outcome = 'pass'
await putIn($, S.agents, agentId, after)
await log($, entry)
$.ui.status(`router: ${rec.taskId} ✓ ${rec.tier}, verified by ${verifier}`)
await tell($, `${head}: PASS from verifier ${verifier}. Continue to the PR.`)
return
}
const fails = rec.fails + 1
after.fails = fails
const step = onFail(fails, rec.tier, cfg.maxFails, cfg.allowFable)
await putIn($, S.agents, agentId, after)
if (step.kind === 'resume') {
entry.outcome = 'resume-requested'
await log($, entry)
$.ui.status(`router: ${rec.taskId} ✗ round ${round}, resume ${agentId}`)
await tell(
$,
`${head}: FAIL from verifier ${verifier}.\nIssues:\n${issues}\n` +
(verdict.brief ? `What must change: ${verdict.brief}\n` : '') +
`Resume agent ${agentId} with SendMessage, passing these issues. Do not fix it yourself.`,
)
return
}
if (step.kind === 'new-agent') {
await putIn<Pending>($, S.pending, rec.taskId, {
prompt:
`${rec.prompt.split('\n\n## Previous attempts rejected')[0]}\n\n## Previous attempts rejected\n` +
`Agents on ${rec.tier} were rejected ${fails} times by an independent verifier (${verifier}). Their work is in the clone ${cwd}: continue from it, fix or replace it, do not start over elsewhere.\n` +
`Issues:\n${issues}\n` +
(verdict.brief ? `What must change: ${verdict.brief}\n` : ''),
tier: step.tier,
fails,
critical: rec.critical,
})
entry.outcome = 'rewrite-requested'
await log($, entry)
$.ui.status(`router: ${rec.taskId} ✗ again, new agent on ${step.tier}`)
await tell(
$,
`${head}: FAIL again from verifier ${verifier}.\nIssues:\n${issues}\n` +
`Stop agent ${agentId}, then start a new agent on ${step.tier} in the same clone: call Agent with prompt [router-rewrite ${rec.taskId}] (same subagent_type).`,
)
return
}
entry.outcome = 'handed-back'
await log($, entry)
$.ui.toast(`model-router: ${rec.taskId} failed verification ${fails}x - handed back`)
await tell(
$,
`${head}: FAIL from verifier ${verifier}, ${fails} failed verdicts. Handed back: stop and show the person these issues.\n${issues}` +
(verdict.brief ? `\nVerifier's brief: ${verdict.brief}` : ''),
)
}
/** Logs an agent run that is not verified; tells the manager when a dev agent waits on a verdict. */
async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, outcome: Outcome) {
await log($, {
id: rec.taskId,
agentId,
at: await $.clock.now(),
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
tier: rec.tier,
role: rec.role,
reason: outcome,
outcome,
})
}
/**
* One finished run of a tracked agent. Decides on the model that actually ran,
* says at once what happens next for a dev, then verifies or logs the skip.
*/
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, reported: string | undefined, cfg: Settings) {
const routed = rec.routed ?? rec.tier
const ran = ranTier(reported, routed)
const round: AgentRec = { ...rec, routed, tier: ran.tier }
const skip = skipReason(round, ended)
const line = handBackLine({ taskId: rec.taskId, role: rec.role, ran: ran.tier, routed, skip, ended })
if (line) await tell($, line)
if (skip) await skipRound($, agentId, round, skip)
else await verifyRound($, agentId, round, cfg)
}
// ---------- later instructions (#6) ----------
const FOLLOW_UPS_KEPT = 12
/** Adds a later instruction to a tracked agent's record; the same text twice counts once. */
async function noteFollowUp($: EngineInterface, agentId: string, text: string) {
const t = text.trim()
if (!t) return
await updateIn<AgentRec>($, S.agents, agentId, old => {
if (!old) return undefined
const list = old.followUps ?? []
if (list.includes(clip(t, 4000))) return old
return { ...old, followUps: [...list, clip(t, 4000)].slice(-FOLLOW_UPS_KEPT) }
})
}
/** The tracked agent a SendMessage `to` names: its id, or the name its Agent call gave it. */
async function agentAddressed($: EngineInterface, to: string) {
const recs = await getMap<AgentRec>($, S.agents)
if (recs[to]) return to
return Object.keys(recs).find(id => recs[id]?.name === to)
}
// ---------- waiting for the real hand-back ----------
// The newest turn end per agent. A turn end that is only an interim stop (the
// agent waits on its own background work) must not post a line or start the
// verifier: only the run's end does. Module state: a reload drops the watches.
const lastTurn = new Map<string, { token: number; ended: string; reported?: string }>()
let tokens = 0
function pause($: EngineInterface, ms: number) {
return new Promise<void>(resolve => {
$.clock.after(ms, () => resolve())
})
}
async function statusOf($: EngineInterface, agentId: string) {
try {
return (await $.agent.list()).find(a => a.id === agentId)?.status
} catch {
return undefined
}
}
/**
* After a tracked agent's turn ends: runs the round once its run has ended,
* and not for an interim stop. A newer turn end of the same agent takes over.
*/
async function awaitHandBack($: EngineInterface, agentId: string, token: number, cfg: Settings) {
const started = await $.clock.now()
for (;;) {
if (lastTurn.get(agentId)?.token !== token) return // a newer turn end took over
if (turnEndKind(await statusOf($, agentId)) === 'handback') break
if ((await $.clock.now()) - started > WATCH_FOR_MS) return
await pause($, WATCH_EVERY_MS)
}
const turn = lastTurn.get(agentId)
if (!turn || turn.token !== token) return
lastTurn.delete(agentId)
// read the record now: an earlier round may have changed it meanwhile
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
if (rec) await runRound($, agentId, rec, turn.ended, turn.reported, cfg)
}
// ---------- the hooks ----------
export const register: Register = (on, options) => {
const patterns = parsePatterns(String(options.safetyPatterns ?? ''))
const respectExplicit = options.respectExplicitModel === true
const cfg: Settings = {
testCommand: String(options.testCommand ?? '').trim(),
testTimeoutSec: validTimeout(options.testTimeoutSec),
testFileCommand: String(options.testFileCommand ?? '').trim(),
testFileTimeoutSec: positiveInt(options.testFileTimeoutSec, 900),
testRerunJobs: positiveInt(options.testRerunJobs, 6),
testRerunMax: positiveInt(options.testRerunMax, 40),
testIssueRe: issueRegex(String(options.testIssuePattern ?? '')),
prBaseCommand: String(options.prBaseCommand ?? '').trim(),
defaultBaseBranch: String(options.defaultBaseBranch ?? 'release/1.0').trim(),
maxFails: Number(options.maxRewrites ?? 2),
allowFable: options.allowFable === true,
prices: parsePrices(String(options.prices ?? '')),
}
on('session.start', async ($, e, next) => {
await $.command.register({
name: 'router',
description: 'Model router: decisions, verdicts and savings vs Opus (`/router savings [Tn]`, `/router clear`)',
})
return next(e)
})
on('prompt.compose', async ($, e, next) => {
const out = await next(e)
if (e.traits.includes('bare')) return out
return { sections: [...out.sections, { id: 'model-router:note', text: ORCHESTRATOR_NOTE, scope: 'session' as const }] }
})
// Routing: every agent the model starts, background ones included.
on('agent.spawn', async ($, e, next) => {
if (e.fork || e.isTeammate || e.workflow) return next(e)
// A resume continues a known agent: same task, same routing, same counts.
// Its message is a later instruction the verifier must see (#4, #6).
if (e.resume) {
const resumed = await next(e)
if (resumed.deny === undefined) await noteFollowUp($, e.resume.agentId, e.prompt)
return resumed
}
let taskId: string
let prompt = e.prompt
let tier: Tier
let reason: string
let fails = 0
let critical: boolean
let opusRequested = false
let classifierUsage: Usage | undefined
// Agents whose model the router chose; kept models (review, test) are not a saving.
let isRouted = true
const markId = e.prompt.match(REWRITE_MARK)?.[1]
const pending = markId ? (await getMap<Pending>($, S.pending))[markId] : undefined
if (markId && pending) {
taskId = markId
const extra = e.prompt.replace(REWRITE_MARK, '').trim()
prompt = pending.prompt + (extra ? `\n\nAdditional guidance from the orchestrator:\n${extra}` : '')
tier = pending.tier
fails = pending.fails
critical = pending.critical
reason = `rewrite of ${markId} after ${fails} failed verdicts`
} else if (markId) {
// Starting it would hand an agent nothing but the marker.
return {
deny:
`model-router: no stored rewrite for ${markId} (already started, or cleared). ` +
'Start the agent with the full brief instead, or check /router.',
}
} else {
taskId = await nextId($)
const brief = parseBrief(e.prompt)
critical = safetyHit(`${e.description}\n${e.prompt}`, patterns) !== undefined
const d = decide({
role: brief.role,
named: asTier(e.model),
subagentType: e.subagentType,
prompt: e.prompt,
respectExplicit,
isAmbiguous: brief.isAmbiguous,
})
opusRequested = d.opusRequested === true
isRouted =
brief.role !== 'review' &&
brief.role !== 'test' &&
!(d.kind === 'fixed' && d.reason === 'model chosen by orchestrator')
if (d.kind === 'fixed') {
tier = d.tier
reason = d.reason
} else {
const c = await classify($, e.prompt)
tier = c.tier
reason = c.reason
classifierUsage = c.usage
}
}
const brief = parseBrief(prompt)
const ran = await next({ ...e, prompt, model: tier })
if (ran.deny !== undefined || !ran.agentId) return ran
// the stored rewrite is used up only once its agent has started
if (markId && pending) await putIn<Pending>($, S.pending, markId, undefined)
// label against what the engine resolved, not what the router asked for
const resolved = asTier(ran.model) ?? tier
if (resolved !== tier) reason += ` (asked ${tier}, engine resolved ${resolved})`
// Learn what the alias resolves to, so router calls are priced on the real model.
if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model)
if (isRouted) {
await putIn($, S.agentTask, ran.agentId, { taskId, task: e.description })
await updateIn<TaskCost>($, S.costs, taskId, old => old ?? emptyTask(e.description))
await chargeOverhead($, taskId, 'haiku', classifierUsage, cfg.prices)
}
// Agents with a role (dev, review, test) get a record; only dev rounds are verified.
if (brief.role)
await putIn<AgentRec>($, S.agents, ran.agentId, {
taskId,
tier: resolved,
routed: resolved,
name: e.name,
role: brief.role,
clone: brief.clone,
baseBranch: brief.baseBranch,
critical,
prompt,
fails,
rounds: 0,
})
await log($, {
id: taskId,
agentId: ran.agentId,
at: await $.clock.now(),
task: e.description,
tier: resolved,
role: brief.role,
reason,
outcome: 'spawned',
opusRequested,
})
$.ui.status(`router: ${taskId} ${brief.role ?? 'agent'} → ${tier}`)
return ran
}).catch(($, e, next) => {
// Fail open: a router bug must never block delegation.
$.ui.status('router: error, agent ran unrouted')
return next(e)
})
// Verification: each finished run of a tracked agent, a resumed SendMessage round included.
on('turn.complete', async ($, e, next) => {
const out = await next(e)
const agentId = e.agentId
if (!agentId) return out
const owner = (await getMap<{ taskId: string; task: string }>($, S.agentTask))[agentId]
if (owner && e.usage) {
const usage = e.usage
await updateIn<TaskCost>($, S.costs, owner.taskId, old =>
addWriterRun(old ?? emptyTask(owner.task), usage.model, usage, cfg.prices),
)
}
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
if (!rec) return out
// Only the run's end is a hand-back; wait for it after this event has gone on.
const token = ++tokens
lastTurn.set(agentId, { token, ended: e.reason, reported: e.usage?.model })
$.clock.after(0, () => void awaitHandBack($, agentId, token, cfg))
return out
})
// A message to a running agent is a later instruction too (#6).
on('tool.call', { tool: 'SendMessage' }, async ($, e, next) => {
const out = await next(e)
if (out.deny !== undefined || out.isError) return out
const to = typeof e.to === 'string' ? e.to : ''
const id = to ? await agentAddressed($, to) : undefined
if (id && typeof e.message === 'string') await noteFollowUp($, id, e.message)
return out
})
on('command.run', { command: 'router' }, async ($, e) => {
if (e.args.trim() === 'clear') {
await serial(async () => {
await $.store.set(S.log, [])
await $.store.set(S.pending, {})
await $.store.set(S.agents, {})
await $.store.set(S.costs, {})
await $.store.set(S.agentTask, {})
})
return { text: 'model-router: log, agents, costs and pending rewrites cleared.' }
}
const [sub, arg] = e.args.trim().split(/\s+/)
if (sub === 'savings') return { text: savingsReport(await getMap<TaskCost>($, S.costs), Infinity, arg) }
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
if (list.length === 0) return { text: 'model-router: nothing routed yet.' }
const count = (key: (l: LogEntry) => string | undefined) => {
const r: Record<string, number> = {}
for (const l of list) {
const k = key(l)
if (k) r[k] = (r[k] ?? 0) + 1
}
return Object.entries(r)
.map(([k, n]) => `${k} ${n}`)
.join(' · ')
}
const rows = list
.slice(-15)
.map(
l =>
`${l.id.padEnd(5)} ${(l.role ?? '-').padEnd(6)} ${l.tier.padEnd(6)} ${(l.verifier ? '→' + l.verifier : '').padEnd(8)} ${l.outcome.padEnd(17)} ${(l.task ?? '').slice(0, 40)}${l.opusRequested ? ' [opus requested]' : ''}`,
)
.join('\n')
const pending = Object.keys(await getMap<Pending>($, S.pending))
return {
text:
`model-router - last ${Math.min(15, list.length)} of ${list.length}\n` +
`writers: ${count(l => (l.outcome === 'spawned' ? `${l.role ?? 'other'}:${l.tier}` : undefined))}\n` +
`results: ${count(l => (l.outcome === 'spawned' ? undefined : l.outcome))}\n` +
`opus requested on dev briefs: ${list.filter(l => l.opusRequested).length}\n` +
(pending.length ? `pending rewrites: ${pending.join(', ')}\n` : '') +
`\n${rows}\n\n` +
savingsReport(await getMap<TaskCost>($, S.costs), 5),
}
})
}