Resuming an agent (SendMessage to a finished agent) raises agent.spawn again, with resume.agentId set. The router treated every resume as a new task: new Tn, fresh routing, the brief replaced by the resume message, the failure count and round base reset. Hence T322's "routed to Sonnet, ran on Opus" for an Opus agent that was only resumed (#4), and verifiers that judged later rounds against the wrong text (#6). - A resume passes through untouched; its message is kept as a later instruction to that agent. Task, routing, failures and rounds stay. - A SendMessage to a tracked agent (by id or by the name its Agent call gave it) that was delivered is kept as a later instruction too. - The verifier prompt lists later instructions after the brief and judges scope added there as in scope. - [router-rewrite Tn]: the stored rewrite is used up only once its agent has started (a failed start kept nothing before: T332 was then routed afresh, to Haiku). With nothing stored the call is refused instead of starting an agent with no brief. The task keeps Tn. - The routing label compares with the model the engine resolved at start, not the one the router asked for. - 11 new tests; each new rule was also broken on purpose and caught. Version 0.4.8. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
833 lines
32 KiB
TypeScript
833 lines
32 KiB
TypeScript
import type { EngineInterface, Register } from 'claude-code'
|
|
|
|
import {
|
|
addOverhead,
|
|
addWriterRun,
|
|
emptyTask,
|
|
parsePrices,
|
|
savingsReport,
|
|
type PriceRow,
|
|
type TaskCost,
|
|
type Usage,
|
|
} from './pricing'
|
|
import {
|
|
REWRITE_MARK,
|
|
asTier,
|
|
clip,
|
|
decide,
|
|
onFail,
|
|
parseBrief,
|
|
parsePatterns,
|
|
parseVerdict,
|
|
handBackLine,
|
|
ranTier,
|
|
turnEndKind,
|
|
WATCH_EVERY_MS,
|
|
WATCH_FOR_MS,
|
|
safetyHit,
|
|
skipReason,
|
|
validTimeout,
|
|
verifierFor,
|
|
type Role,
|
|
type Tier,
|
|
} from './router'
|
|
|
|
// ---------- stored records ----------
|
|
|
|
/** One agent the router started, kept from spawn to its last verified round. */
|
|
type AgentRec = {
|
|
taskId: string
|
|
/** The model the last round actually ran on (the routing choice until a round reports one). */
|
|
tier: Tier
|
|
/** The model the engine resolved at spawn; tier can differ when the run used another. */
|
|
routed?: Tier
|
|
/** The name the Agent call gave it, which SendMessage may address it by. */
|
|
name?: string
|
|
/** Later instructions to this agent (resumes, SendMessage), oldest first: part of its task. */
|
|
followUps?: string[]
|
|
role?: Role
|
|
clone?: string
|
|
baseBranch?: string
|
|
critical: boolean
|
|
prompt: string
|
|
/** Failed verdicts in this task's chain, carried over to a rewrite agent. */
|
|
fails: number
|
|
rounds: number
|
|
/** The tree after the last verified round: the next round diffs from here. */
|
|
lastSnap?: string
|
|
lastUntracked?: string[]
|
|
lastIssues?: string[]
|
|
}
|
|
|
|
type Pending = { prompt: string; tier: Tier; fails: number; critical: boolean }
|
|
|
|
type Outcome =
|
|
| 'pass'
|
|
| 'resume-requested'
|
|
| 'rewrite-requested'
|
|
| 'handed-back'
|
|
| 'no-changes'
|
|
| 'no-base'
|
|
| 'tests-timeout'
|
|
| 'tests-error'
|
|
| 'verify-error'
|
|
| 'verdict-unreadable'
|
|
| 'test-agent'
|
|
| 'review-agent'
|
|
| 'no-clone'
|
|
| 'opus-unverified'
|
|
| 'agent-ended'
|
|
|
|
type LogEntry = {
|
|
id: string
|
|
agentId?: string
|
|
at: number
|
|
task: string
|
|
tier: Tier
|
|
role?: Role
|
|
reason: string
|
|
round?: number
|
|
verifier?: Tier
|
|
outcome: Outcome | 'spawned'
|
|
opusRequested?: boolean
|
|
}
|
|
|
|
const S = {
|
|
counter: 'counter',
|
|
pending: 'pending',
|
|
agents: 'agents',
|
|
log: 'log',
|
|
costs: 'costs',
|
|
agentTask: 'agentTask',
|
|
aliases: 'aliases',
|
|
} as const
|
|
|
|
/** Model ids assumed for an alias until an agent run shows what it resolves to. */
|
|
const ALIAS_DEFAULT: Record<Tier, string> = {
|
|
haiku: 'claude-haiku-4-5',
|
|
sonnet: 'claude-sonnet-5-5',
|
|
opus: 'claude-opus-5-5',
|
|
fable: 'claude-fable-5-1',
|
|
}
|
|
const LOG_LIMIT = 400
|
|
|
|
// ---------- prompts ----------
|
|
|
|
const CLASSIFIER_SYSTEM = `You route coding tasks to the smallest model that will do them right.
|
|
Judge how well-defined the task is, NOT how important or low-level the code is. In this codebase nearly everything touches control loops, asyncio and RS485; that alone never means opus. A stronger model verifies every change below Opus, and a review agent reviews every PR.
|
|
Answer with exactly one word: haiku, sonnet or opus.
|
|
|
|
haiku: read-only lookups and summaries, mechanical changes fully spelled out.
|
|
sonnet: the approach is decided: the brief says what to change, or gives Expected behaviour, acceptance criteria or a test to make pass. Size and domain do not matter.
|
|
opus: the approach is still open: a design or trade-off to decide, a bug whose cause is unknown, or a brief so vague the writer must invent the plan.
|
|
|
|
When unsure between sonnet and opus, pick sonnet: a failed verification escalates.`
|
|
|
|
const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not.
|
|
Judge only whether the change correctly and completely does the task, without breaking anything nearby.
|
|
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
|
|
Scope added in "Later instructions" was asked for after the brief: judge it as in scope, not as unrequested work.
|
|
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
|
|
|
|
Reply with JSON only:
|
|
{"verdict": "PASS" | "FAIL", "issues": ["concrete issue with file:line where possible", ...], "rewrite_brief": "what must change, empty on PASS"}`
|
|
|
|
const ORCHESTRATOR_NOTE = `## Model router (model-router mod)
|
|
The model-router mod picks each subagent's model and verifies dev work before it reaches a PR.
|
|
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
|
|
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
|
|
- Review and test agents keep the model you name (Opus when you name none).
|
|
- Every dev hand-back gets one "[model-router] Tn" line at once; an interim stop while the dev waits on its own background work gets none. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on.
|
|
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person.
|
|
- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself.`
|
|
|
|
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
|
|
|
// Agents start and finish in parallel. Every read-modify-write of the store
|
|
// goes through this queue, one at a time, or parallel hooks overwrite each
|
|
// other's writes (and hand out the same task id).
|
|
let queue: Promise<unknown> = Promise.resolve()
|
|
|
|
function serial<T>(work: () => Promise<T>): Promise<T> {
|
|
const run = queue.then(work)
|
|
queue = run.catch(() => undefined)
|
|
return run
|
|
}
|
|
|
|
async function getMap<T>($: EngineInterface, key: string) {
|
|
return ((await $.store.get(key)) ?? {}) as Record<string, T>
|
|
}
|
|
|
|
/** Replaces one entry of a stored map, atomically with respect to this plugin. */
|
|
async function updateIn<T>($: EngineInterface, key: string, id: string, change: (old: T | undefined) => T | undefined) {
|
|
return serial(async () => {
|
|
const all = await getMap<T>($, key)
|
|
const value = change(all[id])
|
|
if (value === undefined) delete all[id]
|
|
else all[id] = value
|
|
await $.store.set(key, all)
|
|
return value
|
|
})
|
|
}
|
|
|
|
async function putIn<T>($: EngineInterface, key: string, id: string, value: T | undefined) {
|
|
await updateIn<T>($, key, id, () => value)
|
|
}
|
|
|
|
async function nextId($: EngineInterface) {
|
|
return serial(async () => {
|
|
const n = Number((await $.store.get(S.counter)) ?? 0) + 1
|
|
await $.store.set(S.counter, n)
|
|
return `T${n}`
|
|
})
|
|
}
|
|
|
|
async function log($: EngineInterface, entry: LogEntry) {
|
|
await serial(async () => {
|
|
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
|
list.push(entry)
|
|
await $.store.set(S.log, list.slice(-LOG_LIMIT))
|
|
})
|
|
}
|
|
|
|
async function sh($: EngineInterface, argv: string[], cwd: string, timeoutMs = 30_000) {
|
|
try {
|
|
const r = await $.process.run(argv, { cwd, timeoutMs })
|
|
return { ok: r.exitCode === 0, out: r.stdout, err: r.stderr, code: r.exitCode }
|
|
} catch (err) {
|
|
return { ok: false, out: '', err: String(err), code: -1 }
|
|
}
|
|
}
|
|
|
|
async function untracked($: EngineInterface, cwd: string) {
|
|
const r = await sh($, ['git', 'ls-files', '--others', '--exclude-standard'], cwd)
|
|
return r.ok ? r.out.split('\n').filter(Boolean) : []
|
|
}
|
|
|
|
/** A commit holding the clone's tree as it is now (HEAD when clean). */
|
|
async function snapshot($: EngineInterface, cwd: string) {
|
|
const stash = await sh($, ['git', 'stash', 'create'], cwd)
|
|
if (stash.ok && stash.out.trim()) return stash.out.trim()
|
|
const head = await sh($, ['git', 'rev-parse', 'HEAD'], cwd)
|
|
return head.ok ? head.out.trim() : undefined
|
|
}
|
|
|
|
/** Where a first round diffs from: the merge base with the PR's base branch. */
|
|
async function firstBase($: EngineInterface, rec: AgentRec, prBaseCommand: string, fallback: string) {
|
|
const cwd = rec.clone as string
|
|
let branch: string | undefined
|
|
if (prBaseCommand) {
|
|
const r = await sh($, ['sh', '-c', prBaseCommand], cwd)
|
|
const b = r.out.trim().split('\n')[0]?.trim()
|
|
if (r.ok && b) branch = b
|
|
}
|
|
branch = branch ?? rec.baseBranch ?? fallback
|
|
for (const ref of [`origin/${branch}`, branch]) {
|
|
const r = await sh($, ['git', 'merge-base', ref, 'HEAD'], cwd)
|
|
if (r.ok && r.out.trim()) return { base: r.out.trim(), branch }
|
|
}
|
|
return { base: undefined, branch }
|
|
}
|
|
|
|
async function classify($: EngineInterface, prompt: string): Promise<{ tier: Tier; reason: string; usage?: Usage }> {
|
|
const r = await $.model.complete({
|
|
model: 'haiku',
|
|
system: CLASSIFIER_SYSTEM,
|
|
prompt: `Task:\n${clip(prompt, 6000)}`,
|
|
maxTokens: 5,
|
|
timeoutMs: 15_000,
|
|
})
|
|
const tier = r.isAnswered ? asTier(r.text) : undefined
|
|
if (tier && tier !== 'fable') return { tier, reason: 'classifier', usage: r.usage }
|
|
return { tier: 'sonnet', reason: 'classifier unavailable, default', usage: r.usage }
|
|
}
|
|
|
|
async function tell($: EngineInterface, text: string) {
|
|
await $.prompt.submit({ text })
|
|
}
|
|
|
|
// ---------- cost tracking ----------
|
|
|
|
async function modelIdFor($: EngineInterface, alias: Tier) {
|
|
const seen = (await getMap<string>($, S.aliases))[alias]
|
|
return seen ?? ALIAS_DEFAULT[alias]
|
|
}
|
|
|
|
/** Adds a router call (classifier, verifier) to a task's overhead. */
|
|
async function chargeOverhead($: EngineInterface, taskId: string, alias: Tier, u: Usage | undefined, prices: readonly PriceRow[]) {
|
|
if (!u) return
|
|
const modelId = await modelIdFor($, alias)
|
|
await updateIn<TaskCost>($, S.costs, taskId, old => addOverhead(old ?? emptyTask(taskId), modelId, u, prices))
|
|
}
|
|
|
|
type TestRun = { status: 'ran' | 'timeout' | 'error'; code: number | null; tail: string; seconds: number }
|
|
|
|
/**
|
|
* Runs the test command in the clone, streamed: `$.process.run` stops at ten
|
|
* minutes, a stream has no limit of its own. GNU `timeout` does the kill.
|
|
* A timeout or a run that could not start is no verdict on the code.
|
|
*/
|
|
async function runTests($: EngineInterface, cmd: string, cwd: string, timeoutSec: number): Promise<TestRun> {
|
|
const started = await $.clock.now()
|
|
const seconds = async () => Math.round(((await $.clock.now()) - started) / 1000)
|
|
let out = ''
|
|
let errTail = ''
|
|
try {
|
|
const stream = $.process.spawn({
|
|
argv: ['timeout', '--verbose', '--kill-after=30', String(Math.max(1, Math.round(timeoutSec))), 'sh', '-c', cmd],
|
|
cwd,
|
|
})
|
|
for await (const piece of stream) {
|
|
out = (out + piece.text).slice(-65_536)
|
|
if (piece.stream === 'stderr') errTail = (errTail + piece.text).slice(-65_536)
|
|
}
|
|
// GNU timeout --verbose says so on stderr when, and only when, it fires.
|
|
// Read from the joined stderr, so a line split across pieces still counts.
|
|
const isFired = errTail.includes('timeout: sending signal')
|
|
const end = await stream.result
|
|
const tail = clip(out.trim().split('\n').slice(-80).join('\n'), 8000)
|
|
const took = await seconds()
|
|
// Only a timeout that fired is a timeout: a 137 without it is the code's own
|
|
// (e.g. out of memory) and goes to the verifier as a failed run.
|
|
if (isFired) return { status: 'timeout', code: end.code, tail, seconds: took }
|
|
if (end.code === 126 || end.code === 127) return { status: 'error', code: end.code, tail, seconds: took }
|
|
return { status: 'ran', code: end.code, tail, seconds: took }
|
|
} catch (err) {
|
|
return { status: 'error', code: null, tail: String(err).slice(0, 500), seconds: await seconds() }
|
|
}
|
|
}
|
|
|
|
type Settings = {
|
|
testCommand: string
|
|
testTimeoutSec: number
|
|
prBaseCommand: string
|
|
defaultBaseBranch: string
|
|
maxFails: number
|
|
allowFable: boolean
|
|
prices: readonly PriceRow[]
|
|
}
|
|
|
|
/** Verifies one finished round of a dev agent and tells the manager. */
|
|
async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, cfg: Settings) {
|
|
const cwd = rec.clone as string
|
|
const round = rec.rounds + 1
|
|
const entry: LogEntry = {
|
|
id: rec.taskId,
|
|
agentId,
|
|
at: await $.clock.now(),
|
|
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
|
tier: rec.tier,
|
|
role: rec.role,
|
|
reason: `round ${round}`,
|
|
round,
|
|
outcome: 'verify-error',
|
|
}
|
|
const head = `[model-router] ${rec.taskId} (agent ${agentId}, ${rec.tier}, round ${round})`
|
|
|
|
// 1. What this round changed.
|
|
let base = rec.lastSnap
|
|
let baseLabel = 'the previous verified round'
|
|
if (!base) {
|
|
const fb = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch)
|
|
base = fb.base
|
|
baseLabel = `merge-base with ${fb.branch}`
|
|
if (!base) {
|
|
entry.outcome = 'no-base'
|
|
await log($, entry)
|
|
await tell($, `${head}: NOT VERIFIED - no merge base with ${fb.branch} in ${cwd}. Review the change yourself before the PR.`)
|
|
return
|
|
}
|
|
}
|
|
const diff = await sh($, ['git', 'diff', base], cwd, 60_000)
|
|
const known = new Set(rec.lastUntracked ?? [])
|
|
const fresh = (await untracked($, cwd)).filter(f => !known.has(f))
|
|
let added = ''
|
|
for (const f of fresh.slice(0, 20)) {
|
|
try {
|
|
added += `\n--- new file: ${f}\n${clip(await $.fs.read(`${cwd}/${f}`), 8000)}\n`
|
|
} catch {
|
|
added += `\n--- new file: ${f} (unreadable)\n`
|
|
}
|
|
}
|
|
const changes = (diff.out + added).trim()
|
|
if (!changes) {
|
|
entry.outcome = 'no-changes'
|
|
await log($, entry)
|
|
await tell($, `${head}: no changes since ${baseLabel}; nothing to verify.`)
|
|
return
|
|
}
|
|
|
|
// 2. Tests, in the clone.
|
|
let tests = ''
|
|
if (cfg.testCommand) {
|
|
$.ui.status(`router: ${rec.taskId} round ${round} running tests`)
|
|
const t = await runTests($, cfg.testCommand, cwd, cfg.testTimeoutSec)
|
|
if (t.status !== 'ran') {
|
|
// No verdict on the code: no verifier call, no fail counted, no rewrite.
|
|
entry.outcome = t.status === 'timeout' ? 'tests-timeout' : 'tests-error'
|
|
await log($, entry)
|
|
const why =
|
|
t.status === 'timeout'
|
|
? `the tests did not finish within ${cfg.testTimeoutSec} s (testTimeoutSec)`
|
|
: `the test command could not run (exit ${t.code ?? 'none'})`
|
|
await tell(
|
|
$,
|
|
`${head}: NOT VERIFIED - ${why}. This says nothing about the code: do not resume or rewrite the agent for it. ` +
|
|
`Fix the test setup or raise testTimeoutSec, or review the change yourself.\nLast output:\n${clip(t.tail, 2000)}`,
|
|
)
|
|
return
|
|
}
|
|
tests = `exit code ${t.code} after ${t.seconds} s\n${t.tail}`
|
|
}
|
|
|
|
// 3. The verdict, from another model.
|
|
const verifier = verifierFor(rec.tier)
|
|
entry.verifier = verifier
|
|
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
|
|
const question = [
|
|
`## Task\n${clip(rec.prompt, 12_000)}`,
|
|
rec.followUps?.length
|
|
? `## Later instructions to this agent (after the brief; part of the task, in scope)\n${rec.followUps
|
|
.map((f, i) => `${i + 1}. ${f}`)
|
|
.join('\n\n')}`
|
|
: '',
|
|
rec.lastIssues?.length
|
|
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
|
|
: '',
|
|
`## Changes (${baseLabel})\n${clip(changes, 60_000)}`,
|
|
cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '',
|
|
]
|
|
.filter(Boolean)
|
|
.join('\n\n')
|
|
const v = await $.model.complete({ model: verifier, system: VERIFIER_SYSTEM, prompt: question, maxTokens: 2000, timeoutMs: 180_000 })
|
|
await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices)
|
|
if (!v.isAnswered) {
|
|
await log($, entry)
|
|
await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`)
|
|
return
|
|
}
|
|
let reading = parseVerdict(v.text)
|
|
// the first reply is where the verifier judged the code: quote that one
|
|
const firstPreview = v.text.slice(0, 300)
|
|
if (!reading.isReadable) {
|
|
// Ask once more, for the JSON alone. A reply that still cannot be read is
|
|
// no verdict on the code: hand it back, never count it as a FAIL.
|
|
$.ui.status(`router: ${rec.taskId} round ${round} verdict unreadable, asking ${verifier} again`)
|
|
const again = await $.model.complete({
|
|
model: verifier,
|
|
system: VERIFIER_SYSTEM,
|
|
prompt:
|
|
`${question}\n\n## Your previous reply could not be read\n${clip(v.text, 4000)}\n\n` +
|
|
'Reply again with the JSON object only: no code fence, no text before or after it.',
|
|
maxTokens: 2000,
|
|
timeoutMs: 180_000,
|
|
})
|
|
await chargeOverhead($, rec.taskId, verifier, again.usage, cfg.prices)
|
|
if (again.isAnswered) reading = parseVerdict(again.text)
|
|
}
|
|
if (!reading.isReadable) {
|
|
entry.outcome = 'verdict-unreadable'
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} verdict unreadable, handed back`)
|
|
await tell(
|
|
$,
|
|
`${head}: NOT VERIFIED - verdict unreadable. The verifier (${verifier}) did not give a readable PASS or FAIL, also when asked again. ` +
|
|
`This says nothing about the code: do not resume or rewrite the agent for it. Review the change yourself.\n` +
|
|
`Start of its reply: ${firstPreview}`,
|
|
)
|
|
return
|
|
}
|
|
const verdict = reading
|
|
const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)'
|
|
|
|
// 4. Remember where this round ended.
|
|
const after: AgentRec = {
|
|
...rec,
|
|
rounds: round,
|
|
lastSnap: (await snapshot($, cwd)) ?? rec.lastSnap,
|
|
lastUntracked: await untracked($, cwd),
|
|
lastIssues: verdict.pass ? [] : verdict.issues,
|
|
}
|
|
|
|
if (verdict.pass) {
|
|
entry.outcome = 'pass'
|
|
await putIn($, S.agents, agentId, after)
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} ✓ ${rec.tier}, verified by ${verifier}`)
|
|
await tell($, `${head}: PASS from verifier ${verifier}. Continue to the PR.`)
|
|
return
|
|
}
|
|
|
|
const fails = rec.fails + 1
|
|
after.fails = fails
|
|
const step = onFail(fails, rec.tier, cfg.maxFails, cfg.allowFable)
|
|
await putIn($, S.agents, agentId, after)
|
|
|
|
if (step.kind === 'resume') {
|
|
entry.outcome = 'resume-requested'
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} ✗ round ${round}, resume ${agentId}`)
|
|
await tell(
|
|
$,
|
|
`${head}: FAIL from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
|
(verdict.brief ? `What must change: ${verdict.brief}\n` : '') +
|
|
`Resume agent ${agentId} with SendMessage, passing these issues. Do not fix it yourself.`,
|
|
)
|
|
return
|
|
}
|
|
|
|
if (step.kind === 'new-agent') {
|
|
await putIn<Pending>($, S.pending, rec.taskId, {
|
|
prompt:
|
|
`${rec.prompt.split('\n\n## Previous attempts rejected')[0]}\n\n## Previous attempts rejected\n` +
|
|
`Agents on ${rec.tier} were rejected ${fails} times by an independent verifier (${verifier}). Their work is in the clone ${cwd}: continue from it, fix or replace it, do not start over elsewhere.\n` +
|
|
`Issues:\n${issues}\n` +
|
|
(verdict.brief ? `What must change: ${verdict.brief}\n` : ''),
|
|
tier: step.tier,
|
|
fails,
|
|
critical: rec.critical,
|
|
})
|
|
entry.outcome = 'rewrite-requested'
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} ✗ again, new agent on ${step.tier}`)
|
|
await tell(
|
|
$,
|
|
`${head}: FAIL again from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
|
`Stop agent ${agentId}, then start a new agent on ${step.tier} in the same clone: call Agent with prompt [router-rewrite ${rec.taskId}] (same subagent_type).`,
|
|
)
|
|
return
|
|
}
|
|
|
|
entry.outcome = 'handed-back'
|
|
await log($, entry)
|
|
$.ui.toast(`model-router: ${rec.taskId} failed verification ${fails}x - handed back`)
|
|
await tell(
|
|
$,
|
|
`${head}: FAIL from verifier ${verifier}, ${fails} failed verdicts. Handed back: stop and show the person these issues.\n${issues}` +
|
|
(verdict.brief ? `\nVerifier's brief: ${verdict.brief}` : ''),
|
|
)
|
|
}
|
|
|
|
/** Logs an agent run that is not verified; tells the manager when a dev agent waits on a verdict. */
|
|
async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, outcome: Outcome) {
|
|
await log($, {
|
|
id: rec.taskId,
|
|
agentId,
|
|
at: await $.clock.now(),
|
|
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
|
tier: rec.tier,
|
|
role: rec.role,
|
|
reason: outcome,
|
|
outcome,
|
|
})
|
|
}
|
|
|
|
/**
|
|
* One finished run of a tracked agent. Decides on the model that actually ran,
|
|
* says at once what happens next for a dev, then verifies or logs the skip.
|
|
*/
|
|
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, reported: string | undefined, cfg: Settings) {
|
|
const routed = rec.routed ?? rec.tier
|
|
const ran = ranTier(reported, routed)
|
|
const round: AgentRec = { ...rec, routed, tier: ran.tier }
|
|
const skip = skipReason(round, ended)
|
|
const line = handBackLine({ taskId: rec.taskId, role: rec.role, ran: ran.tier, routed, skip, ended })
|
|
if (line) await tell($, line)
|
|
if (skip) await skipRound($, agentId, round, skip)
|
|
else await verifyRound($, agentId, round, cfg)
|
|
}
|
|
|
|
// ---------- later instructions (#6) ----------
|
|
|
|
const FOLLOW_UPS_KEPT = 12
|
|
|
|
/** Adds a later instruction to a tracked agent's record; the same text twice counts once. */
|
|
async function noteFollowUp($: EngineInterface, agentId: string, text: string) {
|
|
const t = text.trim()
|
|
if (!t) return
|
|
await updateIn<AgentRec>($, S.agents, agentId, old => {
|
|
if (!old) return undefined
|
|
const list = old.followUps ?? []
|
|
if (list.includes(clip(t, 4000))) return old
|
|
return { ...old, followUps: [...list, clip(t, 4000)].slice(-FOLLOW_UPS_KEPT) }
|
|
})
|
|
}
|
|
|
|
/** The tracked agent a SendMessage `to` names: its id, or the name its Agent call gave it. */
|
|
async function agentAddressed($: EngineInterface, to: string) {
|
|
const recs = await getMap<AgentRec>($, S.agents)
|
|
if (recs[to]) return to
|
|
return Object.keys(recs).find(id => recs[id]?.name === to)
|
|
}
|
|
|
|
// ---------- waiting for the real hand-back ----------
|
|
|
|
// The newest turn end per agent. A turn end that is only an interim stop (the
|
|
// agent waits on its own background work) must not post a line or start the
|
|
// verifier: only the run's end does. Module state: a reload drops the watches.
|
|
const lastTurn = new Map<string, { token: number; ended: string; reported?: string }>()
|
|
let tokens = 0
|
|
|
|
function pause($: EngineInterface, ms: number) {
|
|
return new Promise<void>(resolve => {
|
|
$.clock.after(ms, () => resolve())
|
|
})
|
|
}
|
|
|
|
async function statusOf($: EngineInterface, agentId: string) {
|
|
try {
|
|
return (await $.agent.list()).find(a => a.id === agentId)?.status
|
|
} catch {
|
|
return undefined
|
|
}
|
|
}
|
|
|
|
/**
|
|
* After a tracked agent's turn ends: runs the round once its run has ended,
|
|
* and not for an interim stop. A newer turn end of the same agent takes over.
|
|
*/
|
|
async function awaitHandBack($: EngineInterface, agentId: string, token: number, cfg: Settings) {
|
|
const started = await $.clock.now()
|
|
for (;;) {
|
|
if (lastTurn.get(agentId)?.token !== token) return // a newer turn end took over
|
|
if (turnEndKind(await statusOf($, agentId)) === 'handback') break
|
|
if ((await $.clock.now()) - started > WATCH_FOR_MS) return
|
|
await pause($, WATCH_EVERY_MS)
|
|
}
|
|
const turn = lastTurn.get(agentId)
|
|
if (!turn || turn.token !== token) return
|
|
lastTurn.delete(agentId)
|
|
// read the record now: an earlier round may have changed it meanwhile
|
|
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
|
if (rec) await runRound($, agentId, rec, turn.ended, turn.reported, cfg)
|
|
}
|
|
|
|
// ---------- the hooks ----------
|
|
|
|
export const register: Register = (on, options) => {
|
|
const patterns = parsePatterns(String(options.safetyPatterns ?? ''))
|
|
const respectExplicit = options.respectExplicitModel === true
|
|
const cfg: Settings = {
|
|
testCommand: String(options.testCommand ?? '').trim(),
|
|
testTimeoutSec: validTimeout(options.testTimeoutSec),
|
|
prBaseCommand: String(options.prBaseCommand ?? '').trim(),
|
|
defaultBaseBranch: String(options.defaultBaseBranch ?? 'release/1.0').trim(),
|
|
maxFails: Number(options.maxRewrites ?? 2),
|
|
allowFable: options.allowFable === true,
|
|
prices: parsePrices(String(options.prices ?? '')),
|
|
}
|
|
|
|
on('session.start', async ($, e, next) => {
|
|
await $.command.register({
|
|
name: 'router',
|
|
description: 'Model router: decisions, verdicts and savings vs Opus (`/router savings [Tn]`, `/router clear`)',
|
|
})
|
|
return next(e)
|
|
})
|
|
|
|
on('prompt.compose', async ($, e, next) => {
|
|
const out = await next(e)
|
|
if (e.traits.includes('bare')) return out
|
|
return { sections: [...out.sections, { id: 'model-router:note', text: ORCHESTRATOR_NOTE, scope: 'session' as const }] }
|
|
})
|
|
|
|
// Routing: every agent the model starts, background ones included.
|
|
on('agent.spawn', async ($, e, next) => {
|
|
if (e.fork || e.isTeammate || e.workflow) return next(e)
|
|
|
|
// A resume continues a known agent: same task, same routing, same counts.
|
|
// Its message is a later instruction the verifier must see (#4, #6).
|
|
if (e.resume) {
|
|
const resumed = await next(e)
|
|
if (resumed.deny === undefined) await noteFollowUp($, e.resume.agentId, e.prompt)
|
|
return resumed
|
|
}
|
|
|
|
let taskId: string
|
|
let prompt = e.prompt
|
|
let tier: Tier
|
|
let reason: string
|
|
let fails = 0
|
|
let critical: boolean
|
|
let opusRequested = false
|
|
let classifierUsage: Usage | undefined
|
|
// Agents whose model the router chose; kept models (review, test) are not a saving.
|
|
let isRouted = true
|
|
|
|
const markId = e.prompt.match(REWRITE_MARK)?.[1]
|
|
const pending = markId ? (await getMap<Pending>($, S.pending))[markId] : undefined
|
|
|
|
if (markId && pending) {
|
|
taskId = markId
|
|
const extra = e.prompt.replace(REWRITE_MARK, '').trim()
|
|
prompt = pending.prompt + (extra ? `\n\nAdditional guidance from the orchestrator:\n${extra}` : '')
|
|
tier = pending.tier
|
|
fails = pending.fails
|
|
critical = pending.critical
|
|
reason = `rewrite of ${markId} after ${fails} failed verdicts`
|
|
} else if (markId) {
|
|
// Starting it would hand an agent nothing but the marker.
|
|
return {
|
|
deny:
|
|
`model-router: no stored rewrite for ${markId} (already started, or cleared). ` +
|
|
'Start the agent with the full brief instead, or check /router.',
|
|
}
|
|
} else {
|
|
taskId = await nextId($)
|
|
const brief = parseBrief(e.prompt)
|
|
critical = safetyHit(`${e.description}\n${e.prompt}`, patterns) !== undefined
|
|
const d = decide({
|
|
role: brief.role,
|
|
named: asTier(e.model),
|
|
subagentType: e.subagentType,
|
|
prompt: e.prompt,
|
|
respectExplicit,
|
|
isAmbiguous: brief.isAmbiguous,
|
|
})
|
|
opusRequested = d.opusRequested === true
|
|
isRouted =
|
|
brief.role !== 'review' &&
|
|
brief.role !== 'test' &&
|
|
!(d.kind === 'fixed' && d.reason === 'model chosen by orchestrator')
|
|
if (d.kind === 'fixed') {
|
|
tier = d.tier
|
|
reason = d.reason
|
|
} else {
|
|
const c = await classify($, e.prompt)
|
|
tier = c.tier
|
|
reason = c.reason
|
|
classifierUsage = c.usage
|
|
}
|
|
}
|
|
|
|
const brief = parseBrief(prompt)
|
|
const ran = await next({ ...e, prompt, model: tier })
|
|
if (ran.deny !== undefined || !ran.agentId) return ran
|
|
// the stored rewrite is used up only once its agent has started
|
|
if (markId && pending) await putIn<Pending>($, S.pending, markId, undefined)
|
|
// label against what the engine resolved, not what the router asked for
|
|
const resolved = asTier(ran.model) ?? tier
|
|
if (resolved !== tier) reason += ` (asked ${tier}, engine resolved ${resolved})`
|
|
|
|
// Learn what the alias resolves to, so router calls are priced on the real model.
|
|
if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model)
|
|
if (isRouted) {
|
|
await putIn($, S.agentTask, ran.agentId, { taskId, task: e.description })
|
|
await updateIn<TaskCost>($, S.costs, taskId, old => old ?? emptyTask(e.description))
|
|
await chargeOverhead($, taskId, 'haiku', classifierUsage, cfg.prices)
|
|
}
|
|
|
|
// Agents with a role (dev, review, test) get a record; only dev rounds are verified.
|
|
if (brief.role)
|
|
await putIn<AgentRec>($, S.agents, ran.agentId, {
|
|
taskId,
|
|
tier: resolved,
|
|
routed: resolved,
|
|
name: e.name,
|
|
role: brief.role,
|
|
clone: brief.clone,
|
|
baseBranch: brief.baseBranch,
|
|
critical,
|
|
prompt,
|
|
fails,
|
|
rounds: 0,
|
|
})
|
|
|
|
await log($, {
|
|
id: taskId,
|
|
agentId: ran.agentId,
|
|
at: await $.clock.now(),
|
|
task: e.description,
|
|
tier: resolved,
|
|
role: brief.role,
|
|
reason,
|
|
outcome: 'spawned',
|
|
opusRequested,
|
|
})
|
|
$.ui.status(`router: ${taskId} ${brief.role ?? 'agent'} → ${tier}`)
|
|
return ran
|
|
}).catch(($, e, next) => {
|
|
// Fail open: a router bug must never block delegation.
|
|
$.ui.status('router: error, agent ran unrouted')
|
|
return next(e)
|
|
})
|
|
|
|
// Verification: each finished run of a tracked agent, a resumed SendMessage round included.
|
|
on('turn.complete', async ($, e, next) => {
|
|
const out = await next(e)
|
|
const agentId = e.agentId
|
|
if (!agentId) return out
|
|
|
|
const owner = (await getMap<{ taskId: string; task: string }>($, S.agentTask))[agentId]
|
|
if (owner && e.usage) {
|
|
const usage = e.usage
|
|
await updateIn<TaskCost>($, S.costs, owner.taskId, old =>
|
|
addWriterRun(old ?? emptyTask(owner.task), usage.model, usage, cfg.prices),
|
|
)
|
|
}
|
|
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
|
if (!rec) return out
|
|
// Only the run's end is a hand-back; wait for it after this event has gone on.
|
|
const token = ++tokens
|
|
lastTurn.set(agentId, { token, ended: e.reason, reported: e.usage?.model })
|
|
$.clock.after(0, () => void awaitHandBack($, agentId, token, cfg))
|
|
return out
|
|
})
|
|
|
|
// A message to a running agent is a later instruction too (#6).
|
|
on('tool.call', { tool: 'SendMessage' }, async ($, e, next) => {
|
|
const out = await next(e)
|
|
if (out.deny !== undefined || out.isError) return out
|
|
const to = typeof e.to === 'string' ? e.to : ''
|
|
const id = to ? await agentAddressed($, to) : undefined
|
|
if (id && typeof e.message === 'string') await noteFollowUp($, id, e.message)
|
|
return out
|
|
})
|
|
|
|
on('command.run', { command: 'router' }, async ($, e) => {
|
|
if (e.args.trim() === 'clear') {
|
|
await serial(async () => {
|
|
await $.store.set(S.log, [])
|
|
await $.store.set(S.pending, {})
|
|
await $.store.set(S.agents, {})
|
|
await $.store.set(S.costs, {})
|
|
await $.store.set(S.agentTask, {})
|
|
})
|
|
return { text: 'model-router: log, agents, costs and pending rewrites cleared.' }
|
|
}
|
|
const [sub, arg] = e.args.trim().split(/\s+/)
|
|
if (sub === 'savings') return { text: savingsReport(await getMap<TaskCost>($, S.costs), Infinity, arg) }
|
|
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
|
if (list.length === 0) return { text: 'model-router: nothing routed yet.' }
|
|
|
|
const count = (key: (l: LogEntry) => string | undefined) => {
|
|
const r: Record<string, number> = {}
|
|
for (const l of list) {
|
|
const k = key(l)
|
|
if (k) r[k] = (r[k] ?? 0) + 1
|
|
}
|
|
return Object.entries(r)
|
|
.map(([k, n]) => `${k} ${n}`)
|
|
.join(' · ')
|
|
}
|
|
const rows = list
|
|
.slice(-15)
|
|
.map(
|
|
l =>
|
|
`${l.id.padEnd(5)} ${(l.role ?? '-').padEnd(6)} ${l.tier.padEnd(6)} ${(l.verifier ? '→' + l.verifier : '').padEnd(8)} ${l.outcome.padEnd(17)} ${(l.task ?? '').slice(0, 40)}${l.opusRequested ? ' [opus requested]' : ''}`,
|
|
)
|
|
.join('\n')
|
|
const pending = Object.keys(await getMap<Pending>($, S.pending))
|
|
return {
|
|
text:
|
|
`model-router - last ${Math.min(15, list.length)} of ${list.length}\n` +
|
|
`writers: ${count(l => (l.outcome === 'spawned' ? `${l.role ?? 'other'}:${l.tier}` : undefined))}\n` +
|
|
`results: ${count(l => (l.outcome === 'spawned' ? undefined : l.outcome))}\n` +
|
|
`opus requested on dev briefs: ${list.filter(l => l.opusRequested).length}\n` +
|
|
(pending.length ? `pending rewrites: ${pending.join(', ')}\n` : '') +
|
|
`\n${rows}\n\n` +
|
|
savingsReport(await getMap<TaskCost>($, S.costs), 5),
|
|
}
|
|
})
|
|
}
|