adminandClaude Opus 5.5
92bc4dc348
Claude Code plugin: routes subagents by how well-defined the brief is, verifies dev work on a second model in the agent's own clone, and estimates savings against Opus 5.5. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
682 lines
25 KiB
TypeScript
682 lines
25 KiB
TypeScript
import type { EngineInterface, Register } from 'claude-code'
|
|
|
|
import {
|
|
addOverhead,
|
|
addWriterRun,
|
|
emptyTask,
|
|
parsePrices,
|
|
savingsReport,
|
|
type PriceRow,
|
|
type TaskCost,
|
|
type Usage,
|
|
} from './pricing'
|
|
import {
|
|
REWRITE_MARK,
|
|
asTier,
|
|
clip,
|
|
decide,
|
|
onFail,
|
|
parseBrief,
|
|
parsePatterns,
|
|
parseVerdict,
|
|
safetyHit,
|
|
skipReason,
|
|
validTimeout,
|
|
verifierFor,
|
|
type Role,
|
|
type Tier,
|
|
} from './router'
|
|
|
|
// ---------- stored records ----------
|
|
|
|
/** One agent the router started, kept from spawn to its last verified round. */
|
|
type AgentRec = {
|
|
taskId: string
|
|
tier: Tier
|
|
role?: Role
|
|
clone?: string
|
|
baseBranch?: string
|
|
critical: boolean
|
|
prompt: string
|
|
/** Failed verdicts in this task's chain, carried over to a rewrite agent. */
|
|
fails: number
|
|
rounds: number
|
|
/** The tree after the last verified round: the next round diffs from here. */
|
|
lastSnap?: string
|
|
lastUntracked?: string[]
|
|
lastIssues?: string[]
|
|
}
|
|
|
|
type Pending = { prompt: string; tier: Tier; fails: number; critical: boolean }
|
|
|
|
type Outcome =
|
|
| 'pass'
|
|
| 'resume-requested'
|
|
| 'rewrite-requested'
|
|
| 'handed-back'
|
|
| 'no-changes'
|
|
| 'no-base'
|
|
| 'tests-timeout'
|
|
| 'tests-error'
|
|
| 'verify-error'
|
|
| 'test-agent'
|
|
| 'review-agent'
|
|
| 'no-clone'
|
|
| 'opus-unverified'
|
|
| 'agent-ended'
|
|
|
|
type LogEntry = {
|
|
id: string
|
|
agentId?: string
|
|
at: number
|
|
task: string
|
|
tier: Tier
|
|
role?: Role
|
|
reason: string
|
|
round?: number
|
|
verifier?: Tier
|
|
outcome: Outcome | 'spawned'
|
|
opusRequested?: boolean
|
|
}
|
|
|
|
const S = {
|
|
counter: 'counter',
|
|
pending: 'pending',
|
|
agents: 'agents',
|
|
log: 'log',
|
|
costs: 'costs',
|
|
agentTask: 'agentTask',
|
|
aliases: 'aliases',
|
|
} as const
|
|
|
|
/** Model ids assumed for an alias until an agent run shows what it resolves to. */
|
|
const ALIAS_DEFAULT: Record<Tier, string> = {
|
|
haiku: 'claude-haiku-4-5',
|
|
sonnet: 'claude-sonnet-5-5',
|
|
opus: 'claude-opus-5-5',
|
|
fable: 'claude-fable-5-1',
|
|
}
|
|
const LOG_LIMIT = 400
|
|
|
|
// ---------- prompts ----------
|
|
|
|
const CLASSIFIER_SYSTEM = `You route coding tasks to the smallest model that will do them right.
|
|
Judge how well-defined the task is, NOT how important or low-level the code is. In this codebase nearly everything touches control loops, asyncio and RS485; that alone never means opus. A stronger model verifies every change below Opus, and a review agent reviews every PR.
|
|
Answer with exactly one word: haiku, sonnet or opus.
|
|
|
|
haiku: read-only lookups and summaries, mechanical changes fully spelled out.
|
|
sonnet: the approach is decided: the brief says what to change, or gives Expected behaviour, acceptance criteria or a test to make pass. Size and domain do not matter.
|
|
opus: the approach is still open: a design or trade-off to decide, a bug whose cause is unknown, or a brief so vague the writer must invent the plan.
|
|
|
|
When unsure between sonnet and opus, pick sonnet: a failed verification escalates.`
|
|
|
|
const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not.
|
|
Judge only whether the change correctly and completely does the task, without breaking anything nearby.
|
|
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
|
|
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
|
|
|
|
Reply with JSON only:
|
|
{"verdict": "PASS" | "FAIL", "issues": ["concrete issue with file:line where possible", ...], "rewrite_brief": "what must change, empty on PASS"}`
|
|
|
|
const ORCHESTRATOR_NOTE = `## Model router (model-router mod)
|
|
The model-router mod picks each subagent's model and verifies dev work before it reaches a PR.
|
|
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
|
|
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
|
|
- Review and test agents keep the model you name (Opus when you name none).
|
|
- When a dev agent below Opus finishes a round, the router verifies its clone's diff on Opus and posts a "[model-router]" verdict. Wait for that verdict before opening or advancing the PR.
|
|
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. Do not fix rejected work yourself.`
|
|
|
|
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
|
|
|
// Agents start and finish in parallel. Every read-modify-write of the store
|
|
// goes through this queue, one at a time, or parallel hooks overwrite each
|
|
// other's writes (and hand out the same task id).
|
|
let queue: Promise<unknown> = Promise.resolve()
|
|
|
|
function serial<T>(work: () => Promise<T>): Promise<T> {
|
|
const run = queue.then(work)
|
|
queue = run.catch(() => undefined)
|
|
return run
|
|
}
|
|
|
|
async function getMap<T>($: EngineInterface, key: string) {
|
|
return ((await $.store.get(key)) ?? {}) as Record<string, T>
|
|
}
|
|
|
|
/** Replaces one entry of a stored map, atomically with respect to this plugin. */
|
|
async function updateIn<T>($: EngineInterface, key: string, id: string, change: (old: T | undefined) => T | undefined) {
|
|
return serial(async () => {
|
|
const all = await getMap<T>($, key)
|
|
const value = change(all[id])
|
|
if (value === undefined) delete all[id]
|
|
else all[id] = value
|
|
await $.store.set(key, all)
|
|
return value
|
|
})
|
|
}
|
|
|
|
async function putIn<T>($: EngineInterface, key: string, id: string, value: T | undefined) {
|
|
await updateIn<T>($, key, id, () => value)
|
|
}
|
|
|
|
async function nextId($: EngineInterface) {
|
|
return serial(async () => {
|
|
const n = Number((await $.store.get(S.counter)) ?? 0) + 1
|
|
await $.store.set(S.counter, n)
|
|
return `T${n}`
|
|
})
|
|
}
|
|
|
|
async function log($: EngineInterface, entry: LogEntry) {
|
|
await serial(async () => {
|
|
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
|
list.push(entry)
|
|
await $.store.set(S.log, list.slice(-LOG_LIMIT))
|
|
})
|
|
}
|
|
|
|
async function sh($: EngineInterface, argv: string[], cwd: string, timeoutMs = 30_000) {
|
|
try {
|
|
const r = await $.process.run(argv, { cwd, timeoutMs })
|
|
return { ok: r.exitCode === 0, out: r.stdout, err: r.stderr, code: r.exitCode }
|
|
} catch (err) {
|
|
return { ok: false, out: '', err: String(err), code: -1 }
|
|
}
|
|
}
|
|
|
|
async function untracked($: EngineInterface, cwd: string) {
|
|
const r = await sh($, ['git', 'ls-files', '--others', '--exclude-standard'], cwd)
|
|
return r.ok ? r.out.split('\n').filter(Boolean) : []
|
|
}
|
|
|
|
/** A commit holding the clone's tree as it is now (HEAD when clean). */
|
|
async function snapshot($: EngineInterface, cwd: string) {
|
|
const stash = await sh($, ['git', 'stash', 'create'], cwd)
|
|
if (stash.ok && stash.out.trim()) return stash.out.trim()
|
|
const head = await sh($, ['git', 'rev-parse', 'HEAD'], cwd)
|
|
return head.ok ? head.out.trim() : undefined
|
|
}
|
|
|
|
/** Where a first round diffs from: the merge base with the PR's base branch. */
|
|
async function firstBase($: EngineInterface, rec: AgentRec, prBaseCommand: string, fallback: string) {
|
|
const cwd = rec.clone as string
|
|
let branch: string | undefined
|
|
if (prBaseCommand) {
|
|
const r = await sh($, ['sh', '-c', prBaseCommand], cwd)
|
|
const b = r.out.trim().split('\n')[0]?.trim()
|
|
if (r.ok && b) branch = b
|
|
}
|
|
branch = branch ?? rec.baseBranch ?? fallback
|
|
for (const ref of [`origin/${branch}`, branch]) {
|
|
const r = await sh($, ['git', 'merge-base', ref, 'HEAD'], cwd)
|
|
if (r.ok && r.out.trim()) return { base: r.out.trim(), branch }
|
|
}
|
|
return { base: undefined, branch }
|
|
}
|
|
|
|
async function classify($: EngineInterface, prompt: string): Promise<{ tier: Tier; reason: string; usage?: Usage }> {
|
|
const r = await $.model.complete({
|
|
model: 'haiku',
|
|
system: CLASSIFIER_SYSTEM,
|
|
prompt: `Task:\n${clip(prompt, 6000)}`,
|
|
maxTokens: 5,
|
|
timeoutMs: 15_000,
|
|
})
|
|
const tier = r.isAnswered ? asTier(r.text) : undefined
|
|
if (tier && tier !== 'fable') return { tier, reason: 'classifier', usage: r.usage }
|
|
return { tier: 'sonnet', reason: 'classifier unavailable, default', usage: r.usage }
|
|
}
|
|
|
|
async function tell($: EngineInterface, text: string) {
|
|
await $.prompt.submit({ text })
|
|
}
|
|
|
|
// ---------- cost tracking ----------
|
|
|
|
async function modelIdFor($: EngineInterface, alias: Tier) {
|
|
const seen = (await getMap<string>($, S.aliases))[alias]
|
|
return seen ?? ALIAS_DEFAULT[alias]
|
|
}
|
|
|
|
/** Adds a router call (classifier, verifier) to a task's overhead. */
|
|
async function chargeOverhead($: EngineInterface, taskId: string, alias: Tier, u: Usage | undefined, prices: readonly PriceRow[]) {
|
|
if (!u) return
|
|
const modelId = await modelIdFor($, alias)
|
|
await updateIn<TaskCost>($, S.costs, taskId, old => addOverhead(old ?? emptyTask(taskId), modelId, u, prices))
|
|
}
|
|
|
|
type TestRun = { status: 'ran' | 'timeout' | 'error'; code: number | null; tail: string; seconds: number }
|
|
|
|
/**
|
|
* Runs the test command in the clone, streamed: `$.process.run` stops at ten
|
|
* minutes, a stream has no limit of its own. GNU `timeout` does the kill.
|
|
* A timeout or a run that could not start is no verdict on the code.
|
|
*/
|
|
async function runTests($: EngineInterface, cmd: string, cwd: string, timeoutSec: number): Promise<TestRun> {
|
|
const started = await $.clock.now()
|
|
const seconds = async () => Math.round(((await $.clock.now()) - started) / 1000)
|
|
let out = ''
|
|
let errTail = ''
|
|
try {
|
|
const stream = $.process.spawn({
|
|
argv: ['timeout', '--verbose', '--kill-after=30', String(Math.max(1, Math.round(timeoutSec))), 'sh', '-c', cmd],
|
|
cwd,
|
|
})
|
|
for await (const piece of stream) {
|
|
out = (out + piece.text).slice(-65_536)
|
|
if (piece.stream === 'stderr') errTail = (errTail + piece.text).slice(-65_536)
|
|
}
|
|
// GNU timeout --verbose says so on stderr when, and only when, it fires.
|
|
// Read from the joined stderr, so a line split across pieces still counts.
|
|
const isFired = errTail.includes('timeout: sending signal')
|
|
const end = await stream.result
|
|
const tail = clip(out.trim().split('\n').slice(-80).join('\n'), 8000)
|
|
const took = await seconds()
|
|
// Only a timeout that fired is a timeout: a 137 without it is the code's own
|
|
// (e.g. out of memory) and goes to the verifier as a failed run.
|
|
if (isFired) return { status: 'timeout', code: end.code, tail, seconds: took }
|
|
if (end.code === 126 || end.code === 127) return { status: 'error', code: end.code, tail, seconds: took }
|
|
return { status: 'ran', code: end.code, tail, seconds: took }
|
|
} catch (err) {
|
|
return { status: 'error', code: null, tail: String(err).slice(0, 500), seconds: await seconds() }
|
|
}
|
|
}
|
|
|
|
type Settings = {
|
|
testCommand: string
|
|
testTimeoutSec: number
|
|
prBaseCommand: string
|
|
defaultBaseBranch: string
|
|
maxFails: number
|
|
allowFable: boolean
|
|
prices: readonly PriceRow[]
|
|
}
|
|
|
|
/** Verifies one finished round of a dev agent and tells the manager. */
|
|
async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, cfg: Settings) {
|
|
const cwd = rec.clone as string
|
|
const round = rec.rounds + 1
|
|
const entry: LogEntry = {
|
|
id: rec.taskId,
|
|
agentId,
|
|
at: await $.clock.now(),
|
|
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
|
tier: rec.tier,
|
|
role: rec.role,
|
|
reason: `round ${round}`,
|
|
round,
|
|
outcome: 'verify-error',
|
|
}
|
|
const head = `[model-router] ${rec.taskId} (agent ${agentId}, ${rec.tier}, round ${round})`
|
|
|
|
// 1. What this round changed.
|
|
let base = rec.lastSnap
|
|
let baseLabel = 'the previous verified round'
|
|
if (!base) {
|
|
const fb = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch)
|
|
base = fb.base
|
|
baseLabel = `merge-base with ${fb.branch}`
|
|
if (!base) {
|
|
entry.outcome = 'no-base'
|
|
await log($, entry)
|
|
await tell($, `${head}: NOT VERIFIED - no merge base with ${fb.branch} in ${cwd}. Review the change yourself before the PR.`)
|
|
return
|
|
}
|
|
}
|
|
const diff = await sh($, ['git', 'diff', base], cwd, 60_000)
|
|
const known = new Set(rec.lastUntracked ?? [])
|
|
const fresh = (await untracked($, cwd)).filter(f => !known.has(f))
|
|
let added = ''
|
|
for (const f of fresh.slice(0, 20)) {
|
|
try {
|
|
added += `\n--- new file: ${f}\n${clip(await $.fs.read(`${cwd}/${f}`), 8000)}\n`
|
|
} catch {
|
|
added += `\n--- new file: ${f} (unreadable)\n`
|
|
}
|
|
}
|
|
const changes = (diff.out + added).trim()
|
|
if (!changes) {
|
|
entry.outcome = 'no-changes'
|
|
await log($, entry)
|
|
await tell($, `${head}: no changes since ${baseLabel}; nothing to verify.`)
|
|
return
|
|
}
|
|
|
|
// 2. Tests, in the clone.
|
|
let tests = ''
|
|
if (cfg.testCommand) {
|
|
$.ui.status(`router: ${rec.taskId} round ${round} running tests`)
|
|
const t = await runTests($, cfg.testCommand, cwd, cfg.testTimeoutSec)
|
|
if (t.status !== 'ran') {
|
|
// No verdict on the code: no verifier call, no fail counted, no rewrite.
|
|
entry.outcome = t.status === 'timeout' ? 'tests-timeout' : 'tests-error'
|
|
await log($, entry)
|
|
const why =
|
|
t.status === 'timeout'
|
|
? `the tests did not finish within ${cfg.testTimeoutSec} s (testTimeoutSec)`
|
|
: `the test command could not run (exit ${t.code ?? 'none'})`
|
|
await tell(
|
|
$,
|
|
`${head}: NOT VERIFIED - ${why}. This says nothing about the code: do not resume or rewrite the agent for it. ` +
|
|
`Fix the test setup or raise testTimeoutSec, or review the change yourself.\nLast output:\n${clip(t.tail, 2000)}`,
|
|
)
|
|
return
|
|
}
|
|
tests = `exit code ${t.code} after ${t.seconds} s\n${t.tail}`
|
|
}
|
|
|
|
// 3. The verdict, from another model.
|
|
const verifier = verifierFor(rec.tier)
|
|
entry.verifier = verifier
|
|
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
|
|
const v = await $.model.complete({
|
|
model: verifier,
|
|
system: VERIFIER_SYSTEM,
|
|
prompt: [
|
|
`## Task\n${clip(rec.prompt, 12_000)}`,
|
|
rec.lastIssues?.length
|
|
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
|
|
: '',
|
|
`## Changes (${baseLabel})\n${clip(changes, 60_000)}`,
|
|
cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '',
|
|
]
|
|
.filter(Boolean)
|
|
.join('\n\n'),
|
|
maxTokens: 2000,
|
|
timeoutMs: 180_000,
|
|
})
|
|
await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices)
|
|
if (!v.isAnswered) {
|
|
await log($, entry)
|
|
await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`)
|
|
return
|
|
}
|
|
const verdict = parseVerdict(v.text)
|
|
const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)'
|
|
|
|
// 4. Remember where this round ended.
|
|
const after: AgentRec = {
|
|
...rec,
|
|
rounds: round,
|
|
lastSnap: (await snapshot($, cwd)) ?? rec.lastSnap,
|
|
lastUntracked: await untracked($, cwd),
|
|
lastIssues: verdict.pass ? [] : verdict.issues,
|
|
}
|
|
|
|
if (verdict.pass) {
|
|
entry.outcome = 'pass'
|
|
await putIn($, S.agents, agentId, after)
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} ✓ ${rec.tier}, verified by ${verifier}`)
|
|
await tell($, `${head}: PASS from verifier ${verifier}. Continue to the PR.`)
|
|
return
|
|
}
|
|
|
|
const fails = rec.fails + 1
|
|
after.fails = fails
|
|
const step = onFail(fails, rec.tier, cfg.maxFails, cfg.allowFable)
|
|
await putIn($, S.agents, agentId, after)
|
|
|
|
if (step.kind === 'resume') {
|
|
entry.outcome = 'resume-requested'
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} ✗ round ${round}, resume ${agentId}`)
|
|
await tell(
|
|
$,
|
|
`${head}: FAIL from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
|
(verdict.brief ? `What must change: ${verdict.brief}\n` : '') +
|
|
`Resume agent ${agentId} with SendMessage, passing these issues. Do not fix it yourself.`,
|
|
)
|
|
return
|
|
}
|
|
|
|
if (step.kind === 'new-agent') {
|
|
await putIn<Pending>($, S.pending, rec.taskId, {
|
|
prompt:
|
|
`${rec.prompt.split('\n\n## Previous attempts rejected')[0]}\n\n## Previous attempts rejected\n` +
|
|
`Agents on ${rec.tier} were rejected ${fails} times by an independent verifier (${verifier}). Their work is in the clone ${cwd}: continue from it, fix or replace it, do not start over elsewhere.\n` +
|
|
`Issues:\n${issues}\n` +
|
|
(verdict.brief ? `What must change: ${verdict.brief}\n` : ''),
|
|
tier: step.tier,
|
|
fails,
|
|
critical: rec.critical,
|
|
})
|
|
entry.outcome = 'rewrite-requested'
|
|
await log($, entry)
|
|
$.ui.status(`router: ${rec.taskId} ✗ again, new agent on ${step.tier}`)
|
|
await tell(
|
|
$,
|
|
`${head}: FAIL again from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
|
`Stop agent ${agentId}, then start a new agent on ${step.tier} in the same clone: call Agent with prompt [router-rewrite ${rec.taskId}] (same subagent_type).`,
|
|
)
|
|
return
|
|
}
|
|
|
|
entry.outcome = 'handed-back'
|
|
await log($, entry)
|
|
$.ui.toast(`model-router: ${rec.taskId} failed verification ${fails}x - handed back`)
|
|
await tell(
|
|
$,
|
|
`${head}: FAIL from verifier ${verifier}, ${fails} failed verdicts. Handed back: stop and show the person these issues.\n${issues}` +
|
|
(verdict.brief ? `\nVerifier's brief: ${verdict.brief}` : ''),
|
|
)
|
|
}
|
|
|
|
/** Logs an agent run that is not verified; tells the manager when a dev agent waits on a verdict. */
|
|
async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, outcome: Outcome) {
|
|
await log($, {
|
|
id: rec.taskId,
|
|
agentId,
|
|
at: await $.clock.now(),
|
|
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
|
tier: rec.tier,
|
|
role: rec.role,
|
|
reason: outcome,
|
|
outcome,
|
|
})
|
|
if (rec.role === 'dev' && outcome === 'no-clone')
|
|
await tell($, `[model-router] ${rec.taskId} (agent ${agentId}): NOT VERIFIED - the brief names no clone path.`)
|
|
}
|
|
|
|
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, cfg: Settings) {
|
|
const skip = skipReason(rec, ended)
|
|
if (skip) await skipRound($, agentId, rec, skip)
|
|
else await verifyRound($, agentId, rec, cfg)
|
|
}
|
|
|
|
// ---------- the hooks ----------
|
|
|
|
export const register: Register = (on, options) => {
|
|
const patterns = parsePatterns(String(options.safetyPatterns ?? ''))
|
|
const respectExplicit = options.respectExplicitModel === true
|
|
const cfg: Settings = {
|
|
testCommand: String(options.testCommand ?? '').trim(),
|
|
testTimeoutSec: validTimeout(options.testTimeoutSec),
|
|
prBaseCommand: String(options.prBaseCommand ?? '').trim(),
|
|
defaultBaseBranch: String(options.defaultBaseBranch ?? 'release/1.0').trim(),
|
|
maxFails: Number(options.maxRewrites ?? 2),
|
|
allowFable: options.allowFable === true,
|
|
prices: parsePrices(String(options.prices ?? '')),
|
|
}
|
|
|
|
on('session.start', async ($, e, next) => {
|
|
await $.command.register({
|
|
name: 'router',
|
|
description: 'Model router: decisions, verdicts and savings vs Opus (`/router savings [Tn]`, `/router clear`)',
|
|
})
|
|
return next(e)
|
|
})
|
|
|
|
on('prompt.compose', async ($, e, next) => {
|
|
const out = await next(e)
|
|
if (e.traits.includes('bare')) return out
|
|
return { sections: [...out.sections, { id: 'model-router:note', text: ORCHESTRATOR_NOTE, scope: 'session' as const }] }
|
|
})
|
|
|
|
// Routing: every agent the model starts, background ones included.
|
|
on('agent.spawn', async ($, e, next) => {
|
|
if (e.fork || e.isTeammate || e.workflow) return next(e)
|
|
|
|
let taskId: string
|
|
let prompt = e.prompt
|
|
let tier: Tier
|
|
let reason: string
|
|
let fails = 0
|
|
let critical: boolean
|
|
let opusRequested = false
|
|
let classifierUsage: Usage | undefined
|
|
// Agents whose model the router chose; kept models (review, test) are not a saving.
|
|
let isRouted = true
|
|
|
|
const markId = e.prompt.match(REWRITE_MARK)?.[1]
|
|
const pending = markId ? (await getMap<Pending>($, S.pending))[markId] : undefined
|
|
|
|
if (markId && pending) {
|
|
taskId = markId
|
|
const extra = e.prompt.replace(REWRITE_MARK, '').trim()
|
|
prompt = pending.prompt + (extra ? `\n\nAdditional guidance from the orchestrator:\n${extra}` : '')
|
|
tier = pending.tier
|
|
fails = pending.fails
|
|
critical = pending.critical
|
|
reason = `new agent after ${fails} failed verdicts`
|
|
await putIn<Pending>($, S.pending, markId, undefined)
|
|
} else {
|
|
taskId = await nextId($)
|
|
const brief = parseBrief(e.prompt)
|
|
critical = safetyHit(`${e.description}\n${e.prompt}`, patterns) !== undefined
|
|
const d = decide({
|
|
role: brief.role,
|
|
named: asTier(e.model),
|
|
subagentType: e.subagentType,
|
|
prompt: e.prompt,
|
|
respectExplicit,
|
|
isAmbiguous: brief.isAmbiguous,
|
|
})
|
|
opusRequested = d.opusRequested === true
|
|
isRouted =
|
|
brief.role !== 'review' &&
|
|
brief.role !== 'test' &&
|
|
!(d.kind === 'fixed' && d.reason === 'model chosen by orchestrator')
|
|
if (d.kind === 'fixed') {
|
|
tier = d.tier
|
|
reason = d.reason
|
|
} else {
|
|
const c = await classify($, e.prompt)
|
|
tier = c.tier
|
|
reason = c.reason
|
|
classifierUsage = c.usage
|
|
}
|
|
}
|
|
|
|
const brief = parseBrief(prompt)
|
|
const ran = await next({ ...e, prompt, model: tier })
|
|
if (ran.deny !== undefined || !ran.agentId) return ran
|
|
|
|
// Learn what the alias resolves to, so router calls are priced on the real model.
|
|
if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model)
|
|
if (isRouted) {
|
|
await putIn($, S.agentTask, ran.agentId, { taskId, task: e.description })
|
|
await updateIn<TaskCost>($, S.costs, taskId, old => old ?? emptyTask(e.description))
|
|
await chargeOverhead($, taskId, 'haiku', classifierUsage, cfg.prices)
|
|
}
|
|
|
|
// Agents with a role (dev, review, test) get a record; only dev rounds are verified.
|
|
if (brief.role)
|
|
await putIn<AgentRec>($, S.agents, ran.agentId, {
|
|
taskId,
|
|
tier,
|
|
role: brief.role,
|
|
clone: brief.clone,
|
|
baseBranch: brief.baseBranch,
|
|
critical,
|
|
prompt,
|
|
fails,
|
|
rounds: 0,
|
|
})
|
|
|
|
await log($, {
|
|
id: taskId,
|
|
agentId: ran.agentId,
|
|
at: await $.clock.now(),
|
|
task: e.description,
|
|
tier,
|
|
role: brief.role,
|
|
reason,
|
|
outcome: 'spawned',
|
|
opusRequested,
|
|
})
|
|
$.ui.status(`router: ${taskId} ${brief.role ?? 'agent'} → ${tier}`)
|
|
return ran
|
|
}).catch(($, e, next) => {
|
|
// Fail open: a router bug must never block delegation.
|
|
$.ui.status('router: error, agent ran unrouted')
|
|
return next(e)
|
|
})
|
|
|
|
// Verification: each finished run of a tracked agent, a resumed SendMessage round included.
|
|
on('turn.complete', async ($, e, next) => {
|
|
const out = await next(e)
|
|
const agentId = e.agentId
|
|
if (!agentId) return out
|
|
|
|
const owner = (await getMap<{ taskId: string; task: string }>($, S.agentTask))[agentId]
|
|
if (owner && e.usage) {
|
|
const usage = e.usage
|
|
await updateIn<TaskCost>($, S.costs, owner.taskId, old =>
|
|
addWriterRun(old ?? emptyTask(owner.task), usage.model, usage, cfg.prices),
|
|
)
|
|
}
|
|
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
|
if (!rec) return out
|
|
// Verification takes minutes: run it after this event has gone on.
|
|
$.clock.after(0, () => void runRound($, agentId, rec, e.reason, cfg))
|
|
return out
|
|
})
|
|
|
|
on('command.run', { command: 'router' }, async ($, e) => {
|
|
if (e.args.trim() === 'clear') {
|
|
await serial(async () => {
|
|
await $.store.set(S.log, [])
|
|
await $.store.set(S.pending, {})
|
|
await $.store.set(S.agents, {})
|
|
await $.store.set(S.costs, {})
|
|
await $.store.set(S.agentTask, {})
|
|
})
|
|
return { text: 'model-router: log, agents, costs and pending rewrites cleared.' }
|
|
}
|
|
const [sub, arg] = e.args.trim().split(/\s+/)
|
|
if (sub === 'savings') return { text: savingsReport(await getMap<TaskCost>($, S.costs), Infinity, arg) }
|
|
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
|
if (list.length === 0) return { text: 'model-router: nothing routed yet.' }
|
|
|
|
const count = (key: (l: LogEntry) => string | undefined) => {
|
|
const r: Record<string, number> = {}
|
|
for (const l of list) {
|
|
const k = key(l)
|
|
if (k) r[k] = (r[k] ?? 0) + 1
|
|
}
|
|
return Object.entries(r)
|
|
.map(([k, n]) => `${k} ${n}`)
|
|
.join(' · ')
|
|
}
|
|
const rows = list
|
|
.slice(-15)
|
|
.map(
|
|
l =>
|
|
`${l.id.padEnd(5)} ${(l.role ?? '-').padEnd(6)} ${l.tier.padEnd(6)} ${(l.verifier ? '→' + l.verifier : '').padEnd(8)} ${l.outcome.padEnd(17)} ${(l.task ?? '').slice(0, 40)}${l.opusRequested ? ' [opus requested]' : ''}`,
|
|
)
|
|
.join('\n')
|
|
const pending = Object.keys(await getMap<Pending>($, S.pending))
|
|
return {
|
|
text:
|
|
`model-router - last ${Math.min(15, list.length)} of ${list.length}\n` +
|
|
`writers: ${count(l => (l.outcome === 'spawned' ? `${l.role ?? 'other'}:${l.tier}` : undefined))}\n` +
|
|
`results: ${count(l => (l.outcome === 'spawned' ? undefined : l.outcome))}\n` +
|
|
`opus requested on dev briefs: ${list.filter(l => l.opusRequested).length}\n` +
|
|
(pending.length ? `pending rewrites: ${pending.join(', ')}\n` : '') +
|
|
`\n${rows}\n\n` +
|
|
savingsReport(await getMap<TaskCost>($, S.costs), 5),
|
|
}
|
|
})
|
|
}
|