Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
  adminandClaude Opus 5.5
|
eba7a16b32 | ||
|
adminandClaude Opus 5.5
|
05f9663129 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "model-router",
|
||||
"version": "0.4.4",
|
||||
"version": "0.4.6",
|
||||
"description": "Routes each subagent to Haiku, Sonnet or Opus by how well-defined its brief is, verifies dev work below Opus in its own clone on a different model, sends failures back to the same agent then a stronger one, and estimates the savings against Opus.",
|
||||
"author": {
|
||||
"name": "Glenn Schrooyen"
|
||||
|
||||
@@ -8,8 +8,9 @@ A Claude Code plugin that picks a model for every subagent, checks dev work on a
|
||||
|---|---|
|
||||
| Route | Dev agents run on **Sonnet** when the brief decides the approach (Expected / acceptance criteria), on **Opus** when the design is still open. Review and test agents keep the model they are given (Opus when none). |
|
||||
| Verify | When a dev agent below Opus finishes a round, the router diffs its clone against the merge base, runs the test command in that clone, and has **Opus** judge the change. |
|
||||
| Hand-back line | Every dev hand-back gets one line at once: `[model-router] Tn verifying on Opus...` (a verdict follows), or `[model-router] Tn no verification: dev ran on Opus. Go straight to review.` Decided by the model the run actually used, not only the routing choice; a difference is shown as `(routed to X, ran on Y)`. |
|
||||
| On FAIL | 1st: resume the same agent with the issues. 2nd: a new agent one model up, in the same clone. After that: handed back to the manager. |
|
||||
| Not verified | A test timeout, a test command that cannot run, a missing clone or merge base: reported as NOT VERIFIED, never as a FAIL. |
|
||||
| Not verified | A test timeout, a test command that cannot run, a missing clone or merge base, or a verdict that cannot be read even after asking once more: reported as NOT VERIFIED, never as a FAIL. |
|
||||
| Savings | `/router savings` prices every run on the model it ran on and compares it with the same tokens on Opus 5.5. |
|
||||
|
||||
## Briefs
|
||||
@@ -58,7 +59,7 @@ claude plugin validate .
|
||||
claude plugin test .
|
||||
```
|
||||
|
||||
44 tests. Every routing, verification, cost and timeout rule was also checked by breaking it on purpose (66 mutations, all caught by a test).
|
||||
62 tests. Every routing, verification, cost and timeout rule was also checked by breaking it on purpose (66 mutations in v0.4.4, plus 13 for the verdict reader in v0.4.5 and 12 for the hand-back line in v0.4.6; all caught by a test).
|
||||
|
||||
## Known limits
|
||||
|
||||
|
||||
+58
-17
@@ -19,6 +19,8 @@ import {
|
||||
parseBrief,
|
||||
parsePatterns,
|
||||
parseVerdict,
|
||||
handBackLine,
|
||||
ranTier,
|
||||
safetyHit,
|
||||
skipReason,
|
||||
validTimeout,
|
||||
@@ -32,7 +34,10 @@ import {
|
||||
/** One agent the router started, kept from spawn to its last verified round. */
|
||||
type AgentRec = {
|
||||
taskId: string
|
||||
/** The model the last round actually ran on (the routing choice until a round reports one). */
|
||||
tier: Tier
|
||||
/** The model the router chose at spawn; tier can differ when the run used another. */
|
||||
routed?: Tier
|
||||
role?: Role
|
||||
clone?: string
|
||||
baseBranch?: string
|
||||
@@ -59,6 +64,7 @@ type Outcome =
|
||||
| 'tests-timeout'
|
||||
| 'tests-error'
|
||||
| 'verify-error'
|
||||
| 'verdict-unreadable'
|
||||
| 'test-agent'
|
||||
| 'review-agent'
|
||||
| 'no-clone'
|
||||
@@ -123,7 +129,7 @@ The model-router mod picks each subagent's model and verifies dev work before it
|
||||
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
|
||||
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
|
||||
- Review and test agents keep the model you name (Opus when you name none).
|
||||
- When a dev agent below Opus finishes a round, the router verifies its clone's diff on Opus and posts a "[model-router]" verdict. Wait for that verdict before opening or advancing the PR.
|
||||
- Every dev hand-back gets one "[model-router] Tn" line at once. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on.
|
||||
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. Do not fix rejected work yourself.`
|
||||
|
||||
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
||||
@@ -369,10 +375,7 @@ async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, c
|
||||
const verifier = verifierFor(rec.tier)
|
||||
entry.verifier = verifier
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
|
||||
const v = await $.model.complete({
|
||||
model: verifier,
|
||||
system: VERIFIER_SYSTEM,
|
||||
prompt: [
|
||||
const question = [
|
||||
`## Task\n${clip(rec.prompt, 12_000)}`,
|
||||
rec.lastIssues?.length
|
||||
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
|
||||
@@ -381,17 +384,46 @@ async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, c
|
||||
cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n\n'),
|
||||
maxTokens: 2000,
|
||||
timeoutMs: 180_000,
|
||||
})
|
||||
.join('\n\n')
|
||||
const v = await $.model.complete({ model: verifier, system: VERIFIER_SYSTEM, prompt: question, maxTokens: 2000, timeoutMs: 180_000 })
|
||||
await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices)
|
||||
if (!v.isAnswered) {
|
||||
await log($, entry)
|
||||
await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`)
|
||||
return
|
||||
}
|
||||
const verdict = parseVerdict(v.text)
|
||||
let reading = parseVerdict(v.text)
|
||||
// the first reply is where the verifier judged the code: quote that one
|
||||
const firstPreview = v.text.slice(0, 300)
|
||||
if (!reading.isReadable) {
|
||||
// Ask once more, for the JSON alone. A reply that still cannot be read is
|
||||
// no verdict on the code: hand it back, never count it as a FAIL.
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} verdict unreadable, asking ${verifier} again`)
|
||||
const again = await $.model.complete({
|
||||
model: verifier,
|
||||
system: VERIFIER_SYSTEM,
|
||||
prompt:
|
||||
`${question}\n\n## Your previous reply could not be read\n${clip(v.text, 4000)}\n\n` +
|
||||
'Reply again with the JSON object only: no code fence, no text before or after it.',
|
||||
maxTokens: 2000,
|
||||
timeoutMs: 180_000,
|
||||
})
|
||||
await chargeOverhead($, rec.taskId, verifier, again.usage, cfg.prices)
|
||||
if (again.isAnswered) reading = parseVerdict(again.text)
|
||||
}
|
||||
if (!reading.isReadable) {
|
||||
entry.outcome = 'verdict-unreadable'
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} verdict unreadable, handed back`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: NOT VERIFIED - verdict unreadable. The verifier (${verifier}) did not give a readable PASS or FAIL, also when asked again. ` +
|
||||
`This says nothing about the code: do not resume or rewrite the agent for it. Review the change yourself.\n` +
|
||||
`Start of its reply: ${firstPreview}`,
|
||||
)
|
||||
return
|
||||
}
|
||||
const verdict = reading
|
||||
const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)'
|
||||
|
||||
// 4. Remember where this round ended.
|
||||
@@ -474,14 +506,21 @@ async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, out
|
||||
reason: outcome,
|
||||
outcome,
|
||||
})
|
||||
if (rec.role === 'dev' && outcome === 'no-clone')
|
||||
await tell($, `[model-router] ${rec.taskId} (agent ${agentId}): NOT VERIFIED - the brief names no clone path.`)
|
||||
}
|
||||
|
||||
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, cfg: Settings) {
|
||||
const skip = skipReason(rec, ended)
|
||||
if (skip) await skipRound($, agentId, rec, skip)
|
||||
else await verifyRound($, agentId, rec, cfg)
|
||||
/**
|
||||
* One finished run of a tracked agent. Decides on the model that actually ran,
|
||||
* says at once what happens next for a dev, then verifies or logs the skip.
|
||||
*/
|
||||
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, reported: string | undefined, cfg: Settings) {
|
||||
const routed = rec.routed ?? rec.tier
|
||||
const ran = ranTier(reported, routed)
|
||||
const round: AgentRec = { ...rec, routed, tier: ran.tier }
|
||||
const skip = skipReason(round, ended)
|
||||
const line = handBackLine({ taskId: rec.taskId, role: rec.role, ran: ran.tier, routed, skip, ended })
|
||||
if (line) await tell($, line)
|
||||
if (skip) await skipRound($, agentId, round, skip)
|
||||
else await verifyRound($, agentId, round, cfg)
|
||||
}
|
||||
|
||||
// ---------- the hooks ----------
|
||||
@@ -585,6 +624,7 @@ export const register: Register = (on, options) => {
|
||||
await putIn<AgentRec>($, S.agents, ran.agentId, {
|
||||
taskId,
|
||||
tier,
|
||||
routed: tier,
|
||||
role: brief.role,
|
||||
clone: brief.clone,
|
||||
baseBranch: brief.baseBranch,
|
||||
@@ -629,7 +669,8 @@ export const register: Register = (on, options) => {
|
||||
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
||||
if (!rec) return out
|
||||
// Verification takes minutes: run it after this event has gone on.
|
||||
$.clock.after(0, () => void runRound($, agentId, rec, e.reason, cfg))
|
||||
const reported = e.usage?.model
|
||||
$.clock.after(0, () => void runRound($, agentId, rec, e.reason, reported, cfg))
|
||||
return out
|
||||
})
|
||||
|
||||
|
||||
+101
-12
@@ -182,20 +182,73 @@ export const onFail = (fails: number, writer: Tier, maxFails: number, allowFable
|
||||
|
||||
export type Verdict = { pass: boolean; issues: string[]; brief: string }
|
||||
|
||||
/** Reads the verifier's reply; anything unreadable counts as a failure. */
|
||||
export const parseVerdict = (text: string): Verdict => {
|
||||
const match = text.match(/\{[\s\S]*\}/)
|
||||
if (match) {
|
||||
try {
|
||||
const v = JSON.parse(match[0]) as { verdict?: string; issues?: unknown; rewrite_brief?: unknown }
|
||||
const issues = Array.isArray(v.issues) ? v.issues.map(String) : []
|
||||
const pass = String(v.verdict ?? '').toUpperCase() === 'PASS'
|
||||
return { pass, issues, brief: typeof v.rewrite_brief === 'string' ? v.rewrite_brief : '' }
|
||||
} catch {
|
||||
// fall through
|
||||
/** A verifier reply: a verdict that could be read, or the reason it could not. */
|
||||
export type Reading = ({ isReadable: true } & Verdict) | { isReadable: false; preview: string }
|
||||
|
||||
/** A JSON object with a PASS or FAIL verdict, or undefined. */
|
||||
const asVerdict = (json: string): Verdict | undefined => {
|
||||
let v: unknown
|
||||
try {
|
||||
v = JSON.parse(json)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
if (!v || typeof v !== 'object' || Array.isArray(v)) return undefined
|
||||
const o = v as { verdict?: unknown; issues?: unknown; rewrite_brief?: unknown }
|
||||
const word = typeof o.verdict === 'string' ? o.verdict.trim().toUpperCase() : ''
|
||||
if (word !== 'PASS' && word !== 'FAIL') return undefined
|
||||
return {
|
||||
pass: word === 'PASS',
|
||||
issues: Array.isArray(o.issues) ? o.issues.map(String) : [],
|
||||
brief: typeof o.rewrite_brief === 'string' ? o.rewrite_brief : '',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every balanced {...} in the text, in order of where it starts. Braces inside
|
||||
* JSON strings do not count, so `"a } b"` stays inside its object.
|
||||
*/
|
||||
export const balancedObjects = (text: string): string[] => {
|
||||
const found: string[] = []
|
||||
for (let start = text.indexOf('{'); start !== -1; start = text.indexOf('{', start + 1)) {
|
||||
let depth = 0
|
||||
let isInString = false
|
||||
for (let i = start; i < text.length; i++) {
|
||||
const ch = text[i]
|
||||
if (isInString) {
|
||||
if (ch === '\\') i++
|
||||
else if (ch === '"') isInString = false
|
||||
continue
|
||||
}
|
||||
if (ch === '"') isInString = true
|
||||
else if (ch === '{') depth++
|
||||
else if (ch === '}' && --depth === 0) {
|
||||
found.push(text.slice(start, i + 1))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return { pass: false, issues: ['Verifier reply was not valid JSON: ' + text.slice(0, 300)], brief: '' }
|
||||
return found
|
||||
}
|
||||
|
||||
const FENCE = /```[ \t]*(?:json)?[ \t]*\r?\n([\s\S]*?)```/gi
|
||||
|
||||
/**
|
||||
* Reads the verifier's reply, tolerating prose before and after: a fenced
|
||||
* ```json block first, then the first balanced {...} that parses and has a
|
||||
* PASS or FAIL verdict. Anything else is unreadable, which is not a FAIL.
|
||||
*/
|
||||
export const parseVerdict = (text: string): Reading => {
|
||||
for (const m of text.matchAll(FENCE)) {
|
||||
const body = m[1] ?? ''
|
||||
const v = asVerdict(body.trim()) ?? balancedObjects(body).map(asVerdict).find(Boolean)
|
||||
if (v) return { isReadable: true, ...v }
|
||||
}
|
||||
for (const candidate of balancedObjects(text)) {
|
||||
const v = asVerdict(candidate)
|
||||
if (v) return { isReadable: true, ...v }
|
||||
}
|
||||
return { isReadable: false, preview: text.slice(0, 300) }
|
||||
}
|
||||
|
||||
export const REWRITE_MARK = /^\s*\[router-rewrite (T\d+)\]/
|
||||
@@ -209,3 +262,39 @@ export function validTimeout(raw: unknown): number {
|
||||
return Number.isFinite(n) && n >= 1 ? Math.round(n) : 1800
|
||||
}
|
||||
|
||||
|
||||
// ---------- the hand-back line (TOOL-12) ----------
|
||||
|
||||
const LABEL: Record<Tier, string> = { haiku: 'Haiku', sonnet: 'Sonnet', opus: 'Opus', fable: 'Fable' }
|
||||
|
||||
/**
|
||||
* The model a run actually used, from the model id its end reports; the
|
||||
* routing choice only when the run reported none.
|
||||
*/
|
||||
export const ranTier = (reportedModel: string | undefined, routed: Tier): { tier: Tier; isMismatch: boolean } => {
|
||||
const seen = asTier(reportedModel)
|
||||
return seen ? { tier: seen, isMismatch: seen !== routed } : { tier: routed, isMismatch: false }
|
||||
}
|
||||
|
||||
/**
|
||||
* The one line every dev hand-back gets at once, so silence never stands for
|
||||
* both "still verifying" and "will never verify". Undefined: no line (not a dev).
|
||||
*/
|
||||
export const handBackLine = (args: {
|
||||
taskId: string
|
||||
role?: Role
|
||||
ran: Tier
|
||||
routed: Tier
|
||||
skip?: Skip
|
||||
ended: string
|
||||
}): string | undefined => {
|
||||
const { taskId, role, ran, routed, skip, ended } = args
|
||||
if (role !== 'dev') return undefined
|
||||
const note = ran !== routed ? ` (routed to ${LABEL[routed]}, ran on ${LABEL[ran]})` : ''
|
||||
const head = `[model-router] ${taskId}`
|
||||
if (!skip) return `${head} verifying on ${LABEL[verifierFor(ran)]}...${note}`
|
||||
if (skip === 'opus-unverified') return `${head} no verification: dev ran on ${LABEL[ran]}. Go straight to review.${note}`
|
||||
if (skip === 'agent-ended') return `${head} no verification: the dev run ended without an answer (${ended}).${note}`
|
||||
if (skip === 'no-clone') return `${head} no verification: the brief names no clone path.${note}`
|
||||
return undefined
|
||||
}
|
||||
|
||||
+203
-9
@@ -11,6 +11,8 @@ import {
|
||||
skipReason,
|
||||
validTimeout,
|
||||
verifierFor,
|
||||
handBackLine,
|
||||
ranTier,
|
||||
} from '../hooks/router'
|
||||
|
||||
const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev'
|
||||
@@ -129,11 +131,80 @@ test('C5: first fail resumes, second starts a stronger agent, then hand back', a
|
||||
expect(rewriteTier('haiku', false)).toBe('sonnet')
|
||||
})
|
||||
|
||||
test('verdict parsing fails closed', async () => {
|
||||
expect(parseVerdict('{"verdict":"PASS","issues":[]}').pass).toBe(true)
|
||||
expect(parseVerdict('text {"verdict":"FAIL","issues":["a.py:3"]}').issues).toEqual(['a.py:3'])
|
||||
expect(parseVerdict('looks fine').pass).toBe(false)
|
||||
expect(parseVerdict('verdict: pass').pass).toBe(false)
|
||||
// ---------- reading the verdict (TOOL-02) ----------
|
||||
|
||||
// The SAFETY-129 reply shape: a fenced block, then prose that contains a `}`.
|
||||
const SAFETY_129 =
|
||||
'Checked the diff against the brief.\n\n```json\n{"verdict": "PASS", "issues": [], "rewrite_brief": ""}\n```\n\n' +
|
||||
'Notes: the guard keeps `if (stale) { hold() }` as before, and the new test covers it.'
|
||||
|
||||
test('TOOL-02: fenced JSON followed by prose with a } reads as PASS', async () => {
|
||||
expect(parseVerdict(SAFETY_129)).toEqual({ isReadable: true, pass: true, issues: [], brief: '' })
|
||||
})
|
||||
|
||||
test('TOOL-02: the fenced answer wins over an earlier object in the prose', async () => {
|
||||
const r = parseVerdict('Last round said {"verdict":"FAIL","issues":["x"]}. Fixed now:\n```json\n{"verdict":"PASS","issues":[]}\n```')
|
||||
expect(r).toMatchObject({ isReadable: true, pass: true })
|
||||
})
|
||||
|
||||
test('TOOL-02: bare JSON reads, with or without prose around it', async () => {
|
||||
expect(parseVerdict('{"verdict":"PASS","issues":[]}')).toMatchObject({ isReadable: true, pass: true })
|
||||
expect(parseVerdict('Verdict below. {"verdict": "pass"} That is all {really}.')).toMatchObject({ isReadable: true, pass: true })
|
||||
})
|
||||
|
||||
test('TOOL-02: FAIL keeps its issues and brief', async () => {
|
||||
const r = parseVerdict('```json\n{"verdict":"FAIL","issues":["loop.py:40 no clamp"],"rewrite_brief":"clamp it"}\n```')
|
||||
expect(r).toEqual({ isReadable: true, pass: false, issues: ['loop.py:40 no clamp'], brief: 'clamp it' })
|
||||
})
|
||||
|
||||
test('TOOL-02: braces inside strings do not end the object', async () => {
|
||||
const r = parseVerdict('{"verdict":"FAIL","issues":["handler } has a stray brace","quote \\" and }"]} trailing {x}')
|
||||
expect(r).toMatchObject({ isReadable: true, pass: false })
|
||||
if (r.isReadable) expect(r.issues).toEqual(['handler } has a stray brace', 'quote " and }'])
|
||||
})
|
||||
|
||||
test('TOOL-02: the first object with a verdict wins; other objects are skipped', async () => {
|
||||
const r = parseVerdict('Config seen: {"limit": 2000}. Verdict: {"verdict":"FAIL","issues":["x"]} and later {"verdict":"PASS"}')
|
||||
expect(r).toMatchObject({ isReadable: true, pass: false })
|
||||
})
|
||||
|
||||
test('TOOL-02: a fence without a verdict falls through to the text', async () => {
|
||||
const r = parseVerdict('```json\n{"limit": 2000}\n```\nResult: {"verdict":"PASS"}')
|
||||
expect(r).toMatchObject({ isReadable: true, pass: true })
|
||||
})
|
||||
|
||||
test('TOOL-02: garbage, prose and odd verdicts are unreadable, never a FAIL', async () => {
|
||||
for (const t of ['looks fine', 'verdict: pass', '{"verdict":"MAYBE"}', '{"issues":[]}', '{broken json', '']) {
|
||||
const r = parseVerdict(t)
|
||||
expect(r.isReadable).toBe(false)
|
||||
}
|
||||
const long = parseVerdict('x'.repeat(500))
|
||||
expect(long.isReadable ? '' : long.preview).toBe('x'.repeat(300))
|
||||
})
|
||||
|
||||
// ---------- the hand-back line (TOOL-12) ----------
|
||||
|
||||
test('TOOL-12: the model that ran wins over the routing choice', async () => {
|
||||
expect(ranTier('claude-opus-5-5', 'sonnet')).toEqual({ tier: 'opus', isMismatch: true })
|
||||
expect(ranTier('claude-sonnet-5-5', 'opus')).toEqual({ tier: 'sonnet', isMismatch: true })
|
||||
expect(ranTier('claude-sonnet-5-5', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
||||
expect(ranTier(undefined, 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
||||
expect(ranTier('some-other-model', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
||||
})
|
||||
|
||||
test('TOOL-12: the exact lines', async () => {
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe('[model-router] T7 verifying on Opus...')
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'opus', skip: 'opus-unverified', ended: 'answer' })).toBe(
|
||||
'[model-router] T7 no verification: dev ran on Opus. Go straight to review.',
|
||||
)
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'sonnet', skip: 'opus-unverified', ended: 'answer' })).toBe(
|
||||
'[model-router] T7 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)',
|
||||
)
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'agent-ended', ended: 'error' })).toContain('ended without an answer (error)')
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'no-clone', ended: 'answer' })).toContain('no clone path')
|
||||
expect(handBackLine({ taskId: 'T7', role: 'review', ran: 'opus', routed: 'opus', skip: 'review-agent', ended: 'answer' })).toBe(undefined)
|
||||
expect(handBackLine({ taskId: 'T7', role: 'test', ran: 'opus', routed: 'opus', skip: 'test-agent', ended: 'answer' })).toBe(undefined)
|
||||
expect(handBackLine({ taskId: 'T7', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe(undefined)
|
||||
})
|
||||
|
||||
// ---------- the hooks, with the engine mocked beneath ----------
|
||||
@@ -143,6 +214,10 @@ type World = {
|
||||
verifier: string[]
|
||||
verdicts: string[]
|
||||
told: string[]
|
||||
/** TOOL-12 hand-back lines, kept apart so `told` holds the verdicts. */
|
||||
handBacks: string[]
|
||||
/** Every message in the order it was sent. */
|
||||
all: string[]
|
||||
diffs: string[]
|
||||
diffBases: string[]
|
||||
testCwd: string[]
|
||||
@@ -206,16 +281,21 @@ function world(on: On, w: World) {
|
||||
})
|
||||
on('turn.complete', async (_$, e) => ({ text: e.answer }))
|
||||
on('prompt.submit', async (_$, e) => {
|
||||
w.told.push(e.text)
|
||||
w.all.push(e.text)
|
||||
if (/^\[model-router\] T\d+ (verifying on|no verification:)/.test(e.text)) w.handBacks.push(e.text)
|
||||
else w.told.push(e.text)
|
||||
return { text: e.text }
|
||||
})
|
||||
return clock
|
||||
}
|
||||
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], handBacks: [], all: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
|
||||
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string) {
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason: 'answer' })
|
||||
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string, ranModel?: string, reason = 'answer') {
|
||||
const usage = ranModel
|
||||
? { usage: { model: ranModel, input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } }
|
||||
: {}
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason, ...usage })
|
||||
await clock.advance(1)
|
||||
}
|
||||
|
||||
@@ -449,3 +529,117 @@ test('the timeout line on stdout (a test printing it) is not a fired timeout', {
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
})
|
||||
|
||||
// ---------- TOOL-02 end to end ----------
|
||||
|
||||
test('TOOL-02: the SAFETY-129 reply passes the round, no rewrite', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.verdicts = [SAFETY_129]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
expect(w.told[0]).toContain('PASS from verifier opus')
|
||||
})
|
||||
|
||||
test('TOOL-02: an unreadable reply is asked again once, and a readable answer then counts', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.verdicts = ['I think it is fine overall.', '{"verdict":"FAIL","issues":["x"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus', 'opus'])
|
||||
expect(w.told[0]).toContain('Resume agent')
|
||||
})
|
||||
|
||||
test('TOOL-02: unreadable twice is handed back, no fail counted, no rewrite', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff', 'diff 2']
|
||||
w.verdicts = ['Looks good to me.', 'Still prose, sorry.', '{"verdict":"FAIL","issues":["y"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus', 'opus'])
|
||||
expect(w.told[0]).toContain('NOT VERIFIED - verdict unreadable')
|
||||
expect(w.told[0]).toContain('Start of its reply: Looks good to me.')
|
||||
expect(w.told[0]).not.toContain('[router-rewrite')
|
||||
expect(w.told[0]).not.toContain('Resume agent')
|
||||
// the next round's FAIL is the first one counted: it resumes, it does not rewrite
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.told[1]).toContain('Resume agent')
|
||||
expect(w.told[1]).not.toContain('[router-rewrite')
|
||||
})
|
||||
|
||||
// ---------- TOOL-12 end to end ----------
|
||||
|
||||
const opusBrief = () =>
|
||||
`SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/\nBase branch: release/1.0.`
|
||||
|
||||
test('TOOL-12 acceptance 1: an Opus-routed dev gets the skip line at hand-back, and no verdict later', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('opus')
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
await clock.advance(60_000)
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('TOOL-12 acceptance 2: a Sonnet-routed dev gets "verifying", then the verdict', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('sonnet')
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus...')
|
||||
expect(w.all[1]).toContain('PASS from verifier opus')
|
||||
expect(w.all.length).toBe(2)
|
||||
})
|
||||
|
||||
test('TOOL-12 acceptance 3a: routed Sonnet but ran on Opus is reported and handled as Opus', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('TOOL-12 acceptance 3b: routed Opus but ran on Sonnet is verified, and labelled Sonnet', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
expect(w.told[0]).toContain('(agent agent-1, sonnet, round 1)')
|
||||
// the resumed round still reports against the original routing choice
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
|
||||
})
|
||||
|
||||
test('TOOL-12: a dev run that ends with an error says so, no verification', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5', 'error')
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: the dev run ended without an answer (error).'])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('TOOL-12: review and test agents get no hand-back line', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: `Review. Clone: ${REVIEW}/`, description: 'review', model: 'opus' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.all).toEqual([])
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user