Files
model-router/tests/router.test.ts
T
adminandClaude Opus 5.5 d5c762b339 Fix #5: rerun reported files alone, and on base, before judging
The full suite run can time out files that pass alone (an overloaded
shared host, a suite-wide deadline in the runner): MAINT-28 T307 was
judged on 30 TIMEOUTs, T325 on a file that passes alone in 230 s.

- With testFileCommand set, every file the full run reports as TIMEOUT
  or FAIL (testIssuePattern, default ^(TIMEOUT|FAIL)\s+(\S+); lines are
  read as the run streams) is rerun alone in the clone,
  testRerunJobs at a time, testFileTimeoutSec each, at most
  testRerunMax files.
- A file that still fails or times out alone runs again on the base,
  in a git worktree next to the clone (<clone>-verifier-base), which is
  removed and pruned afterwards. Without a worktree, base stays "not
  run".
- The verifier gets a table (file, full run, alone on head, on base,
  meaning) and is told: passes alone does not count; fails on base too
  is not caused by the change; name every file it counts.
- Only file names of letters, digits and _ . - / reach the shell; no
  ".." parts, no leading "-".
- 12 new tests; each of the 23 new rules was also broken on purpose and
  caught.

Version 0.4.9.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
2026-11-14 00:09:18 +00:00

1142 lines
55 KiB
TypeScript

import type { On } from 'claude-code'
import { expect, mock, test, type Engine } from 'claude-code/testing'
import {
decide,
heuristic,
onFail,
parseBrief,
parseVerdict,
rewriteTier,
skipReason,
validTimeout,
verifierFor,
handBackLine,
isSafeFile,
issueRegex,
parseIssueLine,
rerunVerdict,
fileCommand,
ranTier,
turnEndKind,
} from '../hooks/router'
const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev'
const REVIEW = '/home/homelab/projects/goodwe/clones/SAFETY-95-review'
const TEST = '/home/homelab/projects/goodwe/clones/TEST-23-test'
// ---------- the brief (C2) ----------
test('brief: clone path, ticket, role and base branch are read', async () => {
const b = parseBrief(`Work in ${DEV}/ only. Base branch: release/1.0.\nExpected: ...`)
expect(b).toEqual({ clone: DEV, ticket: 'SAFETY-95', role: 'dev', baseBranch: 'release/1.0' })
expect(parseBrief(`Clone: \`${TEST}\``).role).toBe('test')
expect(parseBrief('PR against `release/1.1`, no clone given').baseBranch).toBe('release/1.1')
expect(parseBrief('just look around').clone).toBe(undefined)
})
test('fix 1: tickets with extra parts parse, the role is the last part', async () => {
const p = '/home/homelab/projects/goodwe/clones/OPT-11-P4-dev'
expect(parseBrief(`Work in ${p}/ only.`)).toMatchObject({ clone: p, ticket: 'OPT-11-P4', role: 'dev' })
expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').ticket).toBe('OPT-11-P4-R2')
expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').role).toBe('test')
})
test('fix 2: the Clone: line decides, whatever the brief mentions first', async () => {
const brief = `Verify the work of ${DEV}/ on the rig.\nClone: ${TEST}/\nBase branch: release/1.0.`
expect(parseBrief(brief)).toMatchObject({ clone: TEST, role: 'test', baseBranch: 'release/1.0' })
// an explicit role wins over the path
expect(parseBrief(`Clone: \`/x/clones/OPT-11-P4\` (Role: review)`)).toMatchObject({ clone: '/x/clones/OPT-11-P4', role: 'review' })
// (Role: ...) wins over the role in the path
expect(parseBrief(`Clone: ${DEV}/ (Role: review)`).role).toBe('review')
// a list item or quote marker in front of the line is fine
expect(parseBrief(`- Clone: ${DEV}`).role).toBe('dev')
// "Clone:" inside a sentence is not the line, even at a line's end
expect(parseBrief(`Results are in the dev agent's Clone: ${DEV}\nRun the rig in ${TEST}/`).isAmbiguous).toBe(true)
})
test('fix 2: several clones and no Clone: line gives no role, and routes to opus', async () => {
const b = parseBrief(`Compare ${DEV}/ with ${TEST}/.`)
expect(b.role).toBe(undefined)
expect(b.isAmbiguous).toBe(true)
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'haiku' })).toMatchObject({ kind: 'fixed', tier: 'opus' })
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'sonnet' })).toMatchObject({ tier: 'opus' })
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'fable' })).toMatchObject({ tier: 'fable' })
// the same path twice is one clone, not ambiguous
expect(parseBrief(`Work in ${DEV}/. Commit in ${DEV}.`).role).toBe('dev')
})
// ---------- who picks the writer (C4) ----------
test('C4: review and test agents keep the model they are given', async () => {
for (const role of ['review', 'test'] as const)
for (const named of ['haiku', 'sonnet', 'opus'] as const)
expect(decide({ role, named, prompt: 'x', respectExplicit: false })).toEqual({
kind: 'fixed',
tier: named,
reason: `named model kept for ${role} agent`,
})
})
test('C4: unnamed review and test agents run on opus', async () => {
expect(decide({ role: 'test', prompt: 'run the rig', respectExplicit: false })).toMatchObject({ tier: 'opus' })
expect(decide({ role: 'review', prompt: 'review', respectExplicit: false })).toMatchObject({ tier: 'opus' })
})
test('C4: dev agents are routed even when opus is named, and that is logged', async () => {
const d = decide({ role: 'dev', named: 'opus', prompt: 'Expected: clamp to 2000 W', respectExplicit: true })
expect(d.kind).toBe('classify')
expect(d.opusRequested).toBe(true)
expect(decide({ role: 'dev', named: 'sonnet', prompt: 'x', respectExplicit: true }).opusRequested).toBe(false)
})
test('C4: respectExplicit only applies to agents without a role', async () => {
expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: true })).toMatchObject({ kind: 'fixed', tier: 'haiku' })
expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: false }).kind).toBe('classify')
})
test('C4: open design goes to opus, a decided brief is left to the classifier', async () => {
expect(heuristic(undefined, 'MAINT-13: decide how the staleness guard should recover', 'dev')?.tier).toBe('opus')
expect(heuristic(undefined, 'In the asyncio control loop, clamp the integrator; Expected: test_clamp passes', 'dev')).toBe(undefined)
})
test('C4: narrowed light patterns: "docs" or "run tests" in a brief never mean haiku', async () => {
expect(heuristic(undefined, 'Update the docs and run the tests after changing the setpoint logic')).toBe(undefined)
expect(heuristic(undefined, 'Find where the MQTT discovery payload is built')?.tier).toBe('haiku')
// a dev brief never takes the light path, however light it reads
expect(heuristic(undefined, 'Find where the MQTT discovery payload is built', 'dev')).toBe(undefined)
expect(heuristic('Explore', 'anything', 'dev')).toBe(undefined)
})
// ---------- what gets verified (C1, C5) ----------
test('only finished dev rounds below opus with a clone are verified', async () => {
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'answer')).toBe(undefined)
expect(skipReason({ role: 'dev', clone: DEV, tier: 'haiku' }, 'answer')).toBe(undefined)
expect(skipReason({ role: 'dev', clone: DEV, tier: 'opus' }, 'answer')).toBe('opus-unverified')
expect(skipReason({ role: 'dev', tier: 'sonnet' }, 'answer')).toBe('no-clone')
expect(skipReason({ role: 'test', clone: TEST, tier: 'sonnet' }, 'answer')).toBe('test-agent')
expect(skipReason({ role: 'review', clone: REVIEW, tier: 'sonnet' }, 'answer')).toBe('review-agent')
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'error')).toBe('agent-ended')
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'aborted')).toBe('agent-ended')
})
test('the verifier is opus below opus, and never the writer', async () => {
expect(verifierFor('haiku')).toBe('opus')
expect(verifierFor('sonnet')).toBe('opus')
for (const w of ['haiku', 'sonnet', 'opus', 'fable'] as const) expect(verifierFor(w)).not.toBe(w)
})
test('C5: first fail resumes, second starts a stronger agent, then hand back', async () => {
expect(onFail(1, 'sonnet', 2, false)).toEqual({ kind: 'resume' })
expect(onFail(2, 'sonnet', 2, false)).toEqual({ kind: 'new-agent', tier: 'opus' })
expect(onFail(3, 'opus', 2, false)).toEqual({ kind: 'hand-back' })
expect(onFail(2, 'opus', 2, false)).toEqual({ kind: 'hand-back' }) // nothing above opus
expect(onFail(2, 'opus', 2, true)).toEqual({ kind: 'new-agent', tier: 'fable' })
expect(onFail(1, 'sonnet', 0, false)).toEqual({ kind: 'hand-back' })
expect(rewriteTier('haiku', false)).toBe('sonnet')
})
// ---------- reading the verdict (TOOL-02) ----------
// The SAFETY-129 reply shape: a fenced block, then prose that contains a `}`.
const SAFETY_129 =
'Checked the diff against the brief.\n\n```json\n{"verdict": "PASS", "issues": [], "rewrite_brief": ""}\n```\n\n' +
'Notes: the guard keeps `if (stale) { hold() }` as before, and the new test covers it.'
test('TOOL-02: fenced JSON followed by prose with a } reads as PASS', async () => {
expect(parseVerdict(SAFETY_129)).toEqual({ isReadable: true, pass: true, issues: [], brief: '' })
})
test('TOOL-02: the fenced answer wins over an earlier object in the prose', async () => {
const r = parseVerdict('Last round said {"verdict":"FAIL","issues":["x"]}. Fixed now:\n```json\n{"verdict":"PASS","issues":[]}\n```')
expect(r).toMatchObject({ isReadable: true, pass: true })
})
test('TOOL-02: bare JSON reads, with or without prose around it', async () => {
expect(parseVerdict('{"verdict":"PASS","issues":[]}')).toMatchObject({ isReadable: true, pass: true })
expect(parseVerdict('Verdict below. {"verdict": "pass"} That is all {really}.')).toMatchObject({ isReadable: true, pass: true })
})
test('TOOL-02: FAIL keeps its issues and brief', async () => {
const r = parseVerdict('```json\n{"verdict":"FAIL","issues":["loop.py:40 no clamp"],"rewrite_brief":"clamp it"}\n```')
expect(r).toEqual({ isReadable: true, pass: false, issues: ['loop.py:40 no clamp'], brief: 'clamp it' })
})
test('TOOL-02: braces inside strings do not end the object', async () => {
const r = parseVerdict('{"verdict":"FAIL","issues":["handler } has a stray brace","quote \\" and }"]} trailing {x}')
expect(r).toMatchObject({ isReadable: true, pass: false })
if (r.isReadable) expect(r.issues).toEqual(['handler } has a stray brace', 'quote " and }'])
})
test('TOOL-02: the first object with a verdict wins; other objects are skipped', async () => {
const r = parseVerdict('Config seen: {"limit": 2000}. Verdict: {"verdict":"FAIL","issues":["x"]} and later {"verdict":"PASS"}')
expect(r).toMatchObject({ isReadable: true, pass: false })
})
test('TOOL-02: a fence without a verdict falls through to the text', async () => {
const r = parseVerdict('```json\n{"limit": 2000}\n```\nResult: {"verdict":"PASS"}')
expect(r).toMatchObject({ isReadable: true, pass: true })
})
test('TOOL-02: garbage, prose and odd verdicts are unreadable, never a FAIL', async () => {
for (const t of ['looks fine', 'verdict: pass', '{"verdict":"MAYBE"}', '{"issues":[]}', '{broken json', '']) {
const r = parseVerdict(t)
expect(r.isReadable).toBe(false)
}
const long = parseVerdict('x'.repeat(500))
expect(long.isReadable ? '' : long.preview).toBe('x'.repeat(300))
})
// ---------- the hand-back line (TOOL-12) ----------
test('TOOL-12: the model that ran wins over the routing choice', async () => {
expect(ranTier('claude-opus-5-5', 'sonnet')).toEqual({ tier: 'opus', isMismatch: true })
expect(ranTier('claude-sonnet-5-5', 'opus')).toEqual({ tier: 'sonnet', isMismatch: true })
expect(ranTier('claude-sonnet-5-5', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
expect(ranTier(undefined, 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
expect(ranTier('some-other-model', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
})
test('TOOL-12: the exact lines', async () => {
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe('[model-router] T7 verifying on Opus...')
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'opus', skip: 'opus-unverified', ended: 'answer' })).toBe(
'[model-router] T7 no verification: dev ran on Opus. Go straight to review.',
)
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'sonnet', skip: 'opus-unverified', ended: 'answer' })).toBe(
'[model-router] T7 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)',
)
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'agent-ended', ended: 'error' })).toContain('ended without an answer (error)')
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'no-clone', ended: 'answer' })).toContain('no clone path')
expect(handBackLine({ taskId: 'T7', role: 'review', ran: 'opus', routed: 'opus', skip: 'review-agent', ended: 'answer' })).toBe(undefined)
expect(handBackLine({ taskId: 'T7', role: 'test', ran: 'opus', routed: 'opus', skip: 'test-agent', ended: 'answer' })).toBe(undefined)
expect(handBackLine({ taskId: 'T7', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe(undefined)
})
// ---------- the hooks, with the engine mocked beneath ----------
type World = {
spawned: { model?: string; prompt: string }[]
verifier: string[]
verdicts: string[]
told: string[]
/** TOOL-12 hand-back lines, kept apart so `told` holds the verdicts. */
handBacks: string[]
/** Every message in the order it was sent. */
all: string[]
/** Resumes that reached the spawn hook. */
resumed: { agentId: string; prompt: string; model?: string }[]
/** How many coming new spawns the engine refuses. */
denySpawns: number
/** The model the engine resolves for new spawns, whatever was asked. */
engineModel?: string
verifierPrompts: string[]
verifierSystems: string[]
/** SendMessage calls fail (nothing delivered). */
sendFails?: boolean
/** Pieces the full test run prints to stdout. */
fullRunOutput: string[]
/** How a file run alone ends, keyed `head:<file>` or `base:<file>`; absent: pass. */
fileOutcomes: Record<string, 'pass' | 'fail' | 'timeout'>
/** Files run alone, as `head:<file>` or `base:<file>`. */
fileRuns: string[]
/** git worktree commands run. */
worktree: string[]
worktreeFails?: boolean
running: number
maxParallel: number
/** Each agent's loop status as $.agent.list() reports it; absent means completed. */
status: Record<string, string>
diffs: string[]
diffBases: string[]
testCwd: string[]
testArgv: string[][]
/** How the next test run ends: exit code (124 = timed out), or a throw. */
testEnds: (number | 'throw')[]
/** Whether `timeout` fired on each test run (it then says so on stderr). */
testFired: boolean[]
/** The timeout line arrives in two stderr pieces. */
testSplitLine?: boolean
/** A test itself prints the timeout line, on stdout. */
testLineOnStdout?: boolean
snap: number
}
const USAGE = { input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }
const ok = (stdout: string, exitCode = 0) => ({
value: { exitCode, stdout, stderr: '', isStdoutTruncated: false, isStderrTruncated: false },
})
function world(on: On, w: World) {
mock.store(on)
const clock = mock.clock(on)
on('ui.status', async () => ({ value: undefined }))
on('ui.toast', async () => ({ value: undefined }))
on('model.complete', async (_$, e) => {
if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } }
w.verifier.push(e.model)
w.verifierPrompts.push(e.prompt)
w.verifierSystems.push(e.system ?? '')
return { value: { isAnswered: true as const, text: w.verdicts.shift() ?? '{"verdict":"PASS"}', usage: USAGE } }
})
on('process.run', async (_$, e) => {
const a = e.argv.join(' ')
const cwd = String(e.init?.cwd ?? '')
if (cwd !== DEV) return ok('', 128) // not a git repo: the session folder
if (a === 'git merge-base origin/release/1.0 HEAD') return ok('mergebase\n')
if (a === 'git stash create') return ok(`snap${++w.snap}\n`)
if (a.startsWith('git diff ')) {
w.diffBases.push(e.argv[2] ?? '')
return ok(w.diffs.shift() ?? '')
}
if (a.includes(' worktree ')) {
w.worktree.push(a)
if (a.includes(' worktree add ') && w.worktreeFails) return ok('', 128)
}
return ok('')
})
on('process.spawn', async function* (_$, e) {
const cmd = e.argv[e.argv.length - 1] ?? ''
if (cmd.startsWith('runone ')) {
// one file alone: `runone <file>`, in the clone or in the base worktree
const file = cmd.slice('runone '.length)
const where = String(e.cwd ?? '').endsWith('-verifier-base') ? 'base' : 'head'
w.fileRuns.push(`${where}:${file}`)
w.running++
w.maxParallel = Math.max(w.maxParallel, w.running)
for (let i = 0; i < 5; i++) await Promise.resolve() // let other reruns start
w.running--
const outcome = w.fileOutcomes[`${where}:${file}`] ?? 'pass'
if (outcome === 'timeout') yield { stream: 'stderr' as const, text: "timeout: sending signal TERM to command 'sh'\n" }
return { value: { code: outcome === 'pass' ? 0 : outcome === 'timeout' ? 124 : 1, signal: null } }
}
w.testCwd.push(String(e.cwd ?? ''))
w.testArgv.push([...e.argv])
const end = w.testEnds.shift() ?? 0
const fired = w.testFired.shift() ?? false
if (end === 'throw') throw new Error('docker: not found')
yield { stream: 'stdout' as const, text: end === 0 ? '67 passed\n' : 'test_modes_p4 ... still running\n' }
// the full run's report comes last, so a last line without a newline stays last
for (const piece of w.fullRunOutput) yield { stream: 'stdout' as const, text: piece }
if (fired) yield { stream: 'stderr' as const, text: "timeout: sending signal TERM to command 'sh'\n" }
if (w.testSplitLine) {
yield { stream: 'stderr' as const, text: 'timeout: sending sig' }
yield { stream: 'stderr' as const, text: "nal TERM to command 'sh'\n" }
}
if (w.testLineOnStdout) yield { stream: 'stdout' as const, text: 'assert "timeout: sending signal" in log\n' }
return { value: { code: end, signal: null } }
})
on('agent.spawn', async (_$, e) => {
if (e.resume) {
w.resumed.push({ agentId: e.resume.agentId, prompt: e.prompt, model: e.model })
return { model: 'claude-opus-5-5', agentId: e.resume.agentId }
}
if (w.denySpawns > 0) {
w.denySpawns--
return { deny: 'spawn failed' }
}
w.spawned.push({ model: e.model, prompt: e.prompt })
return { model: w.engineModel ?? e.model ?? 'inherit', agentId: `agent-${w.spawned.length}` }
})
on('tool.call', { tool: 'SendMessage' }, async () =>
(w.sendFails ? { result: 'No agent by that name is running', isError: true } : { result: { success: true } }) as never,
)
on('turn.complete', async (_$, e) => ({ text: e.answer }))
on('agent.list', async () => ({
value: w.spawned
.map((_, i) => `agent-${i + 1}`)
.filter(id => w.status[id] !== 'gone') // 'gone': the engine dropped it
.map(id => ({ id, description: '', type: 'general-purpose', status: (w.status[id] ?? 'completed') as never })),
}))
on('prompt.submit', async (_$, e) => {
w.all.push(e.text)
if (/^\[model-router\] T\d+ (verifying on|no verification:)/.test(e.text)) w.handBacks.push(e.text)
else w.told.push(e.text)
return { text: e.text }
})
return clock
}
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], handBacks: [], all: [], resumed: [], denySpawns: 0, verifierPrompts: [], verifierSystems: [], fullRunOutput: [], fileOutcomes: {}, fileRuns: [], worktree: [], running: 0, maxParallel: 0, status: {}, diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string, ranModel?: string, reason = 'answer') {
const usage = ranModel
? { usage: { model: ranModel, input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } }
: {}
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason, ...usage })
await clock.advance(1)
}
const devBrief = (extra = '') => `SAFETY-95: clamp the integrator. Work in ${DEV}/. Base branch: release/1.0.\nExpected: test_clamp passes.${extra}`
test('acceptance 1+2: a background dev agent is verified in its clone, each round on its own diff', { options: { testCommand: 'pytest -q' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff --git a/loop.py', 'diff --git a/loop.py (round 2)']
w.verdicts = ['{"verdict":"FAIL","issues":["loop.py:40 no clamp on negative"]}', '{"verdict":"PASS"}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp', model: 'opus' } as never)
expect(w.spawned[0]?.model).toBe('sonnet') // routed despite the named opus
await finish($, clock, r.agentId as string)
expect(w.diffBases[0]).toBe('mergebase') // first round: from the merge base, in the clone
expect(w.testCwd).toEqual([DEV])
expect(w.verifier).toEqual(['opus'])
expect(w.told[0]).toContain('FAIL')
// round 2: the resumed agent finishes again (same agentId)
await finish($, clock, r.agentId as string)
expect(w.diffBases[1]).toBe('snap1') // only what this round added
expect(w.told[1]).toContain('PASS')
})
test('acceptance 3: review and test agents keep a named opus; dev briefs below opus are verified by opus', async ($, on) => {
const w = fresh()
const clock = world(on, w)
await $.agent.spawn({ prompt: `Review PR. Clone ${REVIEW}/`, description: 'review', model: 'opus' } as never)
await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig', model: 'opus' } as never)
const t = await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig2' } as never)
expect(w.spawned.map(s => s.model)).toEqual(['opus', 'opus', 'opus'])
await finish($, clock, t.agentId as string)
expect(w.verifier).toEqual([]) // test agents are not verified
expect(w.told).toEqual([])
})
test('acceptance 4: the first FAIL asks to resume the same agent', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.told[0]).toContain(`Resume agent ${r.agentId} with SendMessage`)
expect(w.told[0]).not.toContain('[router-rewrite')
})
test('C5: the second FAIL starts a stronger agent in the same clone', async ($, on) => {
const w = fresh()
w.diffs = ['d1', 'd2']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
await finish($, clock, r.agentId as string)
const id = w.told[1]?.match(/\[router-rewrite (T\d+)\]/)?.[1]
expect(id).toBeDefined()
await $.agent.spawn({ prompt: `[router-rewrite ${id}]`, description: 'rewrite' } as never)
expect(w.spawned[1]?.model).toBe('opus')
expect(w.spawned[1]?.prompt).toContain(`Their work is in the clone ${DEV}: continue from it`)
expect(w.spawned[1]?.prompt).toContain('- y')
})
test('C2: a dev brief with no clone is never diffed in the session folder', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: 'Clamp the integrator in clones/ somewhere. Expected: ...', description: 'x' } as never)
await finish($, clock, r.agentId as string)
expect(w.diffBases).toEqual([])
expect(w.verifier).toEqual([])
})
test('fix 2 end to end: a test brief that mentions the dev clone first keeps its model and is not verified', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({
prompt: `Run the OPT-11 P4 rig test against the work in ${DEV}/.\nClone: /home/homelab/projects/goodwe/clones/OPT-11-P4-test/`,
description: 'rig',
model: 'opus',
} as never)
expect(w.spawned[0]?.model).toBe('opus')
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.diffBases).toEqual([])
})
test('fix 2 end to end: two clones and no Clone: line run on opus, unverified', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: `Compare ${DEV}/ with ${TEST}/ and report.`, description: 'cmp', model: 'haiku' } as never)
expect(w.spawned[0]?.model).toBe('opus')
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
})
test('a brief without a role is routed but never tracked or verified', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: 'Find where the MQTT discovery payload is built', description: 'look' } as never)
expect(w.spawned[0]?.model).toBe('haiku')
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.told).toEqual([])
})
// ---------- the test run ----------
test('tests stream with a real time limit, not the ten-minute command cap', { options: { testCommand: 'make verifier-tests', testTimeoutSec: 1800 } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.testArgv[0]).toEqual(['timeout', '--verbose', '--kill-after=30', '1800', 'sh', '-c', 'make verifier-tests'])
expect(w.testCwd).toEqual([DEV])
expect(w.verifier).toEqual(['opus'])
})
test('a test timeout is NOT VERIFIED, never a FAIL: no verifier call, no resume, no fail counted', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff', 'diff 2']
w.testEnds = [124, 0]
w.testFired = [true, false]
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.told[0]).toContain('NOT VERIFIED - the tests did not finish within 60 s')
expect(w.told[0]).not.toContain('Resume agent')
// the next round's FAIL is the first one counted: it resumes, it does not rewrite
await finish($, clock, r.agentId as string)
expect(w.told[1]).toContain('Resume agent')
})
test('a kill after the grace period also counts as a timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [137]
w.testFired = [true]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.told[0]).toContain('did not finish')
})
test('a test command that cannot run (exit 127) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [127]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.told[0]).toContain('could not run')
})
test('a test command that cannot run (a start failure) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = ['throw' as const]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.told[0]).toContain('could not run')
})
test('failing tests are evidence: the verifier sees them and decides', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus'])
})
test('an early 137 (out of memory) is a failed run for the verifier, not a timeout', { options: { testCommand: 'make t', testTimeoutSec: 1800 } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [137]
w.testFired = [false]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus'])
expect(w.told[0] ?? '').not.toContain('NOT VERIFIED')
})
test('an invalid testTimeoutSec falls back to 1800', async () => {
for (const bad of [0, -5, Number.NaN, 'abc', undefined, null]) expect(validTimeout(bad)).toBe(1800)
expect(validTimeout(900)).toBe(900)
expect(validTimeout('600')).toBe(600)
})
test('a command that exits 124 by itself is a failed run, not a timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [124]
w.testFired = [false]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus'])
})
test('the timeout line still counts when it arrives split across pieces', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [124]
w.testSplitLine = true
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual([])
expect(w.told[0]).toContain('did not finish')
})
test('the timeout line on stdout (a test printing it) is not a fired timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.testLineOnStdout = true
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus'])
})
// ---------- TOOL-02 end to end ----------
test('TOOL-02: the SAFETY-129 reply passes the round, no rewrite', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.verdicts = [SAFETY_129]
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus'])
expect(w.told[0]).toContain('PASS from verifier opus')
})
test('TOOL-02: an unreadable reply is asked again once, and a readable answer then counts', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.verdicts = ['I think it is fine overall.', '{"verdict":"FAIL","issues":["x"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus', 'opus'])
expect(w.told[0]).toContain('Resume agent')
})
test('TOOL-02: unreadable twice is handed back, no fail counted, no rewrite', async ($, on) => {
const w = fresh()
w.diffs = ['diff', 'diff 2']
w.verdicts = ['Looks good to me.', 'Still prose, sorry.', '{"verdict":"FAIL","issues":["y"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string)
expect(w.verifier).toEqual(['opus', 'opus'])
expect(w.told[0]).toContain('NOT VERIFIED - verdict unreadable')
expect(w.told[0]).toContain('Start of its reply: Looks good to me.')
expect(w.told[0]).not.toContain('[router-rewrite')
expect(w.told[0]).not.toContain('Resume agent')
// the next round's FAIL is the first one counted: it resumes, it does not rewrite
await finish($, clock, r.agentId as string)
expect(w.told[1]).toContain('Resume agent')
expect(w.told[1]).not.toContain('[router-rewrite')
})
// ---------- TOOL-12 end to end ----------
const opusBrief = () =>
`SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/\nBase branch: release/1.0.`
test('TOOL-12 acceptance 1: an Opus-routed dev gets the skip line at hand-back, and no verdict later', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
expect(w.spawned[0]?.model).toBe('opus')
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
await clock.advance(60_000)
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
expect(w.verifier).toEqual([])
})
test('TOOL-12 acceptance 2: a Sonnet-routed dev gets "verifying", then the verdict', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
expect(w.spawned[0]?.model).toBe('sonnet')
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus...')
expect(w.all[1]).toContain('PASS from verifier opus')
expect(w.all.length).toBe(2)
})
test('TOOL-12 acceptance 3a: routed Sonnet but ran on Opus is reported and handled as Opus', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
expect(w.verifier).toEqual([])
})
test('TOOL-12 acceptance 3b: routed Opus but ran on Sonnet is verified, and labelled Sonnet', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
expect(w.verifier).toEqual(['opus'])
expect(w.told[0]).toContain('(agent agent-1, sonnet, round 1)')
// the resumed round still reports against the original routing choice
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
})
test('TOOL-12: a dev run that ends with an error says so, no verification', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5', 'error')
expect(w.all).toEqual(['[model-router] T1 no verification: the dev run ended without an answer (error).'])
expect(w.verifier).toEqual([])
})
test('TOOL-12: review and test agents get no hand-back line', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: `Review. Clone: ${REVIEW}/`, description: 'review', model: 'opus' } as never)
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
expect(w.all).toEqual([])
})
// ---------- interim stops (TOOL-12 follow-up) ----------
/** A turn end of the agent while it is still alive: its status says so. */
async function interimStop($: Engine, clock: { advance: (ms: number) => Promise<void> }, w: World, agentId: string, status = 'waiting') {
w.status[agentId] = status
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({
answer: 'Waiting for the test suite.',
durationMs: 1,
isAborted: false,
turnId: 't',
agentId,
reason: 'answer',
usage: { model: 'claude-sonnet-5-5', input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
})
await clock.advance(1)
}
test('follow-up acceptance: 3 interim stops give exactly one verifying line and one verdict, both after the hand-back', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28 dev' } as never)
const id = r.agentId as string
for (let i = 0; i < 3; i++) {
await interimStop($, clock, w, id)
await clock.advance(40 * 60 * 1000) // the test suite runs 35-45 min
expect(w.all).toEqual([])
expect(w.verifier).toEqual([])
expect(w.diffBases).toEqual([])
}
w.status[id] = 'completed'
await finish($, clock, id, 'claude-sonnet-5-5')
await clock.advance(60_000)
expect(w.handBacks).toEqual(['[model-router] T1 verifying on Opus...'])
expect(w.verifier).toEqual(['opus'])
expect(w.told.length).toBe(1)
expect(w.all[1]).toContain('PASS from verifier opus')
})
test('follow-up: an interim stop of an Opus dev posts nothing; its hand-back posts the skip line once', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: `SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/`, description: 'dev' } as never)
const id = r.agentId as string
await interimStop($, clock, w, id)
await interimStop($, clock, w, id, 'idle')
await clock.advance(10 * 60 * 1000)
expect(w.all).toEqual([])
w.status[id] = 'completed'
await finish($, clock, id, 'claude-opus-5-5')
await clock.advance(60_000)
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
})
test('follow-up: a status that turns ended a moment after the turn is still caught', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
const id = r.agentId as string
w.status[id] = 'running' // not yet updated when the turn ends
await finish($, clock, id, 'claude-sonnet-5-5')
expect(w.all).toEqual([])
w.status[id] = 'completed'
await clock.advance(16_000)
expect(w.handBacks).toEqual(['[model-router] T1 verifying on Opus...'])
expect(w.verifier).toEqual(['opus'])
})
test('follow-up: an agent the engine already dropped counts as handed back', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
const id = r.agentId as string
w.status[id] = 'gone'
await finish($, clock, id, 'claude-sonnet-5-5')
await clock.advance(1)
expect(w.handBacks.length).toBe(1)
})
test('follow-up: a failed or killed run is a hand-back too', async () => {
for (const s of ['completed', 'failed', 'killed', undefined] as const) expect(turnEndKind(s)).toBe('handback')
for (const s of ['waiting', 'idle', 'running', 'pending'] as const) expect(turnEndKind(s)).toBe('interim')
})
test('follow-up: the hand-back uses the last turn: its model and how it ended', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
const id = r.agentId as string
await interimStop($, clock, w, id) // reported Sonnet
w.status[id] = 'completed'
await finish($, clock, id, 'claude-opus-5-5') // the last turn ran on Opus
await clock.advance(60_000)
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
})
test('follow-up: each agent is judged by its own status, not by that of another agent', async ($, on) => {
const w = fresh()
w.diffs = ['diff']
const clock = world(on, w)
await $.agent.spawn({ prompt: devBrief(), description: 'finished dev' } as never) // agent-1: completed
const r = await $.agent.spawn({ prompt: devBrief(), description: 'busy dev' } as never)
await interimStop($, clock, w, r.agentId as string) // agent-2: waiting
await clock.advance(60_000)
expect(w.all).toEqual([])
expect(w.verifier).toEqual([])
})
// ---------- resumes, follow-ups and rewrites (#4, #6) ----------
const sendMessage = ($: Engine, to: string, message: string) =>
$.tool.call({ tool: 'SendMessage', to, message } as never)
const resume = ($: Engine, agentId: string, prompt: string) =>
$.agent.spawn({ prompt, description: 'resume', resume: { agentId }, model: 'sonnet' } as never)
test('#4: a resume keeps its task, routing and counts: no new Tn, no new routing', async ($, on) => {
const w = fresh()
w.diffs = ['d1', 'd2']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
const id = r.agentId as string
await finish($, clock, id, 'claude-sonnet-5-5')
expect(w.told[0]).toContain(`Resume agent ${id}`)
await resume($, id, 'Fix the issues: x')
expect(w.resumed[0]?.model).toBe('sonnet') // passed through untouched
expect(w.spawned.length).toBe(1)
await finish($, clock, id, 'claude-sonnet-5-5')
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus...')
expect(w.diffBases[1]).toBe('snap1') // round 2 diffs from the last verified round
expect(w.told[1]).toContain('round 2')
expect(w.told[1]).toContain('[router-rewrite T1]') // second fail counted: escalates
})
test('#4: a resumed Opus dev is labelled as Opus, with no "routed to" note', async ($, on) => {
const w = fresh()
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: `SAFETY-135: decide how the guard should recover. Clone: ${DEV}/`, description: 'dev' } as never)
const id = r.agentId as string
await finish($, clock, id, 'claude-opus-5-5')
await resume($, id, 'Docs-only round: update the README.')
await finish($, clock, id, 'claude-opus-5-5')
expect(w.handBacks).toEqual([
'[model-router] T1 no verification: dev ran on Opus. Go straight to review.',
'[model-router] T1 no verification: dev ran on Opus. Go straight to review.',
])
})
test('#4: the label compares with what the engine resolved at start, not what the router asked', async ($, on) => {
const w = fresh()
w.engineModel = 'claude-opus-5-5' // e.g. the agent type sets its own model
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
expect(w.spawned[0]?.model).toBe('sonnet')
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
expect(w.handBacks).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
const log = (await $.command.run({ command: 'router', args: '' } as never)).text ?? ''
expect(log).toContain('opus')
})
test('#4: a router rewrite keeps the task number and its model', async ($, on) => {
const w = fresh()
w.diffs = ['d1', 'd2']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
const rw = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite', model: 'opus' } as never)
expect(w.spawned[1]?.model).toBe('opus')
await finish($, clock, rw.agentId as string, 'claude-opus-5-5')
expect(w.handBacks[2]).toBe('[model-router] T1 no verification: dev ran on Opus. Go straight to review.')
})
test('#4: a rewrite whose first start fails can be started again', async ($, on) => {
const w = fresh()
w.diffs = ['d1', 'd2']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
w.denySpawns = 1
const first = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
expect(first.deny).toBe('spawn failed')
const second = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
expect(second.deny).toBe(undefined)
expect(w.spawned[1]?.model).toBe('opus')
expect(w.spawned[1]?.prompt).toContain('Previous attempts rejected')
})
test('#4: a rewrite marker with nothing stored is refused, not started without a brief', async ($, on) => {
const w = fresh()
world(on, w)
const r = await $.agent.spawn({ prompt: '[router-rewrite T325]', description: 'rewrite', model: 'opus' } as never)
expect(r.deny).toContain('no stored rewrite for T325')
expect(w.spawned).toEqual([])
})
test('#6: a resume message reaches the verifier as a later instruction, in scope', async ($, on) => {
const w = fresh()
w.diffs = ['d1', 'd2']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-24' } as never)
const id = r.agentId as string
await finish($, clock, id, 'claude-sonnet-5-5')
await resume($, id, 'Owner: add MAINT-25 to this PR as well.')
await finish($, clock, id, 'claude-sonnet-5-5')
const p = w.verifierPrompts[1] ?? ''
expect(p).toContain('## Later instructions to this agent')
expect(p).toContain('1. Owner: add MAINT-25 to this PR as well.')
expect(p).toContain('SAFETY-95: clamp the integrator') // the original brief stays
})
test('#6: SendMessage to a running agent, by id or by name, is a later instruction too', async ($, on) => {
const w = fresh()
w.diffs = ['d1']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28', name: 'maint28-dev' } as never)
const id = r.agentId as string
await sendMessage($, id, 'Owner decided: cap is 1.')
await sendMessage($, 'maint28-dev', 'Also cover the empty list.')
await sendMessage($, 'maint28-dev', 'Also cover the empty list.') // the same text counts once
await sendMessage($, 'someone-else', 'Not for this agent.')
await finish($, clock, id, 'claude-sonnet-5-5')
const p = w.verifierPrompts[0] ?? ''
expect(p).toContain('1. Owner decided: cap is 1.')
expect(p).toContain('2. Also cover the empty list.')
expect(p).not.toContain('3.')
expect(p).not.toContain('Not for this agent.')
})
test('#6: without later instructions the section is left out', async ($, on) => {
const w = fresh()
w.diffs = ['d1']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.verifierPrompts[0] ?? '').not.toContain('Later instructions')
})
test('#4: a rewrite that started is used up: the same marker again is refused', async ($, on) => {
const w = fresh()
w.diffs = ['d1', 'd2']
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
const again = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
expect(again.deny).toContain('no stored rewrite for T1')
expect(w.spawned.length).toBe(2)
})
test('#6: a SendMessage that was not delivered is not a later instruction', async ($, on) => {
const w = fresh()
w.diffs = ['d1']
w.sendFails = true
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
await sendMessage($, r.agentId as string, 'Never arrived.')
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.verifierPrompts[0] ?? '').not.toContain('Never arrived.')
})
// ---------- reruns of reported files (#5) ----------
const RERUN = { testCommand: 'make t', testFileCommand: 'runone {file}' }
const T307_OUTPUT = [
'PASS test_a.py 12s\nTIMEOUT test_tariff24_weather.py 230s\nFAIL test_loop.py\nTIME',
'OUT test_slow.py 1500s\n', // a line split across pieces
'--- test_tariff24_weather.py (TIMEOUT)\n<tail>\n77 pass, 1 fail, 2 timeout (limit 1500 s)\n',
]
test('#5: reported lines are read, also when split across pieces; one entry per file', async () => {
const re = issueRegex('')
expect(parseIssueLine('TIMEOUT test_tariff24_weather.py 230s', re)).toEqual({ kind: 'TIMEOUT', file: 'test_tariff24_weather.py' })
expect(parseIssueLine('FAIL test_loop.py', re)).toEqual({ kind: 'FAIL', file: 'test_loop.py' })
expect(parseIssueLine('--- test_tariff24_weather.py (TIMEOUT)', re)).toBe(undefined)
expect(parseIssueLine('77 pass, 0 fail, 30 timeout (limit 1500 s)', re)).toBe(undefined)
expect(parseIssueLine('PASS test_a.py', re)).toBe(undefined)
// a custom pattern, and a broken one falls back to the default
expect(parseIssueLine('[timeout] x.py', issueRegex('^\\[(timeout|fail)\\]\\s+(\\S+)'))?.file).toBe('x.py')
expect(parseIssueLine('TIMEOUT y.py', issueRegex('(unclosed'))?.file).toBe('y.py')
})
test('#5: only safe file names reach the shell', async () => {
for (const ok of ['test_a.py', 'tests/test_b.py', 'a-b.c_d.py']) expect(isSafeFile(ok)).toBe(true)
for (const bad of ['a;rm -rf ~', '$(id).py', '../x.py', 'a/../b.py', '-rf', 'a b.py', 'x`y`.py', "a'.py"]) expect(isSafeFile(bad)).toBe(false)
expect(fileCommand('docker run img python3 {file} && echo {file}', 'test_a.py')).toBe('docker run img python3 test_a.py && echo test_a.py')
})
test('#5: what each rerun outcome means', async () => {
expect(rerunVerdict({ file: 'f', full: 'TIMEOUT', head: 'pass', base: 'not run' })).toContain('does not count')
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'fail', base: 'fail' })).toContain('not caused by this change')
expect(rerunVerdict({ file: 'f', full: 'TIMEOUT', head: 'timeout', base: 'timeout' })).toContain('also times out on base')
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'fail', base: 'pass' })).toContain('counts against the change')
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'fail', base: 'not run' })).toContain('base not checked')
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'error', base: 'not run' })).toContain('could not be rerun')
})
test('#5 acceptance: timed-out and failed files are rerun alone, failures again on base, and the verifier gets the table', { options: RERUN }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = T307_OUTPUT
w.fileOutcomes = { 'head:test_loop.py': 'fail', 'base:test_loop.py': 'fail', 'head:test_slow.py': 'timeout', 'base:test_slow.py': 'pass' }
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect([...w.fileRuns].sort()).toEqual([
'base:test_loop.py',
'base:test_slow.py',
'head:test_loop.py',
'head:test_slow.py',
'head:test_tariff24_weather.py',
])
const p = w.verifierPrompts[0] ?? ''
expect(p).toContain('### Files the full run reported, rerun one by one')
expect(p).toContain('| test_tariff24_weather.py | TIMEOUT | pass | not run | passes alone: does not count against the change |')
expect(p).toContain('| test_loop.py | FAIL | fail | fail | also fails on base: not caused by this change |')
expect(p).toContain('| test_slow.py | TIMEOUT | timeout | pass | times out alone, passes on base: counts against the change |')
// the base worktree sits next to the clone, and is removed and pruned afterwards
expect(w.worktree).toContain(`git -C ${DEV} worktree add --detach ${DEV}-verifier-base mergebase`)
expect(w.worktree.at(-2)).toBe(`git -C ${DEV} worktree remove --force ${DEV}-verifier-base`)
expect(w.worktree.at(-1)).toBe(`git -C ${DEV} worktree prune`)
})
test('#5: when every reported file passes alone, no base worktree is made', { options: RERUN }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = ['TIMEOUT test_tariff24_weather.py 230s\n']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.fileRuns).toEqual(['head:test_tariff24_weather.py'])
expect(w.worktree.filter(c => c.includes(' add '))).toEqual([])
})
test('#5: a base worktree that cannot be made leaves base as not run', { options: RERUN }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.worktreeFails = true
w.fullRunOutput = ['FAIL test_loop.py\n']
w.fileOutcomes = { 'head:test_loop.py': 'fail' }
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.fileRuns).toEqual(['head:test_loop.py'])
expect(w.verifierPrompts[0]).toContain('| test_loop.py | FAIL | fail | not run | fails alone; base not checked |')
})
test('#5: unsafe names and files past testRerunMax are not rerun, and said so', { options: { ...RERUN, testRerunMax: 1 } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = ['FAIL a;rm.py\nTIMEOUT test_a.py 1s\nTIMEOUT test_b.py 1s\n']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.fileRuns).toEqual(['head:test_a.py'])
expect(w.verifierPrompts[0]).toContain('(2 more reported file(s) not rerun')
})
test('#5: without testFileCommand nothing is rerun', { options: { testCommand: 'make t' } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = T307_OUTPUT
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.fileRuns).toEqual([])
expect(w.verifierPrompts[0] ?? '').not.toContain('rerun one by one')
})
test('#5: reruns run at most testRerunJobs at a time', { options: { ...RERUN, testRerunJobs: 2 } }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = ['TIMEOUT t1.py 1s\nTIMEOUT t2.py 1s\nTIMEOUT t3.py 1s\nTIMEOUT t4.py 1s\nTIMEOUT t5.py 1s\n']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect(w.fileRuns.length).toBe(5)
expect(w.maxParallel).toBe(2)
})
test('#5: a custom pattern that also matches other lines still keeps only TIMEOUT and FAIL', async () => {
const re = issueRegex('^(\\w+)\\s+(\\S+)')
expect(parseIssueLine('PASS test_a.py', re)).toBe(undefined)
expect(parseIssueLine('timeout test_b.py', re)).toEqual({ kind: 'TIMEOUT', file: 'test_b.py' })
})
test('#5: a last line without a newline is read; a file reported twice is rerun once', { options: RERUN }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = ['FAIL test_twice.py\nFAIL test_twice.py\n', 'TIMEOUT test_last.py 1500s']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
expect([...w.fileRuns].sort()).toEqual(['head:test_last.py', 'head:test_twice.py'])
})
test('#5: the verifier is told how to read the rerun table', { options: RERUN }, async ($, on) => {
const w = fresh()
w.diffs = ['diff']
w.testEnds = [1]
w.fullRunOutput = ['TIMEOUT test_a.py 1s\n']
const clock = world(on, w)
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
const sys = w.verifierSystems[0] ?? ''
expect(sys).toContain('When a table of files rerun one by one is given')
expect(sys).toContain('a file that passes alone does not count against the change')
expect(sys).toContain('not caused by this change')
expect(sys).toContain('Name every file you count against the change')
})