The full suite run can time out files that pass alone (an overloaded shared host, a suite-wide deadline in the runner): MAINT-28 T307 was judged on 30 TIMEOUTs, T325 on a file that passes alone in 230 s. - With testFileCommand set, every file the full run reports as TIMEOUT or FAIL (testIssuePattern, default ^(TIMEOUT|FAIL)\s+(\S+); lines are read as the run streams) is rerun alone in the clone, testRerunJobs at a time, testFileTimeoutSec each, at most testRerunMax files. - A file that still fails or times out alone runs again on the base, in a git worktree next to the clone (<clone>-verifier-base), which is removed and pruned afterwards. Without a worktree, base stays "not run". - The verifier gets a table (file, full run, alone on head, on base, meaning) and is told: passes alone does not count; fails on base too is not caused by the change; name every file it counts. - Only file names of letters, digits and _ . - / reach the shell; no ".." parts, no leading "-". - 12 new tests; each of the 23 new rules was also broken on purpose and caught. Version 0.4.9. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
1142 lines
55 KiB
TypeScript
1142 lines
55 KiB
TypeScript
import type { On } from 'claude-code'
|
|
import { expect, mock, test, type Engine } from 'claude-code/testing'
|
|
|
|
import {
|
|
decide,
|
|
heuristic,
|
|
onFail,
|
|
parseBrief,
|
|
parseVerdict,
|
|
rewriteTier,
|
|
skipReason,
|
|
validTimeout,
|
|
verifierFor,
|
|
handBackLine,
|
|
isSafeFile,
|
|
issueRegex,
|
|
parseIssueLine,
|
|
rerunVerdict,
|
|
fileCommand,
|
|
ranTier,
|
|
turnEndKind,
|
|
} from '../hooks/router'
|
|
|
|
const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev'
|
|
const REVIEW = '/home/homelab/projects/goodwe/clones/SAFETY-95-review'
|
|
const TEST = '/home/homelab/projects/goodwe/clones/TEST-23-test'
|
|
|
|
// ---------- the brief (C2) ----------
|
|
|
|
test('brief: clone path, ticket, role and base branch are read', async () => {
|
|
const b = parseBrief(`Work in ${DEV}/ only. Base branch: release/1.0.\nExpected: ...`)
|
|
expect(b).toEqual({ clone: DEV, ticket: 'SAFETY-95', role: 'dev', baseBranch: 'release/1.0' })
|
|
expect(parseBrief(`Clone: \`${TEST}\``).role).toBe('test')
|
|
expect(parseBrief('PR against `release/1.1`, no clone given').baseBranch).toBe('release/1.1')
|
|
expect(parseBrief('just look around').clone).toBe(undefined)
|
|
})
|
|
|
|
test('fix 1: tickets with extra parts parse, the role is the last part', async () => {
|
|
const p = '/home/homelab/projects/goodwe/clones/OPT-11-P4-dev'
|
|
expect(parseBrief(`Work in ${p}/ only.`)).toMatchObject({ clone: p, ticket: 'OPT-11-P4', role: 'dev' })
|
|
expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').ticket).toBe('OPT-11-P4-R2')
|
|
expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').role).toBe('test')
|
|
})
|
|
|
|
test('fix 2: the Clone: line decides, whatever the brief mentions first', async () => {
|
|
const brief = `Verify the work of ${DEV}/ on the rig.\nClone: ${TEST}/\nBase branch: release/1.0.`
|
|
expect(parseBrief(brief)).toMatchObject({ clone: TEST, role: 'test', baseBranch: 'release/1.0' })
|
|
// an explicit role wins over the path
|
|
expect(parseBrief(`Clone: \`/x/clones/OPT-11-P4\` (Role: review)`)).toMatchObject({ clone: '/x/clones/OPT-11-P4', role: 'review' })
|
|
// (Role: ...) wins over the role in the path
|
|
expect(parseBrief(`Clone: ${DEV}/ (Role: review)`).role).toBe('review')
|
|
// a list item or quote marker in front of the line is fine
|
|
expect(parseBrief(`- Clone: ${DEV}`).role).toBe('dev')
|
|
// "Clone:" inside a sentence is not the line, even at a line's end
|
|
expect(parseBrief(`Results are in the dev agent's Clone: ${DEV}\nRun the rig in ${TEST}/`).isAmbiguous).toBe(true)
|
|
})
|
|
|
|
test('fix 2: several clones and no Clone: line gives no role, and routes to opus', async () => {
|
|
const b = parseBrief(`Compare ${DEV}/ with ${TEST}/.`)
|
|
expect(b.role).toBe(undefined)
|
|
expect(b.isAmbiguous).toBe(true)
|
|
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'haiku' })).toMatchObject({ kind: 'fixed', tier: 'opus' })
|
|
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'sonnet' })).toMatchObject({ tier: 'opus' })
|
|
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'fable' })).toMatchObject({ tier: 'fable' })
|
|
// the same path twice is one clone, not ambiguous
|
|
expect(parseBrief(`Work in ${DEV}/. Commit in ${DEV}.`).role).toBe('dev')
|
|
})
|
|
|
|
// ---------- who picks the writer (C4) ----------
|
|
|
|
test('C4: review and test agents keep the model they are given', async () => {
|
|
for (const role of ['review', 'test'] as const)
|
|
for (const named of ['haiku', 'sonnet', 'opus'] as const)
|
|
expect(decide({ role, named, prompt: 'x', respectExplicit: false })).toEqual({
|
|
kind: 'fixed',
|
|
tier: named,
|
|
reason: `named model kept for ${role} agent`,
|
|
})
|
|
})
|
|
|
|
test('C4: unnamed review and test agents run on opus', async () => {
|
|
expect(decide({ role: 'test', prompt: 'run the rig', respectExplicit: false })).toMatchObject({ tier: 'opus' })
|
|
expect(decide({ role: 'review', prompt: 'review', respectExplicit: false })).toMatchObject({ tier: 'opus' })
|
|
})
|
|
|
|
test('C4: dev agents are routed even when opus is named, and that is logged', async () => {
|
|
const d = decide({ role: 'dev', named: 'opus', prompt: 'Expected: clamp to 2000 W', respectExplicit: true })
|
|
expect(d.kind).toBe('classify')
|
|
expect(d.opusRequested).toBe(true)
|
|
expect(decide({ role: 'dev', named: 'sonnet', prompt: 'x', respectExplicit: true }).opusRequested).toBe(false)
|
|
})
|
|
|
|
test('C4: respectExplicit only applies to agents without a role', async () => {
|
|
expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: true })).toMatchObject({ kind: 'fixed', tier: 'haiku' })
|
|
expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: false }).kind).toBe('classify')
|
|
})
|
|
|
|
test('C4: open design goes to opus, a decided brief is left to the classifier', async () => {
|
|
expect(heuristic(undefined, 'MAINT-13: decide how the staleness guard should recover', 'dev')?.tier).toBe('opus')
|
|
expect(heuristic(undefined, 'In the asyncio control loop, clamp the integrator; Expected: test_clamp passes', 'dev')).toBe(undefined)
|
|
})
|
|
|
|
test('C4: narrowed light patterns: "docs" or "run tests" in a brief never mean haiku', async () => {
|
|
expect(heuristic(undefined, 'Update the docs and run the tests after changing the setpoint logic')).toBe(undefined)
|
|
expect(heuristic(undefined, 'Find where the MQTT discovery payload is built')?.tier).toBe('haiku')
|
|
// a dev brief never takes the light path, however light it reads
|
|
expect(heuristic(undefined, 'Find where the MQTT discovery payload is built', 'dev')).toBe(undefined)
|
|
expect(heuristic('Explore', 'anything', 'dev')).toBe(undefined)
|
|
})
|
|
|
|
// ---------- what gets verified (C1, C5) ----------
|
|
|
|
test('only finished dev rounds below opus with a clone are verified', async () => {
|
|
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'answer')).toBe(undefined)
|
|
expect(skipReason({ role: 'dev', clone: DEV, tier: 'haiku' }, 'answer')).toBe(undefined)
|
|
expect(skipReason({ role: 'dev', clone: DEV, tier: 'opus' }, 'answer')).toBe('opus-unverified')
|
|
expect(skipReason({ role: 'dev', tier: 'sonnet' }, 'answer')).toBe('no-clone')
|
|
expect(skipReason({ role: 'test', clone: TEST, tier: 'sonnet' }, 'answer')).toBe('test-agent')
|
|
expect(skipReason({ role: 'review', clone: REVIEW, tier: 'sonnet' }, 'answer')).toBe('review-agent')
|
|
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'error')).toBe('agent-ended')
|
|
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'aborted')).toBe('agent-ended')
|
|
})
|
|
|
|
test('the verifier is opus below opus, and never the writer', async () => {
|
|
expect(verifierFor('haiku')).toBe('opus')
|
|
expect(verifierFor('sonnet')).toBe('opus')
|
|
for (const w of ['haiku', 'sonnet', 'opus', 'fable'] as const) expect(verifierFor(w)).not.toBe(w)
|
|
})
|
|
|
|
test('C5: first fail resumes, second starts a stronger agent, then hand back', async () => {
|
|
expect(onFail(1, 'sonnet', 2, false)).toEqual({ kind: 'resume' })
|
|
expect(onFail(2, 'sonnet', 2, false)).toEqual({ kind: 'new-agent', tier: 'opus' })
|
|
expect(onFail(3, 'opus', 2, false)).toEqual({ kind: 'hand-back' })
|
|
expect(onFail(2, 'opus', 2, false)).toEqual({ kind: 'hand-back' }) // nothing above opus
|
|
expect(onFail(2, 'opus', 2, true)).toEqual({ kind: 'new-agent', tier: 'fable' })
|
|
expect(onFail(1, 'sonnet', 0, false)).toEqual({ kind: 'hand-back' })
|
|
expect(rewriteTier('haiku', false)).toBe('sonnet')
|
|
})
|
|
|
|
// ---------- reading the verdict (TOOL-02) ----------
|
|
|
|
// The SAFETY-129 reply shape: a fenced block, then prose that contains a `}`.
|
|
const SAFETY_129 =
|
|
'Checked the diff against the brief.\n\n```json\n{"verdict": "PASS", "issues": [], "rewrite_brief": ""}\n```\n\n' +
|
|
'Notes: the guard keeps `if (stale) { hold() }` as before, and the new test covers it.'
|
|
|
|
test('TOOL-02: fenced JSON followed by prose with a } reads as PASS', async () => {
|
|
expect(parseVerdict(SAFETY_129)).toEqual({ isReadable: true, pass: true, issues: [], brief: '' })
|
|
})
|
|
|
|
test('TOOL-02: the fenced answer wins over an earlier object in the prose', async () => {
|
|
const r = parseVerdict('Last round said {"verdict":"FAIL","issues":["x"]}. Fixed now:\n```json\n{"verdict":"PASS","issues":[]}\n```')
|
|
expect(r).toMatchObject({ isReadable: true, pass: true })
|
|
})
|
|
|
|
test('TOOL-02: bare JSON reads, with or without prose around it', async () => {
|
|
expect(parseVerdict('{"verdict":"PASS","issues":[]}')).toMatchObject({ isReadable: true, pass: true })
|
|
expect(parseVerdict('Verdict below. {"verdict": "pass"} That is all {really}.')).toMatchObject({ isReadable: true, pass: true })
|
|
})
|
|
|
|
test('TOOL-02: FAIL keeps its issues and brief', async () => {
|
|
const r = parseVerdict('```json\n{"verdict":"FAIL","issues":["loop.py:40 no clamp"],"rewrite_brief":"clamp it"}\n```')
|
|
expect(r).toEqual({ isReadable: true, pass: false, issues: ['loop.py:40 no clamp'], brief: 'clamp it' })
|
|
})
|
|
|
|
test('TOOL-02: braces inside strings do not end the object', async () => {
|
|
const r = parseVerdict('{"verdict":"FAIL","issues":["handler } has a stray brace","quote \\" and }"]} trailing {x}')
|
|
expect(r).toMatchObject({ isReadable: true, pass: false })
|
|
if (r.isReadable) expect(r.issues).toEqual(['handler } has a stray brace', 'quote " and }'])
|
|
})
|
|
|
|
test('TOOL-02: the first object with a verdict wins; other objects are skipped', async () => {
|
|
const r = parseVerdict('Config seen: {"limit": 2000}. Verdict: {"verdict":"FAIL","issues":["x"]} and later {"verdict":"PASS"}')
|
|
expect(r).toMatchObject({ isReadable: true, pass: false })
|
|
})
|
|
|
|
test('TOOL-02: a fence without a verdict falls through to the text', async () => {
|
|
const r = parseVerdict('```json\n{"limit": 2000}\n```\nResult: {"verdict":"PASS"}')
|
|
expect(r).toMatchObject({ isReadable: true, pass: true })
|
|
})
|
|
|
|
test('TOOL-02: garbage, prose and odd verdicts are unreadable, never a FAIL', async () => {
|
|
for (const t of ['looks fine', 'verdict: pass', '{"verdict":"MAYBE"}', '{"issues":[]}', '{broken json', '']) {
|
|
const r = parseVerdict(t)
|
|
expect(r.isReadable).toBe(false)
|
|
}
|
|
const long = parseVerdict('x'.repeat(500))
|
|
expect(long.isReadable ? '' : long.preview).toBe('x'.repeat(300))
|
|
})
|
|
|
|
// ---------- the hand-back line (TOOL-12) ----------
|
|
|
|
test('TOOL-12: the model that ran wins over the routing choice', async () => {
|
|
expect(ranTier('claude-opus-5-5', 'sonnet')).toEqual({ tier: 'opus', isMismatch: true })
|
|
expect(ranTier('claude-sonnet-5-5', 'opus')).toEqual({ tier: 'sonnet', isMismatch: true })
|
|
expect(ranTier('claude-sonnet-5-5', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
|
expect(ranTier(undefined, 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
|
expect(ranTier('some-other-model', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
|
})
|
|
|
|
test('TOOL-12: the exact lines', async () => {
|
|
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe('[model-router] T7 verifying on Opus...')
|
|
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'opus', skip: 'opus-unverified', ended: 'answer' })).toBe(
|
|
'[model-router] T7 no verification: dev ran on Opus. Go straight to review.',
|
|
)
|
|
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'sonnet', skip: 'opus-unverified', ended: 'answer' })).toBe(
|
|
'[model-router] T7 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)',
|
|
)
|
|
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'agent-ended', ended: 'error' })).toContain('ended without an answer (error)')
|
|
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'no-clone', ended: 'answer' })).toContain('no clone path')
|
|
expect(handBackLine({ taskId: 'T7', role: 'review', ran: 'opus', routed: 'opus', skip: 'review-agent', ended: 'answer' })).toBe(undefined)
|
|
expect(handBackLine({ taskId: 'T7', role: 'test', ran: 'opus', routed: 'opus', skip: 'test-agent', ended: 'answer' })).toBe(undefined)
|
|
expect(handBackLine({ taskId: 'T7', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe(undefined)
|
|
})
|
|
|
|
// ---------- the hooks, with the engine mocked beneath ----------
|
|
|
|
type World = {
|
|
spawned: { model?: string; prompt: string }[]
|
|
verifier: string[]
|
|
verdicts: string[]
|
|
told: string[]
|
|
/** TOOL-12 hand-back lines, kept apart so `told` holds the verdicts. */
|
|
handBacks: string[]
|
|
/** Every message in the order it was sent. */
|
|
all: string[]
|
|
/** Resumes that reached the spawn hook. */
|
|
resumed: { agentId: string; prompt: string; model?: string }[]
|
|
/** How many coming new spawns the engine refuses. */
|
|
denySpawns: number
|
|
/** The model the engine resolves for new spawns, whatever was asked. */
|
|
engineModel?: string
|
|
verifierPrompts: string[]
|
|
verifierSystems: string[]
|
|
/** SendMessage calls fail (nothing delivered). */
|
|
sendFails?: boolean
|
|
/** Pieces the full test run prints to stdout. */
|
|
fullRunOutput: string[]
|
|
/** How a file run alone ends, keyed `head:<file>` or `base:<file>`; absent: pass. */
|
|
fileOutcomes: Record<string, 'pass' | 'fail' | 'timeout'>
|
|
/** Files run alone, as `head:<file>` or `base:<file>`. */
|
|
fileRuns: string[]
|
|
/** git worktree commands run. */
|
|
worktree: string[]
|
|
worktreeFails?: boolean
|
|
running: number
|
|
maxParallel: number
|
|
/** Each agent's loop status as $.agent.list() reports it; absent means completed. */
|
|
status: Record<string, string>
|
|
diffs: string[]
|
|
diffBases: string[]
|
|
testCwd: string[]
|
|
testArgv: string[][]
|
|
/** How the next test run ends: exit code (124 = timed out), or a throw. */
|
|
testEnds: (number | 'throw')[]
|
|
/** Whether `timeout` fired on each test run (it then says so on stderr). */
|
|
testFired: boolean[]
|
|
/** The timeout line arrives in two stderr pieces. */
|
|
testSplitLine?: boolean
|
|
/** A test itself prints the timeout line, on stdout. */
|
|
testLineOnStdout?: boolean
|
|
snap: number
|
|
}
|
|
|
|
const USAGE = { input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }
|
|
const ok = (stdout: string, exitCode = 0) => ({
|
|
value: { exitCode, stdout, stderr: '', isStdoutTruncated: false, isStderrTruncated: false },
|
|
})
|
|
|
|
function world(on: On, w: World) {
|
|
mock.store(on)
|
|
const clock = mock.clock(on)
|
|
on('ui.status', async () => ({ value: undefined }))
|
|
on('ui.toast', async () => ({ value: undefined }))
|
|
on('model.complete', async (_$, e) => {
|
|
if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } }
|
|
w.verifier.push(e.model)
|
|
w.verifierPrompts.push(e.prompt)
|
|
w.verifierSystems.push(e.system ?? '')
|
|
return { value: { isAnswered: true as const, text: w.verdicts.shift() ?? '{"verdict":"PASS"}', usage: USAGE } }
|
|
})
|
|
on('process.run', async (_$, e) => {
|
|
const a = e.argv.join(' ')
|
|
const cwd = String(e.init?.cwd ?? '')
|
|
if (cwd !== DEV) return ok('', 128) // not a git repo: the session folder
|
|
if (a === 'git merge-base origin/release/1.0 HEAD') return ok('mergebase\n')
|
|
if (a === 'git stash create') return ok(`snap${++w.snap}\n`)
|
|
if (a.startsWith('git diff ')) {
|
|
w.diffBases.push(e.argv[2] ?? '')
|
|
return ok(w.diffs.shift() ?? '')
|
|
}
|
|
if (a.includes(' worktree ')) {
|
|
w.worktree.push(a)
|
|
if (a.includes(' worktree add ') && w.worktreeFails) return ok('', 128)
|
|
}
|
|
return ok('')
|
|
})
|
|
on('process.spawn', async function* (_$, e) {
|
|
const cmd = e.argv[e.argv.length - 1] ?? ''
|
|
if (cmd.startsWith('runone ')) {
|
|
// one file alone: `runone <file>`, in the clone or in the base worktree
|
|
const file = cmd.slice('runone '.length)
|
|
const where = String(e.cwd ?? '').endsWith('-verifier-base') ? 'base' : 'head'
|
|
w.fileRuns.push(`${where}:${file}`)
|
|
w.running++
|
|
w.maxParallel = Math.max(w.maxParallel, w.running)
|
|
for (let i = 0; i < 5; i++) await Promise.resolve() // let other reruns start
|
|
w.running--
|
|
const outcome = w.fileOutcomes[`${where}:${file}`] ?? 'pass'
|
|
if (outcome === 'timeout') yield { stream: 'stderr' as const, text: "timeout: sending signal TERM to command 'sh'\n" }
|
|
return { value: { code: outcome === 'pass' ? 0 : outcome === 'timeout' ? 124 : 1, signal: null } }
|
|
}
|
|
w.testCwd.push(String(e.cwd ?? ''))
|
|
w.testArgv.push([...e.argv])
|
|
const end = w.testEnds.shift() ?? 0
|
|
const fired = w.testFired.shift() ?? false
|
|
if (end === 'throw') throw new Error('docker: not found')
|
|
yield { stream: 'stdout' as const, text: end === 0 ? '67 passed\n' : 'test_modes_p4 ... still running\n' }
|
|
// the full run's report comes last, so a last line without a newline stays last
|
|
for (const piece of w.fullRunOutput) yield { stream: 'stdout' as const, text: piece }
|
|
if (fired) yield { stream: 'stderr' as const, text: "timeout: sending signal TERM to command 'sh'\n" }
|
|
if (w.testSplitLine) {
|
|
yield { stream: 'stderr' as const, text: 'timeout: sending sig' }
|
|
yield { stream: 'stderr' as const, text: "nal TERM to command 'sh'\n" }
|
|
}
|
|
if (w.testLineOnStdout) yield { stream: 'stdout' as const, text: 'assert "timeout: sending signal" in log\n' }
|
|
return { value: { code: end, signal: null } }
|
|
})
|
|
on('agent.spawn', async (_$, e) => {
|
|
if (e.resume) {
|
|
w.resumed.push({ agentId: e.resume.agentId, prompt: e.prompt, model: e.model })
|
|
return { model: 'claude-opus-5-5', agentId: e.resume.agentId }
|
|
}
|
|
if (w.denySpawns > 0) {
|
|
w.denySpawns--
|
|
return { deny: 'spawn failed' }
|
|
}
|
|
w.spawned.push({ model: e.model, prompt: e.prompt })
|
|
return { model: w.engineModel ?? e.model ?? 'inherit', agentId: `agent-${w.spawned.length}` }
|
|
})
|
|
on('tool.call', { tool: 'SendMessage' }, async () =>
|
|
(w.sendFails ? { result: 'No agent by that name is running', isError: true } : { result: { success: true } }) as never,
|
|
)
|
|
on('turn.complete', async (_$, e) => ({ text: e.answer }))
|
|
on('agent.list', async () => ({
|
|
value: w.spawned
|
|
.map((_, i) => `agent-${i + 1}`)
|
|
.filter(id => w.status[id] !== 'gone') // 'gone': the engine dropped it
|
|
.map(id => ({ id, description: '', type: 'general-purpose', status: (w.status[id] ?? 'completed') as never })),
|
|
}))
|
|
on('prompt.submit', async (_$, e) => {
|
|
w.all.push(e.text)
|
|
if (/^\[model-router\] T\d+ (verifying on|no verification:)/.test(e.text)) w.handBacks.push(e.text)
|
|
else w.told.push(e.text)
|
|
return { text: e.text }
|
|
})
|
|
return clock
|
|
}
|
|
|
|
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], handBacks: [], all: [], resumed: [], denySpawns: 0, verifierPrompts: [], verifierSystems: [], fullRunOutput: [], fileOutcomes: {}, fileRuns: [], worktree: [], running: 0, maxParallel: 0, status: {}, diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
|
|
|
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string, ranModel?: string, reason = 'answer') {
|
|
const usage = ranModel
|
|
? { usage: { model: ranModel, input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } }
|
|
: {}
|
|
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason, ...usage })
|
|
await clock.advance(1)
|
|
}
|
|
|
|
const devBrief = (extra = '') => `SAFETY-95: clamp the integrator. Work in ${DEV}/. Base branch: release/1.0.\nExpected: test_clamp passes.${extra}`
|
|
|
|
test('acceptance 1+2: a background dev agent is verified in its clone, each round on its own diff', { options: { testCommand: 'pytest -q' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff --git a/loop.py', 'diff --git a/loop.py (round 2)']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["loop.py:40 no clamp on negative"]}', '{"verdict":"PASS"}']
|
|
const clock = world(on, w)
|
|
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp', model: 'opus' } as never)
|
|
expect(w.spawned[0]?.model).toBe('sonnet') // routed despite the named opus
|
|
await finish($, clock, r.agentId as string)
|
|
|
|
expect(w.diffBases[0]).toBe('mergebase') // first round: from the merge base, in the clone
|
|
expect(w.testCwd).toEqual([DEV])
|
|
expect(w.verifier).toEqual(['opus'])
|
|
expect(w.told[0]).toContain('FAIL')
|
|
|
|
// round 2: the resumed agent finishes again (same agentId)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.diffBases[1]).toBe('snap1') // only what this round added
|
|
expect(w.told[1]).toContain('PASS')
|
|
})
|
|
|
|
test('acceptance 3: review and test agents keep a named opus; dev briefs below opus are verified by opus', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
await $.agent.spawn({ prompt: `Review PR. Clone ${REVIEW}/`, description: 'review', model: 'opus' } as never)
|
|
await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig', model: 'opus' } as never)
|
|
const t = await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig2' } as never)
|
|
expect(w.spawned.map(s => s.model)).toEqual(['opus', 'opus', 'opus'])
|
|
await finish($, clock, t.agentId as string)
|
|
expect(w.verifier).toEqual([]) // test agents are not verified
|
|
expect(w.told).toEqual([])
|
|
})
|
|
|
|
test('acceptance 4: the first FAIL asks to resume the same agent', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.told[0]).toContain(`Resume agent ${r.agentId} with SendMessage`)
|
|
expect(w.told[0]).not.toContain('[router-rewrite')
|
|
})
|
|
|
|
test('C5: the second FAIL starts a stronger agent in the same clone', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1', 'd2']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
await finish($, clock, r.agentId as string)
|
|
const id = w.told[1]?.match(/\[router-rewrite (T\d+)\]/)?.[1]
|
|
expect(id).toBeDefined()
|
|
|
|
await $.agent.spawn({ prompt: `[router-rewrite ${id}]`, description: 'rewrite' } as never)
|
|
expect(w.spawned[1]?.model).toBe('opus')
|
|
expect(w.spawned[1]?.prompt).toContain(`Their work is in the clone ${DEV}: continue from it`)
|
|
expect(w.spawned[1]?.prompt).toContain('- y')
|
|
})
|
|
|
|
test('C2: a dev brief with no clone is never diffed in the session folder', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: 'Clamp the integrator in clones/ somewhere. Expected: ...', description: 'x' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.diffBases).toEqual([])
|
|
expect(w.verifier).toEqual([])
|
|
})
|
|
|
|
test('fix 2 end to end: a test brief that mentions the dev clone first keeps its model and is not verified', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({
|
|
prompt: `Run the OPT-11 P4 rig test against the work in ${DEV}/.\nClone: /home/homelab/projects/goodwe/clones/OPT-11-P4-test/`,
|
|
description: 'rig',
|
|
model: 'opus',
|
|
} as never)
|
|
expect(w.spawned[0]?.model).toBe('opus')
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.diffBases).toEqual([])
|
|
})
|
|
|
|
test('fix 2 end to end: two clones and no Clone: line run on opus, unverified', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: `Compare ${DEV}/ with ${TEST}/ and report.`, description: 'cmp', model: 'haiku' } as never)
|
|
expect(w.spawned[0]?.model).toBe('opus')
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
})
|
|
|
|
test('a brief without a role is routed but never tracked or verified', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: 'Find where the MQTT discovery payload is built', description: 'look' } as never)
|
|
expect(w.spawned[0]?.model).toBe('haiku')
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.told).toEqual([])
|
|
})
|
|
|
|
// ---------- the test run ----------
|
|
|
|
test('tests stream with a real time limit, not the ten-minute command cap', { options: { testCommand: 'make verifier-tests', testTimeoutSec: 1800 } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.testArgv[0]).toEqual(['timeout', '--verbose', '--kill-after=30', '1800', 'sh', '-c', 'make verifier-tests'])
|
|
expect(w.testCwd).toEqual([DEV])
|
|
expect(w.verifier).toEqual(['opus'])
|
|
})
|
|
|
|
test('a test timeout is NOT VERIFIED, never a FAIL: no verifier call, no resume, no fail counted', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff', 'diff 2']
|
|
w.testEnds = [124, 0]
|
|
w.testFired = [true, false]
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.told[0]).toContain('NOT VERIFIED - the tests did not finish within 60 s')
|
|
expect(w.told[0]).not.toContain('Resume agent')
|
|
// the next round's FAIL is the first one counted: it resumes, it does not rewrite
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.told[1]).toContain('Resume agent')
|
|
})
|
|
|
|
test('a kill after the grace period also counts as a timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [137]
|
|
w.testFired = [true]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.told[0]).toContain('did not finish')
|
|
})
|
|
|
|
test('a test command that cannot run (exit 127) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [127]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.told[0]).toContain('could not run')
|
|
})
|
|
|
|
test('a test command that cannot run (a start failure) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = ['throw' as const]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.told[0]).toContain('could not run')
|
|
})
|
|
|
|
test('failing tests are evidence: the verifier sees them and decides', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus'])
|
|
})
|
|
|
|
test('an early 137 (out of memory) is a failed run for the verifier, not a timeout', { options: { testCommand: 'make t', testTimeoutSec: 1800 } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [137]
|
|
w.testFired = [false]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus'])
|
|
expect(w.told[0] ?? '').not.toContain('NOT VERIFIED')
|
|
})
|
|
|
|
test('an invalid testTimeoutSec falls back to 1800', async () => {
|
|
for (const bad of [0, -5, Number.NaN, 'abc', undefined, null]) expect(validTimeout(bad)).toBe(1800)
|
|
expect(validTimeout(900)).toBe(900)
|
|
expect(validTimeout('600')).toBe(600)
|
|
})
|
|
|
|
test('a command that exits 124 by itself is a failed run, not a timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [124]
|
|
w.testFired = [false]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus'])
|
|
})
|
|
|
|
test('the timeout line still counts when it arrives split across pieces', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [124]
|
|
w.testSplitLine = true
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.told[0]).toContain('did not finish')
|
|
})
|
|
|
|
test('the timeout line on stdout (a test printing it) is not a fired timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.testLineOnStdout = true
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus'])
|
|
})
|
|
|
|
// ---------- TOOL-02 end to end ----------
|
|
|
|
test('TOOL-02: the SAFETY-129 reply passes the round, no rewrite', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.verdicts = [SAFETY_129]
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus'])
|
|
expect(w.told[0]).toContain('PASS from verifier opus')
|
|
})
|
|
|
|
test('TOOL-02: an unreadable reply is asked again once, and a readable answer then counts', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.verdicts = ['I think it is fine overall.', '{"verdict":"FAIL","issues":["x"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus', 'opus'])
|
|
expect(w.told[0]).toContain('Resume agent')
|
|
})
|
|
|
|
test('TOOL-02: unreadable twice is handed back, no fail counted, no rewrite', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff', 'diff 2']
|
|
w.verdicts = ['Looks good to me.', 'Still prose, sorry.', '{"verdict":"FAIL","issues":["y"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.verifier).toEqual(['opus', 'opus'])
|
|
expect(w.told[0]).toContain('NOT VERIFIED - verdict unreadable')
|
|
expect(w.told[0]).toContain('Start of its reply: Looks good to me.')
|
|
expect(w.told[0]).not.toContain('[router-rewrite')
|
|
expect(w.told[0]).not.toContain('Resume agent')
|
|
// the next round's FAIL is the first one counted: it resumes, it does not rewrite
|
|
await finish($, clock, r.agentId as string)
|
|
expect(w.told[1]).toContain('Resume agent')
|
|
expect(w.told[1]).not.toContain('[router-rewrite')
|
|
})
|
|
|
|
// ---------- TOOL-12 end to end ----------
|
|
|
|
const opusBrief = () =>
|
|
`SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/\nBase branch: release/1.0.`
|
|
|
|
test('TOOL-12 acceptance 1: an Opus-routed dev gets the skip line at hand-back, and no verdict later', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
|
|
expect(w.spawned[0]?.model).toBe('opus')
|
|
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
|
await clock.advance(60_000)
|
|
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
|
expect(w.verifier).toEqual([])
|
|
})
|
|
|
|
test('TOOL-12 acceptance 2: a Sonnet-routed dev gets "verifying", then the verdict', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
expect(w.spawned[0]?.model).toBe('sonnet')
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus...')
|
|
expect(w.all[1]).toContain('PASS from verifier opus')
|
|
expect(w.all.length).toBe(2)
|
|
})
|
|
|
|
test('TOOL-12 acceptance 3a: routed Sonnet but ran on Opus is reported and handled as Opus', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
|
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
|
|
expect(w.verifier).toEqual([])
|
|
})
|
|
|
|
test('TOOL-12 acceptance 3b: routed Opus but ran on Sonnet is verified, and labelled Sonnet', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
|
|
expect(w.verifier).toEqual(['opus'])
|
|
expect(w.told[0]).toContain('(agent agent-1, sonnet, round 1)')
|
|
// the resumed round still reports against the original routing choice
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
|
|
})
|
|
|
|
test('TOOL-12: a dev run that ends with an error says so, no verification', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5', 'error')
|
|
expect(w.all).toEqual(['[model-router] T1 no verification: the dev run ended without an answer (error).'])
|
|
expect(w.verifier).toEqual([])
|
|
})
|
|
|
|
test('TOOL-12: review and test agents get no hand-back line', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: `Review. Clone: ${REVIEW}/`, description: 'review', model: 'opus' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
|
expect(w.all).toEqual([])
|
|
})
|
|
|
|
// ---------- interim stops (TOOL-12 follow-up) ----------
|
|
|
|
/** A turn end of the agent while it is still alive: its status says so. */
|
|
async function interimStop($: Engine, clock: { advance: (ms: number) => Promise<void> }, w: World, agentId: string, status = 'waiting') {
|
|
w.status[agentId] = status
|
|
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({
|
|
answer: 'Waiting for the test suite.',
|
|
durationMs: 1,
|
|
isAborted: false,
|
|
turnId: 't',
|
|
agentId,
|
|
reason: 'answer',
|
|
usage: { model: 'claude-sonnet-5-5', input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
|
})
|
|
await clock.advance(1)
|
|
}
|
|
|
|
test('follow-up acceptance: 3 interim stops give exactly one verifying line and one verdict, both after the hand-back', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28 dev' } as never)
|
|
const id = r.agentId as string
|
|
for (let i = 0; i < 3; i++) {
|
|
await interimStop($, clock, w, id)
|
|
await clock.advance(40 * 60 * 1000) // the test suite runs 35-45 min
|
|
expect(w.all).toEqual([])
|
|
expect(w.verifier).toEqual([])
|
|
expect(w.diffBases).toEqual([])
|
|
}
|
|
w.status[id] = 'completed'
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
await clock.advance(60_000)
|
|
expect(w.handBacks).toEqual(['[model-router] T1 verifying on Opus...'])
|
|
expect(w.verifier).toEqual(['opus'])
|
|
expect(w.told.length).toBe(1)
|
|
expect(w.all[1]).toContain('PASS from verifier opus')
|
|
})
|
|
|
|
test('follow-up: an interim stop of an Opus dev posts nothing; its hand-back posts the skip line once', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: `SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/`, description: 'dev' } as never)
|
|
const id = r.agentId as string
|
|
await interimStop($, clock, w, id)
|
|
await interimStop($, clock, w, id, 'idle')
|
|
await clock.advance(10 * 60 * 1000)
|
|
expect(w.all).toEqual([])
|
|
w.status[id] = 'completed'
|
|
await finish($, clock, id, 'claude-opus-5-5')
|
|
await clock.advance(60_000)
|
|
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
|
})
|
|
|
|
test('follow-up: a status that turns ended a moment after the turn is still caught', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
const id = r.agentId as string
|
|
w.status[id] = 'running' // not yet updated when the turn ends
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
expect(w.all).toEqual([])
|
|
w.status[id] = 'completed'
|
|
await clock.advance(16_000)
|
|
expect(w.handBacks).toEqual(['[model-router] T1 verifying on Opus...'])
|
|
expect(w.verifier).toEqual(['opus'])
|
|
})
|
|
|
|
test('follow-up: an agent the engine already dropped counts as handed back', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
const id = r.agentId as string
|
|
w.status[id] = 'gone'
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
await clock.advance(1)
|
|
expect(w.handBacks.length).toBe(1)
|
|
})
|
|
|
|
test('follow-up: a failed or killed run is a hand-back too', async () => {
|
|
for (const s of ['completed', 'failed', 'killed', undefined] as const) expect(turnEndKind(s)).toBe('handback')
|
|
for (const s of ['waiting', 'idle', 'running', 'pending'] as const) expect(turnEndKind(s)).toBe('interim')
|
|
})
|
|
|
|
test('follow-up: the hand-back uses the last turn: its model and how it ended', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
const id = r.agentId as string
|
|
await interimStop($, clock, w, id) // reported Sonnet
|
|
w.status[id] = 'completed'
|
|
await finish($, clock, id, 'claude-opus-5-5') // the last turn ran on Opus
|
|
await clock.advance(60_000)
|
|
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
|
|
})
|
|
|
|
test('follow-up: each agent is judged by its own status, not by that of another agent', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
const clock = world(on, w)
|
|
await $.agent.spawn({ prompt: devBrief(), description: 'finished dev' } as never) // agent-1: completed
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'busy dev' } as never)
|
|
await interimStop($, clock, w, r.agentId as string) // agent-2: waiting
|
|
await clock.advance(60_000)
|
|
expect(w.all).toEqual([])
|
|
expect(w.verifier).toEqual([])
|
|
})
|
|
|
|
// ---------- resumes, follow-ups and rewrites (#4, #6) ----------
|
|
|
|
const sendMessage = ($: Engine, to: string, message: string) =>
|
|
$.tool.call({ tool: 'SendMessage', to, message } as never)
|
|
|
|
const resume = ($: Engine, agentId: string, prompt: string) =>
|
|
$.agent.spawn({ prompt, description: 'resume', resume: { agentId }, model: 'sonnet' } as never)
|
|
|
|
test('#4: a resume keeps its task, routing and counts: no new Tn, no new routing', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1', 'd2']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
const id = r.agentId as string
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
expect(w.told[0]).toContain(`Resume agent ${id}`)
|
|
await resume($, id, 'Fix the issues: x')
|
|
expect(w.resumed[0]?.model).toBe('sonnet') // passed through untouched
|
|
expect(w.spawned.length).toBe(1)
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus...')
|
|
expect(w.diffBases[1]).toBe('snap1') // round 2 diffs from the last verified round
|
|
expect(w.told[1]).toContain('round 2')
|
|
expect(w.told[1]).toContain('[router-rewrite T1]') // second fail counted: escalates
|
|
})
|
|
|
|
test('#4: a resumed Opus dev is labelled as Opus, with no "routed to" note', async ($, on) => {
|
|
const w = fresh()
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: `SAFETY-135: decide how the guard should recover. Clone: ${DEV}/`, description: 'dev' } as never)
|
|
const id = r.agentId as string
|
|
await finish($, clock, id, 'claude-opus-5-5')
|
|
await resume($, id, 'Docs-only round: update the README.')
|
|
await finish($, clock, id, 'claude-opus-5-5')
|
|
expect(w.handBacks).toEqual([
|
|
'[model-router] T1 no verification: dev ran on Opus. Go straight to review.',
|
|
'[model-router] T1 no verification: dev ran on Opus. Go straight to review.',
|
|
])
|
|
})
|
|
|
|
test('#4: the label compares with what the engine resolved at start, not what the router asked', async ($, on) => {
|
|
const w = fresh()
|
|
w.engineModel = 'claude-opus-5-5' // e.g. the agent type sets its own model
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
expect(w.spawned[0]?.model).toBe('sonnet')
|
|
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
|
expect(w.handBacks).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
|
const log = (await $.command.run({ command: 'router', args: '' } as never)).text ?? ''
|
|
expect(log).toContain('opus')
|
|
})
|
|
|
|
test('#4: a router rewrite keeps the task number and its model', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1', 'd2']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
const rw = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite', model: 'opus' } as never)
|
|
expect(w.spawned[1]?.model).toBe('opus')
|
|
await finish($, clock, rw.agentId as string, 'claude-opus-5-5')
|
|
expect(w.handBacks[2]).toBe('[model-router] T1 no verification: dev ran on Opus. Go straight to review.')
|
|
})
|
|
|
|
test('#4: a rewrite whose first start fails can be started again', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1', 'd2']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
w.denySpawns = 1
|
|
const first = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
|
expect(first.deny).toBe('spawn failed')
|
|
const second = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
|
expect(second.deny).toBe(undefined)
|
|
expect(w.spawned[1]?.model).toBe('opus')
|
|
expect(w.spawned[1]?.prompt).toContain('Previous attempts rejected')
|
|
})
|
|
|
|
test('#4: a rewrite marker with nothing stored is refused, not started without a brief', async ($, on) => {
|
|
const w = fresh()
|
|
world(on, w)
|
|
const r = await $.agent.spawn({ prompt: '[router-rewrite T325]', description: 'rewrite', model: 'opus' } as never)
|
|
expect(r.deny).toContain('no stored rewrite for T325')
|
|
expect(w.spawned).toEqual([])
|
|
})
|
|
|
|
test('#6: a resume message reaches the verifier as a later instruction, in scope', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1', 'd2']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-24' } as never)
|
|
const id = r.agentId as string
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
await resume($, id, 'Owner: add MAINT-25 to this PR as well.')
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
const p = w.verifierPrompts[1] ?? ''
|
|
expect(p).toContain('## Later instructions to this agent')
|
|
expect(p).toContain('1. Owner: add MAINT-25 to this PR as well.')
|
|
expect(p).toContain('SAFETY-95: clamp the integrator') // the original brief stays
|
|
})
|
|
|
|
test('#6: SendMessage to a running agent, by id or by name, is a later instruction too', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28', name: 'maint28-dev' } as never)
|
|
const id = r.agentId as string
|
|
await sendMessage($, id, 'Owner decided: cap is 1.')
|
|
await sendMessage($, 'maint28-dev', 'Also cover the empty list.')
|
|
await sendMessage($, 'maint28-dev', 'Also cover the empty list.') // the same text counts once
|
|
await sendMessage($, 'someone-else', 'Not for this agent.')
|
|
await finish($, clock, id, 'claude-sonnet-5-5')
|
|
const p = w.verifierPrompts[0] ?? ''
|
|
expect(p).toContain('1. Owner decided: cap is 1.')
|
|
expect(p).toContain('2. Also cover the empty list.')
|
|
expect(p).not.toContain('3.')
|
|
expect(p).not.toContain('Not for this agent.')
|
|
})
|
|
|
|
test('#6: without later instructions the section is left out', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.verifierPrompts[0] ?? '').not.toContain('Later instructions')
|
|
})
|
|
|
|
test('#4: a rewrite that started is used up: the same marker again is refused', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1', 'd2']
|
|
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
|
const again = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
|
expect(again.deny).toContain('no stored rewrite for T1')
|
|
expect(w.spawned.length).toBe(2)
|
|
})
|
|
|
|
test('#6: a SendMessage that was not delivered is not a later instruction', async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['d1']
|
|
w.sendFails = true
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
|
await sendMessage($, r.agentId as string, 'Never arrived.')
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.verifierPrompts[0] ?? '').not.toContain('Never arrived.')
|
|
})
|
|
|
|
// ---------- reruns of reported files (#5) ----------
|
|
|
|
const RERUN = { testCommand: 'make t', testFileCommand: 'runone {file}' }
|
|
const T307_OUTPUT = [
|
|
'PASS test_a.py 12s\nTIMEOUT test_tariff24_weather.py 230s\nFAIL test_loop.py\nTIME',
|
|
'OUT test_slow.py 1500s\n', // a line split across pieces
|
|
'--- test_tariff24_weather.py (TIMEOUT)\n<tail>\n77 pass, 1 fail, 2 timeout (limit 1500 s)\n',
|
|
]
|
|
|
|
test('#5: reported lines are read, also when split across pieces; one entry per file', async () => {
|
|
const re = issueRegex('')
|
|
expect(parseIssueLine('TIMEOUT test_tariff24_weather.py 230s', re)).toEqual({ kind: 'TIMEOUT', file: 'test_tariff24_weather.py' })
|
|
expect(parseIssueLine('FAIL test_loop.py', re)).toEqual({ kind: 'FAIL', file: 'test_loop.py' })
|
|
expect(parseIssueLine('--- test_tariff24_weather.py (TIMEOUT)', re)).toBe(undefined)
|
|
expect(parseIssueLine('77 pass, 0 fail, 30 timeout (limit 1500 s)', re)).toBe(undefined)
|
|
expect(parseIssueLine('PASS test_a.py', re)).toBe(undefined)
|
|
// a custom pattern, and a broken one falls back to the default
|
|
expect(parseIssueLine('[timeout] x.py', issueRegex('^\\[(timeout|fail)\\]\\s+(\\S+)'))?.file).toBe('x.py')
|
|
expect(parseIssueLine('TIMEOUT y.py', issueRegex('(unclosed'))?.file).toBe('y.py')
|
|
})
|
|
|
|
test('#5: only safe file names reach the shell', async () => {
|
|
for (const ok of ['test_a.py', 'tests/test_b.py', 'a-b.c_d.py']) expect(isSafeFile(ok)).toBe(true)
|
|
for (const bad of ['a;rm -rf ~', '$(id).py', '../x.py', 'a/../b.py', '-rf', 'a b.py', 'x`y`.py', "a'.py"]) expect(isSafeFile(bad)).toBe(false)
|
|
expect(fileCommand('docker run img python3 {file} && echo {file}', 'test_a.py')).toBe('docker run img python3 test_a.py && echo test_a.py')
|
|
})
|
|
|
|
test('#5: what each rerun outcome means', async () => {
|
|
expect(rerunVerdict({ file: 'f', full: 'TIMEOUT', head: 'pass', base: 'not run' })).toContain('does not count')
|
|
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'fail', base: 'fail' })).toContain('not caused by this change')
|
|
expect(rerunVerdict({ file: 'f', full: 'TIMEOUT', head: 'timeout', base: 'timeout' })).toContain('also times out on base')
|
|
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'fail', base: 'pass' })).toContain('counts against the change')
|
|
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'fail', base: 'not run' })).toContain('base not checked')
|
|
expect(rerunVerdict({ file: 'f', full: 'FAIL', head: 'error', base: 'not run' })).toContain('could not be rerun')
|
|
})
|
|
|
|
test('#5 acceptance: timed-out and failed files are rerun alone, failures again on base, and the verifier gets the table', { options: RERUN }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = T307_OUTPUT
|
|
w.fileOutcomes = { 'head:test_loop.py': 'fail', 'base:test_loop.py': 'fail', 'head:test_slow.py': 'timeout', 'base:test_slow.py': 'pass' }
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect([...w.fileRuns].sort()).toEqual([
|
|
'base:test_loop.py',
|
|
'base:test_slow.py',
|
|
'head:test_loop.py',
|
|
'head:test_slow.py',
|
|
'head:test_tariff24_weather.py',
|
|
])
|
|
const p = w.verifierPrompts[0] ?? ''
|
|
expect(p).toContain('### Files the full run reported, rerun one by one')
|
|
expect(p).toContain('| test_tariff24_weather.py | TIMEOUT | pass | not run | passes alone: does not count against the change |')
|
|
expect(p).toContain('| test_loop.py | FAIL | fail | fail | also fails on base: not caused by this change |')
|
|
expect(p).toContain('| test_slow.py | TIMEOUT | timeout | pass | times out alone, passes on base: counts against the change |')
|
|
// the base worktree sits next to the clone, and is removed and pruned afterwards
|
|
expect(w.worktree).toContain(`git -C ${DEV} worktree add --detach ${DEV}-verifier-base mergebase`)
|
|
expect(w.worktree.at(-2)).toBe(`git -C ${DEV} worktree remove --force ${DEV}-verifier-base`)
|
|
expect(w.worktree.at(-1)).toBe(`git -C ${DEV} worktree prune`)
|
|
})
|
|
|
|
test('#5: when every reported file passes alone, no base worktree is made', { options: RERUN }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = ['TIMEOUT test_tariff24_weather.py 230s\n']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.fileRuns).toEqual(['head:test_tariff24_weather.py'])
|
|
expect(w.worktree.filter(c => c.includes(' add '))).toEqual([])
|
|
})
|
|
|
|
test('#5: a base worktree that cannot be made leaves base as not run', { options: RERUN }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.worktreeFails = true
|
|
w.fullRunOutput = ['FAIL test_loop.py\n']
|
|
w.fileOutcomes = { 'head:test_loop.py': 'fail' }
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.fileRuns).toEqual(['head:test_loop.py'])
|
|
expect(w.verifierPrompts[0]).toContain('| test_loop.py | FAIL | fail | not run | fails alone; base not checked |')
|
|
})
|
|
|
|
test('#5: unsafe names and files past testRerunMax are not rerun, and said so', { options: { ...RERUN, testRerunMax: 1 } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = ['FAIL a;rm.py\nTIMEOUT test_a.py 1s\nTIMEOUT test_b.py 1s\n']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.fileRuns).toEqual(['head:test_a.py'])
|
|
expect(w.verifierPrompts[0]).toContain('(2 more reported file(s) not rerun')
|
|
})
|
|
|
|
test('#5: without testFileCommand nothing is rerun', { options: { testCommand: 'make t' } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = T307_OUTPUT
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.fileRuns).toEqual([])
|
|
expect(w.verifierPrompts[0] ?? '').not.toContain('rerun one by one')
|
|
})
|
|
|
|
test('#5: reruns run at most testRerunJobs at a time', { options: { ...RERUN, testRerunJobs: 2 } }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = ['TIMEOUT t1.py 1s\nTIMEOUT t2.py 1s\nTIMEOUT t3.py 1s\nTIMEOUT t4.py 1s\nTIMEOUT t5.py 1s\n']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect(w.fileRuns.length).toBe(5)
|
|
expect(w.maxParallel).toBe(2)
|
|
})
|
|
|
|
test('#5: a custom pattern that also matches other lines still keeps only TIMEOUT and FAIL', async () => {
|
|
const re = issueRegex('^(\\w+)\\s+(\\S+)')
|
|
expect(parseIssueLine('PASS test_a.py', re)).toBe(undefined)
|
|
expect(parseIssueLine('timeout test_b.py', re)).toEqual({ kind: 'TIMEOUT', file: 'test_b.py' })
|
|
})
|
|
|
|
test('#5: a last line without a newline is read; a file reported twice is rerun once', { options: RERUN }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = ['FAIL test_twice.py\nFAIL test_twice.py\n', 'TIMEOUT test_last.py 1500s']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
expect([...w.fileRuns].sort()).toEqual(['head:test_last.py', 'head:test_twice.py'])
|
|
})
|
|
|
|
test('#5: the verifier is told how to read the rerun table', { options: RERUN }, async ($, on) => {
|
|
const w = fresh()
|
|
w.diffs = ['diff']
|
|
w.testEnds = [1]
|
|
w.fullRunOutput = ['TIMEOUT test_a.py 1s\n']
|
|
const clock = world(on, w)
|
|
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
|
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
|
const sys = w.verifierSystems[0] ?? ''
|
|
expect(sys).toContain('When a table of files rerun one by one is given')
|
|
expect(sys).toContain('a file that passes alone does not count against the change')
|
|
expect(sys).toContain('not caused by this change')
|
|
expect(sys).toContain('Name every file you count against the change')
|
|
})
|