Fix TOOL-12: one line on every dev hand-back, by the model that ran
An Opus-run dev got no verification and no message, so silence meant both "still verifying" and "will never verify" (SAFETY-135 waited overnight, SAFETY-126 8 h). Verdicts were also labelled with the routing choice, not the model the run used. - Every dev hand-back posts one line at once: "[model-router] Tn verifying on Opus..." (a verdict follows), or "[model-router] Tn no verification: dev ran on Opus. Go straight to review." A run that ends with an error or names no clone says so. - Each round is decided and labelled by the model the run reported (its end usage), the routing choice only as fallback. A difference is shown as "(routed to X, ran on Y)", also on later rounds. - 8 new tests; each new rule was also broken on purpose and caught, apart from one equivalent change. Version 0.4.6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
This commit is contained in:
Claude Opus 5.5
+112
-4
@@ -11,6 +11,8 @@ import {
|
||||
skipReason,
|
||||
validTimeout,
|
||||
verifierFor,
|
||||
handBackLine,
|
||||
ranTier,
|
||||
} from '../hooks/router'
|
||||
|
||||
const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev'
|
||||
@@ -180,6 +182,31 @@ test('TOOL-02: garbage, prose and odd verdicts are unreadable, never a FAIL', as
|
||||
expect(long.isReadable ? '' : long.preview).toBe('x'.repeat(300))
|
||||
})
|
||||
|
||||
// ---------- the hand-back line (TOOL-12) ----------
|
||||
|
||||
test('TOOL-12: the model that ran wins over the routing choice', async () => {
|
||||
expect(ranTier('claude-opus-5-5', 'sonnet')).toEqual({ tier: 'opus', isMismatch: true })
|
||||
expect(ranTier('claude-sonnet-5-5', 'opus')).toEqual({ tier: 'sonnet', isMismatch: true })
|
||||
expect(ranTier('claude-sonnet-5-5', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
||||
expect(ranTier(undefined, 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
||||
expect(ranTier('some-other-model', 'sonnet')).toEqual({ tier: 'sonnet', isMismatch: false })
|
||||
})
|
||||
|
||||
test('TOOL-12: the exact lines', async () => {
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe('[model-router] T7 verifying on Opus...')
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'opus', skip: 'opus-unverified', ended: 'answer' })).toBe(
|
||||
'[model-router] T7 no verification: dev ran on Opus. Go straight to review.',
|
||||
)
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'opus', routed: 'sonnet', skip: 'opus-unverified', ended: 'answer' })).toBe(
|
||||
'[model-router] T7 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)',
|
||||
)
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'agent-ended', ended: 'error' })).toContain('ended without an answer (error)')
|
||||
expect(handBackLine({ taskId: 'T7', role: 'dev', ran: 'sonnet', routed: 'sonnet', skip: 'no-clone', ended: 'answer' })).toContain('no clone path')
|
||||
expect(handBackLine({ taskId: 'T7', role: 'review', ran: 'opus', routed: 'opus', skip: 'review-agent', ended: 'answer' })).toBe(undefined)
|
||||
expect(handBackLine({ taskId: 'T7', role: 'test', ran: 'opus', routed: 'opus', skip: 'test-agent', ended: 'answer' })).toBe(undefined)
|
||||
expect(handBackLine({ taskId: 'T7', ran: 'sonnet', routed: 'sonnet', ended: 'answer' })).toBe(undefined)
|
||||
})
|
||||
|
||||
// ---------- the hooks, with the engine mocked beneath ----------
|
||||
|
||||
type World = {
|
||||
@@ -187,6 +214,10 @@ type World = {
|
||||
verifier: string[]
|
||||
verdicts: string[]
|
||||
told: string[]
|
||||
/** TOOL-12 hand-back lines, kept apart so `told` holds the verdicts. */
|
||||
handBacks: string[]
|
||||
/** Every message in the order it was sent. */
|
||||
all: string[]
|
||||
diffs: string[]
|
||||
diffBases: string[]
|
||||
testCwd: string[]
|
||||
@@ -250,16 +281,21 @@ function world(on: On, w: World) {
|
||||
})
|
||||
on('turn.complete', async (_$, e) => ({ text: e.answer }))
|
||||
on('prompt.submit', async (_$, e) => {
|
||||
w.told.push(e.text)
|
||||
w.all.push(e.text)
|
||||
if (/^\[model-router\] T\d+ (verifying on|no verification:)/.test(e.text)) w.handBacks.push(e.text)
|
||||
else w.told.push(e.text)
|
||||
return { text: e.text }
|
||||
})
|
||||
return clock
|
||||
}
|
||||
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], handBacks: [], all: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
|
||||
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string) {
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason: 'answer' })
|
||||
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string, ranModel?: string, reason = 'answer') {
|
||||
const usage = ranModel
|
||||
? { usage: { model: ranModel, input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } }
|
||||
: {}
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason, ...usage })
|
||||
await clock.advance(1)
|
||||
}
|
||||
|
||||
@@ -535,3 +571,75 @@ test('TOOL-02: unreadable twice is handed back, no fail counted, no rewrite', as
|
||||
expect(w.told[1]).toContain('Resume agent')
|
||||
expect(w.told[1]).not.toContain('[router-rewrite')
|
||||
})
|
||||
|
||||
// ---------- TOOL-12 end to end ----------
|
||||
|
||||
const opusBrief = () =>
|
||||
`SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/\nBase branch: release/1.0.`
|
||||
|
||||
test('TOOL-12 acceptance 1: an Opus-routed dev gets the skip line at hand-back, and no verdict later', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('opus')
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
await clock.advance(60_000)
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('TOOL-12 acceptance 2: a Sonnet-routed dev gets "verifying", then the verdict', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('sonnet')
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus...')
|
||||
expect(w.all[1]).toContain('PASS from verifier opus')
|
||||
expect(w.all.length).toBe(2)
|
||||
})
|
||||
|
||||
test('TOOL-12 acceptance 3a: routed Sonnet but ran on Opus is reported and handled as Opus', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('TOOL-12 acceptance 3b: routed Opus but ran on Sonnet is verified, and labelled Sonnet', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: opusBrief(), description: 'SAFETY-135 dev' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.all[0]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
expect(w.told[0]).toContain('(agent agent-1, sonnet, round 1)')
|
||||
// the resumed round still reports against the original routing choice
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus... (routed to Opus, ran on Sonnet)')
|
||||
})
|
||||
|
||||
test('TOOL-12: a dev run that ends with an error says so, no verification', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5', 'error')
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: the dev run ended without an answer (error).'])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('TOOL-12: review and test agents get no hand-back line', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: `Review. Clone: ${REVIEW}/`, description: 'review', model: 'opus' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.all).toEqual([])
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user