Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
  adminandClaude Opus 5.5
|
fd75c183ba | ||
|
adminandClaude Opus 5.5
|
fc41ad21aa |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "model-router",
|
||||
"version": "0.4.6",
|
||||
"version": "0.4.8",
|
||||
"description": "Routes each subagent to Haiku, Sonnet or Opus by how well-defined its brief is, verifies dev work below Opus in its own clone on a different model, sends failures back to the same agent then a stronger one, and estimates the savings against Opus.",
|
||||
"author": {
|
||||
"name": "Glenn Schrooyen"
|
||||
|
||||
@@ -7,8 +7,9 @@ A Claude Code plugin that picks a model for every subagent, checks dev work on a
|
||||
| Step | Behaviour |
|
||||
|---|---|
|
||||
| Route | Dev agents run on **Sonnet** when the brief decides the approach (Expected / acceptance criteria), on **Opus** when the design is still open. Review and test agents keep the model they are given (Opus when none). |
|
||||
| Later instructions | Every message sent to a dev agent after its brief (a SendMessage, or the message that resumes it) reaches the verifier as part of the task, so scope added later is judged as in scope. A resumed agent keeps its task number, routing and failure count. |
|
||||
| Verify | When a dev agent below Opus finishes a round, the router diffs its clone against the merge base, runs the test command in that clone, and has **Opus** judge the change. |
|
||||
| Hand-back line | Every dev hand-back gets one line at once: `[model-router] Tn verifying on Opus...` (a verdict follows), or `[model-router] Tn no verification: dev ran on Opus. Go straight to review.` Decided by the model the run actually used, not only the routing choice; a difference is shown as `(routed to X, ran on Y)`. |
|
||||
| Hand-back line | Every dev hand-back gets one line at once: `[model-router] Tn verifying on Opus...` (a verdict follows), or `[model-router] Tn no verification: dev ran on Opus. Go straight to review.` Only the real hand-back counts: an interim stop while the dev waits on its own background work (e.g. a long test suite) posts nothing and starts no verifier. Decided by the model the run actually used, not only the routing choice; a difference is shown as `(routed to X, ran on Y)`. |
|
||||
| On FAIL | 1st: resume the same agent with the issues. 2nd: a new agent one model up, in the same clone. After that: handed back to the manager. |
|
||||
| Not verified | A test timeout, a test command that cannot run, a missing clone or merge base, or a verdict that cannot be read even after asking once more: reported as NOT VERIFIED, never as a FAIL. |
|
||||
| Savings | `/router savings` prices every run on the model it ran on and compares it with the same tokens on Opus 5.5. |
|
||||
@@ -59,7 +60,7 @@ claude plugin validate .
|
||||
claude plugin test .
|
||||
```
|
||||
|
||||
62 tests. Every routing, verification, cost and timeout rule was also checked by breaking it on purpose (66 mutations in v0.4.4, plus 13 for the verdict reader in v0.4.5 and 12 for the hand-back line in v0.4.6; all caught by a test).
|
||||
80 tests. Every routing, verification, cost and timeout rule was also checked by breaking it on purpose (66 mutations in v0.4.4, plus 13 for the verdict reader in v0.4.5 12 for the hand-back line in v0.4.6 10 for interim stops in v0.4.7 and 14 for resumes, later instructions and rewrites in v0.4.8; all caught by a test).
|
||||
|
||||
## Known limits
|
||||
|
||||
|
||||
+121
-11
@@ -21,6 +21,9 @@ import {
|
||||
parseVerdict,
|
||||
handBackLine,
|
||||
ranTier,
|
||||
turnEndKind,
|
||||
WATCH_EVERY_MS,
|
||||
WATCH_FOR_MS,
|
||||
safetyHit,
|
||||
skipReason,
|
||||
validTimeout,
|
||||
@@ -36,8 +39,12 @@ type AgentRec = {
|
||||
taskId: string
|
||||
/** The model the last round actually ran on (the routing choice until a round reports one). */
|
||||
tier: Tier
|
||||
/** The model the router chose at spawn; tier can differ when the run used another. */
|
||||
/** The model the engine resolved at spawn; tier can differ when the run used another. */
|
||||
routed?: Tier
|
||||
/** The name the Agent call gave it, which SendMessage may address it by. */
|
||||
name?: string
|
||||
/** Later instructions to this agent (resumes, SendMessage), oldest first: part of its task. */
|
||||
followUps?: string[]
|
||||
role?: Role
|
||||
clone?: string
|
||||
baseBranch?: string
|
||||
@@ -119,6 +126,7 @@ When unsure between sonnet and opus, pick sonnet: a failed verification escalate
|
||||
const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not.
|
||||
Judge only whether the change correctly and completely does the task, without breaking anything nearby.
|
||||
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
|
||||
Scope added in "Later instructions" was asked for after the brief: judge it as in scope, not as unrequested work.
|
||||
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
|
||||
|
||||
Reply with JSON only:
|
||||
@@ -129,8 +137,9 @@ The model-router mod picks each subagent's model and verifies dev work before it
|
||||
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
|
||||
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
|
||||
- Review and test agents keep the model you name (Opus when you name none).
|
||||
- Every dev hand-back gets one "[model-router] Tn" line at once. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on.
|
||||
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. Do not fix rejected work yourself.`
|
||||
- Every dev hand-back gets one "[model-router] Tn" line at once; an interim stop while the dev waits on its own background work gets none. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on.
|
||||
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person.
|
||||
- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself.`
|
||||
|
||||
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
||||
|
||||
@@ -377,6 +386,11 @@ async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, c
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
|
||||
const question = [
|
||||
`## Task\n${clip(rec.prompt, 12_000)}`,
|
||||
rec.followUps?.length
|
||||
? `## Later instructions to this agent (after the brief; part of the task, in scope)\n${rec.followUps
|
||||
.map((f, i) => `${i + 1}. ${f}`)
|
||||
.join('\n\n')}`
|
||||
: '',
|
||||
rec.lastIssues?.length
|
||||
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
|
||||
: '',
|
||||
@@ -523,6 +537,71 @@ async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ende
|
||||
else await verifyRound($, agentId, round, cfg)
|
||||
}
|
||||
|
||||
// ---------- later instructions (#6) ----------
|
||||
|
||||
const FOLLOW_UPS_KEPT = 12
|
||||
|
||||
/** Adds a later instruction to a tracked agent's record; the same text twice counts once. */
|
||||
async function noteFollowUp($: EngineInterface, agentId: string, text: string) {
|
||||
const t = text.trim()
|
||||
if (!t) return
|
||||
await updateIn<AgentRec>($, S.agents, agentId, old => {
|
||||
if (!old) return undefined
|
||||
const list = old.followUps ?? []
|
||||
if (list.includes(clip(t, 4000))) return old
|
||||
return { ...old, followUps: [...list, clip(t, 4000)].slice(-FOLLOW_UPS_KEPT) }
|
||||
})
|
||||
}
|
||||
|
||||
/** The tracked agent a SendMessage `to` names: its id, or the name its Agent call gave it. */
|
||||
async function agentAddressed($: EngineInterface, to: string) {
|
||||
const recs = await getMap<AgentRec>($, S.agents)
|
||||
if (recs[to]) return to
|
||||
return Object.keys(recs).find(id => recs[id]?.name === to)
|
||||
}
|
||||
|
||||
// ---------- waiting for the real hand-back ----------
|
||||
|
||||
// The newest turn end per agent. A turn end that is only an interim stop (the
|
||||
// agent waits on its own background work) must not post a line or start the
|
||||
// verifier: only the run's end does. Module state: a reload drops the watches.
|
||||
const lastTurn = new Map<string, { token: number; ended: string; reported?: string }>()
|
||||
let tokens = 0
|
||||
|
||||
function pause($: EngineInterface, ms: number) {
|
||||
return new Promise<void>(resolve => {
|
||||
$.clock.after(ms, () => resolve())
|
||||
})
|
||||
}
|
||||
|
||||
async function statusOf($: EngineInterface, agentId: string) {
|
||||
try {
|
||||
return (await $.agent.list()).find(a => a.id === agentId)?.status
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* After a tracked agent's turn ends: runs the round once its run has ended,
|
||||
* and not for an interim stop. A newer turn end of the same agent takes over.
|
||||
*/
|
||||
async function awaitHandBack($: EngineInterface, agentId: string, token: number, cfg: Settings) {
|
||||
const started = await $.clock.now()
|
||||
for (;;) {
|
||||
if (lastTurn.get(agentId)?.token !== token) return // a newer turn end took over
|
||||
if (turnEndKind(await statusOf($, agentId)) === 'handback') break
|
||||
if ((await $.clock.now()) - started > WATCH_FOR_MS) return
|
||||
await pause($, WATCH_EVERY_MS)
|
||||
}
|
||||
const turn = lastTurn.get(agentId)
|
||||
if (!turn || turn.token !== token) return
|
||||
lastTurn.delete(agentId)
|
||||
// read the record now: an earlier round may have changed it meanwhile
|
||||
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
||||
if (rec) await runRound($, agentId, rec, turn.ended, turn.reported, cfg)
|
||||
}
|
||||
|
||||
// ---------- the hooks ----------
|
||||
|
||||
export const register: Register = (on, options) => {
|
||||
@@ -556,6 +635,14 @@ export const register: Register = (on, options) => {
|
||||
on('agent.spawn', async ($, e, next) => {
|
||||
if (e.fork || e.isTeammate || e.workflow) return next(e)
|
||||
|
||||
// A resume continues a known agent: same task, same routing, same counts.
|
||||
// Its message is a later instruction the verifier must see (#4, #6).
|
||||
if (e.resume) {
|
||||
const resumed = await next(e)
|
||||
if (resumed.deny === undefined) await noteFollowUp($, e.resume.agentId, e.prompt)
|
||||
return resumed
|
||||
}
|
||||
|
||||
let taskId: string
|
||||
let prompt = e.prompt
|
||||
let tier: Tier
|
||||
@@ -577,8 +664,14 @@ export const register: Register = (on, options) => {
|
||||
tier = pending.tier
|
||||
fails = pending.fails
|
||||
critical = pending.critical
|
||||
reason = `new agent after ${fails} failed verdicts`
|
||||
await putIn<Pending>($, S.pending, markId, undefined)
|
||||
reason = `rewrite of ${markId} after ${fails} failed verdicts`
|
||||
} else if (markId) {
|
||||
// Starting it would hand an agent nothing but the marker.
|
||||
return {
|
||||
deny:
|
||||
`model-router: no stored rewrite for ${markId} (already started, or cleared). ` +
|
||||
'Start the agent with the full brief instead, or check /router.',
|
||||
}
|
||||
} else {
|
||||
taskId = await nextId($)
|
||||
const brief = parseBrief(e.prompt)
|
||||
@@ -610,6 +703,11 @@ export const register: Register = (on, options) => {
|
||||
const brief = parseBrief(prompt)
|
||||
const ran = await next({ ...e, prompt, model: tier })
|
||||
if (ran.deny !== undefined || !ran.agentId) return ran
|
||||
// the stored rewrite is used up only once its agent has started
|
||||
if (markId && pending) await putIn<Pending>($, S.pending, markId, undefined)
|
||||
// label against what the engine resolved, not what the router asked for
|
||||
const resolved = asTier(ran.model) ?? tier
|
||||
if (resolved !== tier) reason += ` (asked ${tier}, engine resolved ${resolved})`
|
||||
|
||||
// Learn what the alias resolves to, so router calls are priced on the real model.
|
||||
if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model)
|
||||
@@ -623,8 +721,9 @@ export const register: Register = (on, options) => {
|
||||
if (brief.role)
|
||||
await putIn<AgentRec>($, S.agents, ran.agentId, {
|
||||
taskId,
|
||||
tier,
|
||||
routed: tier,
|
||||
tier: resolved,
|
||||
routed: resolved,
|
||||
name: e.name,
|
||||
role: brief.role,
|
||||
clone: brief.clone,
|
||||
baseBranch: brief.baseBranch,
|
||||
@@ -639,7 +738,7 @@ export const register: Register = (on, options) => {
|
||||
agentId: ran.agentId,
|
||||
at: await $.clock.now(),
|
||||
task: e.description,
|
||||
tier,
|
||||
tier: resolved,
|
||||
role: brief.role,
|
||||
reason,
|
||||
outcome: 'spawned',
|
||||
@@ -668,9 +767,20 @@ export const register: Register = (on, options) => {
|
||||
}
|
||||
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
||||
if (!rec) return out
|
||||
// Verification takes minutes: run it after this event has gone on.
|
||||
const reported = e.usage?.model
|
||||
$.clock.after(0, () => void runRound($, agentId, rec, e.reason, reported, cfg))
|
||||
// Only the run's end is a hand-back; wait for it after this event has gone on.
|
||||
const token = ++tokens
|
||||
lastTurn.set(agentId, { token, ended: e.reason, reported: e.usage?.model })
|
||||
$.clock.after(0, () => void awaitHandBack($, agentId, token, cfg))
|
||||
return out
|
||||
})
|
||||
|
||||
// A message to a running agent is a later instruction too (#6).
|
||||
on('tool.call', { tool: 'SendMessage' }, async ($, e, next) => {
|
||||
const out = await next(e)
|
||||
if (out.deny !== undefined || out.isError) return out
|
||||
const to = typeof e.to === 'string' ? e.to : ''
|
||||
const id = to ? await agentAddressed($, to) : undefined
|
||||
if (id && typeof e.message === 'string') await noteFollowUp($, id, e.message)
|
||||
return out
|
||||
})
|
||||
|
||||
|
||||
@@ -298,3 +298,22 @@ export const handBackLine = (args: {
|
||||
if (skip === 'no-clone') return `${head} no verification: the brief names no clone path.${note}`
|
||||
return undefined
|
||||
}
|
||||
|
||||
// ---------- interim stops vs the hand-back (TOOL-12 follow-up) ----------
|
||||
|
||||
/** An agent's loop status, as `$.agent.list()` reports it. */
|
||||
export type LoopStatus = 'pending' | 'running' | 'waiting' | 'idle' | 'completed' | 'failed' | 'killed'
|
||||
|
||||
/**
|
||||
* What a turn's end means for the agent, read from its status afterwards.
|
||||
* - `handback`: its run ended (completed, failed, killed), or the engine
|
||||
* already dropped it from the list: the report went back to its spawner.
|
||||
* - `interim`: it is held on its own background work (`waiting`), between
|
||||
* turns (`idle`), or already in its next turn (`running`, `pending`).
|
||||
*/
|
||||
export const turnEndKind = (status: LoopStatus | undefined): 'handback' | 'interim' =>
|
||||
status === undefined || status === 'completed' || status === 'failed' || status === 'killed' ? 'handback' : 'interim'
|
||||
|
||||
/** How often an interim agent's status is checked again, and for how long at most. */
|
||||
export const WATCH_EVERY_MS = 15_000
|
||||
export const WATCH_FOR_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
+303
-2
@@ -13,6 +13,7 @@ import {
|
||||
verifierFor,
|
||||
handBackLine,
|
||||
ranTier,
|
||||
turnEndKind,
|
||||
} from '../hooks/router'
|
||||
|
||||
const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev'
|
||||
@@ -218,6 +219,17 @@ type World = {
|
||||
handBacks: string[]
|
||||
/** Every message in the order it was sent. */
|
||||
all: string[]
|
||||
/** Resumes that reached the spawn hook. */
|
||||
resumed: { agentId: string; prompt: string; model?: string }[]
|
||||
/** How many coming new spawns the engine refuses. */
|
||||
denySpawns: number
|
||||
/** The model the engine resolves for new spawns, whatever was asked. */
|
||||
engineModel?: string
|
||||
verifierPrompts: string[]
|
||||
/** SendMessage calls fail (nothing delivered). */
|
||||
sendFails?: boolean
|
||||
/** Each agent's loop status as $.agent.list() reports it; absent means completed. */
|
||||
status: Record<string, string>
|
||||
diffs: string[]
|
||||
diffBases: string[]
|
||||
testCwd: string[]
|
||||
@@ -246,6 +258,7 @@ function world(on: On, w: World) {
|
||||
on('model.complete', async (_$, e) => {
|
||||
if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } }
|
||||
w.verifier.push(e.model)
|
||||
w.verifierPrompts.push(e.prompt)
|
||||
return { value: { isAnswered: true as const, text: w.verdicts.shift() ?? '{"verdict":"PASS"}', usage: USAGE } }
|
||||
})
|
||||
on('process.run', async (_$, e) => {
|
||||
@@ -276,10 +289,27 @@ function world(on: On, w: World) {
|
||||
return { value: { code: end, signal: null } }
|
||||
})
|
||||
on('agent.spawn', async (_$, e) => {
|
||||
if (e.resume) {
|
||||
w.resumed.push({ agentId: e.resume.agentId, prompt: e.prompt, model: e.model })
|
||||
return { model: 'claude-opus-5-5', agentId: e.resume.agentId }
|
||||
}
|
||||
if (w.denySpawns > 0) {
|
||||
w.denySpawns--
|
||||
return { deny: 'spawn failed' }
|
||||
}
|
||||
w.spawned.push({ model: e.model, prompt: e.prompt })
|
||||
return { model: e.model ?? 'inherit', agentId: `agent-${w.spawned.length}` }
|
||||
return { model: w.engineModel ?? e.model ?? 'inherit', agentId: `agent-${w.spawned.length}` }
|
||||
})
|
||||
on('tool.call', { tool: 'SendMessage' }, async () =>
|
||||
(w.sendFails ? { result: 'No agent by that name is running', isError: true } : { result: { success: true } }) as never,
|
||||
)
|
||||
on('turn.complete', async (_$, e) => ({ text: e.answer }))
|
||||
on('agent.list', async () => ({
|
||||
value: w.spawned
|
||||
.map((_, i) => `agent-${i + 1}`)
|
||||
.filter(id => w.status[id] !== 'gone') // 'gone': the engine dropped it
|
||||
.map(id => ({ id, description: '', type: 'general-purpose', status: (w.status[id] ?? 'completed') as never })),
|
||||
}))
|
||||
on('prompt.submit', async (_$, e) => {
|
||||
w.all.push(e.text)
|
||||
if (/^\[model-router\] T\d+ (verifying on|no verification:)/.test(e.text)) w.handBacks.push(e.text)
|
||||
@@ -289,7 +319,7 @@ function world(on: On, w: World) {
|
||||
return clock
|
||||
}
|
||||
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], handBacks: [], all: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], handBacks: [], all: [], resumed: [], denySpawns: 0, verifierPrompts: [], status: {}, diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
|
||||
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string, ranModel?: string, reason = 'answer') {
|
||||
const usage = ranModel
|
||||
@@ -643,3 +673,274 @@ test('TOOL-12: review and test agents get no hand-back line', async ($, on) => {
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.all).toEqual([])
|
||||
})
|
||||
|
||||
// ---------- interim stops (TOOL-12 follow-up) ----------
|
||||
|
||||
/** A turn end of the agent while it is still alive: its status says so. */
|
||||
async function interimStop($: Engine, clock: { advance: (ms: number) => Promise<void> }, w: World, agentId: string, status = 'waiting') {
|
||||
w.status[agentId] = status
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({
|
||||
answer: 'Waiting for the test suite.',
|
||||
durationMs: 1,
|
||||
isAborted: false,
|
||||
turnId: 't',
|
||||
agentId,
|
||||
reason: 'answer',
|
||||
usage: { model: 'claude-sonnet-5-5', input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||
})
|
||||
await clock.advance(1)
|
||||
}
|
||||
|
||||
test('follow-up acceptance: 3 interim stops give exactly one verifying line and one verdict, both after the hand-back', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28 dev' } as never)
|
||||
const id = r.agentId as string
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await interimStop($, clock, w, id)
|
||||
await clock.advance(40 * 60 * 1000) // the test suite runs 35-45 min
|
||||
expect(w.all).toEqual([])
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.diffBases).toEqual([])
|
||||
}
|
||||
w.status[id] = 'completed'
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
await clock.advance(60_000)
|
||||
expect(w.handBacks).toEqual(['[model-router] T1 verifying on Opus...'])
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
expect(w.told.length).toBe(1)
|
||||
expect(w.all[1]).toContain('PASS from verifier opus')
|
||||
})
|
||||
|
||||
test('follow-up: an interim stop of an Opus dev posts nothing; its hand-back posts the skip line once', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: `SAFETY-135: decide how the staleness guard should recover. Clone: ${DEV}/`, description: 'dev' } as never)
|
||||
const id = r.agentId as string
|
||||
await interimStop($, clock, w, id)
|
||||
await interimStop($, clock, w, id, 'idle')
|
||||
await clock.advance(10 * 60 * 1000)
|
||||
expect(w.all).toEqual([])
|
||||
w.status[id] = 'completed'
|
||||
await finish($, clock, id, 'claude-opus-5-5')
|
||||
await clock.advance(60_000)
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
||||
})
|
||||
|
||||
test('follow-up: a status that turns ended a moment after the turn is still caught', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
||||
const id = r.agentId as string
|
||||
w.status[id] = 'running' // not yet updated when the turn ends
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
expect(w.all).toEqual([])
|
||||
w.status[id] = 'completed'
|
||||
await clock.advance(16_000)
|
||||
expect(w.handBacks).toEqual(['[model-router] T1 verifying on Opus...'])
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
})
|
||||
|
||||
test('follow-up: an agent the engine already dropped counts as handed back', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
||||
const id = r.agentId as string
|
||||
w.status[id] = 'gone'
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
await clock.advance(1)
|
||||
expect(w.handBacks.length).toBe(1)
|
||||
})
|
||||
|
||||
test('follow-up: a failed or killed run is a hand-back too', async () => {
|
||||
for (const s of ['completed', 'failed', 'killed', undefined] as const) expect(turnEndKind(s)).toBe('handback')
|
||||
for (const s of ['waiting', 'idle', 'running', 'pending'] as const) expect(turnEndKind(s)).toBe('interim')
|
||||
})
|
||||
|
||||
test('follow-up: the hand-back uses the last turn: its model and how it ended', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'dev' } as never)
|
||||
const id = r.agentId as string
|
||||
await interimStop($, clock, w, id) // reported Sonnet
|
||||
w.status[id] = 'completed'
|
||||
await finish($, clock, id, 'claude-opus-5-5') // the last turn ran on Opus
|
||||
await clock.advance(60_000)
|
||||
expect(w.all).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review. (routed to Sonnet, ran on Opus)'])
|
||||
})
|
||||
|
||||
test('follow-up: each agent is judged by its own status, not by that of another agent', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
await $.agent.spawn({ prompt: devBrief(), description: 'finished dev' } as never) // agent-1: completed
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'busy dev' } as never)
|
||||
await interimStop($, clock, w, r.agentId as string) // agent-2: waiting
|
||||
await clock.advance(60_000)
|
||||
expect(w.all).toEqual([])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
// ---------- resumes, follow-ups and rewrites (#4, #6) ----------
|
||||
|
||||
const sendMessage = ($: Engine, to: string, message: string) =>
|
||||
$.tool.call({ tool: 'SendMessage', to, message } as never)
|
||||
|
||||
const resume = ($: Engine, agentId: string, prompt: string) =>
|
||||
$.agent.spawn({ prompt, description: 'resume', resume: { agentId }, model: 'sonnet' } as never)
|
||||
|
||||
test('#4: a resume keeps its task, routing and counts: no new Tn, no new routing', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1', 'd2']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
const id = r.agentId as string
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
expect(w.told[0]).toContain(`Resume agent ${id}`)
|
||||
await resume($, id, 'Fix the issues: x')
|
||||
expect(w.resumed[0]?.model).toBe('sonnet') // passed through untouched
|
||||
expect(w.spawned.length).toBe(1)
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
expect(w.handBacks[1]).toBe('[model-router] T1 verifying on Opus...')
|
||||
expect(w.diffBases[1]).toBe('snap1') // round 2 diffs from the last verified round
|
||||
expect(w.told[1]).toContain('round 2')
|
||||
expect(w.told[1]).toContain('[router-rewrite T1]') // second fail counted: escalates
|
||||
})
|
||||
|
||||
test('#4: a resumed Opus dev is labelled as Opus, with no "routed to" note', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: `SAFETY-135: decide how the guard should recover. Clone: ${DEV}/`, description: 'dev' } as never)
|
||||
const id = r.agentId as string
|
||||
await finish($, clock, id, 'claude-opus-5-5')
|
||||
await resume($, id, 'Docs-only round: update the README.')
|
||||
await finish($, clock, id, 'claude-opus-5-5')
|
||||
expect(w.handBacks).toEqual([
|
||||
'[model-router] T1 no verification: dev ran on Opus. Go straight to review.',
|
||||
'[model-router] T1 no verification: dev ran on Opus. Go straight to review.',
|
||||
])
|
||||
})
|
||||
|
||||
test('#4: the label compares with what the engine resolved at start, not what the router asked', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.engineModel = 'claude-opus-5-5' // e.g. the agent type sets its own model
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('sonnet')
|
||||
await finish($, clock, r.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.handBacks).toEqual(['[model-router] T1 no verification: dev ran on Opus. Go straight to review.'])
|
||||
const log = (await $.command.run({ command: 'router', args: '' } as never)).text ?? ''
|
||||
expect(log).toContain('opus')
|
||||
})
|
||||
|
||||
test('#4: a router rewrite keeps the task number and its model', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1', 'd2']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
const rw = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite', model: 'opus' } as never)
|
||||
expect(w.spawned[1]?.model).toBe('opus')
|
||||
await finish($, clock, rw.agentId as string, 'claude-opus-5-5')
|
||||
expect(w.handBacks[2]).toBe('[model-router] T1 no verification: dev ran on Opus. Go straight to review.')
|
||||
})
|
||||
|
||||
test('#4: a rewrite whose first start fails can be started again', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1', 'd2']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
w.denySpawns = 1
|
||||
const first = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
||||
expect(first.deny).toBe('spawn failed')
|
||||
const second = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
||||
expect(second.deny).toBe(undefined)
|
||||
expect(w.spawned[1]?.model).toBe('opus')
|
||||
expect(w.spawned[1]?.prompt).toContain('Previous attempts rejected')
|
||||
})
|
||||
|
||||
test('#4: a rewrite marker with nothing stored is refused, not started without a brief', async ($, on) => {
|
||||
const w = fresh()
|
||||
world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: '[router-rewrite T325]', description: 'rewrite', model: 'opus' } as never)
|
||||
expect(r.deny).toContain('no stored rewrite for T325')
|
||||
expect(w.spawned).toEqual([])
|
||||
})
|
||||
|
||||
test('#6: a resume message reaches the verifier as a later instruction, in scope', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1', 'd2']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-24' } as never)
|
||||
const id = r.agentId as string
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
await resume($, id, 'Owner: add MAINT-25 to this PR as well.')
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
const p = w.verifierPrompts[1] ?? ''
|
||||
expect(p).toContain('## Later instructions to this agent')
|
||||
expect(p).toContain('1. Owner: add MAINT-25 to this PR as well.')
|
||||
expect(p).toContain('SAFETY-95: clamp the integrator') // the original brief stays
|
||||
})
|
||||
|
||||
test('#6: SendMessage to a running agent, by id or by name, is a later instruction too', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'MAINT-28', name: 'maint28-dev' } as never)
|
||||
const id = r.agentId as string
|
||||
await sendMessage($, id, 'Owner decided: cap is 1.')
|
||||
await sendMessage($, 'maint28-dev', 'Also cover the empty list.')
|
||||
await sendMessage($, 'maint28-dev', 'Also cover the empty list.') // the same text counts once
|
||||
await sendMessage($, 'someone-else', 'Not for this agent.')
|
||||
await finish($, clock, id, 'claude-sonnet-5-5')
|
||||
const p = w.verifierPrompts[0] ?? ''
|
||||
expect(p).toContain('1. Owner decided: cap is 1.')
|
||||
expect(p).toContain('2. Also cover the empty list.')
|
||||
expect(p).not.toContain('3.')
|
||||
expect(p).not.toContain('Not for this agent.')
|
||||
})
|
||||
|
||||
test('#6: without later instructions the section is left out', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.verifierPrompts[0] ?? '').not.toContain('Later instructions')
|
||||
})
|
||||
|
||||
test('#4: a rewrite that started is used up: the same marker again is refused', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1', 'd2']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
||||
const again = await $.agent.spawn({ prompt: '[router-rewrite T1]', description: 'rewrite' } as never)
|
||||
expect(again.deny).toContain('no stored rewrite for T1')
|
||||
expect(w.spawned.length).toBe(2)
|
||||
})
|
||||
|
||||
test('#6: a SendMessage that was not delivered is not a later instruction', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1']
|
||||
w.sendFails = true
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await sendMessage($, r.agentId as string, 'Never arrived.')
|
||||
await finish($, clock, r.agentId as string, 'claude-sonnet-5-5')
|
||||
expect(w.verifierPrompts[0] ?? '').not.toContain('Never arrived.')
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user