model-router v0.4.4
Claude Code plugin: routes subagents by how well-defined the brief is, verifies dev work on a second model in the agent's own clone, and estimates savings against Opus 5.5. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "model-router",
|
||||
"owner": { "name": "Glenn Schrooyen" },
|
||||
"plugins": [
|
||||
{
|
||||
"name": "model-router",
|
||||
"source": "./",
|
||||
"description": "Routes subagents to Haiku, Sonnet or Opus by how well-defined the brief is, verifies dev work on a different model, and estimates savings against Opus."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"name": "model-router",
|
||||
"version": "0.4.4",
|
||||
"description": "Routes each subagent to Haiku, Sonnet or Opus by how well-defined its brief is, verifies dev work below Opus in its own clone on a different model, sends failures back to the same agent then a stronger one, and estimates the savings against Opus.",
|
||||
"author": {
|
||||
"name": "Glenn Schrooyen"
|
||||
},
|
||||
"userConfig": {
|
||||
"testCommand": {
|
||||
"type": "string",
|
||||
"title": "Test command (run in the agent's clone)",
|
||||
"description": "The unit-suite command CI runs, from .gitea/workflows/checks.yml. Its output goes to the verifier. Empty: no tests.",
|
||||
"default": ""
|
||||
},
|
||||
"testTimeoutSec": {
|
||||
"type": "number",
|
||||
"title": "Test time limit (seconds)",
|
||||
"description": "How long the test command may run in the clone. A run that times out is reported as NOT VERIFIED, never as a FAIL.",
|
||||
"default": 1800
|
||||
},
|
||||
"defaultBaseBranch": {
|
||||
"type": "string",
|
||||
"title": "Base branch when the brief names none",
|
||||
"description": "Used after the PR lookup and the brief's own base branch.",
|
||||
"default": "release/1.0"
|
||||
},
|
||||
"prBaseCommand": {
|
||||
"type": "string",
|
||||
"title": "Command that prints the PR's base branch",
|
||||
"description": "Run in the clone; its first output line is the branch. Empty or failing: the brief's base branch, then the default.",
|
||||
"default": ""
|
||||
},
|
||||
"maxRewrites": {
|
||||
"type": "number",
|
||||
"title": "Failed verdicts before handing back",
|
||||
"description": "1st fail: resume the same agent. 2nd: new agent one model up in the same clone. Past this number: handed back to the manager.",
|
||||
"default": 2
|
||||
},
|
||||
"safetyPatterns": {
|
||||
"type": "string",
|
||||
"title": "Safety-critical patterns (logged only)",
|
||||
"description": "Comma-separated. Marks a task as critical in the log. Dev work below Opus is always verified by Opus.",
|
||||
"default": "control loop,integrator,safety,staleness,watchdog,failsafe,fail-safe,deadlock,rs485,modbus,meter emulat,asyncio,setpoint"
|
||||
},
|
||||
"respectExplicitModel": {
|
||||
"type": "boolean",
|
||||
"title": "Keep a named model on agents without a role",
|
||||
"description": "Review and test agents always keep a named model; dev agents never do. This covers everything else.",
|
||||
"default": false
|
||||
},
|
||||
"allowFable": {
|
||||
"type": "boolean",
|
||||
"title": "Allow Fable for rewrites",
|
||||
"description": "A rewrite after Opus fails runs on Fable instead of handing back.",
|
||||
"default": false
|
||||
},
|
||||
"prices": {
|
||||
"type": "string",
|
||||
"title": "Price overrides (JSON, USD per MTok)",
|
||||
"description": "Empty: list prices of 2026-10-08. Else a JSON list of rows, checked before the defaults: [{\"match\":\"sonnet-5-5\",\"price\":{\"in\":2,\"out\":10,\"write\":2.5,\"read\":0.1}}]",
|
||||
"default": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
# written by Claude Code when the plugin loads
|
||||
.claude-plugin/types/
|
||||
node_modules/
|
||||
@@ -0,0 +1,67 @@
|
||||
# model-router
|
||||
|
||||
A Claude Code plugin that picks a model for every subagent, checks dev work on a second model before it reaches a PR, and estimates what that saved compared with running everything on Opus.
|
||||
|
||||
## What it does
|
||||
|
||||
| Step | Behaviour |
|
||||
|---|---|
|
||||
| Route | Dev agents run on **Sonnet** when the brief decides the approach (Expected / acceptance criteria), on **Opus** when the design is still open. Review and test agents keep the model they are given (Opus when none). |
|
||||
| Verify | When a dev agent below Opus finishes a round, the router diffs its clone against the merge base, runs the test command in that clone, and has **Opus** judge the change. |
|
||||
| On FAIL | 1st: resume the same agent with the issues. 2nd: a new agent one model up, in the same clone. After that: handed back to the manager. |
|
||||
| Not verified | A test timeout, a test command that cannot run, a missing clone or merge base: reported as NOT VERIFIED, never as a FAIL. |
|
||||
| Savings | `/router savings` prices every run on the model it ran on and compares it with the same tokens on Opus 5.5. |
|
||||
|
||||
## Briefs
|
||||
|
||||
Every dev, review and test brief carries one line:
|
||||
|
||||
```
|
||||
Clone: /home/homelab/projects/goodwe/clones/<TICKET>-dev/
|
||||
```
|
||||
|
||||
Add `(Role: dev|review|test)` when the path does not end in the role. Tickets with extra parts (`OPT-11-P4`) are fine. Name the base branch (`Base branch: release/1.0`).
|
||||
|
||||
## Install
|
||||
|
||||
```
|
||||
/plugin marketplace add <this repo's git URL>
|
||||
/plugin install model-router@model-router
|
||||
```
|
||||
|
||||
Or for development: `claude --plugin-dir /path/to/model-router`.
|
||||
|
||||
## Settings (`/config`)
|
||||
|
||||
| Setting | Default | Notes |
|
||||
|---|---|---|
|
||||
| `testCommand` | empty | Run in the agent's clone; output goes to the verifier |
|
||||
| `testTimeoutSec` | 1800 | Wrapped in GNU `timeout --verbose`; a timeout is NOT VERIFIED |
|
||||
| `defaultBaseBranch` | `release/1.0` | After the PR lookup and the brief's own base branch |
|
||||
| `prBaseCommand` | empty | Optional command that prints the PR's base branch |
|
||||
| `maxRewrites` | 2 | Failed verdicts before handing back |
|
||||
| `safetyPatterns` | control loop, rs485, ... | Marks tasks as critical in the log |
|
||||
| `respectExplicitModel` | false | Keep a named model on agents without a role |
|
||||
| `allowFable` | false | Rewrite on Fable after Opus fails |
|
||||
| `prices` | empty | JSON price overrides, USD per MTok |
|
||||
|
||||
## Commands
|
||||
|
||||
- `/router`: recent decisions, verdicts and savings
|
||||
- `/router savings [Tn]`: the full savings table, or one task
|
||||
- `/router clear`: reset log, records and costs
|
||||
|
||||
## Tests
|
||||
|
||||
```
|
||||
claude plugin validate .
|
||||
claude plugin test .
|
||||
```
|
||||
|
||||
44 tests. Every routing, verification, cost and timeout rule was also checked by breaking it on purpose (66 mutations, all caught by a test).
|
||||
|
||||
## Known limits
|
||||
|
||||
- Savings are an upper bound: Opus may need fewer tokens than the cheaper model did.
|
||||
- `timeout` stops the `docker` client, not the container: name the container and remove it in a `trap` in the test command.
|
||||
- An Opus rewrite is not verified; the PR review covers it.
|
||||
@@ -0,0 +1 @@
|
||||
{ "modules": ["./register.ts"] }
|
||||
@@ -0,0 +1,185 @@
|
||||
// Cost of a run, and what the same work would have cost on Opus. Pure: no `$`.
|
||||
//
|
||||
// Prices: USD per million tokens, from platform.claude.com/docs/en/about-claude/pricing
|
||||
// (read 2026-10-08). A setting can replace them.
|
||||
|
||||
export type Usage = {
|
||||
input_tokens: number
|
||||
output_tokens: number
|
||||
cache_creation_input_tokens?: number | null
|
||||
cache_read_input_tokens?: number | null
|
||||
}
|
||||
|
||||
/** in, out, cache write (5 min), cache read: USD per MTok. */
|
||||
export type Price = { in: number; out: number; write: number; read: number }
|
||||
|
||||
export type PriceRow = { match: string; price: Price; label: string }
|
||||
|
||||
// First match wins: more specific ids come first.
|
||||
export const DEFAULT_PRICES: readonly PriceRow[] = [
|
||||
{ match: 'fable-5-1', label: 'Fable 5.1', price: { in: 10, out: 50, write: 12.5, read: 0.25 } },
|
||||
{ match: 'mythos-5-1', label: 'Mythos 5.1', price: { in: 10, out: 50, write: 12.5, read: 0.25 } },
|
||||
{ match: 'opus-5-5', label: 'Opus 5.5', price: { in: 4, out: 20, write: 5, read: 0.2 } },
|
||||
{ match: 'sonnet-5-5', label: 'Sonnet 5.5', price: { in: 2, out: 10, write: 2.5, read: 0.1 } },
|
||||
// Haiku 5.5 doubles past 100k-token prompts; a run's summed usage cannot tell, so the low tier is used.
|
||||
{ match: 'haiku-5-5', label: 'Haiku 5.5', price: { in: 0.1, out: 0.5, write: 0.125, read: 0.01 } },
|
||||
{ match: 'haiku-4-5', label: 'Haiku 4.5', price: { in: 1, out: 5, write: 1.25, read: 0.1 } },
|
||||
{ match: 'fable-5', label: 'Fable 5', price: { in: 10, out: 50, write: 12.5, read: 1 } },
|
||||
{ match: 'opus-5', label: 'Opus 5', price: { in: 5, out: 25, write: 6.25, read: 0.5 } },
|
||||
{ match: 'sonnet-5', label: 'Sonnet 5', price: { in: 2, out: 10, write: 2.5, read: 0.2 } },
|
||||
]
|
||||
|
||||
/** The model every saving is measured against. */
|
||||
export const BASELINE = 'opus-5-5'
|
||||
|
||||
/**
|
||||
* Models before Claude 4.7 use a tokenizer that yields ~30% fewer tokens for
|
||||
* the same text, so their counts are scaled up before pricing them as Opus 5.5.
|
||||
*/
|
||||
const OLD_TOKENIZER = /(haiku-4-5|sonnet-4|opus-4-[0-6]|opus-4-1|opus-4(?!-)|haiku-3)/
|
||||
export const tokenizerFactor = (modelId: string) => (OLD_TOKENIZER.test(modelId) ? 1.3 : 1)
|
||||
|
||||
export const priceRow = (modelId: string, table: readonly PriceRow[] = DEFAULT_PRICES) =>
|
||||
table.find(r => modelId.toLowerCase().includes(r.match))
|
||||
|
||||
/** USD for a usage at a price; tokens scaled by `factor`. */
|
||||
export const cost = (u: Usage, p: Price, factor = 1) =>
|
||||
(factor *
|
||||
(u.input_tokens * p.in +
|
||||
u.output_tokens * p.out +
|
||||
(u.cache_creation_input_tokens ?? 0) * p.write +
|
||||
(u.cache_read_input_tokens ?? 0) * p.read)) /
|
||||
1_000_000
|
||||
|
||||
export type Priced = { actual: number; opus: number; known: boolean; label: string }
|
||||
|
||||
/** What a run cost on its own model, and the same tokens on the baseline. */
|
||||
export const priceRun = (modelId: string, u: Usage, table: readonly PriceRow[] = DEFAULT_PRICES): Priced => {
|
||||
const row = priceRow(modelId, table)
|
||||
const base = priceRow(BASELINE, table)
|
||||
if (!row || !base) return { actual: 0, opus: 0, known: false, label: modelId }
|
||||
return {
|
||||
actual: cost(u, row.price),
|
||||
opus: cost(u, base.price, tokenizerFactor(modelId)),
|
||||
known: true,
|
||||
label: row.label,
|
||||
}
|
||||
}
|
||||
|
||||
/** A task's running totals, across every agent and round it took. */
|
||||
export type TaskCost = {
|
||||
task: string
|
||||
/** Writer runs on each model, USD. */
|
||||
writerActual: number
|
||||
/** Writer runs below Opus, priced at Opus: what Opus would have cost for the same tokens. */
|
||||
writerAsOpus: number
|
||||
/** Writer runs that were on Opus already, USD. */
|
||||
writerOnOpus: number
|
||||
/** Classifier and verifier calls the router added, USD. */
|
||||
overhead: number
|
||||
models: string[]
|
||||
unpriced: string[]
|
||||
}
|
||||
|
||||
export const emptyTask = (task: string): TaskCost => ({
|
||||
task,
|
||||
writerActual: 0,
|
||||
writerAsOpus: 0,
|
||||
writerOnOpus: 0,
|
||||
overhead: 0,
|
||||
models: [],
|
||||
unpriced: [],
|
||||
})
|
||||
|
||||
export const addWriterRun = (t: TaskCost, modelId: string, u: Usage, table?: readonly PriceRow[]): TaskCost => {
|
||||
const p = priceRun(modelId, u, table)
|
||||
const next = { ...t, models: [...t.models], unpriced: [...t.unpriced] }
|
||||
if (!p.known) {
|
||||
if (!next.unpriced.includes(modelId)) next.unpriced.push(modelId)
|
||||
return next
|
||||
}
|
||||
if (!next.models.includes(p.label)) next.models.push(p.label)
|
||||
next.writerActual += p.actual
|
||||
if (modelId.toLowerCase().includes(BASELINE)) next.writerOnOpus += p.actual
|
||||
else next.writerAsOpus += p.opus
|
||||
return next
|
||||
}
|
||||
|
||||
export const addOverhead = (t: TaskCost, modelId: string, u: Usage, table?: readonly PriceRow[]): TaskCost => {
|
||||
const p = priceRun(modelId, u, table)
|
||||
return p.known ? { ...t, overhead: t.overhead + p.actual } : t
|
||||
}
|
||||
|
||||
/**
|
||||
* What the task would have cost had it gone to Opus at once.
|
||||
*
|
||||
* When Opus wrote part of it (a rewrite after failed rounds, or an open
|
||||
* design), Opus's own runs are taken as what Opus alone would have needed:
|
||||
* the failed cheaper rounds then count as pure loss. Otherwise every writer
|
||||
* token is priced at Opus, an upper bound, since Opus may need fewer tokens.
|
||||
*/
|
||||
export const counterfactual = (t: TaskCost) => (t.writerOnOpus > 0 ? t.writerOnOpus : t.writerAsOpus)
|
||||
|
||||
export const actualTotal = (t: TaskCost) => t.writerActual + t.overhead
|
||||
|
||||
export const saved = (t: TaskCost) => counterfactual(t) - actualTotal(t)
|
||||
|
||||
export const savedPct = (t: TaskCost) => {
|
||||
const c = counterfactual(t)
|
||||
return c > 0 ? (saved(t) / c) * 100 : 0
|
||||
}
|
||||
|
||||
/** Reads a prices setting: a JSON list of rows, or empty for the defaults. */
|
||||
export const parsePrices = (json: string): readonly PriceRow[] => {
|
||||
if (!json.trim()) return DEFAULT_PRICES
|
||||
try {
|
||||
const rows = JSON.parse(json) as PriceRow[]
|
||||
const ok = Array.isArray(rows) && rows.every(r => typeof r.match === 'string' && typeof r.price?.in === 'number')
|
||||
return ok ? [...rows.map(r => ({ ...r, label: r.label ?? r.match })), ...DEFAULT_PRICES] : DEFAULT_PRICES
|
||||
} catch {
|
||||
return DEFAULT_PRICES
|
||||
}
|
||||
}
|
||||
|
||||
export const usd = (n: number) => (Math.abs(n) < 0.01 && n !== 0 ? `$${n.toFixed(4)}` : `$${n.toFixed(2)}`)
|
||||
|
||||
/**
|
||||
* The savings table: per task, what it cost, what Opus would have cost, and
|
||||
* the difference, newest first; then the total over every task.
|
||||
*/
|
||||
export const savingsReport = (costs: Readonly<Record<string, TaskCost>>, limit: number, only?: string): string => {
|
||||
const ids = Object.keys(costs)
|
||||
.filter(id => !only || id === only)
|
||||
.sort((a, b) => Number(b.slice(1)) - Number(a.slice(1)))
|
||||
if (ids.length === 0) return only ? `model-router: no costs recorded for ${only}.` : 'savings: no agent runs recorded yet.'
|
||||
|
||||
const row = (id: string, t: TaskCost) => {
|
||||
const pct = counterfactual(t) > 0 ? `${savedPct(t).toFixed(0)}%` : '-'
|
||||
const how = t.writerActual === 0 ? 'running' : t.writerOnOpus > 0 ? 'opus ran' : 'upper bound'
|
||||
return `${id.padEnd(5)} ${usd(actualTotal(t)).padStart(9)} ${usd(counterfactual(t)).padStart(9)} ${usd(saved(t)).padStart(9)} ${pct.padStart(5)} ${how.padEnd(11)} ${t.models.join('+').padEnd(20)} ${t.task.slice(0, 32)}${t.unpriced.length ? ` [no price: ${t.unpriced.join(', ')}]` : ''}`
|
||||
}
|
||||
|
||||
let actual = 0
|
||||
let opus = 0
|
||||
for (const id of Object.keys(costs)) {
|
||||
const t = costs[id]
|
||||
// a task whose writer has not finished has no comparison yet
|
||||
if (!t || t.writerActual === 0) continue
|
||||
actual += actualTotal(t)
|
||||
opus += counterfactual(t)
|
||||
}
|
||||
const shown = ids.slice(0, limit)
|
||||
const lines = [
|
||||
`savings vs Opus 5.5 (USD, API list prices)${only ? '' : ` - ${shown.length} of ${ids.length} tasks`}`,
|
||||
`task actual if opus saved % basis models task`,
|
||||
...shown.map(id => row(id, costs[id] as TaskCost)),
|
||||
]
|
||||
if (!only)
|
||||
lines.push(
|
||||
`total ${usd(actual).padStart(9)} ${usd(opus).padStart(9)} ${usd(opus - actual).padStart(9)} ${(opus > 0 ? `${(((opus - actual) / opus) * 100).toFixed(0)}%` : '-').padStart(5)}`,
|
||||
)
|
||||
lines.push(
|
||||
'actual = writer runs + classifier + verifier. "upper bound": all tokens priced at Opus; Opus may need fewer. "opus ran": the Opus run stands for Opus alone, cheaper failed rounds count as loss.',
|
||||
)
|
||||
return lines.join('\n')
|
||||
}
|
||||
@@ -0,0 +1,681 @@
|
||||
import type { EngineInterface, Register } from 'claude-code'
|
||||
|
||||
import {
|
||||
addOverhead,
|
||||
addWriterRun,
|
||||
emptyTask,
|
||||
parsePrices,
|
||||
savingsReport,
|
||||
type PriceRow,
|
||||
type TaskCost,
|
||||
type Usage,
|
||||
} from './pricing'
|
||||
import {
|
||||
REWRITE_MARK,
|
||||
asTier,
|
||||
clip,
|
||||
decide,
|
||||
onFail,
|
||||
parseBrief,
|
||||
parsePatterns,
|
||||
parseVerdict,
|
||||
safetyHit,
|
||||
skipReason,
|
||||
validTimeout,
|
||||
verifierFor,
|
||||
type Role,
|
||||
type Tier,
|
||||
} from './router'
|
||||
|
||||
// ---------- stored records ----------
|
||||
|
||||
/** One agent the router started, kept from spawn to its last verified round. */
|
||||
type AgentRec = {
|
||||
taskId: string
|
||||
tier: Tier
|
||||
role?: Role
|
||||
clone?: string
|
||||
baseBranch?: string
|
||||
critical: boolean
|
||||
prompt: string
|
||||
/** Failed verdicts in this task's chain, carried over to a rewrite agent. */
|
||||
fails: number
|
||||
rounds: number
|
||||
/** The tree after the last verified round: the next round diffs from here. */
|
||||
lastSnap?: string
|
||||
lastUntracked?: string[]
|
||||
lastIssues?: string[]
|
||||
}
|
||||
|
||||
type Pending = { prompt: string; tier: Tier; fails: number; critical: boolean }
|
||||
|
||||
type Outcome =
|
||||
| 'pass'
|
||||
| 'resume-requested'
|
||||
| 'rewrite-requested'
|
||||
| 'handed-back'
|
||||
| 'no-changes'
|
||||
| 'no-base'
|
||||
| 'tests-timeout'
|
||||
| 'tests-error'
|
||||
| 'verify-error'
|
||||
| 'test-agent'
|
||||
| 'review-agent'
|
||||
| 'no-clone'
|
||||
| 'opus-unverified'
|
||||
| 'agent-ended'
|
||||
|
||||
type LogEntry = {
|
||||
id: string
|
||||
agentId?: string
|
||||
at: number
|
||||
task: string
|
||||
tier: Tier
|
||||
role?: Role
|
||||
reason: string
|
||||
round?: number
|
||||
verifier?: Tier
|
||||
outcome: Outcome | 'spawned'
|
||||
opusRequested?: boolean
|
||||
}
|
||||
|
||||
const S = {
|
||||
counter: 'counter',
|
||||
pending: 'pending',
|
||||
agents: 'agents',
|
||||
log: 'log',
|
||||
costs: 'costs',
|
||||
agentTask: 'agentTask',
|
||||
aliases: 'aliases',
|
||||
} as const
|
||||
|
||||
/** Model ids assumed for an alias until an agent run shows what it resolves to. */
|
||||
const ALIAS_DEFAULT: Record<Tier, string> = {
|
||||
haiku: 'claude-haiku-4-5',
|
||||
sonnet: 'claude-sonnet-5-5',
|
||||
opus: 'claude-opus-5-5',
|
||||
fable: 'claude-fable-5-1',
|
||||
}
|
||||
const LOG_LIMIT = 400
|
||||
|
||||
// ---------- prompts ----------
|
||||
|
||||
const CLASSIFIER_SYSTEM = `You route coding tasks to the smallest model that will do them right.
|
||||
Judge how well-defined the task is, NOT how important or low-level the code is. In this codebase nearly everything touches control loops, asyncio and RS485; that alone never means opus. A stronger model verifies every change below Opus, and a review agent reviews every PR.
|
||||
Answer with exactly one word: haiku, sonnet or opus.
|
||||
|
||||
haiku: read-only lookups and summaries, mechanical changes fully spelled out.
|
||||
sonnet: the approach is decided: the brief says what to change, or gives Expected behaviour, acceptance criteria or a test to make pass. Size and domain do not matter.
|
||||
opus: the approach is still open: a design or trade-off to decide, a bug whose cause is unknown, or a brief so vague the writer must invent the plan.
|
||||
|
||||
When unsure between sonnet and opus, pick sonnet: a failed verification escalates.`
|
||||
|
||||
const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not.
|
||||
Judge only whether the change correctly and completely does the task, without breaking anything nearby.
|
||||
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
|
||||
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
|
||||
|
||||
Reply with JSON only:
|
||||
{"verdict": "PASS" | "FAIL", "issues": ["concrete issue with file:line where possible", ...], "rewrite_brief": "what must change, empty on PASS"}`
|
||||
|
||||
const ORCHESTRATOR_NOTE = `## Model router (model-router mod)
|
||||
The model-router mod picks each subagent's model and verifies dev work before it reaches a PR.
|
||||
- Every dev, review and test brief carries one line \`Clone: <path>\` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified.
|
||||
- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch.
|
||||
- Review and test agents keep the model you name (Opus when you name none).
|
||||
- When a dev agent below Opus finishes a round, the router verifies its clone's diff on Opus and posts a "[model-router]" verdict. Wait for that verdict before opening or advancing the PR.
|
||||
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. Do not fix rejected work yourself.`
|
||||
|
||||
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
||||
|
||||
// Agents start and finish in parallel. Every read-modify-write of the store
|
||||
// goes through this queue, one at a time, or parallel hooks overwrite each
|
||||
// other's writes (and hand out the same task id).
|
||||
let queue: Promise<unknown> = Promise.resolve()
|
||||
|
||||
function serial<T>(work: () => Promise<T>): Promise<T> {
|
||||
const run = queue.then(work)
|
||||
queue = run.catch(() => undefined)
|
||||
return run
|
||||
}
|
||||
|
||||
async function getMap<T>($: EngineInterface, key: string) {
|
||||
return ((await $.store.get(key)) ?? {}) as Record<string, T>
|
||||
}
|
||||
|
||||
/** Replaces one entry of a stored map, atomically with respect to this plugin. */
|
||||
async function updateIn<T>($: EngineInterface, key: string, id: string, change: (old: T | undefined) => T | undefined) {
|
||||
return serial(async () => {
|
||||
const all = await getMap<T>($, key)
|
||||
const value = change(all[id])
|
||||
if (value === undefined) delete all[id]
|
||||
else all[id] = value
|
||||
await $.store.set(key, all)
|
||||
return value
|
||||
})
|
||||
}
|
||||
|
||||
async function putIn<T>($: EngineInterface, key: string, id: string, value: T | undefined) {
|
||||
await updateIn<T>($, key, id, () => value)
|
||||
}
|
||||
|
||||
async function nextId($: EngineInterface) {
|
||||
return serial(async () => {
|
||||
const n = Number((await $.store.get(S.counter)) ?? 0) + 1
|
||||
await $.store.set(S.counter, n)
|
||||
return `T${n}`
|
||||
})
|
||||
}
|
||||
|
||||
async function log($: EngineInterface, entry: LogEntry) {
|
||||
await serial(async () => {
|
||||
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
||||
list.push(entry)
|
||||
await $.store.set(S.log, list.slice(-LOG_LIMIT))
|
||||
})
|
||||
}
|
||||
|
||||
async function sh($: EngineInterface, argv: string[], cwd: string, timeoutMs = 30_000) {
|
||||
try {
|
||||
const r = await $.process.run(argv, { cwd, timeoutMs })
|
||||
return { ok: r.exitCode === 0, out: r.stdout, err: r.stderr, code: r.exitCode }
|
||||
} catch (err) {
|
||||
return { ok: false, out: '', err: String(err), code: -1 }
|
||||
}
|
||||
}
|
||||
|
||||
async function untracked($: EngineInterface, cwd: string) {
|
||||
const r = await sh($, ['git', 'ls-files', '--others', '--exclude-standard'], cwd)
|
||||
return r.ok ? r.out.split('\n').filter(Boolean) : []
|
||||
}
|
||||
|
||||
/** A commit holding the clone's tree as it is now (HEAD when clean). */
|
||||
async function snapshot($: EngineInterface, cwd: string) {
|
||||
const stash = await sh($, ['git', 'stash', 'create'], cwd)
|
||||
if (stash.ok && stash.out.trim()) return stash.out.trim()
|
||||
const head = await sh($, ['git', 'rev-parse', 'HEAD'], cwd)
|
||||
return head.ok ? head.out.trim() : undefined
|
||||
}
|
||||
|
||||
/** Where a first round diffs from: the merge base with the PR's base branch. */
|
||||
async function firstBase($: EngineInterface, rec: AgentRec, prBaseCommand: string, fallback: string) {
|
||||
const cwd = rec.clone as string
|
||||
let branch: string | undefined
|
||||
if (prBaseCommand) {
|
||||
const r = await sh($, ['sh', '-c', prBaseCommand], cwd)
|
||||
const b = r.out.trim().split('\n')[0]?.trim()
|
||||
if (r.ok && b) branch = b
|
||||
}
|
||||
branch = branch ?? rec.baseBranch ?? fallback
|
||||
for (const ref of [`origin/${branch}`, branch]) {
|
||||
const r = await sh($, ['git', 'merge-base', ref, 'HEAD'], cwd)
|
||||
if (r.ok && r.out.trim()) return { base: r.out.trim(), branch }
|
||||
}
|
||||
return { base: undefined, branch }
|
||||
}
|
||||
|
||||
async function classify($: EngineInterface, prompt: string): Promise<{ tier: Tier; reason: string; usage?: Usage }> {
|
||||
const r = await $.model.complete({
|
||||
model: 'haiku',
|
||||
system: CLASSIFIER_SYSTEM,
|
||||
prompt: `Task:\n${clip(prompt, 6000)}`,
|
||||
maxTokens: 5,
|
||||
timeoutMs: 15_000,
|
||||
})
|
||||
const tier = r.isAnswered ? asTier(r.text) : undefined
|
||||
if (tier && tier !== 'fable') return { tier, reason: 'classifier', usage: r.usage }
|
||||
return { tier: 'sonnet', reason: 'classifier unavailable, default', usage: r.usage }
|
||||
}
|
||||
|
||||
async function tell($: EngineInterface, text: string) {
|
||||
await $.prompt.submit({ text })
|
||||
}
|
||||
|
||||
// ---------- cost tracking ----------
|
||||
|
||||
async function modelIdFor($: EngineInterface, alias: Tier) {
|
||||
const seen = (await getMap<string>($, S.aliases))[alias]
|
||||
return seen ?? ALIAS_DEFAULT[alias]
|
||||
}
|
||||
|
||||
/** Adds a router call (classifier, verifier) to a task's overhead. */
|
||||
async function chargeOverhead($: EngineInterface, taskId: string, alias: Tier, u: Usage | undefined, prices: readonly PriceRow[]) {
|
||||
if (!u) return
|
||||
const modelId = await modelIdFor($, alias)
|
||||
await updateIn<TaskCost>($, S.costs, taskId, old => addOverhead(old ?? emptyTask(taskId), modelId, u, prices))
|
||||
}
|
||||
|
||||
type TestRun = { status: 'ran' | 'timeout' | 'error'; code: number | null; tail: string; seconds: number }
|
||||
|
||||
/**
|
||||
* Runs the test command in the clone, streamed: `$.process.run` stops at ten
|
||||
* minutes, a stream has no limit of its own. GNU `timeout` does the kill.
|
||||
* A timeout or a run that could not start is no verdict on the code.
|
||||
*/
|
||||
async function runTests($: EngineInterface, cmd: string, cwd: string, timeoutSec: number): Promise<TestRun> {
|
||||
const started = await $.clock.now()
|
||||
const seconds = async () => Math.round(((await $.clock.now()) - started) / 1000)
|
||||
let out = ''
|
||||
let errTail = ''
|
||||
try {
|
||||
const stream = $.process.spawn({
|
||||
argv: ['timeout', '--verbose', '--kill-after=30', String(Math.max(1, Math.round(timeoutSec))), 'sh', '-c', cmd],
|
||||
cwd,
|
||||
})
|
||||
for await (const piece of stream) {
|
||||
out = (out + piece.text).slice(-65_536)
|
||||
if (piece.stream === 'stderr') errTail = (errTail + piece.text).slice(-65_536)
|
||||
}
|
||||
// GNU timeout --verbose says so on stderr when, and only when, it fires.
|
||||
// Read from the joined stderr, so a line split across pieces still counts.
|
||||
const isFired = errTail.includes('timeout: sending signal')
|
||||
const end = await stream.result
|
||||
const tail = clip(out.trim().split('\n').slice(-80).join('\n'), 8000)
|
||||
const took = await seconds()
|
||||
// Only a timeout that fired is a timeout: a 137 without it is the code's own
|
||||
// (e.g. out of memory) and goes to the verifier as a failed run.
|
||||
if (isFired) return { status: 'timeout', code: end.code, tail, seconds: took }
|
||||
if (end.code === 126 || end.code === 127) return { status: 'error', code: end.code, tail, seconds: took }
|
||||
return { status: 'ran', code: end.code, tail, seconds: took }
|
||||
} catch (err) {
|
||||
return { status: 'error', code: null, tail: String(err).slice(0, 500), seconds: await seconds() }
|
||||
}
|
||||
}
|
||||
|
||||
type Settings = {
|
||||
testCommand: string
|
||||
testTimeoutSec: number
|
||||
prBaseCommand: string
|
||||
defaultBaseBranch: string
|
||||
maxFails: number
|
||||
allowFable: boolean
|
||||
prices: readonly PriceRow[]
|
||||
}
|
||||
|
||||
/** Verifies one finished round of a dev agent and tells the manager. */
|
||||
async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, cfg: Settings) {
|
||||
const cwd = rec.clone as string
|
||||
const round = rec.rounds + 1
|
||||
const entry: LogEntry = {
|
||||
id: rec.taskId,
|
||||
agentId,
|
||||
at: await $.clock.now(),
|
||||
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
||||
tier: rec.tier,
|
||||
role: rec.role,
|
||||
reason: `round ${round}`,
|
||||
round,
|
||||
outcome: 'verify-error',
|
||||
}
|
||||
const head = `[model-router] ${rec.taskId} (agent ${agentId}, ${rec.tier}, round ${round})`
|
||||
|
||||
// 1. What this round changed.
|
||||
let base = rec.lastSnap
|
||||
let baseLabel = 'the previous verified round'
|
||||
if (!base) {
|
||||
const fb = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch)
|
||||
base = fb.base
|
||||
baseLabel = `merge-base with ${fb.branch}`
|
||||
if (!base) {
|
||||
entry.outcome = 'no-base'
|
||||
await log($, entry)
|
||||
await tell($, `${head}: NOT VERIFIED - no merge base with ${fb.branch} in ${cwd}. Review the change yourself before the PR.`)
|
||||
return
|
||||
}
|
||||
}
|
||||
const diff = await sh($, ['git', 'diff', base], cwd, 60_000)
|
||||
const known = new Set(rec.lastUntracked ?? [])
|
||||
const fresh = (await untracked($, cwd)).filter(f => !known.has(f))
|
||||
let added = ''
|
||||
for (const f of fresh.slice(0, 20)) {
|
||||
try {
|
||||
added += `\n--- new file: ${f}\n${clip(await $.fs.read(`${cwd}/${f}`), 8000)}\n`
|
||||
} catch {
|
||||
added += `\n--- new file: ${f} (unreadable)\n`
|
||||
}
|
||||
}
|
||||
const changes = (diff.out + added).trim()
|
||||
if (!changes) {
|
||||
entry.outcome = 'no-changes'
|
||||
await log($, entry)
|
||||
await tell($, `${head}: no changes since ${baseLabel}; nothing to verify.`)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Tests, in the clone.
|
||||
let tests = ''
|
||||
if (cfg.testCommand) {
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} running tests`)
|
||||
const t = await runTests($, cfg.testCommand, cwd, cfg.testTimeoutSec)
|
||||
if (t.status !== 'ran') {
|
||||
// No verdict on the code: no verifier call, no fail counted, no rewrite.
|
||||
entry.outcome = t.status === 'timeout' ? 'tests-timeout' : 'tests-error'
|
||||
await log($, entry)
|
||||
const why =
|
||||
t.status === 'timeout'
|
||||
? `the tests did not finish within ${cfg.testTimeoutSec} s (testTimeoutSec)`
|
||||
: `the test command could not run (exit ${t.code ?? 'none'})`
|
||||
await tell(
|
||||
$,
|
||||
`${head}: NOT VERIFIED - ${why}. This says nothing about the code: do not resume or rewrite the agent for it. ` +
|
||||
`Fix the test setup or raise testTimeoutSec, or review the change yourself.\nLast output:\n${clip(t.tail, 2000)}`,
|
||||
)
|
||||
return
|
||||
}
|
||||
tests = `exit code ${t.code} after ${t.seconds} s\n${t.tail}`
|
||||
}
|
||||
|
||||
// 3. The verdict, from another model.
|
||||
const verifier = verifierFor(rec.tier)
|
||||
entry.verifier = verifier
|
||||
$.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`)
|
||||
const v = await $.model.complete({
|
||||
model: verifier,
|
||||
system: VERIFIER_SYSTEM,
|
||||
prompt: [
|
||||
`## Task\n${clip(rec.prompt, 12_000)}`,
|
||||
rec.lastIssues?.length
|
||||
? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}`
|
||||
: '',
|
||||
`## Changes (${baseLabel})\n${clip(changes, 60_000)}`,
|
||||
cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n\n'),
|
||||
maxTokens: 2000,
|
||||
timeoutMs: 180_000,
|
||||
})
|
||||
await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices)
|
||||
if (!v.isAnswered) {
|
||||
await log($, entry)
|
||||
await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`)
|
||||
return
|
||||
}
|
||||
const verdict = parseVerdict(v.text)
|
||||
const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)'
|
||||
|
||||
// 4. Remember where this round ended.
|
||||
const after: AgentRec = {
|
||||
...rec,
|
||||
rounds: round,
|
||||
lastSnap: (await snapshot($, cwd)) ?? rec.lastSnap,
|
||||
lastUntracked: await untracked($, cwd),
|
||||
lastIssues: verdict.pass ? [] : verdict.issues,
|
||||
}
|
||||
|
||||
if (verdict.pass) {
|
||||
entry.outcome = 'pass'
|
||||
await putIn($, S.agents, agentId, after)
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} ✓ ${rec.tier}, verified by ${verifier}`)
|
||||
await tell($, `${head}: PASS from verifier ${verifier}. Continue to the PR.`)
|
||||
return
|
||||
}
|
||||
|
||||
const fails = rec.fails + 1
|
||||
after.fails = fails
|
||||
const step = onFail(fails, rec.tier, cfg.maxFails, cfg.allowFable)
|
||||
await putIn($, S.agents, agentId, after)
|
||||
|
||||
if (step.kind === 'resume') {
|
||||
entry.outcome = 'resume-requested'
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} ✗ round ${round}, resume ${agentId}`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: FAIL from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
||||
(verdict.brief ? `What must change: ${verdict.brief}\n` : '') +
|
||||
`Resume agent ${agentId} with SendMessage, passing these issues. Do not fix it yourself.`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (step.kind === 'new-agent') {
|
||||
await putIn<Pending>($, S.pending, rec.taskId, {
|
||||
prompt:
|
||||
`${rec.prompt.split('\n\n## Previous attempts rejected')[0]}\n\n## Previous attempts rejected\n` +
|
||||
`Agents on ${rec.tier} were rejected ${fails} times by an independent verifier (${verifier}). Their work is in the clone ${cwd}: continue from it, fix or replace it, do not start over elsewhere.\n` +
|
||||
`Issues:\n${issues}\n` +
|
||||
(verdict.brief ? `What must change: ${verdict.brief}\n` : ''),
|
||||
tier: step.tier,
|
||||
fails,
|
||||
critical: rec.critical,
|
||||
})
|
||||
entry.outcome = 'rewrite-requested'
|
||||
await log($, entry)
|
||||
$.ui.status(`router: ${rec.taskId} ✗ again, new agent on ${step.tier}`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: FAIL again from verifier ${verifier}.\nIssues:\n${issues}\n` +
|
||||
`Stop agent ${agentId}, then start a new agent on ${step.tier} in the same clone: call Agent with prompt [router-rewrite ${rec.taskId}] (same subagent_type).`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
entry.outcome = 'handed-back'
|
||||
await log($, entry)
|
||||
$.ui.toast(`model-router: ${rec.taskId} failed verification ${fails}x - handed back`)
|
||||
await tell(
|
||||
$,
|
||||
`${head}: FAIL from verifier ${verifier}, ${fails} failed verdicts. Handed back: stop and show the person these issues.\n${issues}` +
|
||||
(verdict.brief ? `\nVerifier's brief: ${verdict.brief}` : ''),
|
||||
)
|
||||
}
|
||||
|
||||
/** Logs an agent run that is not verified; tells the manager when a dev agent waits on a verdict. */
|
||||
async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, outcome: Outcome) {
|
||||
await log($, {
|
||||
id: rec.taskId,
|
||||
agentId,
|
||||
at: await $.clock.now(),
|
||||
task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '',
|
||||
tier: rec.tier,
|
||||
role: rec.role,
|
||||
reason: outcome,
|
||||
outcome,
|
||||
})
|
||||
if (rec.role === 'dev' && outcome === 'no-clone')
|
||||
await tell($, `[model-router] ${rec.taskId} (agent ${agentId}): NOT VERIFIED - the brief names no clone path.`)
|
||||
}
|
||||
|
||||
async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, cfg: Settings) {
|
||||
const skip = skipReason(rec, ended)
|
||||
if (skip) await skipRound($, agentId, rec, skip)
|
||||
else await verifyRound($, agentId, rec, cfg)
|
||||
}
|
||||
|
||||
// ---------- the hooks ----------
|
||||
|
||||
export const register: Register = (on, options) => {
|
||||
const patterns = parsePatterns(String(options.safetyPatterns ?? ''))
|
||||
const respectExplicit = options.respectExplicitModel === true
|
||||
const cfg: Settings = {
|
||||
testCommand: String(options.testCommand ?? '').trim(),
|
||||
testTimeoutSec: validTimeout(options.testTimeoutSec),
|
||||
prBaseCommand: String(options.prBaseCommand ?? '').trim(),
|
||||
defaultBaseBranch: String(options.defaultBaseBranch ?? 'release/1.0').trim(),
|
||||
maxFails: Number(options.maxRewrites ?? 2),
|
||||
allowFable: options.allowFable === true,
|
||||
prices: parsePrices(String(options.prices ?? '')),
|
||||
}
|
||||
|
||||
on('session.start', async ($, e, next) => {
|
||||
await $.command.register({
|
||||
name: 'router',
|
||||
description: 'Model router: decisions, verdicts and savings vs Opus (`/router savings [Tn]`, `/router clear`)',
|
||||
})
|
||||
return next(e)
|
||||
})
|
||||
|
||||
on('prompt.compose', async ($, e, next) => {
|
||||
const out = await next(e)
|
||||
if (e.traits.includes('bare')) return out
|
||||
return { sections: [...out.sections, { id: 'model-router:note', text: ORCHESTRATOR_NOTE, scope: 'session' as const }] }
|
||||
})
|
||||
|
||||
// Routing: every agent the model starts, background ones included.
|
||||
on('agent.spawn', async ($, e, next) => {
|
||||
if (e.fork || e.isTeammate || e.workflow) return next(e)
|
||||
|
||||
let taskId: string
|
||||
let prompt = e.prompt
|
||||
let tier: Tier
|
||||
let reason: string
|
||||
let fails = 0
|
||||
let critical: boolean
|
||||
let opusRequested = false
|
||||
let classifierUsage: Usage | undefined
|
||||
// Agents whose model the router chose; kept models (review, test) are not a saving.
|
||||
let isRouted = true
|
||||
|
||||
const markId = e.prompt.match(REWRITE_MARK)?.[1]
|
||||
const pending = markId ? (await getMap<Pending>($, S.pending))[markId] : undefined
|
||||
|
||||
if (markId && pending) {
|
||||
taskId = markId
|
||||
const extra = e.prompt.replace(REWRITE_MARK, '').trim()
|
||||
prompt = pending.prompt + (extra ? `\n\nAdditional guidance from the orchestrator:\n${extra}` : '')
|
||||
tier = pending.tier
|
||||
fails = pending.fails
|
||||
critical = pending.critical
|
||||
reason = `new agent after ${fails} failed verdicts`
|
||||
await putIn<Pending>($, S.pending, markId, undefined)
|
||||
} else {
|
||||
taskId = await nextId($)
|
||||
const brief = parseBrief(e.prompt)
|
||||
critical = safetyHit(`${e.description}\n${e.prompt}`, patterns) !== undefined
|
||||
const d = decide({
|
||||
role: brief.role,
|
||||
named: asTier(e.model),
|
||||
subagentType: e.subagentType,
|
||||
prompt: e.prompt,
|
||||
respectExplicit,
|
||||
isAmbiguous: brief.isAmbiguous,
|
||||
})
|
||||
opusRequested = d.opusRequested === true
|
||||
isRouted =
|
||||
brief.role !== 'review' &&
|
||||
brief.role !== 'test' &&
|
||||
!(d.kind === 'fixed' && d.reason === 'model chosen by orchestrator')
|
||||
if (d.kind === 'fixed') {
|
||||
tier = d.tier
|
||||
reason = d.reason
|
||||
} else {
|
||||
const c = await classify($, e.prompt)
|
||||
tier = c.tier
|
||||
reason = c.reason
|
||||
classifierUsage = c.usage
|
||||
}
|
||||
}
|
||||
|
||||
const brief = parseBrief(prompt)
|
||||
const ran = await next({ ...e, prompt, model: tier })
|
||||
if (ran.deny !== undefined || !ran.agentId) return ran
|
||||
|
||||
// Learn what the alias resolves to, so router calls are priced on the real model.
|
||||
if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model)
|
||||
if (isRouted) {
|
||||
await putIn($, S.agentTask, ran.agentId, { taskId, task: e.description })
|
||||
await updateIn<TaskCost>($, S.costs, taskId, old => old ?? emptyTask(e.description))
|
||||
await chargeOverhead($, taskId, 'haiku', classifierUsage, cfg.prices)
|
||||
}
|
||||
|
||||
// Agents with a role (dev, review, test) get a record; only dev rounds are verified.
|
||||
if (brief.role)
|
||||
await putIn<AgentRec>($, S.agents, ran.agentId, {
|
||||
taskId,
|
||||
tier,
|
||||
role: brief.role,
|
||||
clone: brief.clone,
|
||||
baseBranch: brief.baseBranch,
|
||||
critical,
|
||||
prompt,
|
||||
fails,
|
||||
rounds: 0,
|
||||
})
|
||||
|
||||
await log($, {
|
||||
id: taskId,
|
||||
agentId: ran.agentId,
|
||||
at: await $.clock.now(),
|
||||
task: e.description,
|
||||
tier,
|
||||
role: brief.role,
|
||||
reason,
|
||||
outcome: 'spawned',
|
||||
opusRequested,
|
||||
})
|
||||
$.ui.status(`router: ${taskId} ${brief.role ?? 'agent'} → ${tier}`)
|
||||
return ran
|
||||
}).catch(($, e, next) => {
|
||||
// Fail open: a router bug must never block delegation.
|
||||
$.ui.status('router: error, agent ran unrouted')
|
||||
return next(e)
|
||||
})
|
||||
|
||||
// Verification: each finished run of a tracked agent, a resumed SendMessage round included.
|
||||
on('turn.complete', async ($, e, next) => {
|
||||
const out = await next(e)
|
||||
const agentId = e.agentId
|
||||
if (!agentId) return out
|
||||
|
||||
const owner = (await getMap<{ taskId: string; task: string }>($, S.agentTask))[agentId]
|
||||
if (owner && e.usage) {
|
||||
const usage = e.usage
|
||||
await updateIn<TaskCost>($, S.costs, owner.taskId, old =>
|
||||
addWriterRun(old ?? emptyTask(owner.task), usage.model, usage, cfg.prices),
|
||||
)
|
||||
}
|
||||
const rec = (await getMap<AgentRec>($, S.agents))[agentId]
|
||||
if (!rec) return out
|
||||
// Verification takes minutes: run it after this event has gone on.
|
||||
$.clock.after(0, () => void runRound($, agentId, rec, e.reason, cfg))
|
||||
return out
|
||||
})
|
||||
|
||||
on('command.run', { command: 'router' }, async ($, e) => {
|
||||
if (e.args.trim() === 'clear') {
|
||||
await serial(async () => {
|
||||
await $.store.set(S.log, [])
|
||||
await $.store.set(S.pending, {})
|
||||
await $.store.set(S.agents, {})
|
||||
await $.store.set(S.costs, {})
|
||||
await $.store.set(S.agentTask, {})
|
||||
})
|
||||
return { text: 'model-router: log, agents, costs and pending rewrites cleared.' }
|
||||
}
|
||||
const [sub, arg] = e.args.trim().split(/\s+/)
|
||||
if (sub === 'savings') return { text: savingsReport(await getMap<TaskCost>($, S.costs), Infinity, arg) }
|
||||
const list = ((await $.store.get(S.log)) ?? []) as LogEntry[]
|
||||
if (list.length === 0) return { text: 'model-router: nothing routed yet.' }
|
||||
|
||||
const count = (key: (l: LogEntry) => string | undefined) => {
|
||||
const r: Record<string, number> = {}
|
||||
for (const l of list) {
|
||||
const k = key(l)
|
||||
if (k) r[k] = (r[k] ?? 0) + 1
|
||||
}
|
||||
return Object.entries(r)
|
||||
.map(([k, n]) => `${k} ${n}`)
|
||||
.join(' · ')
|
||||
}
|
||||
const rows = list
|
||||
.slice(-15)
|
||||
.map(
|
||||
l =>
|
||||
`${l.id.padEnd(5)} ${(l.role ?? '-').padEnd(6)} ${l.tier.padEnd(6)} ${(l.verifier ? '→' + l.verifier : '').padEnd(8)} ${l.outcome.padEnd(17)} ${(l.task ?? '').slice(0, 40)}${l.opusRequested ? ' [opus requested]' : ''}`,
|
||||
)
|
||||
.join('\n')
|
||||
const pending = Object.keys(await getMap<Pending>($, S.pending))
|
||||
return {
|
||||
text:
|
||||
`model-router - last ${Math.min(15, list.length)} of ${list.length}\n` +
|
||||
`writers: ${count(l => (l.outcome === 'spawned' ? `${l.role ?? 'other'}:${l.tier}` : undefined))}\n` +
|
||||
`results: ${count(l => (l.outcome === 'spawned' ? undefined : l.outcome))}\n` +
|
||||
`opus requested on dev briefs: ${list.filter(l => l.opusRequested).length}\n` +
|
||||
(pending.length ? `pending rewrites: ${pending.join(', ')}\n` : '') +
|
||||
`\n${rows}\n\n` +
|
||||
savingsReport(await getMap<TaskCost>($, S.costs), 5),
|
||||
}
|
||||
})
|
||||
}
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
// Pure routing logic: no `$`, so every rule here is unit-tested.
|
||||
|
||||
export type Tier = 'haiku' | 'sonnet' | 'opus' | 'fable'
|
||||
export type Role = 'dev' | 'review' | 'test'
|
||||
|
||||
export const TIERS: readonly Tier[] = ['haiku', 'sonnet', 'opus', 'fable']
|
||||
|
||||
export const rank = (t: Tier) => TIERS.indexOf(t)
|
||||
|
||||
export const asTier = (s: string | undefined): Tier | undefined => {
|
||||
if (!s) return undefined
|
||||
const lower = s.toLowerCase()
|
||||
return TIERS.find(t => lower.includes(t))
|
||||
}
|
||||
|
||||
export const parsePatterns = (csv: string): string[] =>
|
||||
csv
|
||||
.split(',')
|
||||
.map(s => s.trim().toLowerCase())
|
||||
.filter(s => s.length > 0)
|
||||
|
||||
/** The first safety pattern the text mentions, or undefined. */
|
||||
export const safetyHit = (text: string, patterns: readonly string[]): string | undefined => {
|
||||
const lower = text.toLowerCase()
|
||||
return patterns.find(p => lower.includes(p))
|
||||
}
|
||||
|
||||
// ---------- the brief ----------
|
||||
|
||||
export type Brief = {
|
||||
clone?: string
|
||||
ticket?: string
|
||||
role?: Role
|
||||
baseBranch?: string
|
||||
/** Several clone paths and no `Clone:` line: the role cannot be told. */
|
||||
isAmbiguous?: boolean
|
||||
}
|
||||
|
||||
// A ticket is LETTERS-digits plus any extra parts (OPT-11-P4); the role is the last part.
|
||||
const TICKET = '[A-Za-z]+-\\d+(?:-[A-Za-z0-9]+)*?'
|
||||
const CLONE_PATH = new RegExp(`(\\S*?\\/clones\\/(${TICKET})-(dev|review|test))\\/?`)
|
||||
const CLONE_PATHS = new RegExp(CLONE_PATH.source, 'g')
|
||||
// The explicit line every brief carries: `Clone: <path>` with an optional `(Role: dev)`.
|
||||
const CLONE_LINE = /^[ \t>*-]*Clone:[ \t]*`?([^\s`]+?)\/?`?[ \t]*(?:\(Role:[ \t]*(dev|review|test)\))?[ \t]*$/im
|
||||
const BASE =
|
||||
/\b(?:base(?:\s+branch)?|target(?:\s+branch)?|merge\s+into|PR\s+(?:against|into))\s*[:=]?\s*`?((?:release|main|master|develop)[\w./-]*)/i
|
||||
|
||||
const trimPath = (p: string) => p.replace(/^[`'"(]+/, '').replace(/[`'")]+$/, '').replace(/\/+$/, '')
|
||||
|
||||
/**
|
||||
* Reads the clone path, ticket, role and base branch out of a brief.
|
||||
*
|
||||
* The `Clone:` line decides when present: its `(Role: ...)`, else the role in
|
||||
* its path. Without one, a brief that names one clone is read from that path;
|
||||
* one that names several is ambiguous and gets no role.
|
||||
*/
|
||||
export const parseBrief = (prompt: string): Brief => {
|
||||
const b = prompt.match(BASE)
|
||||
// a sentence's full stop or a trailing slash is no part of the branch
|
||||
const baseBranch = b?.[1]?.replace(/[./-]+$/, '')
|
||||
|
||||
const line = prompt.match(CLONE_LINE)
|
||||
if (line?.[1]) {
|
||||
const clone = trimPath(line[1])
|
||||
const fromPath = clone.match(CLONE_PATH)
|
||||
const role = (line[2] ?? fromPath?.[3]) as Role | undefined
|
||||
return { clone, ticket: fromPath?.[2], role, baseBranch }
|
||||
}
|
||||
|
||||
const paths = [...new Set([...prompt.matchAll(CLONE_PATHS)].map(m => trimPath(m[1] ?? '')))]
|
||||
if (paths.length > 1) return { baseBranch, isAmbiguous: true }
|
||||
const only = paths[0]?.match(CLONE_PATH)
|
||||
if (!only) return { baseBranch }
|
||||
return { clone: trimPath(only[1] ?? ''), ticket: only[2], role: only[3] as Role, baseBranch }
|
||||
}
|
||||
|
||||
// ---------- routing ----------
|
||||
|
||||
const READ_ONLY_TYPES = /^(explore|scout|runner|claude-code-guide|statusline-setup)$|:(scout|runner)$/i
|
||||
|
||||
// Light work, only for tasks outside the dev/review/test roles. No bare
|
||||
// "docs" or "run tests": both appear in nearly every brief.
|
||||
const LIGHT = [
|
||||
/\b(find|locate|grep|where is|which (files|modules)|summari[sz]e|read and report)\b/,
|
||||
/\b(fix (a |the )?typo|update (the )?(readme|changelog)|add docstrings?)\b/,
|
||||
/\b(run (the )?(unit )?tests? and report|report (the )?test results)\b/,
|
||||
]
|
||||
|
||||
// Opus writes only when the approach itself is still open. Domain words
|
||||
// (control loop, asyncio, RS485) are everywhere in this codebase and say nothing.
|
||||
const HEAVY = [
|
||||
/\b(design|architect|redesign|decide (how|whether)|propose an approach|trade-?offs?)\b/,
|
||||
/\b(root cause|unclear why|no idea why|investigate why|figure out why|intermittent|flaky)\b/,
|
||||
]
|
||||
|
||||
export type Route = { tier: Tier; reason: string }
|
||||
|
||||
/** A clear signal from the text, or undefined (ask the classifier). */
|
||||
export const heuristic = (subagentType: string | undefined, prompt: string, role?: Role): Route | undefined => {
|
||||
if (!role && subagentType && READ_ONLY_TYPES.test(subagentType))
|
||||
return { tier: 'haiku', reason: `read-only agent type ${subagentType}` }
|
||||
const lower = prompt.toLowerCase()
|
||||
const heavy = HEAVY.find(r => r.test(lower))
|
||||
if (heavy) return { tier: 'opus', reason: 'approach still open' }
|
||||
if (role) return undefined // dev briefs never take the light path
|
||||
const light = LIGHT.find(r => r.test(lower))
|
||||
if (light && prompt.length < 1500) return { tier: 'haiku', reason: 'light task' }
|
||||
return undefined
|
||||
}
|
||||
|
||||
export type Decision =
|
||||
| { kind: 'fixed'; tier: Tier; reason: string; opusRequested?: boolean }
|
||||
| { kind: 'classify'; opusRequested?: boolean }
|
||||
|
||||
/**
|
||||
* Who decides the writer's model.
|
||||
* - review and test agents: a named model is kept; unnamed, they run on Opus
|
||||
* (they are the gates, and a test agent's work has no verifier behind it).
|
||||
* - dev agents: always routed; a named Opus is only logged.
|
||||
* - anything else: routed, unless respectExplicit and a model is named.
|
||||
*/
|
||||
export const decide = (args: {
|
||||
role?: Role
|
||||
named?: Tier
|
||||
subagentType?: string
|
||||
prompt: string
|
||||
respectExplicit: boolean
|
||||
isAmbiguous?: boolean
|
||||
}): Decision => {
|
||||
const { role, named, subagentType, prompt, respectExplicit, isAmbiguous } = args
|
||||
// Could be a test or review agent: never risk a weaker model on it.
|
||||
if (isAmbiguous)
|
||||
return { kind: 'fixed', tier: named && named !== 'haiku' && named !== 'sonnet' ? named : 'opus', reason: 'several clone paths and no Clone: line' }
|
||||
if (role === 'review' || role === 'test')
|
||||
return named
|
||||
? { kind: 'fixed', tier: named, reason: `named model kept for ${role} agent` }
|
||||
: { kind: 'fixed', tier: 'opus', reason: `${role} agent default` }
|
||||
if (!role && respectExplicit && named) return { kind: 'fixed', tier: named, reason: 'model chosen by orchestrator' }
|
||||
const opusRequested = role === 'dev' && (named === 'opus' || named === 'fable')
|
||||
const h = heuristic(subagentType, prompt, role)
|
||||
return h ? { kind: 'fixed', ...h, opusRequested } : { kind: 'classify', opusRequested }
|
||||
}
|
||||
|
||||
// ---------- verification ----------
|
||||
|
||||
export type Skip = 'test-agent' | 'review-agent' | 'no-clone' | 'opus-unverified' | 'agent-ended'
|
||||
|
||||
/** Why an agent run is not verified, or undefined when it is. */
|
||||
export const skipReason = (rec: { role?: Role; clone?: string; tier: Tier }, ended: string): Skip | undefined => {
|
||||
if (ended !== 'answer') return 'agent-ended'
|
||||
if (rec.role === 'test') return 'test-agent'
|
||||
if (rec.role === 'review') return 'review-agent'
|
||||
if (!rec.clone) return 'no-clone'
|
||||
if (rank(rec.tier) >= rank('opus')) return 'opus-unverified'
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** The verifier never runs on the writer's model; below Opus it is Opus. */
|
||||
export const verifierFor = (writer: Tier): Tier => (rank(writer) >= rank('opus') ? 'sonnet' : 'opus')
|
||||
|
||||
/** The model a rewrite runs on: one tier up, never the same model. */
|
||||
export const rewriteTier = (writer: Tier, allowFable: boolean): Tier | undefined => {
|
||||
if (writer === 'haiku') return 'sonnet'
|
||||
if (writer === 'sonnet') return 'opus'
|
||||
if (writer === 'opus' && allowFable) return 'fable'
|
||||
return undefined
|
||||
}
|
||||
|
||||
export type FailStep = { kind: 'resume' } | { kind: 'new-agent'; tier: Tier } | { kind: 'hand-back' }
|
||||
|
||||
/**
|
||||
* After the n-th failed verdict in a task's chain: the first resumes the same
|
||||
* agent, the second starts a new agent one model up in the same clone, and
|
||||
* past maxFails (or with no model above) it goes back to the manager.
|
||||
*/
|
||||
export const onFail = (fails: number, writer: Tier, maxFails: number, allowFable: boolean): FailStep => {
|
||||
if (fails > maxFails) return { kind: 'hand-back' }
|
||||
if (fails === 1) return { kind: 'resume' }
|
||||
const up = rewriteTier(writer, allowFable)
|
||||
return up ? { kind: 'new-agent', tier: up } : { kind: 'hand-back' }
|
||||
}
|
||||
|
||||
export type Verdict = { pass: boolean; issues: string[]; brief: string }
|
||||
|
||||
/** Reads the verifier's reply; anything unreadable counts as a failure. */
|
||||
export const parseVerdict = (text: string): Verdict => {
|
||||
const match = text.match(/\{[\s\S]*\}/)
|
||||
if (match) {
|
||||
try {
|
||||
const v = JSON.parse(match[0]) as { verdict?: string; issues?: unknown; rewrite_brief?: unknown }
|
||||
const issues = Array.isArray(v.issues) ? v.issues.map(String) : []
|
||||
const pass = String(v.verdict ?? '').toUpperCase() === 'PASS'
|
||||
return { pass, issues, brief: typeof v.rewrite_brief === 'string' ? v.rewrite_brief : '' }
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
return { pass: false, issues: ['Verifier reply was not valid JSON: ' + text.slice(0, 300)], brief: '' }
|
||||
}
|
||||
|
||||
export const REWRITE_MARK = /^\s*\[router-rewrite (T\d+)\]/
|
||||
|
||||
export const clip = (s: string, n: number) =>
|
||||
s.length <= n ? s : s.slice(0, n) + `\n... [${s.length - n} more characters cut]`
|
||||
|
||||
/** A usable test time limit in seconds: anything not a positive number is the default. */
|
||||
export function validTimeout(raw: unknown): number {
|
||||
const n = Number(raw)
|
||||
return Number.isFinite(n) && n >= 1 ? Math.round(n) : 1800
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import type { On } from 'claude-code'
|
||||
import { expect, mock, test, type Engine } from 'claude-code/testing'
|
||||
|
||||
import {
|
||||
addOverhead,
|
||||
addWriterRun,
|
||||
actualTotal,
|
||||
counterfactual,
|
||||
emptyTask,
|
||||
parsePrices,
|
||||
priceRun,
|
||||
saved,
|
||||
savingsReport,
|
||||
tokenizerFactor,
|
||||
} from '../hooks/pricing'
|
||||
|
||||
const U = (i: number, o: number, w = 0, r = 0) => ({
|
||||
input_tokens: i,
|
||||
output_tokens: o,
|
||||
cache_creation_input_tokens: w,
|
||||
cache_read_input_tokens: r,
|
||||
})
|
||||
const M = 1_000_000
|
||||
const round = (n: number) => Math.round(n * 1e6) / 1e6
|
||||
|
||||
test('prices: each part of the usage at its own rate', async () => {
|
||||
// Sonnet 5.5: in 2, out 10, write 2.5, read 0.1 per MTok
|
||||
expect(priceRun('claude-sonnet-5-5', U(M, M, M, M)).actual).toBe(2 + 10 + 2.5 + 0.1)
|
||||
// the same tokens at Opus 5.5: 4 + 20 + 5 + 0.2
|
||||
expect(round(priceRun('claude-sonnet-5-5', U(M, M, M, M)).opus)).toBe(29.2)
|
||||
expect(priceRun('claude-opus-5-5', U(M, 0)).actual).toBe(4)
|
||||
expect(priceRun('claude-haiku-5-5', U(M, 0)).actual).toBe(0.1)
|
||||
expect(priceRun('claude-haiku-4-5', U(M, 0)).actual).toBe(1)
|
||||
})
|
||||
|
||||
test('prices: the most specific model wins, unknown models are flagged', async () => {
|
||||
expect(priceRun('claude-opus-5-5-20260601', U(M, 0)).label).toBe('Opus 5.5')
|
||||
expect(priceRun('claude-opus-5', U(M, 0)).label).toBe('Opus 5')
|
||||
expect(priceRun('claude-sonnet-5', U(M, 0)).label).toBe('Sonnet 5')
|
||||
expect(priceRun('gpt-9', U(M, 0)).known).toBe(false)
|
||||
})
|
||||
|
||||
test('older tokenizers are scaled before pricing as Opus 5.5', async () => {
|
||||
expect(tokenizerFactor('claude-haiku-4-5')).toBe(1.3)
|
||||
expect(tokenizerFactor('claude-sonnet-5-5')).toBe(1)
|
||||
expect(tokenizerFactor('claude-haiku-5-5')).toBe(1)
|
||||
expect(round(priceRun('claude-haiku-4-5', U(M, 0)).opus)).toBe(5.2)
|
||||
})
|
||||
|
||||
test('a task below opus: every writer token at opus price is the comparison', async () => {
|
||||
let t = emptyTask('clamp')
|
||||
t = addWriterRun(t, 'claude-sonnet-5-5', U(M, 0))
|
||||
t = addWriterRun(t, 'claude-sonnet-5-5', U(M, 0)) // resumed round
|
||||
t = addOverhead(t, 'claude-opus-5-5', U(M / 2, 0)) // verifier
|
||||
expect(t.writerActual).toBe(4)
|
||||
expect(counterfactual(t)).toBe(8)
|
||||
expect(actualTotal(t)).toBe(6)
|
||||
expect(saved(t)).toBe(2)
|
||||
})
|
||||
|
||||
test('a chain that ended on opus: the opus run is the comparison, failed rounds are loss', async () => {
|
||||
let t = emptyTask('clamp')
|
||||
t = addWriterRun(t, 'claude-sonnet-5-5', U(M, 0)) // failed twice
|
||||
t = addWriterRun(t, 'claude-opus-5-5', U(M, 0)) // rewrite
|
||||
expect(counterfactual(t)).toBe(4)
|
||||
expect(actualTotal(t)).toBe(6)
|
||||
expect(saved(t)).toBe(-2)
|
||||
})
|
||||
|
||||
test('unpriced models are listed, never counted as free', async () => {
|
||||
const t = addWriterRun(emptyTask('x'), 'mystery-1', U(M, M))
|
||||
expect(t.unpriced).toEqual(['mystery-1'])
|
||||
expect(t.writerActual).toBe(0)
|
||||
expect(savingsReport({ T1: t }, 5)).toContain('[no price: mystery-1]')
|
||||
})
|
||||
|
||||
test('price overrides come first; bad JSON falls back to list prices', async () => {
|
||||
const table = parsePrices('[{"match":"sonnet-5-5","price":{"in":1,"out":1,"write":1,"read":1}}]')
|
||||
expect(priceRun('claude-sonnet-5-5', U(M, 0), table).actual).toBe(1)
|
||||
expect(priceRun('claude-opus-5-5', U(M, 0), table).actual).toBe(4)
|
||||
expect(priceRun('claude-sonnet-5-5', U(M, 0), parsePrices('{oops')).actual).toBe(2)
|
||||
})
|
||||
|
||||
test('the report: newest first, totals over every task, filter by task', async () => {
|
||||
const a = addWriterRun(emptyTask('first'), 'claude-sonnet-5-5', U(M, 0))
|
||||
const b = addWriterRun(emptyTask('second'), 'claude-sonnet-5-5', U(2 * M, 0))
|
||||
const r = savingsReport({ T1: a, T2: b }, 1)
|
||||
expect(r.indexOf('T2')).toBeGreaterThan(-1)
|
||||
expect(r).not.toContain('T1 ')
|
||||
expect(r).toMatch(/total\s+\$6\.00\s+\$12\.00\s+\$6\.00\s+50%/)
|
||||
expect(savingsReport({ T1: a, T2: b }, 9, 'T1')).toContain('first')
|
||||
expect(savingsReport({ T1: a, T2: b }, 9, 'T1')).not.toContain('second')
|
||||
})
|
||||
|
||||
// ---------- end to end ----------
|
||||
|
||||
const USAGE = { input_tokens: 1000, output_tokens: 10, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }
|
||||
const ok = (stdout: string) => ({ value: { exitCode: 0, stdout, stderr: '', isStdoutTruncated: false, isStderrTruncated: false } })
|
||||
const DEV = '/home/homelab/projects/goodwe/clones/OPT-11-P4-dev'
|
||||
|
||||
function world(on: On, models: string[]) {
|
||||
mock.store(on)
|
||||
const clock = mock.clock(on)
|
||||
on('ui.status', async () => ({ value: undefined }))
|
||||
on('ui.toast', async () => ({ value: undefined }))
|
||||
on('model.complete', async (_$, e) => {
|
||||
if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } }
|
||||
return { value: { isAnswered: true as const, text: '{"verdict":"PASS"}', usage: USAGE } }
|
||||
})
|
||||
on('process.run', async (_$, e) => {
|
||||
const a = e.argv.join(' ')
|
||||
if (a.startsWith('git merge-base')) return ok('base\n')
|
||||
if (a.startsWith('git diff')) return ok('diff')
|
||||
return ok('')
|
||||
})
|
||||
on('agent.spawn', async (_$, e) => {
|
||||
models.push(String(e.model))
|
||||
return { model: e.model === 'sonnet' ? 'claude-sonnet-5-5' : String(e.model), agentId: `a${models.length}` }
|
||||
})
|
||||
on('turn.complete', async (_$, e) => ({ text: e.answer }))
|
||||
on('prompt.submit', async (_$, e) => ({ text: e.text }))
|
||||
return clock
|
||||
}
|
||||
|
||||
async function end($: Engine, agentId: string, model: string, input: number) {
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({
|
||||
answer: 'done',
|
||||
durationMs: 1,
|
||||
isAborted: false,
|
||||
turnId: 't',
|
||||
agentId,
|
||||
reason: 'answer',
|
||||
usage: { model, input_tokens: input, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||
})
|
||||
}
|
||||
|
||||
test('end to end: a routed dev agent is priced on the model it ran on, a kept review agent is not counted', async ($, on) => {
|
||||
const models: string[] = []
|
||||
const clock = world(on, models)
|
||||
const dev = await $.agent.spawn({ prompt: `Clone: ${DEV}/\nExpected: x`, description: 'P4 dev' } as never)
|
||||
await end($, dev.agentId as string, 'claude-sonnet-5-5', M)
|
||||
await clock.advance(1)
|
||||
const review = await $.agent.spawn({ prompt: 'Clone: /x/clones/OPT-11-P4-review/', description: 'P4 review', model: 'opus' } as never)
|
||||
await end($, review.agentId as string, 'claude-opus-5-5', M)
|
||||
await clock.advance(1)
|
||||
|
||||
const r = (await $.command.run({ command: 'router', args: 'savings' } as never)).text ?? ''
|
||||
expect(r).toContain('P4 dev')
|
||||
expect(r).not.toContain('P4 review')
|
||||
// writer $2.00 + classifier and verifier (1000 in + 10 out each, Haiku 4.5 then Opus 5.5)
|
||||
expect(r).toMatch(/T1\s+\$2\.01\s+\$4\.00/)
|
||||
expect(r).toContain('Sonnet 5.5')
|
||||
})
|
||||
|
||||
test('agents started in parallel each get their own task, and every run is counted', async ($, on) => {
|
||||
const models: string[] = []
|
||||
const clock = world(on, models)
|
||||
const ids = (
|
||||
await Promise.all(
|
||||
['A', 'D', 'E', 'F', 'G'].map(c => $.agent.spawn({ prompt: `Scope chunk ${c}: list the open cards`, description: `Scope chunk ${c}` } as never)),
|
||||
)
|
||||
).map(r => r.agentId as string)
|
||||
await Promise.all(ids.map(id => end($, id, 'claude-sonnet-5-5', M)))
|
||||
await clock.advance(1)
|
||||
|
||||
const r = (await $.command.run({ command: 'router', args: 'savings' } as never)).text ?? ''
|
||||
for (const c of ['A', 'D', 'E', 'F', 'G']) expect(r).toContain(`Scope chunk ${c}`)
|
||||
expect(r).toContain('5 of 5 tasks')
|
||||
// five writer runs of $2.00 each, all counted
|
||||
expect(r).toMatch(/total\s+\$10\.\d\d\s+\$20\.00/)
|
||||
})
|
||||
|
||||
test('a task whose writer has not finished shows as running and stays out of the total', async () => {
|
||||
const running = addOverhead(emptyTask('MAINT-14 dev'), 'claude-haiku-4-5', U(M, 0)) // $1 of classifier, no writer yet
|
||||
const done = addWriterRun(emptyTask('done'), 'claude-sonnet-5-5', U(M, 0))
|
||||
const r = savingsReport({ T1: done, T2: running }, 5)
|
||||
expect(r).toMatch(/T2 .* running /)
|
||||
expect(r).toMatch(/total\s+\$2\.00\s+\$4\.00/)
|
||||
})
|
||||
@@ -0,0 +1,451 @@
|
||||
import type { On } from 'claude-code'
|
||||
import { expect, mock, test, type Engine } from 'claude-code/testing'
|
||||
|
||||
import {
|
||||
decide,
|
||||
heuristic,
|
||||
onFail,
|
||||
parseBrief,
|
||||
parseVerdict,
|
||||
rewriteTier,
|
||||
skipReason,
|
||||
validTimeout,
|
||||
verifierFor,
|
||||
} from '../hooks/router'
|
||||
|
||||
const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev'
|
||||
const REVIEW = '/home/homelab/projects/goodwe/clones/SAFETY-95-review'
|
||||
const TEST = '/home/homelab/projects/goodwe/clones/TEST-23-test'
|
||||
|
||||
// ---------- the brief (C2) ----------
|
||||
|
||||
test('brief: clone path, ticket, role and base branch are read', async () => {
|
||||
const b = parseBrief(`Work in ${DEV}/ only. Base branch: release/1.0.\nExpected: ...`)
|
||||
expect(b).toEqual({ clone: DEV, ticket: 'SAFETY-95', role: 'dev', baseBranch: 'release/1.0' })
|
||||
expect(parseBrief(`Clone: \`${TEST}\``).role).toBe('test')
|
||||
expect(parseBrief('PR against `release/1.1`, no clone given').baseBranch).toBe('release/1.1')
|
||||
expect(parseBrief('just look around').clone).toBe(undefined)
|
||||
})
|
||||
|
||||
test('fix 1: tickets with extra parts parse, the role is the last part', async () => {
|
||||
const p = '/home/homelab/projects/goodwe/clones/OPT-11-P4-dev'
|
||||
expect(parseBrief(`Work in ${p}/ only.`)).toMatchObject({ clone: p, ticket: 'OPT-11-P4', role: 'dev' })
|
||||
expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').ticket).toBe('OPT-11-P4-R2')
|
||||
expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').role).toBe('test')
|
||||
})
|
||||
|
||||
test('fix 2: the Clone: line decides, whatever the brief mentions first', async () => {
|
||||
const brief = `Verify the work of ${DEV}/ on the rig.\nClone: ${TEST}/\nBase branch: release/1.0.`
|
||||
expect(parseBrief(brief)).toMatchObject({ clone: TEST, role: 'test', baseBranch: 'release/1.0' })
|
||||
// an explicit role wins over the path
|
||||
expect(parseBrief(`Clone: \`/x/clones/OPT-11-P4\` (Role: review)`)).toMatchObject({ clone: '/x/clones/OPT-11-P4', role: 'review' })
|
||||
// (Role: ...) wins over the role in the path
|
||||
expect(parseBrief(`Clone: ${DEV}/ (Role: review)`).role).toBe('review')
|
||||
// a list item or quote marker in front of the line is fine
|
||||
expect(parseBrief(`- Clone: ${DEV}`).role).toBe('dev')
|
||||
// "Clone:" inside a sentence is not the line, even at a line's end
|
||||
expect(parseBrief(`Results are in the dev agent's Clone: ${DEV}\nRun the rig in ${TEST}/`).isAmbiguous).toBe(true)
|
||||
})
|
||||
|
||||
test('fix 2: several clones and no Clone: line gives no role, and routes to opus', async () => {
|
||||
const b = parseBrief(`Compare ${DEV}/ with ${TEST}/.`)
|
||||
expect(b.role).toBe(undefined)
|
||||
expect(b.isAmbiguous).toBe(true)
|
||||
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'haiku' })).toMatchObject({ kind: 'fixed', tier: 'opus' })
|
||||
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'sonnet' })).toMatchObject({ tier: 'opus' })
|
||||
expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'fable' })).toMatchObject({ tier: 'fable' })
|
||||
// the same path twice is one clone, not ambiguous
|
||||
expect(parseBrief(`Work in ${DEV}/. Commit in ${DEV}.`).role).toBe('dev')
|
||||
})
|
||||
|
||||
// ---------- who picks the writer (C4) ----------
|
||||
|
||||
test('C4: review and test agents keep the model they are given', async () => {
|
||||
for (const role of ['review', 'test'] as const)
|
||||
for (const named of ['haiku', 'sonnet', 'opus'] as const)
|
||||
expect(decide({ role, named, prompt: 'x', respectExplicit: false })).toEqual({
|
||||
kind: 'fixed',
|
||||
tier: named,
|
||||
reason: `named model kept for ${role} agent`,
|
||||
})
|
||||
})
|
||||
|
||||
test('C4: unnamed review and test agents run on opus', async () => {
|
||||
expect(decide({ role: 'test', prompt: 'run the rig', respectExplicit: false })).toMatchObject({ tier: 'opus' })
|
||||
expect(decide({ role: 'review', prompt: 'review', respectExplicit: false })).toMatchObject({ tier: 'opus' })
|
||||
})
|
||||
|
||||
test('C4: dev agents are routed even when opus is named, and that is logged', async () => {
|
||||
const d = decide({ role: 'dev', named: 'opus', prompt: 'Expected: clamp to 2000 W', respectExplicit: true })
|
||||
expect(d.kind).toBe('classify')
|
||||
expect(d.opusRequested).toBe(true)
|
||||
expect(decide({ role: 'dev', named: 'sonnet', prompt: 'x', respectExplicit: true }).opusRequested).toBe(false)
|
||||
})
|
||||
|
||||
test('C4: respectExplicit only applies to agents without a role', async () => {
|
||||
expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: true })).toMatchObject({ kind: 'fixed', tier: 'haiku' })
|
||||
expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: false }).kind).toBe('classify')
|
||||
})
|
||||
|
||||
test('C4: open design goes to opus, a decided brief is left to the classifier', async () => {
|
||||
expect(heuristic(undefined, 'MAINT-13: decide how the staleness guard should recover', 'dev')?.tier).toBe('opus')
|
||||
expect(heuristic(undefined, 'In the asyncio control loop, clamp the integrator; Expected: test_clamp passes', 'dev')).toBe(undefined)
|
||||
})
|
||||
|
||||
test('C4: narrowed light patterns: "docs" or "run tests" in a brief never mean haiku', async () => {
|
||||
expect(heuristic(undefined, 'Update the docs and run the tests after changing the setpoint logic')).toBe(undefined)
|
||||
expect(heuristic(undefined, 'Find where the MQTT discovery payload is built')?.tier).toBe('haiku')
|
||||
// a dev brief never takes the light path, however light it reads
|
||||
expect(heuristic(undefined, 'Find where the MQTT discovery payload is built', 'dev')).toBe(undefined)
|
||||
expect(heuristic('Explore', 'anything', 'dev')).toBe(undefined)
|
||||
})
|
||||
|
||||
// ---------- what gets verified (C1, C5) ----------
|
||||
|
||||
test('only finished dev rounds below opus with a clone are verified', async () => {
|
||||
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'answer')).toBe(undefined)
|
||||
expect(skipReason({ role: 'dev', clone: DEV, tier: 'haiku' }, 'answer')).toBe(undefined)
|
||||
expect(skipReason({ role: 'dev', clone: DEV, tier: 'opus' }, 'answer')).toBe('opus-unverified')
|
||||
expect(skipReason({ role: 'dev', tier: 'sonnet' }, 'answer')).toBe('no-clone')
|
||||
expect(skipReason({ role: 'test', clone: TEST, tier: 'sonnet' }, 'answer')).toBe('test-agent')
|
||||
expect(skipReason({ role: 'review', clone: REVIEW, tier: 'sonnet' }, 'answer')).toBe('review-agent')
|
||||
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'error')).toBe('agent-ended')
|
||||
expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'aborted')).toBe('agent-ended')
|
||||
})
|
||||
|
||||
test('the verifier is opus below opus, and never the writer', async () => {
|
||||
expect(verifierFor('haiku')).toBe('opus')
|
||||
expect(verifierFor('sonnet')).toBe('opus')
|
||||
for (const w of ['haiku', 'sonnet', 'opus', 'fable'] as const) expect(verifierFor(w)).not.toBe(w)
|
||||
})
|
||||
|
||||
test('C5: first fail resumes, second starts a stronger agent, then hand back', async () => {
|
||||
expect(onFail(1, 'sonnet', 2, false)).toEqual({ kind: 'resume' })
|
||||
expect(onFail(2, 'sonnet', 2, false)).toEqual({ kind: 'new-agent', tier: 'opus' })
|
||||
expect(onFail(3, 'opus', 2, false)).toEqual({ kind: 'hand-back' })
|
||||
expect(onFail(2, 'opus', 2, false)).toEqual({ kind: 'hand-back' }) // nothing above opus
|
||||
expect(onFail(2, 'opus', 2, true)).toEqual({ kind: 'new-agent', tier: 'fable' })
|
||||
expect(onFail(1, 'sonnet', 0, false)).toEqual({ kind: 'hand-back' })
|
||||
expect(rewriteTier('haiku', false)).toBe('sonnet')
|
||||
})
|
||||
|
||||
test('verdict parsing fails closed', async () => {
|
||||
expect(parseVerdict('{"verdict":"PASS","issues":[]}').pass).toBe(true)
|
||||
expect(parseVerdict('text {"verdict":"FAIL","issues":["a.py:3"]}').issues).toEqual(['a.py:3'])
|
||||
expect(parseVerdict('looks fine').pass).toBe(false)
|
||||
expect(parseVerdict('verdict: pass').pass).toBe(false)
|
||||
})
|
||||
|
||||
// ---------- the hooks, with the engine mocked beneath ----------
|
||||
|
||||
type World = {
|
||||
spawned: { model?: string; prompt: string }[]
|
||||
verifier: string[]
|
||||
verdicts: string[]
|
||||
told: string[]
|
||||
diffs: string[]
|
||||
diffBases: string[]
|
||||
testCwd: string[]
|
||||
testArgv: string[][]
|
||||
/** How the next test run ends: exit code (124 = timed out), or a throw. */
|
||||
testEnds: (number | 'throw')[]
|
||||
/** Whether `timeout` fired on each test run (it then says so on stderr). */
|
||||
testFired: boolean[]
|
||||
/** The timeout line arrives in two stderr pieces. */
|
||||
testSplitLine?: boolean
|
||||
/** A test itself prints the timeout line, on stdout. */
|
||||
testLineOnStdout?: boolean
|
||||
snap: number
|
||||
}
|
||||
|
||||
const USAGE = { input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }
|
||||
const ok = (stdout: string, exitCode = 0) => ({
|
||||
value: { exitCode, stdout, stderr: '', isStdoutTruncated: false, isStderrTruncated: false },
|
||||
})
|
||||
|
||||
function world(on: On, w: World) {
|
||||
mock.store(on)
|
||||
const clock = mock.clock(on)
|
||||
on('ui.status', async () => ({ value: undefined }))
|
||||
on('ui.toast', async () => ({ value: undefined }))
|
||||
on('model.complete', async (_$, e) => {
|
||||
if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } }
|
||||
w.verifier.push(e.model)
|
||||
return { value: { isAnswered: true as const, text: w.verdicts.shift() ?? '{"verdict":"PASS"}', usage: USAGE } }
|
||||
})
|
||||
on('process.run', async (_$, e) => {
|
||||
const a = e.argv.join(' ')
|
||||
const cwd = String(e.init?.cwd ?? '')
|
||||
if (cwd !== DEV) return ok('', 128) // not a git repo: the session folder
|
||||
if (a === 'git merge-base origin/release/1.0 HEAD') return ok('mergebase\n')
|
||||
if (a === 'git stash create') return ok(`snap${++w.snap}\n`)
|
||||
if (a.startsWith('git diff ')) {
|
||||
w.diffBases.push(e.argv[2] ?? '')
|
||||
return ok(w.diffs.shift() ?? '')
|
||||
}
|
||||
return ok('')
|
||||
})
|
||||
on('process.spawn', async function* (_$, e) {
|
||||
w.testCwd.push(String(e.cwd ?? ''))
|
||||
w.testArgv.push([...e.argv])
|
||||
const end = w.testEnds.shift() ?? 0
|
||||
const fired = w.testFired.shift() ?? false
|
||||
if (end === 'throw') throw new Error('docker: not found')
|
||||
yield { stream: 'stdout' as const, text: end === 0 ? '67 passed\n' : 'test_modes_p4 ... still running\n' }
|
||||
if (fired) yield { stream: 'stderr' as const, text: "timeout: sending signal TERM to command 'sh'\n" }
|
||||
if (w.testSplitLine) {
|
||||
yield { stream: 'stderr' as const, text: 'timeout: sending sig' }
|
||||
yield { stream: 'stderr' as const, text: "nal TERM to command 'sh'\n" }
|
||||
}
|
||||
if (w.testLineOnStdout) yield { stream: 'stdout' as const, text: 'assert "timeout: sending signal" in log\n' }
|
||||
return { value: { code: end, signal: null } }
|
||||
})
|
||||
on('agent.spawn', async (_$, e) => {
|
||||
w.spawned.push({ model: e.model, prompt: e.prompt })
|
||||
return { model: e.model ?? 'inherit', agentId: `agent-${w.spawned.length}` }
|
||||
})
|
||||
on('turn.complete', async (_$, e) => ({ text: e.answer }))
|
||||
on('prompt.submit', async (_$, e) => {
|
||||
w.told.push(e.text)
|
||||
return { text: e.text }
|
||||
})
|
||||
return clock
|
||||
}
|
||||
|
||||
const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 })
|
||||
|
||||
async function finish($: Engine, clock: { advance: (ms: number) => Promise<void> }, agentId: string) {
|
||||
await ($.turn as unknown as { complete: (e: unknown) => Promise<unknown> }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason: 'answer' })
|
||||
await clock.advance(1)
|
||||
}
|
||||
|
||||
const devBrief = (extra = '') => `SAFETY-95: clamp the integrator. Work in ${DEV}/. Base branch: release/1.0.\nExpected: test_clamp passes.${extra}`
|
||||
|
||||
test('acceptance 1+2: a background dev agent is verified in its clone, each round on its own diff', { options: { testCommand: 'pytest -q' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff --git a/loop.py', 'diff --git a/loop.py (round 2)']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["loop.py:40 no clamp on negative"]}', '{"verdict":"PASS"}']
|
||||
const clock = world(on, w)
|
||||
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp', model: 'opus' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('sonnet') // routed despite the named opus
|
||||
await finish($, clock, r.agentId as string)
|
||||
|
||||
expect(w.diffBases[0]).toBe('mergebase') // first round: from the merge base, in the clone
|
||||
expect(w.testCwd).toEqual([DEV])
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
expect(w.told[0]).toContain('FAIL')
|
||||
|
||||
// round 2: the resumed agent finishes again (same agentId)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.diffBases[1]).toBe('snap1') // only what this round added
|
||||
expect(w.told[1]).toContain('PASS')
|
||||
})
|
||||
|
||||
test('acceptance 3: review and test agents keep a named opus; dev briefs below opus are verified by opus', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
await $.agent.spawn({ prompt: `Review PR. Clone ${REVIEW}/`, description: 'review', model: 'opus' } as never)
|
||||
await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig', model: 'opus' } as never)
|
||||
const t = await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig2' } as never)
|
||||
expect(w.spawned.map(s => s.model)).toEqual(['opus', 'opus', 'opus'])
|
||||
await finish($, clock, t.agentId as string)
|
||||
expect(w.verifier).toEqual([]) // test agents are not verified
|
||||
expect(w.told).toEqual([])
|
||||
})
|
||||
|
||||
test('acceptance 4: the first FAIL asks to resume the same agent', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.told[0]).toContain(`Resume agent ${r.agentId} with SendMessage`)
|
||||
expect(w.told[0]).not.toContain('[router-rewrite')
|
||||
})
|
||||
|
||||
test('C5: the second FAIL starts a stronger agent in the same clone', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['d1', 'd2']
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
await finish($, clock, r.agentId as string)
|
||||
const id = w.told[1]?.match(/\[router-rewrite (T\d+)\]/)?.[1]
|
||||
expect(id).toBeDefined()
|
||||
|
||||
await $.agent.spawn({ prompt: `[router-rewrite ${id}]`, description: 'rewrite' } as never)
|
||||
expect(w.spawned[1]?.model).toBe('opus')
|
||||
expect(w.spawned[1]?.prompt).toContain(`Their work is in the clone ${DEV}: continue from it`)
|
||||
expect(w.spawned[1]?.prompt).toContain('- y')
|
||||
})
|
||||
|
||||
test('C2: a dev brief with no clone is never diffed in the session folder', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: 'Clamp the integrator in clones/ somewhere. Expected: ...', description: 'x' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.diffBases).toEqual([])
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('fix 2 end to end: a test brief that mentions the dev clone first keeps its model and is not verified', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({
|
||||
prompt: `Run the OPT-11 P4 rig test against the work in ${DEV}/.\nClone: /home/homelab/projects/goodwe/clones/OPT-11-P4-test/`,
|
||||
description: 'rig',
|
||||
model: 'opus',
|
||||
} as never)
|
||||
expect(w.spawned[0]?.model).toBe('opus')
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.diffBases).toEqual([])
|
||||
})
|
||||
|
||||
test('fix 2 end to end: two clones and no Clone: line run on opus, unverified', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: `Compare ${DEV}/ with ${TEST}/ and report.`, description: 'cmp', model: 'haiku' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('opus')
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
})
|
||||
|
||||
test('a brief without a role is routed but never tracked or verified', async ($, on) => {
|
||||
const w = fresh()
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: 'Find where the MQTT discovery payload is built', description: 'look' } as never)
|
||||
expect(w.spawned[0]?.model).toBe('haiku')
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.told).toEqual([])
|
||||
})
|
||||
|
||||
// ---------- the test run ----------
|
||||
|
||||
test('tests stream with a real time limit, not the ten-minute command cap', { options: { testCommand: 'make verifier-tests', testTimeoutSec: 1800 } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.testArgv[0]).toEqual(['timeout', '--verbose', '--kill-after=30', '1800', 'sh', '-c', 'make verifier-tests'])
|
||||
expect(w.testCwd).toEqual([DEV])
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
})
|
||||
|
||||
test('a test timeout is NOT VERIFIED, never a FAIL: no verifier call, no resume, no fail counted', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff', 'diff 2']
|
||||
w.testEnds = [124, 0]
|
||||
w.testFired = [true, false]
|
||||
w.verdicts = ['{"verdict":"FAIL","issues":["x"]}']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.told[0]).toContain('NOT VERIFIED - the tests did not finish within 60 s')
|
||||
expect(w.told[0]).not.toContain('Resume agent')
|
||||
// the next round's FAIL is the first one counted: it resumes, it does not rewrite
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.told[1]).toContain('Resume agent')
|
||||
})
|
||||
|
||||
test('a kill after the grace period also counts as a timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [137]
|
||||
w.testFired = [true]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.told[0]).toContain('did not finish')
|
||||
})
|
||||
|
||||
test('a test command that cannot run (exit 127) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [127]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.told[0]).toContain('could not run')
|
||||
})
|
||||
|
||||
test('a test command that cannot run (a start failure) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = ['throw' as const]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.told[0]).toContain('could not run')
|
||||
})
|
||||
|
||||
test('failing tests are evidence: the verifier sees them and decides', { options: { testCommand: 'make t' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [1]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
})
|
||||
|
||||
test('an early 137 (out of memory) is a failed run for the verifier, not a timeout', { options: { testCommand: 'make t', testTimeoutSec: 1800 } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [137]
|
||||
w.testFired = [false]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
expect(w.told[0] ?? '').not.toContain('NOT VERIFIED')
|
||||
})
|
||||
|
||||
test('an invalid testTimeoutSec falls back to 1800', async () => {
|
||||
for (const bad of [0, -5, Number.NaN, 'abc', undefined, null]) expect(validTimeout(bad)).toBe(1800)
|
||||
expect(validTimeout(900)).toBe(900)
|
||||
expect(validTimeout('600')).toBe(600)
|
||||
})
|
||||
|
||||
test('a command that exits 124 by itself is a failed run, not a timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [124]
|
||||
w.testFired = [false]
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
})
|
||||
|
||||
test('the timeout line still counts when it arrives split across pieces', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [124]
|
||||
w.testSplitLine = true
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual([])
|
||||
expect(w.told[0]).toContain('did not finish')
|
||||
})
|
||||
|
||||
test('the timeout line on stdout (a test printing it) is not a fired timeout', { options: { testCommand: 'make t' } }, async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff']
|
||||
w.testEnds = [1]
|
||||
w.testLineOnStdout = true
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifier).toEqual(['opus'])
|
||||
})
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"extends": "./.claude-plugin/types/tsconfig.json"
|
||||
}
|
||||
Reference in New Issue
Block a user