From 92bc4dc348fe5d9fb8bd72efcf73a53584f80227 Mon Sep 17 00:00:00 2001 From: Glenn Schrooyen Date: Fri, 6 Nov 2026 23:46:30 +0000 Subject: [PATCH] model-router v0.4.4 Claude Code plugin: routes subagents by how well-defined the brief is, verifies dev work on a second model in the agent's own clone, and estimates savings against Opus 5.5. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MdqPWvuDpbpRvDEdURoAwo --- .claude-plugin/marketplace.json | 11 + .claude-plugin/plugin.json | 64 +++ .gitignore | 3 + README.md | 67 ++++ hooks/hooks.json | 1 + hooks/pricing.ts | 185 +++++++++ hooks/register.ts | 681 ++++++++++++++++++++++++++++++++ hooks/router.ts | 211 ++++++++++ tests/pricing.test.ts | 179 +++++++++ tests/router.test.ts | 451 +++++++++++++++++++++ tsconfig.json | 3 + 11 files changed, 1856 insertions(+) create mode 100644 .claude-plugin/marketplace.json create mode 100644 .claude-plugin/plugin.json create mode 100644 .gitignore create mode 100644 README.md create mode 100644 hooks/hooks.json create mode 100644 hooks/pricing.ts create mode 100644 hooks/register.ts create mode 100644 hooks/router.ts create mode 100644 tests/pricing.test.ts create mode 100644 tests/router.test.ts create mode 100644 tsconfig.json diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..e37d154 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,11 @@ +{ + "name": "model-router", + "owner": { "name": "Glenn Schrooyen" }, + "plugins": [ + { + "name": "model-router", + "source": "./", + "description": "Routes subagents to Haiku, Sonnet or Opus by how well-defined the brief is, verifies dev work on a different model, and estimates savings against Opus." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..c7667d1 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,64 @@ +{ + "name": "model-router", + "version": "0.4.4", + "description": "Routes each subagent to Haiku, Sonnet or Opus by how well-defined its brief is, verifies dev work below Opus in its own clone on a different model, sends failures back to the same agent then a stronger one, and estimates the savings against Opus.", + "author": { + "name": "Glenn Schrooyen" + }, + "userConfig": { + "testCommand": { + "type": "string", + "title": "Test command (run in the agent's clone)", + "description": "The unit-suite command CI runs, from .gitea/workflows/checks.yml. Its output goes to the verifier. Empty: no tests.", + "default": "" + }, + "testTimeoutSec": { + "type": "number", + "title": "Test time limit (seconds)", + "description": "How long the test command may run in the clone. A run that times out is reported as NOT VERIFIED, never as a FAIL.", + "default": 1800 + }, + "defaultBaseBranch": { + "type": "string", + "title": "Base branch when the brief names none", + "description": "Used after the PR lookup and the brief's own base branch.", + "default": "release/1.0" + }, + "prBaseCommand": { + "type": "string", + "title": "Command that prints the PR's base branch", + "description": "Run in the clone; its first output line is the branch. Empty or failing: the brief's base branch, then the default.", + "default": "" + }, + "maxRewrites": { + "type": "number", + "title": "Failed verdicts before handing back", + "description": "1st fail: resume the same agent. 2nd: new agent one model up in the same clone. Past this number: handed back to the manager.", + "default": 2 + }, + "safetyPatterns": { + "type": "string", + "title": "Safety-critical patterns (logged only)", + "description": "Comma-separated. Marks a task as critical in the log. Dev work below Opus is always verified by Opus.", + "default": "control loop,integrator,safety,staleness,watchdog,failsafe,fail-safe,deadlock,rs485,modbus,meter emulat,asyncio,setpoint" + }, + "respectExplicitModel": { + "type": "boolean", + "title": "Keep a named model on agents without a role", + "description": "Review and test agents always keep a named model; dev agents never do. This covers everything else.", + "default": false + }, + "allowFable": { + "type": "boolean", + "title": "Allow Fable for rewrites", + "description": "A rewrite after Opus fails runs on Fable instead of handing back.", + "default": false + }, + "prices": { + "type": "string", + "title": "Price overrides (JSON, USD per MTok)", + "description": "Empty: list prices of 2026-10-08. Else a JSON list of rows, checked before the defaults: [{\"match\":\"sonnet-5-5\",\"price\":{\"in\":2,\"out\":10,\"write\":2.5,\"read\":0.1}}]", + "default": "" + } + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d136fda --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +# written by Claude Code when the plugin loads +.claude-plugin/types/ +node_modules/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..8d1db08 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# model-router + +A Claude Code plugin that picks a model for every subagent, checks dev work on a second model before it reaches a PR, and estimates what that saved compared with running everything on Opus. + +## What it does + +| Step | Behaviour | +|---|---| +| Route | Dev agents run on **Sonnet** when the brief decides the approach (Expected / acceptance criteria), on **Opus** when the design is still open. Review and test agents keep the model they are given (Opus when none). | +| Verify | When a dev agent below Opus finishes a round, the router diffs its clone against the merge base, runs the test command in that clone, and has **Opus** judge the change. | +| On FAIL | 1st: resume the same agent with the issues. 2nd: a new agent one model up, in the same clone. After that: handed back to the manager. | +| Not verified | A test timeout, a test command that cannot run, a missing clone or merge base: reported as NOT VERIFIED, never as a FAIL. | +| Savings | `/router savings` prices every run on the model it ran on and compares it with the same tokens on Opus 5.5. | + +## Briefs + +Every dev, review and test brief carries one line: + +``` +Clone: /home/homelab/projects/goodwe/clones/-dev/ +``` + +Add `(Role: dev|review|test)` when the path does not end in the role. Tickets with extra parts (`OPT-11-P4`) are fine. Name the base branch (`Base branch: release/1.0`). + +## Install + +``` +/plugin marketplace add +/plugin install model-router@model-router +``` + +Or for development: `claude --plugin-dir /path/to/model-router`. + +## Settings (`/config`) + +| Setting | Default | Notes | +|---|---|---| +| `testCommand` | empty | Run in the agent's clone; output goes to the verifier | +| `testTimeoutSec` | 1800 | Wrapped in GNU `timeout --verbose`; a timeout is NOT VERIFIED | +| `defaultBaseBranch` | `release/1.0` | After the PR lookup and the brief's own base branch | +| `prBaseCommand` | empty | Optional command that prints the PR's base branch | +| `maxRewrites` | 2 | Failed verdicts before handing back | +| `safetyPatterns` | control loop, rs485, ... | Marks tasks as critical in the log | +| `respectExplicitModel` | false | Keep a named model on agents without a role | +| `allowFable` | false | Rewrite on Fable after Opus fails | +| `prices` | empty | JSON price overrides, USD per MTok | + +## Commands + +- `/router`: recent decisions, verdicts and savings +- `/router savings [Tn]`: the full savings table, or one task +- `/router clear`: reset log, records and costs + +## Tests + +``` +claude plugin validate . +claude plugin test . +``` + +44 tests. Every routing, verification, cost and timeout rule was also checked by breaking it on purpose (66 mutations, all caught by a test). + +## Known limits + +- Savings are an upper bound: Opus may need fewer tokens than the cheaper model did. +- `timeout` stops the `docker` client, not the container: name the container and remove it in a `trap` in the test command. +- An Opus rewrite is not verified; the PR review covers it. diff --git a/hooks/hooks.json b/hooks/hooks.json new file mode 100644 index 0000000..5fcba7c --- /dev/null +++ b/hooks/hooks.json @@ -0,0 +1 @@ +{ "modules": ["./register.ts"] } diff --git a/hooks/pricing.ts b/hooks/pricing.ts new file mode 100644 index 0000000..920cb07 --- /dev/null +++ b/hooks/pricing.ts @@ -0,0 +1,185 @@ +// Cost of a run, and what the same work would have cost on Opus. Pure: no `$`. +// +// Prices: USD per million tokens, from platform.claude.com/docs/en/about-claude/pricing +// (read 2026-10-08). A setting can replace them. + +export type Usage = { + input_tokens: number + output_tokens: number + cache_creation_input_tokens?: number | null + cache_read_input_tokens?: number | null +} + +/** in, out, cache write (5 min), cache read: USD per MTok. */ +export type Price = { in: number; out: number; write: number; read: number } + +export type PriceRow = { match: string; price: Price; label: string } + +// First match wins: more specific ids come first. +export const DEFAULT_PRICES: readonly PriceRow[] = [ + { match: 'fable-5-1', label: 'Fable 5.1', price: { in: 10, out: 50, write: 12.5, read: 0.25 } }, + { match: 'mythos-5-1', label: 'Mythos 5.1', price: { in: 10, out: 50, write: 12.5, read: 0.25 } }, + { match: 'opus-5-5', label: 'Opus 5.5', price: { in: 4, out: 20, write: 5, read: 0.2 } }, + { match: 'sonnet-5-5', label: 'Sonnet 5.5', price: { in: 2, out: 10, write: 2.5, read: 0.1 } }, + // Haiku 5.5 doubles past 100k-token prompts; a run's summed usage cannot tell, so the low tier is used. + { match: 'haiku-5-5', label: 'Haiku 5.5', price: { in: 0.1, out: 0.5, write: 0.125, read: 0.01 } }, + { match: 'haiku-4-5', label: 'Haiku 4.5', price: { in: 1, out: 5, write: 1.25, read: 0.1 } }, + { match: 'fable-5', label: 'Fable 5', price: { in: 10, out: 50, write: 12.5, read: 1 } }, + { match: 'opus-5', label: 'Opus 5', price: { in: 5, out: 25, write: 6.25, read: 0.5 } }, + { match: 'sonnet-5', label: 'Sonnet 5', price: { in: 2, out: 10, write: 2.5, read: 0.2 } }, +] + +/** The model every saving is measured against. */ +export const BASELINE = 'opus-5-5' + +/** + * Models before Claude 4.7 use a tokenizer that yields ~30% fewer tokens for + * the same text, so their counts are scaled up before pricing them as Opus 5.5. + */ +const OLD_TOKENIZER = /(haiku-4-5|sonnet-4|opus-4-[0-6]|opus-4-1|opus-4(?!-)|haiku-3)/ +export const tokenizerFactor = (modelId: string) => (OLD_TOKENIZER.test(modelId) ? 1.3 : 1) + +export const priceRow = (modelId: string, table: readonly PriceRow[] = DEFAULT_PRICES) => + table.find(r => modelId.toLowerCase().includes(r.match)) + +/** USD for a usage at a price; tokens scaled by `factor`. */ +export const cost = (u: Usage, p: Price, factor = 1) => + (factor * + (u.input_tokens * p.in + + u.output_tokens * p.out + + (u.cache_creation_input_tokens ?? 0) * p.write + + (u.cache_read_input_tokens ?? 0) * p.read)) / + 1_000_000 + +export type Priced = { actual: number; opus: number; known: boolean; label: string } + +/** What a run cost on its own model, and the same tokens on the baseline. */ +export const priceRun = (modelId: string, u: Usage, table: readonly PriceRow[] = DEFAULT_PRICES): Priced => { + const row = priceRow(modelId, table) + const base = priceRow(BASELINE, table) + if (!row || !base) return { actual: 0, opus: 0, known: false, label: modelId } + return { + actual: cost(u, row.price), + opus: cost(u, base.price, tokenizerFactor(modelId)), + known: true, + label: row.label, + } +} + +/** A task's running totals, across every agent and round it took. */ +export type TaskCost = { + task: string + /** Writer runs on each model, USD. */ + writerActual: number + /** Writer runs below Opus, priced at Opus: what Opus would have cost for the same tokens. */ + writerAsOpus: number + /** Writer runs that were on Opus already, USD. */ + writerOnOpus: number + /** Classifier and verifier calls the router added, USD. */ + overhead: number + models: string[] + unpriced: string[] +} + +export const emptyTask = (task: string): TaskCost => ({ + task, + writerActual: 0, + writerAsOpus: 0, + writerOnOpus: 0, + overhead: 0, + models: [], + unpriced: [], +}) + +export const addWriterRun = (t: TaskCost, modelId: string, u: Usage, table?: readonly PriceRow[]): TaskCost => { + const p = priceRun(modelId, u, table) + const next = { ...t, models: [...t.models], unpriced: [...t.unpriced] } + if (!p.known) { + if (!next.unpriced.includes(modelId)) next.unpriced.push(modelId) + return next + } + if (!next.models.includes(p.label)) next.models.push(p.label) + next.writerActual += p.actual + if (modelId.toLowerCase().includes(BASELINE)) next.writerOnOpus += p.actual + else next.writerAsOpus += p.opus + return next +} + +export const addOverhead = (t: TaskCost, modelId: string, u: Usage, table?: readonly PriceRow[]): TaskCost => { + const p = priceRun(modelId, u, table) + return p.known ? { ...t, overhead: t.overhead + p.actual } : t +} + +/** + * What the task would have cost had it gone to Opus at once. + * + * When Opus wrote part of it (a rewrite after failed rounds, or an open + * design), Opus's own runs are taken as what Opus alone would have needed: + * the failed cheaper rounds then count as pure loss. Otherwise every writer + * token is priced at Opus, an upper bound, since Opus may need fewer tokens. + */ +export const counterfactual = (t: TaskCost) => (t.writerOnOpus > 0 ? t.writerOnOpus : t.writerAsOpus) + +export const actualTotal = (t: TaskCost) => t.writerActual + t.overhead + +export const saved = (t: TaskCost) => counterfactual(t) - actualTotal(t) + +export const savedPct = (t: TaskCost) => { + const c = counterfactual(t) + return c > 0 ? (saved(t) / c) * 100 : 0 +} + +/** Reads a prices setting: a JSON list of rows, or empty for the defaults. */ +export const parsePrices = (json: string): readonly PriceRow[] => { + if (!json.trim()) return DEFAULT_PRICES + try { + const rows = JSON.parse(json) as PriceRow[] + const ok = Array.isArray(rows) && rows.every(r => typeof r.match === 'string' && typeof r.price?.in === 'number') + return ok ? [...rows.map(r => ({ ...r, label: r.label ?? r.match })), ...DEFAULT_PRICES] : DEFAULT_PRICES + } catch { + return DEFAULT_PRICES + } +} + +export const usd = (n: number) => (Math.abs(n) < 0.01 && n !== 0 ? `$${n.toFixed(4)}` : `$${n.toFixed(2)}`) + +/** + * The savings table: per task, what it cost, what Opus would have cost, and + * the difference, newest first; then the total over every task. + */ +export const savingsReport = (costs: Readonly>, limit: number, only?: string): string => { + const ids = Object.keys(costs) + .filter(id => !only || id === only) + .sort((a, b) => Number(b.slice(1)) - Number(a.slice(1))) + if (ids.length === 0) return only ? `model-router: no costs recorded for ${only}.` : 'savings: no agent runs recorded yet.' + + const row = (id: string, t: TaskCost) => { + const pct = counterfactual(t) > 0 ? `${savedPct(t).toFixed(0)}%` : '-' + const how = t.writerActual === 0 ? 'running' : t.writerOnOpus > 0 ? 'opus ran' : 'upper bound' + return `${id.padEnd(5)} ${usd(actualTotal(t)).padStart(9)} ${usd(counterfactual(t)).padStart(9)} ${usd(saved(t)).padStart(9)} ${pct.padStart(5)} ${how.padEnd(11)} ${t.models.join('+').padEnd(20)} ${t.task.slice(0, 32)}${t.unpriced.length ? ` [no price: ${t.unpriced.join(', ')}]` : ''}` + } + + let actual = 0 + let opus = 0 + for (const id of Object.keys(costs)) { + const t = costs[id] + // a task whose writer has not finished has no comparison yet + if (!t || t.writerActual === 0) continue + actual += actualTotal(t) + opus += counterfactual(t) + } + const shown = ids.slice(0, limit) + const lines = [ + `savings vs Opus 5.5 (USD, API list prices)${only ? '' : ` - ${shown.length} of ${ids.length} tasks`}`, + `task actual if opus saved % basis models task`, + ...shown.map(id => row(id, costs[id] as TaskCost)), + ] + if (!only) + lines.push( + `total ${usd(actual).padStart(9)} ${usd(opus).padStart(9)} ${usd(opus - actual).padStart(9)} ${(opus > 0 ? `${(((opus - actual) / opus) * 100).toFixed(0)}%` : '-').padStart(5)}`, + ) + lines.push( + 'actual = writer runs + classifier + verifier. "upper bound": all tokens priced at Opus; Opus may need fewer. "opus ran": the Opus run stands for Opus alone, cheaper failed rounds count as loss.', + ) + return lines.join('\n') +} diff --git a/hooks/register.ts b/hooks/register.ts new file mode 100644 index 0000000..0890f7a --- /dev/null +++ b/hooks/register.ts @@ -0,0 +1,681 @@ +import type { EngineInterface, Register } from 'claude-code' + +import { + addOverhead, + addWriterRun, + emptyTask, + parsePrices, + savingsReport, + type PriceRow, + type TaskCost, + type Usage, +} from './pricing' +import { + REWRITE_MARK, + asTier, + clip, + decide, + onFail, + parseBrief, + parsePatterns, + parseVerdict, + safetyHit, + skipReason, + validTimeout, + verifierFor, + type Role, + type Tier, +} from './router' + +// ---------- stored records ---------- + +/** One agent the router started, kept from spawn to its last verified round. */ +type AgentRec = { + taskId: string + tier: Tier + role?: Role + clone?: string + baseBranch?: string + critical: boolean + prompt: string + /** Failed verdicts in this task's chain, carried over to a rewrite agent. */ + fails: number + rounds: number + /** The tree after the last verified round: the next round diffs from here. */ + lastSnap?: string + lastUntracked?: string[] + lastIssues?: string[] +} + +type Pending = { prompt: string; tier: Tier; fails: number; critical: boolean } + +type Outcome = + | 'pass' + | 'resume-requested' + | 'rewrite-requested' + | 'handed-back' + | 'no-changes' + | 'no-base' + | 'tests-timeout' + | 'tests-error' + | 'verify-error' + | 'test-agent' + | 'review-agent' + | 'no-clone' + | 'opus-unverified' + | 'agent-ended' + +type LogEntry = { + id: string + agentId?: string + at: number + task: string + tier: Tier + role?: Role + reason: string + round?: number + verifier?: Tier + outcome: Outcome | 'spawned' + opusRequested?: boolean +} + +const S = { + counter: 'counter', + pending: 'pending', + agents: 'agents', + log: 'log', + costs: 'costs', + agentTask: 'agentTask', + aliases: 'aliases', +} as const + +/** Model ids assumed for an alias until an agent run shows what it resolves to. */ +const ALIAS_DEFAULT: Record = { + haiku: 'claude-haiku-4-5', + sonnet: 'claude-sonnet-5-5', + opus: 'claude-opus-5-5', + fable: 'claude-fable-5-1', +} +const LOG_LIMIT = 400 + +// ---------- prompts ---------- + +const CLASSIFIER_SYSTEM = `You route coding tasks to the smallest model that will do them right. +Judge how well-defined the task is, NOT how important or low-level the code is. In this codebase nearly everything touches control loops, asyncio and RS485; that alone never means opus. A stronger model verifies every change below Opus, and a review agent reviews every PR. +Answer with exactly one word: haiku, sonnet or opus. + +haiku: read-only lookups and summaries, mechanical changes fully spelled out. +sonnet: the approach is decided: the brief says what to change, or gives Expected behaviour, acceptance criteria or a test to make pass. Size and domain do not matter. +opus: the approach is still open: a design or trade-off to decide, a bug whose cause is unknown, or a brief so vague the writer must invent the plan. + +When unsure between sonnet and opus, pick sonnet: a failed verification escalates.` + +const VERIFIER_SYSTEM = `You are an adversarial verifier. Another model wrote the change below. You did not. +Judge only whether the change correctly and completely does the task, without breaking anything nearby. +Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes. +Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate. + +Reply with JSON only: +{"verdict": "PASS" | "FAIL", "issues": ["concrete issue with file:line where possible", ...], "rewrite_brief": "what must change, empty on PASS"}` + +const ORCHESTRATOR_NOTE = `## Model router (model-router mod) +The model-router mod picks each subagent's model and verifies dev work before it reaches a PR. +- Every dev, review and test brief carries one line \`Clone: \` (add \`(Role: dev|review|test)\` if the path does not end in the role). The router reads the role from that line only; a brief naming several clones without it runs on Opus, unverified. +- Dev agents are routed: Sonnet when the brief decides the approach (Expected / acceptance criteria), Opus when the design is still open. Naming a model on a dev brief has no effect. Give each dev brief its clone path and base branch. +- Review and test agents keep the model you name (Opus when you name none). +- When a dev agent below Opus finishes a round, the router verifies its clone's diff on Opus and posts a "[model-router]" verdict. Wait for that verdict before opening or advancing the PR. +- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. Do not fix rejected work yourself.` + +// ---------- store helpers (top-level, as the engine requires for `$`) ---------- + +// Agents start and finish in parallel. Every read-modify-write of the store +// goes through this queue, one at a time, or parallel hooks overwrite each +// other's writes (and hand out the same task id). +let queue: Promise = Promise.resolve() + +function serial(work: () => Promise): Promise { + const run = queue.then(work) + queue = run.catch(() => undefined) + return run +} + +async function getMap($: EngineInterface, key: string) { + return ((await $.store.get(key)) ?? {}) as Record +} + +/** Replaces one entry of a stored map, atomically with respect to this plugin. */ +async function updateIn($: EngineInterface, key: string, id: string, change: (old: T | undefined) => T | undefined) { + return serial(async () => { + const all = await getMap($, key) + const value = change(all[id]) + if (value === undefined) delete all[id] + else all[id] = value + await $.store.set(key, all) + return value + }) +} + +async function putIn($: EngineInterface, key: string, id: string, value: T | undefined) { + await updateIn($, key, id, () => value) +} + +async function nextId($: EngineInterface) { + return serial(async () => { + const n = Number((await $.store.get(S.counter)) ?? 0) + 1 + await $.store.set(S.counter, n) + return `T${n}` + }) +} + +async function log($: EngineInterface, entry: LogEntry) { + await serial(async () => { + const list = ((await $.store.get(S.log)) ?? []) as LogEntry[] + list.push(entry) + await $.store.set(S.log, list.slice(-LOG_LIMIT)) + }) +} + +async function sh($: EngineInterface, argv: string[], cwd: string, timeoutMs = 30_000) { + try { + const r = await $.process.run(argv, { cwd, timeoutMs }) + return { ok: r.exitCode === 0, out: r.stdout, err: r.stderr, code: r.exitCode } + } catch (err) { + return { ok: false, out: '', err: String(err), code: -1 } + } +} + +async function untracked($: EngineInterface, cwd: string) { + const r = await sh($, ['git', 'ls-files', '--others', '--exclude-standard'], cwd) + return r.ok ? r.out.split('\n').filter(Boolean) : [] +} + +/** A commit holding the clone's tree as it is now (HEAD when clean). */ +async function snapshot($: EngineInterface, cwd: string) { + const stash = await sh($, ['git', 'stash', 'create'], cwd) + if (stash.ok && stash.out.trim()) return stash.out.trim() + const head = await sh($, ['git', 'rev-parse', 'HEAD'], cwd) + return head.ok ? head.out.trim() : undefined +} + +/** Where a first round diffs from: the merge base with the PR's base branch. */ +async function firstBase($: EngineInterface, rec: AgentRec, prBaseCommand: string, fallback: string) { + const cwd = rec.clone as string + let branch: string | undefined + if (prBaseCommand) { + const r = await sh($, ['sh', '-c', prBaseCommand], cwd) + const b = r.out.trim().split('\n')[0]?.trim() + if (r.ok && b) branch = b + } + branch = branch ?? rec.baseBranch ?? fallback + for (const ref of [`origin/${branch}`, branch]) { + const r = await sh($, ['git', 'merge-base', ref, 'HEAD'], cwd) + if (r.ok && r.out.trim()) return { base: r.out.trim(), branch } + } + return { base: undefined, branch } +} + +async function classify($: EngineInterface, prompt: string): Promise<{ tier: Tier; reason: string; usage?: Usage }> { + const r = await $.model.complete({ + model: 'haiku', + system: CLASSIFIER_SYSTEM, + prompt: `Task:\n${clip(prompt, 6000)}`, + maxTokens: 5, + timeoutMs: 15_000, + }) + const tier = r.isAnswered ? asTier(r.text) : undefined + if (tier && tier !== 'fable') return { tier, reason: 'classifier', usage: r.usage } + return { tier: 'sonnet', reason: 'classifier unavailable, default', usage: r.usage } +} + +async function tell($: EngineInterface, text: string) { + await $.prompt.submit({ text }) +} + +// ---------- cost tracking ---------- + +async function modelIdFor($: EngineInterface, alias: Tier) { + const seen = (await getMap($, S.aliases))[alias] + return seen ?? ALIAS_DEFAULT[alias] +} + +/** Adds a router call (classifier, verifier) to a task's overhead. */ +async function chargeOverhead($: EngineInterface, taskId: string, alias: Tier, u: Usage | undefined, prices: readonly PriceRow[]) { + if (!u) return + const modelId = await modelIdFor($, alias) + await updateIn($, S.costs, taskId, old => addOverhead(old ?? emptyTask(taskId), modelId, u, prices)) +} + +type TestRun = { status: 'ran' | 'timeout' | 'error'; code: number | null; tail: string; seconds: number } + +/** + * Runs the test command in the clone, streamed: `$.process.run` stops at ten + * minutes, a stream has no limit of its own. GNU `timeout` does the kill. + * A timeout or a run that could not start is no verdict on the code. + */ +async function runTests($: EngineInterface, cmd: string, cwd: string, timeoutSec: number): Promise { + const started = await $.clock.now() + const seconds = async () => Math.round(((await $.clock.now()) - started) / 1000) + let out = '' + let errTail = '' + try { + const stream = $.process.spawn({ + argv: ['timeout', '--verbose', '--kill-after=30', String(Math.max(1, Math.round(timeoutSec))), 'sh', '-c', cmd], + cwd, + }) + for await (const piece of stream) { + out = (out + piece.text).slice(-65_536) + if (piece.stream === 'stderr') errTail = (errTail + piece.text).slice(-65_536) + } + // GNU timeout --verbose says so on stderr when, and only when, it fires. + // Read from the joined stderr, so a line split across pieces still counts. + const isFired = errTail.includes('timeout: sending signal') + const end = await stream.result + const tail = clip(out.trim().split('\n').slice(-80).join('\n'), 8000) + const took = await seconds() + // Only a timeout that fired is a timeout: a 137 without it is the code's own + // (e.g. out of memory) and goes to the verifier as a failed run. + if (isFired) return { status: 'timeout', code: end.code, tail, seconds: took } + if (end.code === 126 || end.code === 127) return { status: 'error', code: end.code, tail, seconds: took } + return { status: 'ran', code: end.code, tail, seconds: took } + } catch (err) { + return { status: 'error', code: null, tail: String(err).slice(0, 500), seconds: await seconds() } + } +} + +type Settings = { + testCommand: string + testTimeoutSec: number + prBaseCommand: string + defaultBaseBranch: string + maxFails: number + allowFable: boolean + prices: readonly PriceRow[] +} + +/** Verifies one finished round of a dev agent and tells the manager. */ +async function verifyRound($: EngineInterface, agentId: string, rec: AgentRec, cfg: Settings) { + const cwd = rec.clone as string + const round = rec.rounds + 1 + const entry: LogEntry = { + id: rec.taskId, + agentId, + at: await $.clock.now(), + task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '', + tier: rec.tier, + role: rec.role, + reason: `round ${round}`, + round, + outcome: 'verify-error', + } + const head = `[model-router] ${rec.taskId} (agent ${agentId}, ${rec.tier}, round ${round})` + + // 1. What this round changed. + let base = rec.lastSnap + let baseLabel = 'the previous verified round' + if (!base) { + const fb = await firstBase($, rec, cfg.prBaseCommand, cfg.defaultBaseBranch) + base = fb.base + baseLabel = `merge-base with ${fb.branch}` + if (!base) { + entry.outcome = 'no-base' + await log($, entry) + await tell($, `${head}: NOT VERIFIED - no merge base with ${fb.branch} in ${cwd}. Review the change yourself before the PR.`) + return + } + } + const diff = await sh($, ['git', 'diff', base], cwd, 60_000) + const known = new Set(rec.lastUntracked ?? []) + const fresh = (await untracked($, cwd)).filter(f => !known.has(f)) + let added = '' + for (const f of fresh.slice(0, 20)) { + try { + added += `\n--- new file: ${f}\n${clip(await $.fs.read(`${cwd}/${f}`), 8000)}\n` + } catch { + added += `\n--- new file: ${f} (unreadable)\n` + } + } + const changes = (diff.out + added).trim() + if (!changes) { + entry.outcome = 'no-changes' + await log($, entry) + await tell($, `${head}: no changes since ${baseLabel}; nothing to verify.`) + return + } + + // 2. Tests, in the clone. + let tests = '' + if (cfg.testCommand) { + $.ui.status(`router: ${rec.taskId} round ${round} running tests`) + const t = await runTests($, cfg.testCommand, cwd, cfg.testTimeoutSec) + if (t.status !== 'ran') { + // No verdict on the code: no verifier call, no fail counted, no rewrite. + entry.outcome = t.status === 'timeout' ? 'tests-timeout' : 'tests-error' + await log($, entry) + const why = + t.status === 'timeout' + ? `the tests did not finish within ${cfg.testTimeoutSec} s (testTimeoutSec)` + : `the test command could not run (exit ${t.code ?? 'none'})` + await tell( + $, + `${head}: NOT VERIFIED - ${why}. This says nothing about the code: do not resume or rewrite the agent for it. ` + + `Fix the test setup or raise testTimeoutSec, or review the change yourself.\nLast output:\n${clip(t.tail, 2000)}`, + ) + return + } + tests = `exit code ${t.code} after ${t.seconds} s\n${t.tail}` + } + + // 3. The verdict, from another model. + const verifier = verifierFor(rec.tier) + entry.verifier = verifier + $.ui.status(`router: ${rec.taskId} round ${round} verifying on ${verifier}`) + const v = await $.model.complete({ + model: verifier, + system: VERIFIER_SYSTEM, + prompt: [ + `## Task\n${clip(rec.prompt, 12_000)}`, + rec.lastIssues?.length + ? `## Issues raised last round (check they are fixed)\n${rec.lastIssues.map(i => `- ${i}`).join('\n')}` + : '', + `## Changes (${baseLabel})\n${clip(changes, 60_000)}`, + cfg.testCommand ? `## Test run in the clone: \`${cfg.testCommand}\`\n${tests}` : '', + ] + .filter(Boolean) + .join('\n\n'), + maxTokens: 2000, + timeoutMs: 180_000, + }) + await chargeOverhead($, rec.taskId, verifier, v.usage, cfg.prices) + if (!v.isAnswered) { + await log($, entry) + await tell($, `${head}: NOT VERIFIED - the verifier (${verifier}) could not run (${v.reason}). Review the change yourself before the PR.`) + return + } + const verdict = parseVerdict(v.text) + const issues = verdict.issues.length ? verdict.issues.map(i => `- ${i}`).join('\n') : '- (none listed)' + + // 4. Remember where this round ended. + const after: AgentRec = { + ...rec, + rounds: round, + lastSnap: (await snapshot($, cwd)) ?? rec.lastSnap, + lastUntracked: await untracked($, cwd), + lastIssues: verdict.pass ? [] : verdict.issues, + } + + if (verdict.pass) { + entry.outcome = 'pass' + await putIn($, S.agents, agentId, after) + await log($, entry) + $.ui.status(`router: ${rec.taskId} ✓ ${rec.tier}, verified by ${verifier}`) + await tell($, `${head}: PASS from verifier ${verifier}. Continue to the PR.`) + return + } + + const fails = rec.fails + 1 + after.fails = fails + const step = onFail(fails, rec.tier, cfg.maxFails, cfg.allowFable) + await putIn($, S.agents, agentId, after) + + if (step.kind === 'resume') { + entry.outcome = 'resume-requested' + await log($, entry) + $.ui.status(`router: ${rec.taskId} ✗ round ${round}, resume ${agentId}`) + await tell( + $, + `${head}: FAIL from verifier ${verifier}.\nIssues:\n${issues}\n` + + (verdict.brief ? `What must change: ${verdict.brief}\n` : '') + + `Resume agent ${agentId} with SendMessage, passing these issues. Do not fix it yourself.`, + ) + return + } + + if (step.kind === 'new-agent') { + await putIn($, S.pending, rec.taskId, { + prompt: + `${rec.prompt.split('\n\n## Previous attempts rejected')[0]}\n\n## Previous attempts rejected\n` + + `Agents on ${rec.tier} were rejected ${fails} times by an independent verifier (${verifier}). Their work is in the clone ${cwd}: continue from it, fix or replace it, do not start over elsewhere.\n` + + `Issues:\n${issues}\n` + + (verdict.brief ? `What must change: ${verdict.brief}\n` : ''), + tier: step.tier, + fails, + critical: rec.critical, + }) + entry.outcome = 'rewrite-requested' + await log($, entry) + $.ui.status(`router: ${rec.taskId} ✗ again, new agent on ${step.tier}`) + await tell( + $, + `${head}: FAIL again from verifier ${verifier}.\nIssues:\n${issues}\n` + + `Stop agent ${agentId}, then start a new agent on ${step.tier} in the same clone: call Agent with prompt [router-rewrite ${rec.taskId}] (same subagent_type).`, + ) + return + } + + entry.outcome = 'handed-back' + await log($, entry) + $.ui.toast(`model-router: ${rec.taskId} failed verification ${fails}x - handed back`) + await tell( + $, + `${head}: FAIL from verifier ${verifier}, ${fails} failed verdicts. Handed back: stop and show the person these issues.\n${issues}` + + (verdict.brief ? `\nVerifier's brief: ${verdict.brief}` : ''), + ) +} + +/** Logs an agent run that is not verified; tells the manager when a dev agent waits on a verdict. */ +async function skipRound($: EngineInterface, agentId: string, rec: AgentRec, outcome: Outcome) { + await log($, { + id: rec.taskId, + agentId, + at: await $.clock.now(), + task: rec.prompt.split('\n')[0]?.slice(0, 60) ?? '', + tier: rec.tier, + role: rec.role, + reason: outcome, + outcome, + }) + if (rec.role === 'dev' && outcome === 'no-clone') + await tell($, `[model-router] ${rec.taskId} (agent ${agentId}): NOT VERIFIED - the brief names no clone path.`) +} + +async function runRound($: EngineInterface, agentId: string, rec: AgentRec, ended: string, cfg: Settings) { + const skip = skipReason(rec, ended) + if (skip) await skipRound($, agentId, rec, skip) + else await verifyRound($, agentId, rec, cfg) +} + +// ---------- the hooks ---------- + +export const register: Register = (on, options) => { + const patterns = parsePatterns(String(options.safetyPatterns ?? '')) + const respectExplicit = options.respectExplicitModel === true + const cfg: Settings = { + testCommand: String(options.testCommand ?? '').trim(), + testTimeoutSec: validTimeout(options.testTimeoutSec), + prBaseCommand: String(options.prBaseCommand ?? '').trim(), + defaultBaseBranch: String(options.defaultBaseBranch ?? 'release/1.0').trim(), + maxFails: Number(options.maxRewrites ?? 2), + allowFable: options.allowFable === true, + prices: parsePrices(String(options.prices ?? '')), + } + + on('session.start', async ($, e, next) => { + await $.command.register({ + name: 'router', + description: 'Model router: decisions, verdicts and savings vs Opus (`/router savings [Tn]`, `/router clear`)', + }) + return next(e) + }) + + on('prompt.compose', async ($, e, next) => { + const out = await next(e) + if (e.traits.includes('bare')) return out + return { sections: [...out.sections, { id: 'model-router:note', text: ORCHESTRATOR_NOTE, scope: 'session' as const }] } + }) + + // Routing: every agent the model starts, background ones included. + on('agent.spawn', async ($, e, next) => { + if (e.fork || e.isTeammate || e.workflow) return next(e) + + let taskId: string + let prompt = e.prompt + let tier: Tier + let reason: string + let fails = 0 + let critical: boolean + let opusRequested = false + let classifierUsage: Usage | undefined + // Agents whose model the router chose; kept models (review, test) are not a saving. + let isRouted = true + + const markId = e.prompt.match(REWRITE_MARK)?.[1] + const pending = markId ? (await getMap($, S.pending))[markId] : undefined + + if (markId && pending) { + taskId = markId + const extra = e.prompt.replace(REWRITE_MARK, '').trim() + prompt = pending.prompt + (extra ? `\n\nAdditional guidance from the orchestrator:\n${extra}` : '') + tier = pending.tier + fails = pending.fails + critical = pending.critical + reason = `new agent after ${fails} failed verdicts` + await putIn($, S.pending, markId, undefined) + } else { + taskId = await nextId($) + const brief = parseBrief(e.prompt) + critical = safetyHit(`${e.description}\n${e.prompt}`, patterns) !== undefined + const d = decide({ + role: brief.role, + named: asTier(e.model), + subagentType: e.subagentType, + prompt: e.prompt, + respectExplicit, + isAmbiguous: brief.isAmbiguous, + }) + opusRequested = d.opusRequested === true + isRouted = + brief.role !== 'review' && + brief.role !== 'test' && + !(d.kind === 'fixed' && d.reason === 'model chosen by orchestrator') + if (d.kind === 'fixed') { + tier = d.tier + reason = d.reason + } else { + const c = await classify($, e.prompt) + tier = c.tier + reason = c.reason + classifierUsage = c.usage + } + } + + const brief = parseBrief(prompt) + const ran = await next({ ...e, prompt, model: tier }) + if (ran.deny !== undefined || !ran.agentId) return ran + + // Learn what the alias resolves to, so router calls are priced on the real model. + if (ran.model && ran.model !== tier && ran.model.includes(tier)) await putIn($, S.aliases, tier, ran.model) + if (isRouted) { + await putIn($, S.agentTask, ran.agentId, { taskId, task: e.description }) + await updateIn($, S.costs, taskId, old => old ?? emptyTask(e.description)) + await chargeOverhead($, taskId, 'haiku', classifierUsage, cfg.prices) + } + + // Agents with a role (dev, review, test) get a record; only dev rounds are verified. + if (brief.role) + await putIn($, S.agents, ran.agentId, { + taskId, + tier, + role: brief.role, + clone: brief.clone, + baseBranch: brief.baseBranch, + critical, + prompt, + fails, + rounds: 0, + }) + + await log($, { + id: taskId, + agentId: ran.agentId, + at: await $.clock.now(), + task: e.description, + tier, + role: brief.role, + reason, + outcome: 'spawned', + opusRequested, + }) + $.ui.status(`router: ${taskId} ${brief.role ?? 'agent'} → ${tier}`) + return ran + }).catch(($, e, next) => { + // Fail open: a router bug must never block delegation. + $.ui.status('router: error, agent ran unrouted') + return next(e) + }) + + // Verification: each finished run of a tracked agent, a resumed SendMessage round included. + on('turn.complete', async ($, e, next) => { + const out = await next(e) + const agentId = e.agentId + if (!agentId) return out + + const owner = (await getMap<{ taskId: string; task: string }>($, S.agentTask))[agentId] + if (owner && e.usage) { + const usage = e.usage + await updateIn($, S.costs, owner.taskId, old => + addWriterRun(old ?? emptyTask(owner.task), usage.model, usage, cfg.prices), + ) + } + const rec = (await getMap($, S.agents))[agentId] + if (!rec) return out + // Verification takes minutes: run it after this event has gone on. + $.clock.after(0, () => void runRound($, agentId, rec, e.reason, cfg)) + return out + }) + + on('command.run', { command: 'router' }, async ($, e) => { + if (e.args.trim() === 'clear') { + await serial(async () => { + await $.store.set(S.log, []) + await $.store.set(S.pending, {}) + await $.store.set(S.agents, {}) + await $.store.set(S.costs, {}) + await $.store.set(S.agentTask, {}) + }) + return { text: 'model-router: log, agents, costs and pending rewrites cleared.' } + } + const [sub, arg] = e.args.trim().split(/\s+/) + if (sub === 'savings') return { text: savingsReport(await getMap($, S.costs), Infinity, arg) } + const list = ((await $.store.get(S.log)) ?? []) as LogEntry[] + if (list.length === 0) return { text: 'model-router: nothing routed yet.' } + + const count = (key: (l: LogEntry) => string | undefined) => { + const r: Record = {} + for (const l of list) { + const k = key(l) + if (k) r[k] = (r[k] ?? 0) + 1 + } + return Object.entries(r) + .map(([k, n]) => `${k} ${n}`) + .join(' · ') + } + const rows = list + .slice(-15) + .map( + l => + `${l.id.padEnd(5)} ${(l.role ?? '-').padEnd(6)} ${l.tier.padEnd(6)} ${(l.verifier ? '→' + l.verifier : '').padEnd(8)} ${l.outcome.padEnd(17)} ${(l.task ?? '').slice(0, 40)}${l.opusRequested ? ' [opus requested]' : ''}`, + ) + .join('\n') + const pending = Object.keys(await getMap($, S.pending)) + return { + text: + `model-router - last ${Math.min(15, list.length)} of ${list.length}\n` + + `writers: ${count(l => (l.outcome === 'spawned' ? `${l.role ?? 'other'}:${l.tier}` : undefined))}\n` + + `results: ${count(l => (l.outcome === 'spawned' ? undefined : l.outcome))}\n` + + `opus requested on dev briefs: ${list.filter(l => l.opusRequested).length}\n` + + (pending.length ? `pending rewrites: ${pending.join(', ')}\n` : '') + + `\n${rows}\n\n` + + savingsReport(await getMap($, S.costs), 5), + } + }) +} diff --git a/hooks/router.ts b/hooks/router.ts new file mode 100644 index 0000000..86be4c5 --- /dev/null +++ b/hooks/router.ts @@ -0,0 +1,211 @@ +// Pure routing logic: no `$`, so every rule here is unit-tested. + +export type Tier = 'haiku' | 'sonnet' | 'opus' | 'fable' +export type Role = 'dev' | 'review' | 'test' + +export const TIERS: readonly Tier[] = ['haiku', 'sonnet', 'opus', 'fable'] + +export const rank = (t: Tier) => TIERS.indexOf(t) + +export const asTier = (s: string | undefined): Tier | undefined => { + if (!s) return undefined + const lower = s.toLowerCase() + return TIERS.find(t => lower.includes(t)) +} + +export const parsePatterns = (csv: string): string[] => + csv + .split(',') + .map(s => s.trim().toLowerCase()) + .filter(s => s.length > 0) + +/** The first safety pattern the text mentions, or undefined. */ +export const safetyHit = (text: string, patterns: readonly string[]): string | undefined => { + const lower = text.toLowerCase() + return patterns.find(p => lower.includes(p)) +} + +// ---------- the brief ---------- + +export type Brief = { + clone?: string + ticket?: string + role?: Role + baseBranch?: string + /** Several clone paths and no `Clone:` line: the role cannot be told. */ + isAmbiguous?: boolean +} + +// A ticket is LETTERS-digits plus any extra parts (OPT-11-P4); the role is the last part. +const TICKET = '[A-Za-z]+-\\d+(?:-[A-Za-z0-9]+)*?' +const CLONE_PATH = new RegExp(`(\\S*?\\/clones\\/(${TICKET})-(dev|review|test))\\/?`) +const CLONE_PATHS = new RegExp(CLONE_PATH.source, 'g') +// The explicit line every brief carries: `Clone: ` with an optional `(Role: dev)`. +const CLONE_LINE = /^[ \t>*-]*Clone:[ \t]*`?([^\s`]+?)\/?`?[ \t]*(?:\(Role:[ \t]*(dev|review|test)\))?[ \t]*$/im +const BASE = + /\b(?:base(?:\s+branch)?|target(?:\s+branch)?|merge\s+into|PR\s+(?:against|into))\s*[:=]?\s*`?((?:release|main|master|develop)[\w./-]*)/i + +const trimPath = (p: string) => p.replace(/^[`'"(]+/, '').replace(/[`'")]+$/, '').replace(/\/+$/, '') + +/** + * Reads the clone path, ticket, role and base branch out of a brief. + * + * The `Clone:` line decides when present: its `(Role: ...)`, else the role in + * its path. Without one, a brief that names one clone is read from that path; + * one that names several is ambiguous and gets no role. + */ +export const parseBrief = (prompt: string): Brief => { + const b = prompt.match(BASE) + // a sentence's full stop or a trailing slash is no part of the branch + const baseBranch = b?.[1]?.replace(/[./-]+$/, '') + + const line = prompt.match(CLONE_LINE) + if (line?.[1]) { + const clone = trimPath(line[1]) + const fromPath = clone.match(CLONE_PATH) + const role = (line[2] ?? fromPath?.[3]) as Role | undefined + return { clone, ticket: fromPath?.[2], role, baseBranch } + } + + const paths = [...new Set([...prompt.matchAll(CLONE_PATHS)].map(m => trimPath(m[1] ?? '')))] + if (paths.length > 1) return { baseBranch, isAmbiguous: true } + const only = paths[0]?.match(CLONE_PATH) + if (!only) return { baseBranch } + return { clone: trimPath(only[1] ?? ''), ticket: only[2], role: only[3] as Role, baseBranch } +} + +// ---------- routing ---------- + +const READ_ONLY_TYPES = /^(explore|scout|runner|claude-code-guide|statusline-setup)$|:(scout|runner)$/i + +// Light work, only for tasks outside the dev/review/test roles. No bare +// "docs" or "run tests": both appear in nearly every brief. +const LIGHT = [ + /\b(find|locate|grep|where is|which (files|modules)|summari[sz]e|read and report)\b/, + /\b(fix (a |the )?typo|update (the )?(readme|changelog)|add docstrings?)\b/, + /\b(run (the )?(unit )?tests? and report|report (the )?test results)\b/, +] + +// Opus writes only when the approach itself is still open. Domain words +// (control loop, asyncio, RS485) are everywhere in this codebase and say nothing. +const HEAVY = [ + /\b(design|architect|redesign|decide (how|whether)|propose an approach|trade-?offs?)\b/, + /\b(root cause|unclear why|no idea why|investigate why|figure out why|intermittent|flaky)\b/, +] + +export type Route = { tier: Tier; reason: string } + +/** A clear signal from the text, or undefined (ask the classifier). */ +export const heuristic = (subagentType: string | undefined, prompt: string, role?: Role): Route | undefined => { + if (!role && subagentType && READ_ONLY_TYPES.test(subagentType)) + return { tier: 'haiku', reason: `read-only agent type ${subagentType}` } + const lower = prompt.toLowerCase() + const heavy = HEAVY.find(r => r.test(lower)) + if (heavy) return { tier: 'opus', reason: 'approach still open' } + if (role) return undefined // dev briefs never take the light path + const light = LIGHT.find(r => r.test(lower)) + if (light && prompt.length < 1500) return { tier: 'haiku', reason: 'light task' } + return undefined +} + +export type Decision = + | { kind: 'fixed'; tier: Tier; reason: string; opusRequested?: boolean } + | { kind: 'classify'; opusRequested?: boolean } + +/** + * Who decides the writer's model. + * - review and test agents: a named model is kept; unnamed, they run on Opus + * (they are the gates, and a test agent's work has no verifier behind it). + * - dev agents: always routed; a named Opus is only logged. + * - anything else: routed, unless respectExplicit and a model is named. + */ +export const decide = (args: { + role?: Role + named?: Tier + subagentType?: string + prompt: string + respectExplicit: boolean + isAmbiguous?: boolean +}): Decision => { + const { role, named, subagentType, prompt, respectExplicit, isAmbiguous } = args + // Could be a test or review agent: never risk a weaker model on it. + if (isAmbiguous) + return { kind: 'fixed', tier: named && named !== 'haiku' && named !== 'sonnet' ? named : 'opus', reason: 'several clone paths and no Clone: line' } + if (role === 'review' || role === 'test') + return named + ? { kind: 'fixed', tier: named, reason: `named model kept for ${role} agent` } + : { kind: 'fixed', tier: 'opus', reason: `${role} agent default` } + if (!role && respectExplicit && named) return { kind: 'fixed', tier: named, reason: 'model chosen by orchestrator' } + const opusRequested = role === 'dev' && (named === 'opus' || named === 'fable') + const h = heuristic(subagentType, prompt, role) + return h ? { kind: 'fixed', ...h, opusRequested } : { kind: 'classify', opusRequested } +} + +// ---------- verification ---------- + +export type Skip = 'test-agent' | 'review-agent' | 'no-clone' | 'opus-unverified' | 'agent-ended' + +/** Why an agent run is not verified, or undefined when it is. */ +export const skipReason = (rec: { role?: Role; clone?: string; tier: Tier }, ended: string): Skip | undefined => { + if (ended !== 'answer') return 'agent-ended' + if (rec.role === 'test') return 'test-agent' + if (rec.role === 'review') return 'review-agent' + if (!rec.clone) return 'no-clone' + if (rank(rec.tier) >= rank('opus')) return 'opus-unverified' + return undefined +} + +/** The verifier never runs on the writer's model; below Opus it is Opus. */ +export const verifierFor = (writer: Tier): Tier => (rank(writer) >= rank('opus') ? 'sonnet' : 'opus') + +/** The model a rewrite runs on: one tier up, never the same model. */ +export const rewriteTier = (writer: Tier, allowFable: boolean): Tier | undefined => { + if (writer === 'haiku') return 'sonnet' + if (writer === 'sonnet') return 'opus' + if (writer === 'opus' && allowFable) return 'fable' + return undefined +} + +export type FailStep = { kind: 'resume' } | { kind: 'new-agent'; tier: Tier } | { kind: 'hand-back' } + +/** + * After the n-th failed verdict in a task's chain: the first resumes the same + * agent, the second starts a new agent one model up in the same clone, and + * past maxFails (or with no model above) it goes back to the manager. + */ +export const onFail = (fails: number, writer: Tier, maxFails: number, allowFable: boolean): FailStep => { + if (fails > maxFails) return { kind: 'hand-back' } + if (fails === 1) return { kind: 'resume' } + const up = rewriteTier(writer, allowFable) + return up ? { kind: 'new-agent', tier: up } : { kind: 'hand-back' } +} + +export type Verdict = { pass: boolean; issues: string[]; brief: string } + +/** Reads the verifier's reply; anything unreadable counts as a failure. */ +export const parseVerdict = (text: string): Verdict => { + const match = text.match(/\{[\s\S]*\}/) + if (match) { + try { + const v = JSON.parse(match[0]) as { verdict?: string; issues?: unknown; rewrite_brief?: unknown } + const issues = Array.isArray(v.issues) ? v.issues.map(String) : [] + const pass = String(v.verdict ?? '').toUpperCase() === 'PASS' + return { pass, issues, brief: typeof v.rewrite_brief === 'string' ? v.rewrite_brief : '' } + } catch { + // fall through + } + } + return { pass: false, issues: ['Verifier reply was not valid JSON: ' + text.slice(0, 300)], brief: '' } +} + +export const REWRITE_MARK = /^\s*\[router-rewrite (T\d+)\]/ + +export const clip = (s: string, n: number) => + s.length <= n ? s : s.slice(0, n) + `\n... [${s.length - n} more characters cut]` + +/** A usable test time limit in seconds: anything not a positive number is the default. */ +export function validTimeout(raw: unknown): number { + const n = Number(raw) + return Number.isFinite(n) && n >= 1 ? Math.round(n) : 1800 +} + diff --git a/tests/pricing.test.ts b/tests/pricing.test.ts new file mode 100644 index 0000000..ad8ce67 --- /dev/null +++ b/tests/pricing.test.ts @@ -0,0 +1,179 @@ +import type { On } from 'claude-code' +import { expect, mock, test, type Engine } from 'claude-code/testing' + +import { + addOverhead, + addWriterRun, + actualTotal, + counterfactual, + emptyTask, + parsePrices, + priceRun, + saved, + savingsReport, + tokenizerFactor, +} from '../hooks/pricing' + +const U = (i: number, o: number, w = 0, r = 0) => ({ + input_tokens: i, + output_tokens: o, + cache_creation_input_tokens: w, + cache_read_input_tokens: r, +}) +const M = 1_000_000 +const round = (n: number) => Math.round(n * 1e6) / 1e6 + +test('prices: each part of the usage at its own rate', async () => { + // Sonnet 5.5: in 2, out 10, write 2.5, read 0.1 per MTok + expect(priceRun('claude-sonnet-5-5', U(M, M, M, M)).actual).toBe(2 + 10 + 2.5 + 0.1) + // the same tokens at Opus 5.5: 4 + 20 + 5 + 0.2 + expect(round(priceRun('claude-sonnet-5-5', U(M, M, M, M)).opus)).toBe(29.2) + expect(priceRun('claude-opus-5-5', U(M, 0)).actual).toBe(4) + expect(priceRun('claude-haiku-5-5', U(M, 0)).actual).toBe(0.1) + expect(priceRun('claude-haiku-4-5', U(M, 0)).actual).toBe(1) +}) + +test('prices: the most specific model wins, unknown models are flagged', async () => { + expect(priceRun('claude-opus-5-5-20260601', U(M, 0)).label).toBe('Opus 5.5') + expect(priceRun('claude-opus-5', U(M, 0)).label).toBe('Opus 5') + expect(priceRun('claude-sonnet-5', U(M, 0)).label).toBe('Sonnet 5') + expect(priceRun('gpt-9', U(M, 0)).known).toBe(false) +}) + +test('older tokenizers are scaled before pricing as Opus 5.5', async () => { + expect(tokenizerFactor('claude-haiku-4-5')).toBe(1.3) + expect(tokenizerFactor('claude-sonnet-5-5')).toBe(1) + expect(tokenizerFactor('claude-haiku-5-5')).toBe(1) + expect(round(priceRun('claude-haiku-4-5', U(M, 0)).opus)).toBe(5.2) +}) + +test('a task below opus: every writer token at opus price is the comparison', async () => { + let t = emptyTask('clamp') + t = addWriterRun(t, 'claude-sonnet-5-5', U(M, 0)) + t = addWriterRun(t, 'claude-sonnet-5-5', U(M, 0)) // resumed round + t = addOverhead(t, 'claude-opus-5-5', U(M / 2, 0)) // verifier + expect(t.writerActual).toBe(4) + expect(counterfactual(t)).toBe(8) + expect(actualTotal(t)).toBe(6) + expect(saved(t)).toBe(2) +}) + +test('a chain that ended on opus: the opus run is the comparison, failed rounds are loss', async () => { + let t = emptyTask('clamp') + t = addWriterRun(t, 'claude-sonnet-5-5', U(M, 0)) // failed twice + t = addWriterRun(t, 'claude-opus-5-5', U(M, 0)) // rewrite + expect(counterfactual(t)).toBe(4) + expect(actualTotal(t)).toBe(6) + expect(saved(t)).toBe(-2) +}) + +test('unpriced models are listed, never counted as free', async () => { + const t = addWriterRun(emptyTask('x'), 'mystery-1', U(M, M)) + expect(t.unpriced).toEqual(['mystery-1']) + expect(t.writerActual).toBe(0) + expect(savingsReport({ T1: t }, 5)).toContain('[no price: mystery-1]') +}) + +test('price overrides come first; bad JSON falls back to list prices', async () => { + const table = parsePrices('[{"match":"sonnet-5-5","price":{"in":1,"out":1,"write":1,"read":1}}]') + expect(priceRun('claude-sonnet-5-5', U(M, 0), table).actual).toBe(1) + expect(priceRun('claude-opus-5-5', U(M, 0), table).actual).toBe(4) + expect(priceRun('claude-sonnet-5-5', U(M, 0), parsePrices('{oops')).actual).toBe(2) +}) + +test('the report: newest first, totals over every task, filter by task', async () => { + const a = addWriterRun(emptyTask('first'), 'claude-sonnet-5-5', U(M, 0)) + const b = addWriterRun(emptyTask('second'), 'claude-sonnet-5-5', U(2 * M, 0)) + const r = savingsReport({ T1: a, T2: b }, 1) + expect(r.indexOf('T2')).toBeGreaterThan(-1) + expect(r).not.toContain('T1 ') + expect(r).toMatch(/total\s+\$6\.00\s+\$12\.00\s+\$6\.00\s+50%/) + expect(savingsReport({ T1: a, T2: b }, 9, 'T1')).toContain('first') + expect(savingsReport({ T1: a, T2: b }, 9, 'T1')).not.toContain('second') +}) + +// ---------- end to end ---------- + +const USAGE = { input_tokens: 1000, output_tokens: 10, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } +const ok = (stdout: string) => ({ value: { exitCode: 0, stdout, stderr: '', isStdoutTruncated: false, isStderrTruncated: false } }) +const DEV = '/home/homelab/projects/goodwe/clones/OPT-11-P4-dev' + +function world(on: On, models: string[]) { + mock.store(on) + const clock = mock.clock(on) + on('ui.status', async () => ({ value: undefined })) + on('ui.toast', async () => ({ value: undefined })) + on('model.complete', async (_$, e) => { + if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } } + return { value: { isAnswered: true as const, text: '{"verdict":"PASS"}', usage: USAGE } } + }) + on('process.run', async (_$, e) => { + const a = e.argv.join(' ') + if (a.startsWith('git merge-base')) return ok('base\n') + if (a.startsWith('git diff')) return ok('diff') + return ok('') + }) + on('agent.spawn', async (_$, e) => { + models.push(String(e.model)) + return { model: e.model === 'sonnet' ? 'claude-sonnet-5-5' : String(e.model), agentId: `a${models.length}` } + }) + on('turn.complete', async (_$, e) => ({ text: e.answer })) + on('prompt.submit', async (_$, e) => ({ text: e.text })) + return clock +} + +async function end($: Engine, agentId: string, model: string, input: number) { + await ($.turn as unknown as { complete: (e: unknown) => Promise }).complete({ + answer: 'done', + durationMs: 1, + isAborted: false, + turnId: 't', + agentId, + reason: 'answer', + usage: { model, input_tokens: input, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, + }) +} + +test('end to end: a routed dev agent is priced on the model it ran on, a kept review agent is not counted', async ($, on) => { + const models: string[] = [] + const clock = world(on, models) + const dev = await $.agent.spawn({ prompt: `Clone: ${DEV}/\nExpected: x`, description: 'P4 dev' } as never) + await end($, dev.agentId as string, 'claude-sonnet-5-5', M) + await clock.advance(1) + const review = await $.agent.spawn({ prompt: 'Clone: /x/clones/OPT-11-P4-review/', description: 'P4 review', model: 'opus' } as never) + await end($, review.agentId as string, 'claude-opus-5-5', M) + await clock.advance(1) + + const r = (await $.command.run({ command: 'router', args: 'savings' } as never)).text ?? '' + expect(r).toContain('P4 dev') + expect(r).not.toContain('P4 review') + // writer $2.00 + classifier and verifier (1000 in + 10 out each, Haiku 4.5 then Opus 5.5) + expect(r).toMatch(/T1\s+\$2\.01\s+\$4\.00/) + expect(r).toContain('Sonnet 5.5') +}) + +test('agents started in parallel each get their own task, and every run is counted', async ($, on) => { + const models: string[] = [] + const clock = world(on, models) + const ids = ( + await Promise.all( + ['A', 'D', 'E', 'F', 'G'].map(c => $.agent.spawn({ prompt: `Scope chunk ${c}: list the open cards`, description: `Scope chunk ${c}` } as never)), + ) + ).map(r => r.agentId as string) + await Promise.all(ids.map(id => end($, id, 'claude-sonnet-5-5', M))) + await clock.advance(1) + + const r = (await $.command.run({ command: 'router', args: 'savings' } as never)).text ?? '' + for (const c of ['A', 'D', 'E', 'F', 'G']) expect(r).toContain(`Scope chunk ${c}`) + expect(r).toContain('5 of 5 tasks') + // five writer runs of $2.00 each, all counted + expect(r).toMatch(/total\s+\$10\.\d\d\s+\$20\.00/) +}) + +test('a task whose writer has not finished shows as running and stays out of the total', async () => { + const running = addOverhead(emptyTask('MAINT-14 dev'), 'claude-haiku-4-5', U(M, 0)) // $1 of classifier, no writer yet + const done = addWriterRun(emptyTask('done'), 'claude-sonnet-5-5', U(M, 0)) + const r = savingsReport({ T1: done, T2: running }, 5) + expect(r).toMatch(/T2 .* running /) + expect(r).toMatch(/total\s+\$2\.00\s+\$4\.00/) +}) diff --git a/tests/router.test.ts b/tests/router.test.ts new file mode 100644 index 0000000..93886cf --- /dev/null +++ b/tests/router.test.ts @@ -0,0 +1,451 @@ +import type { On } from 'claude-code' +import { expect, mock, test, type Engine } from 'claude-code/testing' + +import { + decide, + heuristic, + onFail, + parseBrief, + parseVerdict, + rewriteTier, + skipReason, + validTimeout, + verifierFor, +} from '../hooks/router' + +const DEV = '/home/homelab/projects/goodwe/clones/SAFETY-95-dev' +const REVIEW = '/home/homelab/projects/goodwe/clones/SAFETY-95-review' +const TEST = '/home/homelab/projects/goodwe/clones/TEST-23-test' + +// ---------- the brief (C2) ---------- + +test('brief: clone path, ticket, role and base branch are read', async () => { + const b = parseBrief(`Work in ${DEV}/ only. Base branch: release/1.0.\nExpected: ...`) + expect(b).toEqual({ clone: DEV, ticket: 'SAFETY-95', role: 'dev', baseBranch: 'release/1.0' }) + expect(parseBrief(`Clone: \`${TEST}\``).role).toBe('test') + expect(parseBrief('PR against `release/1.1`, no clone given').baseBranch).toBe('release/1.1') + expect(parseBrief('just look around').clone).toBe(undefined) +}) + +test('fix 1: tickets with extra parts parse, the role is the last part', async () => { + const p = '/home/homelab/projects/goodwe/clones/OPT-11-P4-dev' + expect(parseBrief(`Work in ${p}/ only.`)).toMatchObject({ clone: p, ticket: 'OPT-11-P4', role: 'dev' }) + expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').ticket).toBe('OPT-11-P4-R2') + expect(parseBrief('Clone: /x/clones/OPT-11-P4-R2-test/').role).toBe('test') +}) + +test('fix 2: the Clone: line decides, whatever the brief mentions first', async () => { + const brief = `Verify the work of ${DEV}/ on the rig.\nClone: ${TEST}/\nBase branch: release/1.0.` + expect(parseBrief(brief)).toMatchObject({ clone: TEST, role: 'test', baseBranch: 'release/1.0' }) + // an explicit role wins over the path + expect(parseBrief(`Clone: \`/x/clones/OPT-11-P4\` (Role: review)`)).toMatchObject({ clone: '/x/clones/OPT-11-P4', role: 'review' }) + // (Role: ...) wins over the role in the path + expect(parseBrief(`Clone: ${DEV}/ (Role: review)`).role).toBe('review') + // a list item or quote marker in front of the line is fine + expect(parseBrief(`- Clone: ${DEV}`).role).toBe('dev') + // "Clone:" inside a sentence is not the line, even at a line's end + expect(parseBrief(`Results are in the dev agent's Clone: ${DEV}\nRun the rig in ${TEST}/`).isAmbiguous).toBe(true) +}) + +test('fix 2: several clones and no Clone: line gives no role, and routes to opus', async () => { + const b = parseBrief(`Compare ${DEV}/ with ${TEST}/.`) + expect(b.role).toBe(undefined) + expect(b.isAmbiguous).toBe(true) + expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'haiku' })).toMatchObject({ kind: 'fixed', tier: 'opus' }) + expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'sonnet' })).toMatchObject({ tier: 'opus' }) + expect(decide({ prompt: 'x', respectExplicit: false, isAmbiguous: true, named: 'fable' })).toMatchObject({ tier: 'fable' }) + // the same path twice is one clone, not ambiguous + expect(parseBrief(`Work in ${DEV}/. Commit in ${DEV}.`).role).toBe('dev') +}) + +// ---------- who picks the writer (C4) ---------- + +test('C4: review and test agents keep the model they are given', async () => { + for (const role of ['review', 'test'] as const) + for (const named of ['haiku', 'sonnet', 'opus'] as const) + expect(decide({ role, named, prompt: 'x', respectExplicit: false })).toEqual({ + kind: 'fixed', + tier: named, + reason: `named model kept for ${role} agent`, + }) +}) + +test('C4: unnamed review and test agents run on opus', async () => { + expect(decide({ role: 'test', prompt: 'run the rig', respectExplicit: false })).toMatchObject({ tier: 'opus' }) + expect(decide({ role: 'review', prompt: 'review', respectExplicit: false })).toMatchObject({ tier: 'opus' }) +}) + +test('C4: dev agents are routed even when opus is named, and that is logged', async () => { + const d = decide({ role: 'dev', named: 'opus', prompt: 'Expected: clamp to 2000 W', respectExplicit: true }) + expect(d.kind).toBe('classify') + expect(d.opusRequested).toBe(true) + expect(decide({ role: 'dev', named: 'sonnet', prompt: 'x', respectExplicit: true }).opusRequested).toBe(false) +}) + +test('C4: respectExplicit only applies to agents without a role', async () => { + expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: true })).toMatchObject({ kind: 'fixed', tier: 'haiku' }) + expect(decide({ named: 'haiku', prompt: 'x', respectExplicit: false }).kind).toBe('classify') +}) + +test('C4: open design goes to opus, a decided brief is left to the classifier', async () => { + expect(heuristic(undefined, 'MAINT-13: decide how the staleness guard should recover', 'dev')?.tier).toBe('opus') + expect(heuristic(undefined, 'In the asyncio control loop, clamp the integrator; Expected: test_clamp passes', 'dev')).toBe(undefined) +}) + +test('C4: narrowed light patterns: "docs" or "run tests" in a brief never mean haiku', async () => { + expect(heuristic(undefined, 'Update the docs and run the tests after changing the setpoint logic')).toBe(undefined) + expect(heuristic(undefined, 'Find where the MQTT discovery payload is built')?.tier).toBe('haiku') + // a dev brief never takes the light path, however light it reads + expect(heuristic(undefined, 'Find where the MQTT discovery payload is built', 'dev')).toBe(undefined) + expect(heuristic('Explore', 'anything', 'dev')).toBe(undefined) +}) + +// ---------- what gets verified (C1, C5) ---------- + +test('only finished dev rounds below opus with a clone are verified', async () => { + expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'answer')).toBe(undefined) + expect(skipReason({ role: 'dev', clone: DEV, tier: 'haiku' }, 'answer')).toBe(undefined) + expect(skipReason({ role: 'dev', clone: DEV, tier: 'opus' }, 'answer')).toBe('opus-unverified') + expect(skipReason({ role: 'dev', tier: 'sonnet' }, 'answer')).toBe('no-clone') + expect(skipReason({ role: 'test', clone: TEST, tier: 'sonnet' }, 'answer')).toBe('test-agent') + expect(skipReason({ role: 'review', clone: REVIEW, tier: 'sonnet' }, 'answer')).toBe('review-agent') + expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'error')).toBe('agent-ended') + expect(skipReason({ role: 'dev', clone: DEV, tier: 'sonnet' }, 'aborted')).toBe('agent-ended') +}) + +test('the verifier is opus below opus, and never the writer', async () => { + expect(verifierFor('haiku')).toBe('opus') + expect(verifierFor('sonnet')).toBe('opus') + for (const w of ['haiku', 'sonnet', 'opus', 'fable'] as const) expect(verifierFor(w)).not.toBe(w) +}) + +test('C5: first fail resumes, second starts a stronger agent, then hand back', async () => { + expect(onFail(1, 'sonnet', 2, false)).toEqual({ kind: 'resume' }) + expect(onFail(2, 'sonnet', 2, false)).toEqual({ kind: 'new-agent', tier: 'opus' }) + expect(onFail(3, 'opus', 2, false)).toEqual({ kind: 'hand-back' }) + expect(onFail(2, 'opus', 2, false)).toEqual({ kind: 'hand-back' }) // nothing above opus + expect(onFail(2, 'opus', 2, true)).toEqual({ kind: 'new-agent', tier: 'fable' }) + expect(onFail(1, 'sonnet', 0, false)).toEqual({ kind: 'hand-back' }) + expect(rewriteTier('haiku', false)).toBe('sonnet') +}) + +test('verdict parsing fails closed', async () => { + expect(parseVerdict('{"verdict":"PASS","issues":[]}').pass).toBe(true) + expect(parseVerdict('text {"verdict":"FAIL","issues":["a.py:3"]}').issues).toEqual(['a.py:3']) + expect(parseVerdict('looks fine').pass).toBe(false) + expect(parseVerdict('verdict: pass').pass).toBe(false) +}) + +// ---------- the hooks, with the engine mocked beneath ---------- + +type World = { + spawned: { model?: string; prompt: string }[] + verifier: string[] + verdicts: string[] + told: string[] + diffs: string[] + diffBases: string[] + testCwd: string[] + testArgv: string[][] + /** How the next test run ends: exit code (124 = timed out), or a throw. */ + testEnds: (number | 'throw')[] + /** Whether `timeout` fired on each test run (it then says so on stderr). */ + testFired: boolean[] + /** The timeout line arrives in two stderr pieces. */ + testSplitLine?: boolean + /** A test itself prints the timeout line, on stdout. */ + testLineOnStdout?: boolean + snap: number +} + +const USAGE = { input_tokens: 1, output_tokens: 1, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } +const ok = (stdout: string, exitCode = 0) => ({ + value: { exitCode, stdout, stderr: '', isStdoutTruncated: false, isStderrTruncated: false }, +}) + +function world(on: On, w: World) { + mock.store(on) + const clock = mock.clock(on) + on('ui.status', async () => ({ value: undefined })) + on('ui.toast', async () => ({ value: undefined })) + on('model.complete', async (_$, e) => { + if (e.system?.startsWith('You route')) return { value: { isAnswered: true as const, text: 'sonnet', usage: USAGE } } + w.verifier.push(e.model) + return { value: { isAnswered: true as const, text: w.verdicts.shift() ?? '{"verdict":"PASS"}', usage: USAGE } } + }) + on('process.run', async (_$, e) => { + const a = e.argv.join(' ') + const cwd = String(e.init?.cwd ?? '') + if (cwd !== DEV) return ok('', 128) // not a git repo: the session folder + if (a === 'git merge-base origin/release/1.0 HEAD') return ok('mergebase\n') + if (a === 'git stash create') return ok(`snap${++w.snap}\n`) + if (a.startsWith('git diff ')) { + w.diffBases.push(e.argv[2] ?? '') + return ok(w.diffs.shift() ?? '') + } + return ok('') + }) + on('process.spawn', async function* (_$, e) { + w.testCwd.push(String(e.cwd ?? '')) + w.testArgv.push([...e.argv]) + const end = w.testEnds.shift() ?? 0 + const fired = w.testFired.shift() ?? false + if (end === 'throw') throw new Error('docker: not found') + yield { stream: 'stdout' as const, text: end === 0 ? '67 passed\n' : 'test_modes_p4 ... still running\n' } + if (fired) yield { stream: 'stderr' as const, text: "timeout: sending signal TERM to command 'sh'\n" } + if (w.testSplitLine) { + yield { stream: 'stderr' as const, text: 'timeout: sending sig' } + yield { stream: 'stderr' as const, text: "nal TERM to command 'sh'\n" } + } + if (w.testLineOnStdout) yield { stream: 'stdout' as const, text: 'assert "timeout: sending signal" in log\n' } + return { value: { code: end, signal: null } } + }) + on('agent.spawn', async (_$, e) => { + w.spawned.push({ model: e.model, prompt: e.prompt }) + return { model: e.model ?? 'inherit', agentId: `agent-${w.spawned.length}` } + }) + on('turn.complete', async (_$, e) => ({ text: e.answer })) + on('prompt.submit', async (_$, e) => { + w.told.push(e.text) + return { text: e.text } + }) + return clock +} + +const fresh = (): World => ({ spawned: [], verifier: [], verdicts: [], told: [], diffs: [], diffBases: [], testCwd: [], testArgv: [], testEnds: [], testFired: [], snap: 0 }) + +async function finish($: Engine, clock: { advance: (ms: number) => Promise }, agentId: string) { + await ($.turn as unknown as { complete: (e: unknown) => Promise }).complete({ answer: 'done', durationMs: 1, isAborted: false, turnId: 't', agentId, reason: 'answer' }) + await clock.advance(1) +} + +const devBrief = (extra = '') => `SAFETY-95: clamp the integrator. Work in ${DEV}/. Base branch: release/1.0.\nExpected: test_clamp passes.${extra}` + +test('acceptance 1+2: a background dev agent is verified in its clone, each round on its own diff', { options: { testCommand: 'pytest -q' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff --git a/loop.py', 'diff --git a/loop.py (round 2)'] + w.verdicts = ['{"verdict":"FAIL","issues":["loop.py:40 no clamp on negative"]}', '{"verdict":"PASS"}'] + const clock = world(on, w) + + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp', model: 'opus' } as never) + expect(w.spawned[0]?.model).toBe('sonnet') // routed despite the named opus + await finish($, clock, r.agentId as string) + + expect(w.diffBases[0]).toBe('mergebase') // first round: from the merge base, in the clone + expect(w.testCwd).toEqual([DEV]) + expect(w.verifier).toEqual(['opus']) + expect(w.told[0]).toContain('FAIL') + + // round 2: the resumed agent finishes again (same agentId) + await finish($, clock, r.agentId as string) + expect(w.diffBases[1]).toBe('snap1') // only what this round added + expect(w.told[1]).toContain('PASS') +}) + +test('acceptance 3: review and test agents keep a named opus; dev briefs below opus are verified by opus', async ($, on) => { + const w = fresh() + const clock = world(on, w) + await $.agent.spawn({ prompt: `Review PR. Clone ${REVIEW}/`, description: 'review', model: 'opus' } as never) + await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig', model: 'opus' } as never) + const t = await $.agent.spawn({ prompt: `Run the rig test. Clone ${TEST}/`, description: 'rig2' } as never) + expect(w.spawned.map(s => s.model)).toEqual(['opus', 'opus', 'opus']) + await finish($, clock, t.agentId as string) + expect(w.verifier).toEqual([]) // test agents are not verified + expect(w.told).toEqual([]) +}) + +test('acceptance 4: the first FAIL asks to resume the same agent', async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.verdicts = ['{"verdict":"FAIL","issues":["x"]}'] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.told[0]).toContain(`Resume agent ${r.agentId} with SendMessage`) + expect(w.told[0]).not.toContain('[router-rewrite') +}) + +test('C5: the second FAIL starts a stronger agent in the same clone', async ($, on) => { + const w = fresh() + w.diffs = ['d1', 'd2'] + w.verdicts = ['{"verdict":"FAIL","issues":["x"]}', '{"verdict":"FAIL","issues":["y"]}'] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + await finish($, clock, r.agentId as string) + const id = w.told[1]?.match(/\[router-rewrite (T\d+)\]/)?.[1] + expect(id).toBeDefined() + + await $.agent.spawn({ prompt: `[router-rewrite ${id}]`, description: 'rewrite' } as never) + expect(w.spawned[1]?.model).toBe('opus') + expect(w.spawned[1]?.prompt).toContain(`Their work is in the clone ${DEV}: continue from it`) + expect(w.spawned[1]?.prompt).toContain('- y') +}) + +test('C2: a dev brief with no clone is never diffed in the session folder', async ($, on) => { + const w = fresh() + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: 'Clamp the integrator in clones/ somewhere. Expected: ...', description: 'x' } as never) + await finish($, clock, r.agentId as string) + expect(w.diffBases).toEqual([]) + expect(w.verifier).toEqual([]) +}) + +test('fix 2 end to end: a test brief that mentions the dev clone first keeps its model and is not verified', async ($, on) => { + const w = fresh() + const clock = world(on, w) + const r = await $.agent.spawn({ + prompt: `Run the OPT-11 P4 rig test against the work in ${DEV}/.\nClone: /home/homelab/projects/goodwe/clones/OPT-11-P4-test/`, + description: 'rig', + model: 'opus', + } as never) + expect(w.spawned[0]?.model).toBe('opus') + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.diffBases).toEqual([]) +}) + +test('fix 2 end to end: two clones and no Clone: line run on opus, unverified', async ($, on) => { + const w = fresh() + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: `Compare ${DEV}/ with ${TEST}/ and report.`, description: 'cmp', model: 'haiku' } as never) + expect(w.spawned[0]?.model).toBe('opus') + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) +}) + +test('a brief without a role is routed but never tracked or verified', async ($, on) => { + const w = fresh() + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: 'Find where the MQTT discovery payload is built', description: 'look' } as never) + expect(w.spawned[0]?.model).toBe('haiku') + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.told).toEqual([]) +}) + +// ---------- the test run ---------- + +test('tests stream with a real time limit, not the ten-minute command cap', { options: { testCommand: 'make verifier-tests', testTimeoutSec: 1800 } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.testArgv[0]).toEqual(['timeout', '--verbose', '--kill-after=30', '1800', 'sh', '-c', 'make verifier-tests']) + expect(w.testCwd).toEqual([DEV]) + expect(w.verifier).toEqual(['opus']) +}) + +test('a test timeout is NOT VERIFIED, never a FAIL: no verifier call, no resume, no fail counted', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff', 'diff 2'] + w.testEnds = [124, 0] + w.testFired = [true, false] + w.verdicts = ['{"verdict":"FAIL","issues":["x"]}'] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.told[0]).toContain('NOT VERIFIED - the tests did not finish within 60 s') + expect(w.told[0]).not.toContain('Resume agent') + // the next round's FAIL is the first one counted: it resumes, it does not rewrite + await finish($, clock, r.agentId as string) + expect(w.told[1]).toContain('Resume agent') +}) + +test('a kill after the grace period also counts as a timeout', { options: { testCommand: 'make t' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [137] + w.testFired = [true] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.told[0]).toContain('did not finish') +}) + +test('a test command that cannot run (exit 127) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [127] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.told[0]).toContain('could not run') +}) + +test('a test command that cannot run (a start failure) is NOT VERIFIED, not a FAIL', { options: { testCommand: 'make t' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = ['throw' as const] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.told[0]).toContain('could not run') +}) + +test('failing tests are evidence: the verifier sees them and decides', { options: { testCommand: 'make t' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [1] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual(['opus']) +}) + +test('an early 137 (out of memory) is a failed run for the verifier, not a timeout', { options: { testCommand: 'make t', testTimeoutSec: 1800 } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [137] + w.testFired = [false] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual(['opus']) + expect(w.told[0] ?? '').not.toContain('NOT VERIFIED') +}) + +test('an invalid testTimeoutSec falls back to 1800', async () => { + for (const bad of [0, -5, Number.NaN, 'abc', undefined, null]) expect(validTimeout(bad)).toBe(1800) + expect(validTimeout(900)).toBe(900) + expect(validTimeout('600')).toBe(600) +}) + +test('a command that exits 124 by itself is a failed run, not a timeout', { options: { testCommand: 'make t' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [124] + w.testFired = [false] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual(['opus']) +}) + +test('the timeout line still counts when it arrives split across pieces', { options: { testCommand: 'make t', testTimeoutSec: 60 } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [124] + w.testSplitLine = true + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual([]) + expect(w.told[0]).toContain('did not finish') +}) + +test('the timeout line on stdout (a test printing it) is not a fired timeout', { options: { testCommand: 'make t' } }, async ($, on) => { + const w = fresh() + w.diffs = ['diff'] + w.testEnds = [1] + w.testLineOnStdout = true + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief(), description: 'clamp' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifier).toEqual(['opus']) +}) diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..8681ec0 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./.claude-plugin/types/tsconfig.json" +}