From 86bb9d79cce7b05e6b9d30044a25e70da07fae8c Mon Sep 17 00:00:00 2001 From: glenn schrooyen Date: Tue, 17 Nov 2026 21:28:36 +0100 Subject: [PATCH] 0.5.0: pipeline support for ems-dev-tooling - test-designer and release-reviewer keep a named model (Opus when none, never Haiku) - verifier checks every T-id and N-id of a "## Test plan" section - manager guidance: record each verdict with `ticket` before resume or rewrite Already running on the homelab install (applied from ems-dev-tooling patches/model-router-0.5.0-pipeline.patch). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Bdqr7PiaHcBnzPituu2yTC --- .claude-plugin/plugin.json | 2 +- hooks/register.ts | 4 +++- hooks/router.ts | 9 +++++++++ tests/router.test.ts | 20 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index fe901b8..6ff7aee 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "model-router", - "version": "0.4.9", + "version": "0.5.0", "description": "Routes each subagent to Haiku, Sonnet or Opus by how well-defined its brief is, verifies dev work below Opus in its own clone on a different model, sends failures back to the same agent then a stronger one, and estimates the savings against Opus.", "author": { "name": "Glenn Schrooyen" diff --git a/hooks/register.ts b/hooks/register.ts index 1e6c155..3485b38 100644 --- a/hooks/register.ts +++ b/hooks/register.ts @@ -137,6 +137,7 @@ Judge only whether the change correctly and completely does the task, without br Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes. Scope added in "Later instructions" was asked for after the brief: judge it as in scope, not as unrequested work. When a table of files rerun one by one is given: a file that passes alone does not count against the change (the full run was overloaded or flaky); one that also fails or times out on base is not caused by this change: mention it, do not fail for it. Name every file you count against the change in issues. +When the task has a "## Test plan" section with T-ids and N-ids: every T-id must be implemented by a test that can fail, and every N-id asserted somewhere, unless the developer says why not. A missing or untestable T-id/N-id is a FAIL issue naming the id. A test that cannot fail on the old code where the plan says "red on base" is a FAIL issue. Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate. Reply with JSON only: @@ -149,7 +150,8 @@ The model-router mod picks each subagent's model and verifies dev work before it - Review and test agents keep the model you name (Opus when you name none). - Every dev hand-back gets one "[model-router] Tn" line at once; an interim stop while the dev waits on its own background work gets none. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on. - On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person. -- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself.` +- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself. +- With ems-dev-tooling: record every verdict with \`ticket\` first (PASS: \`ticket advance --to review --sha \`; FAIL: \`ticket fail --stage router\`), then resume or rewrite. pipeline-guard refuses a resume while the ticket is not back in development.` // ---------- store helpers (top-level, as the engine requires for `$`) ---------- diff --git a/hooks/router.ts b/hooks/router.ts index b534d03..84a9379 100644 --- a/hooks/router.ts +++ b/hooks/router.ts @@ -108,6 +108,9 @@ export const heuristic = (subagentType: string | undefined, prompt: string, role return undefined } +/** Agent types that gate rather than write: never routed down, never verified. */ +export const KEEP_MODEL_TYPES = /^(test-designer|release-reviewer)$/i + export type Decision = | { kind: 'fixed'; tier: Tier; reason: string; opusRequested?: boolean } | { kind: 'classify'; opusRequested?: boolean } @@ -135,6 +138,12 @@ export const decide = (args: { return named ? { kind: 'fixed', tier: named, reason: `named model kept for ${role} agent` } : { kind: 'fixed', tier: 'opus', reason: `${role} agent default` } + // Pipeline gates without a clone role (test design, release review) judge, they do not write code: + // like review and test agents they keep a named model, Opus when none. + if (!role && subagentType && KEEP_MODEL_TYPES.test(subagentType)) + return named && named !== 'haiku' + ? { kind: 'fixed', tier: named, reason: `named model kept for ${subagentType}` } + : { kind: 'fixed', tier: 'opus', reason: `${subagentType} default` } if (!role && respectExplicit && named) return { kind: 'fixed', tier: named, reason: 'model chosen by orchestrator' } const opusRequested = role === 'dev' && (named === 'opus' || named === 'fable') const h = heuristic(subagentType, prompt, role) diff --git a/tests/router.test.ts b/tests/router.test.ts index c1b56e8..4cb4cb2 100644 --- a/tests/router.test.ts +++ b/tests/router.test.ts @@ -1139,3 +1139,23 @@ test('#5: the verifier is told how to read the rerun table', { options: RERUN }, expect(sys).toContain('not caused by this change') expect(sys).toContain('Name every file you count against the change') }) + +// ---------- ems-dev-tooling (test design, test-plan check) ---------- + +test('pipeline: a test designer keeps Opus; a named Sonnet is kept, Haiku never', async () => { + expect(decide({ subagentType: 'test-designer', prompt: 'Write the test plan', respectExplicit: false })).toEqual({ kind: 'fixed', tier: 'opus', reason: 'test-designer default' }) + expect(decide({ subagentType: 'test-designer', named: 'sonnet', prompt: 'x', respectExplicit: false }).kind).toBe('fixed') + expect(decide({ subagentType: 'test-designer', named: 'haiku', prompt: 'find x', respectExplicit: false })).toEqual({ kind: 'fixed', tier: 'opus', reason: 'test-designer default' }) + // a design clone path carries no role: no record, so never verified as dev work + expect(parseBrief('Clone: /home/homelab/projects/goodwe/clones/EV-15-design').role).toBe(undefined) +}) + +test('pipeline: the verifier is told to check every T-id and N-id of the test plan', async ($, on) => { + const w = fresh() + w.diffs = ['diff --git a/x.py'] + const clock = world(on, w) + const r = await $.agent.spawn({ prompt: devBrief('\n## Test plan\n| T1 | boundary | batteries above house | no alarm | yes |'), description: 'ev15' } as never) + await finish($, clock, r.agentId as string) + expect(w.verifierSystems[0]).toContain('every T-id must be implemented') + expect(w.verifierPrompts[0]).toContain('## Test plan') +})