Compare commits
1
Commits
v0.4.9
...
pipeline-0.5.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
86bb9d79cc |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "model-router",
|
||||
"version": "0.4.9",
|
||||
"version": "0.5.0",
|
||||
"description": "Routes each subagent to Haiku, Sonnet or Opus by how well-defined its brief is, verifies dev work below Opus in its own clone on a different model, sends failures back to the same agent then a stronger one, and estimates the savings against Opus.",
|
||||
"author": {
|
||||
"name": "Glenn Schrooyen"
|
||||
|
||||
+3
-1
@@ -137,6 +137,7 @@ Judge only whether the change correctly and completely does the task, without br
|
||||
Look for: requirements or acceptance criteria missed, wrong logic, unhandled edge cases, tests that cannot fail, tests weakened or deleted, safety checks bypassed, unrelated changes.
|
||||
Scope added in "Later instructions" was asked for after the brief: judge it as in scope, not as unrequested work.
|
||||
When a table of files rerun one by one is given: a file that passes alone does not count against the change (the full run was overloaded or flaky); one that also fails or times out on base is not caused by this change: mention it, do not fail for it. Name every file you count against the change in issues.
|
||||
When the task has a "## Test plan" section with T-ids and N-ids: every T-id must be implemented by a test that can fail, and every N-id asserted somewhere, unless the developer says why not. A missing or untestable T-id/N-id is a FAIL issue naming the id. A test that cannot fail on the old code where the plan says "red on base" is a FAIL issue.
|
||||
Do not fail a change for style alone. A later PR review and mutation testing follow you; you are the cheap first gate.
|
||||
|
||||
Reply with JSON only:
|
||||
@@ -149,7 +150,8 @@ The model-router mod picks each subagent's model and verifies dev work before it
|
||||
- Review and test agents keep the model you name (Opus when you name none).
|
||||
- Every dev hand-back gets one "[model-router] Tn" line at once; an interim stop while the dev waits on its own background work gets none. "verifying on Opus...": wait for the verdict before advancing the PR. "no verification: dev ran on Opus. Go straight to review.": no verdict will come. Lines are decided by the model the dev actually ran on.
|
||||
- On "[model-router] ... FAIL ... resume agent", resume that same agent with SendMessage, passing the issues. On "start a new agent ... [router-rewrite Tn]", stop the old agent and call Agent with exactly that prompt. On "handed back", stop and show the issues to the person.
|
||||
- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself.`
|
||||
- Scope or decisions added after the brief: send them to the dev agent with SendMessage. The verifier sees every later message to that agent and treats it as in scope. A resumed agent keeps its task number. Do not fix rejected work yourself.
|
||||
- With ems-dev-tooling: record every verdict with \`ticket\` first (PASS: \`ticket advance <T> --to review --sha <sha>\`; FAIL: \`ticket fail <T> --stage router\`), then resume or rewrite. pipeline-guard refuses a resume while the ticket is not back in development.`
|
||||
|
||||
// ---------- store helpers (top-level, as the engine requires for `$`) ----------
|
||||
|
||||
|
||||
@@ -108,6 +108,9 @@ export const heuristic = (subagentType: string | undefined, prompt: string, role
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Agent types that gate rather than write: never routed down, never verified. */
|
||||
export const KEEP_MODEL_TYPES = /^(test-designer|release-reviewer)$/i
|
||||
|
||||
export type Decision =
|
||||
| { kind: 'fixed'; tier: Tier; reason: string; opusRequested?: boolean }
|
||||
| { kind: 'classify'; opusRequested?: boolean }
|
||||
@@ -135,6 +138,12 @@ export const decide = (args: {
|
||||
return named
|
||||
? { kind: 'fixed', tier: named, reason: `named model kept for ${role} agent` }
|
||||
: { kind: 'fixed', tier: 'opus', reason: `${role} agent default` }
|
||||
// Pipeline gates without a clone role (test design, release review) judge, they do not write code:
|
||||
// like review and test agents they keep a named model, Opus when none.
|
||||
if (!role && subagentType && KEEP_MODEL_TYPES.test(subagentType))
|
||||
return named && named !== 'haiku'
|
||||
? { kind: 'fixed', tier: named, reason: `named model kept for ${subagentType}` }
|
||||
: { kind: 'fixed', tier: 'opus', reason: `${subagentType} default` }
|
||||
if (!role && respectExplicit && named) return { kind: 'fixed', tier: named, reason: 'model chosen by orchestrator' }
|
||||
const opusRequested = role === 'dev' && (named === 'opus' || named === 'fable')
|
||||
const h = heuristic(subagentType, prompt, role)
|
||||
|
||||
@@ -1139,3 +1139,23 @@ test('#5: the verifier is told how to read the rerun table', { options: RERUN },
|
||||
expect(sys).toContain('not caused by this change')
|
||||
expect(sys).toContain('Name every file you count against the change')
|
||||
})
|
||||
|
||||
// ---------- ems-dev-tooling (test design, test-plan check) ----------
|
||||
|
||||
test('pipeline: a test designer keeps Opus; a named Sonnet is kept, Haiku never', async () => {
|
||||
expect(decide({ subagentType: 'test-designer', prompt: 'Write the test plan', respectExplicit: false })).toEqual({ kind: 'fixed', tier: 'opus', reason: 'test-designer default' })
|
||||
expect(decide({ subagentType: 'test-designer', named: 'sonnet', prompt: 'x', respectExplicit: false }).kind).toBe('fixed')
|
||||
expect(decide({ subagentType: 'test-designer', named: 'haiku', prompt: 'find x', respectExplicit: false })).toEqual({ kind: 'fixed', tier: 'opus', reason: 'test-designer default' })
|
||||
// a design clone path carries no role: no record, so never verified as dev work
|
||||
expect(parseBrief('Clone: /home/homelab/projects/goodwe/clones/EV-15-design').role).toBe(undefined)
|
||||
})
|
||||
|
||||
test('pipeline: the verifier is told to check every T-id and N-id of the test plan', async ($, on) => {
|
||||
const w = fresh()
|
||||
w.diffs = ['diff --git a/x.py']
|
||||
const clock = world(on, w)
|
||||
const r = await $.agent.spawn({ prompt: devBrief('\n## Test plan\n| T1 | boundary | batteries above house | no alarm | yes |'), description: 'ev15' } as never)
|
||||
await finish($, clock, r.agentId as string)
|
||||
expect(w.verifierSystems[0]).toContain('every T-id must be implemented')
|
||||
expect(w.verifierPrompts[0]).toContain('## Test plan')
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user