test(core): Add routing mode to Instance AI discovery evals (no-changelog)

Add a routing mode to the discovery eval harness. It runs one user message
through the real orchestrator, stops at the first action that commits to a
route, and grades the route against the accepted routes of the case.

- Load routing cases from a directory or from a LangTracer suite. Cases can
  seed workflows, Agents, prior messages and failed runs, and attach a
  workflow or Agent to the message.
- Resolve the route from tool calls. A small judge classifies text-only
  replies and the steer of clarifying questions.
- Write routing results, a markdown summary, and an eval-results.json in the
  LangTracer dispatcher contract.
- Add a grader with per-bucket reports and run comparison, and a push script
  for LangTracer suites.
- Add a manual workflow that runs a routing suite.

.github/WORKFLOWS.md

@@ -421,6 +421,7 @@ Run-workflow dropdown, which only a user with write access can pick.
 | `util-codespace-preview.yml`| Wake, re-serve or delete a PR preview instance by hand   |
 | `util-data-tooling.yml`     | SQLite/PostgreSQL export/import validation (manual)     |
 | `util-probe-registry.yml`   | Diagnose slow npm metadata fetches (temporary)          |
+| `test-evals-routing.yml`    | Instance AI routing evals on a LangTracer suite (`eval:discovery --stop-on-route`) |
 
 ---
 

.github/workflows/test-evals-routing.yml

@@ -0,0 +1,101 @@
+name: 'Test: Instance AI Routing Evals'
+run-name: "Test: Instance AI Routing Evals (branch=${{ inputs.branch || 'master' }}, suite=${{ inputs.suite || 'intent-routing' }}, model=${{ inputs.model || 'anthropic/claude-opus-5-5' }}, iterations=${{ inputs.iterations || '3' }}, filter=${{ inputs.filter || 'all' }})"
+
+# Manual only. LangTracer's dispatcher runs cases with eval:instance-ai, so the
+# routing suite has no scheduled lane until the dispatcher can run eval:discovery.
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'GitHub branch to test'
+        required: false
+        default: 'master'
+      suite:
+        description: 'LangTracer suite slug or id with the routing cases'
+        required: false
+        default: 'intent-routing'
+      model:
+        description: 'Orchestrator model (provider/model)'
+        required: false
+        default: 'anthropic/claude-opus-5-5'
+      iterations:
+        description: 'Trials per case'
+        required: false
+        default: '3'
+      filter:
+        description: 'Comma-separated case-id tokens; an exact id runs only that case. Empty = all cases.'
+        required: false
+        default: ''
+
+permissions:
+  contents: read
+
+jobs:
+  run-routing-evals:
+    name: 'Run Routing Evals'
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    timeout-minutes: 90
+    env:
+      NODE_OPTIONS: '--max-old-space-size=7168'
+      OUTPUT_DIR: ${{ github.workspace }}/routing-eval-output
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.branch || github.ref }}
+          fetch-depth: 1
+
+      - name: Setup Environment
+        uses: ./.github/actions/setup-nodejs
+
+      - name: Export Node Types
+        run: |
+          ./packages/cli/bin/n8n export:nodes --output ./packages/@n8n/ai-workflow-builder.ee/evaluations/nodes.json
+
+      - name: Run Routing Evals
+        working-directory: packages/@n8n/instance-ai
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
+          N8N_ENABLED_MODULES: agents,instance-ai
+          LANGTRACER_URL: ${{ secrets.EVALS_LANGTRACER_URL }}
+          LANGTRACER_API_KEY: ${{ secrets.LANGTRACER_API_KEY }}
+          SUITE: ${{ inputs.suite || 'intent-routing' }}
+          MODEL: ${{ inputs.model || 'anthropic/claude-opus-5-5' }}
+          ITERATIONS: ${{ inputs.iterations || '3' }}
+          FILTER: ${{ inputs.filter }}
+        run: |
+          ARGS=(
+            --source langtracer
+            --suite "$SUITE"
+            --stop-on-route
+            --trials "$ITERATIONS"
+            --model "$MODEL"
+            --scenario-concurrency 4
+            --output-dir "$OUTPUT_DIR"
+            # The cases let evaluations/routing/grade.ts re-grade or compare the artifact.
+            --save-cases-dir "$OUTPUT_DIR/cases"
+          )
+          if [ -n "$FILTER" ]; then
+            ARGS+=(--filter "$FILTER")
+          fi
+          pnpm eval:discovery "${ARGS[@]}"
+
+      - name: Write job summary
+        if: ${{ always() }}
+        run: |
+          SUMMARY_FILE="$OUTPUT_DIR/routing-summary.md"
+          if [ -f "$SUMMARY_FILE" ]; then
+            cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "No routing summary was written. Check the 'Run Routing Evals' step log." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload Results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: instance-ai-routing-eval-results
+          path: routing-eval-output/
+          if-no-files-found: warn
+          retention-days: 14

packages/@n8n/instance-ai/evaluations/README.md

@@ -21,6 +21,7 @@ Sections:
 - [Regression detection](#regression-detection)
 - [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows)
 - [Running discovery evals](#discovery-evals)
+- [Running routing evals](#routing-evals)
 - [Running pairwise evals](#pairwise-evals)
 - [How the e2e harness works](#how-the-e2e-harness-works)
 - [How the workflow-build harness works](#how-the-workflow-build-harness-works)
@@ -564,6 +565,117 @@ For data-table routing, look for `load_skill(skillId="data-table-manager")`
 and `data-tables(action="list")`, and verify there are no planning,
 workflow-builder, or spawned-agent entries in the spawned-agent section.
 
+## Routing evals
+
+Routing evals measure how often the Assistant takes an acceptable path for a
+prompt: build an Agent, build a workflow, do the task once, ask a question, and
+so on. They use the discovery harness in routing mode. With `--stop-on-route`,
+a trial stops at the first call that commits to a route, so each trial costs
+one routing decision, not a full build. The grader then gives each trial a
+route and checks it against the case's accept tokens.
+
+Each case has a `bucket` (its main expected route) and one or more `accepts`
+tokens. A trial passes when its route matches one of the tokens. A case passes
+when at least 2 of 3 trials pass. `routing/grade-resolve.ts` has the exact
+rules and their order.
+
+| Route | The trial | Accept tokens |
+|-------|-----------|---------------|
+| `agent` | calls `build-agent` | `agent` |
+| `workflow` | calls `build-workflow` | `workflow` |
+| `one-off` | calls `build-workflow` with `executionIntent: "one-off"`, executes a node, runs or stops an execution, writes rows, or changes a workflow | `one-off` |
+| `multi` | calls `create-tasks` | `multi` |
+| `debug` | reads executions (`get`, `list`, or `debug`) | `debug` |
+| `clarify` | calls `ask-user`, or replies with a question | `clarify` (any direction), `clarify:agent` (points to an Agent, or to both), `clarify:open` (does not push a workflow only) |
+| `answer` | replies and commits to nothing | `answer` |
+| `decline` | refuses the request | `decline` |
+
+An LLM judge reads `ask-user` cards and text-only replies to tell `clarify`,
+`answer`, and `decline` apart, and to find the direction of a question. An
+`agent` or `workflow` case also passes when the Assistant asks a question that
+points the same way. The report shows this score next to the older "strict"
+score, which does not count those questions.
+
+### Cases in LangTracer
+
+The cases live in the LangTracer suite `intent-routing`, not in this
+repository. A LangTracer case has no routing fields, so the routing labels are
+tags in `evalTags`: `routing`, `bucket:<bucket>`, one `accepts:<token>` for each
+token, `policy-dependent`, `agent-shaped`, `source:<source>`, and `lang:<code>`.
+The case name is the case id. The prompt is the single conversation turn.
+
+Each case also has one process expectation, `Routes to one of: <tokens>`.
+LangTracer uses this text to match a run's result to the case. The runner
+ignores it and reads the accept tokens from the tags. `setKind` is
+`regression` when the case passes on the baseline model, else
+`capability_gap`.
+
+### Run locally
+
+```bash
+# From packages/@n8n/instance-ai/
+ANTHROPIC_API_KEY=... N8N_ENABLED_MODULES=agents,instance-ai \
+  pnpm eval:discovery --source langtracer --suite intent-routing --stop-on-route \
+  --trials 3 --model anthropic/claude-opus-5-5 --scenario-concurrency 4 \
+  --output-dir .data/routing/baseline --save-cases-dir .data/routing/cases
+```
+
+- `--source langtracer --suite intent-routing` pulls the cases from
+  LangTracer. Set `LANGTRACER_URL` and `LANGTRACER_API_KEY`. If they are not
+  set, the `lang-tracer` MCP entry in `~/.claude.json` supplies them.
+- `--cases-dir <dir>` reads the cases from a local directory instead. It reads
+  routing case files (`route-*.json`) and LangTracer suite-export bodies.
+- `--output-dir <dir>` grades each case when it finishes. It writes
+  `routing-results.json` (every trial), `routing-summary.md` (a short summary),
+  and `eval-results.json` (the format LangTracer reads).
+- `--save-cases-dir <dir>` writes the loaded cases, one `<id>.json` file for
+  each case. The grader needs them.
+- `--skill-file <skillId>=<path>` runs the orchestrator with a different
+  `SKILL.md` for one runtime skill, for example
+  `--skill-file intent-recognition=./SKILL.md`. Repeat the flag for more
+  skills. Sub-agents still use the bundled skills.
+- `--filter <csv>` runs only the case ids that contain one of the tokens.
+
+### Grade and compare
+
+```bash
+pnpm tsx evaluations/routing/grade.ts --results .data/routing/baseline/routing-results.json \
+  --cases-dir .data/routing/cases --out .data/routing/baseline/report.md \
+  --compare .data/routing/previous/routing-results.json
+```
+
+The grader writes a markdown report and a JSON summary next to it. The report
+shows the macro accuracy (the mean of the case pass rates of all buckets), the
+score of each bucket, the skill-load rate, a confusion table, and the failing
+cases. `--compare` is optional. It adds the change for each bucket and the
+cases that flipped. The judge needs `ANTHROPIC_API_KEY` unless its cache
+already has every verdict.
+
+### Push cases
+
+```bash
+pnpm tsx evaluations/routing/langtracer-push.ts --suite intent-routing \
+  --cases-dir <dir> --passing-ids <file> --dry-run
+```
+
+The push creates or updates each case by name and leaves unchanged cases as
+they are. `--passing-ids` is a JSON list of the case ids that pass on the
+baseline. Those cases get `setKind: regression`, all others
+`capability_gap`. Remove `--dry-run` to write. After a write, the script reads
+the suite back and fails when a case does not match the local file.
+
+### CI
+
+Dispatch **Test: Instance AI Routing Evals**
+(`.github/workflows/test-evals-routing.yml`) to run a suite on a branch. It has
+no pull-request or schedule trigger. The job writes the summary to the run page
+and uploads the output directory as an artifact.
+
+LangTracer's dispatcher runs every case with `eval:instance-ai`, which builds
+and verifies a workflow. Routing cases need `eval:discovery`. Do not schedule
+the `intent-routing` suite in LangTracer until the dispatcher can run
+`eval:discovery`.
+
 ## Pairwise evals
 
 Pairwise evals score a built workflow against the dataset's `dos` / `donts`

packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts

@@ -5,7 +5,12 @@ import { tmpdir } from 'os';
 import { join } from 'path';
 
 import type { CheckOutcome } from '../binaryChecks/types';
+import { RoutingEvalOutput } from '../discovery/routing-eval-results';
+import type { RoutingTrialRecord } from '../discovery/types';
 import { AGENT_ARTIFACT_CASE_CAP_BYTES } from '../harness/artifacts/agent-artifact';
+import { routingExpectationText } from '../routing/expectation';
+import type { JudgeInput, JudgeResult } from '../routing/judge';
+import { parseRoutingCase } from '../routing/loader';
 import { aggregateResults } from '../run/aggregator';
 import { writeEvalResults } from '../run/persist';
 import type {
@@ -139,15 +144,19 @@ interface DispatcherView {
 		agentArtifactPerRun: Array<Record<string, unknown> | null>;
 		totalRuns: number;
 		workflowChecksPerRun: Array<Record<string, string> | null>;
+		status?: string;
 		buildExpectations: Array<{
 			expectation: string;
 			passCount: number;
 			evaluatedCount: number;
+			passAtK?: number;
+			passHatK?: number;
 		}>;
 		buildExpectationResultsPerRun: Array<Array<{
 			expectation: string;
 			pass: boolean;
 			reason: string;
+			incomplete?: boolean;
 			attribution?: string;
 		}> | null>;
 		buildCostUsdPerRun?: Array<number | null>;
@@ -415,3 +424,134 @@ describe('eval-results.json — dispatcher contract', () => {
 		expect(tc.buildTurnsPerRun).toEqual([5, null]);
 	});
 });
+
+// The discovery CLI's routing mode (`--output-dir`) writes the same file for a
+// LangTracer routing case. The dispatcher matches each expectation to the
+// case's stored process expectation by trimmed text, and passes it on a strict
+// majority of the evaluated (not `incomplete`) runs.
+describe('eval-results.json — dispatcher contract (routing)', () => {
+	const parsed = parseRoutingCase({
+		id: 'route-agent-support',
+		bucket: 'agent',
+		userMessage: 'Set up AI support in our group.',
+		accepts: ['agent', 'clarify:agent'],
+		source: 'synthetic',
+	});
+	if (!parsed.success) throw new Error(parsed.issues.join('; '));
+	const routingCase = parsed.data;
+
+	function trial(n: number, overrides: Partial<RoutingTrialRecord>): RoutingTrialRecord {
+		return {
+			trial: n,
+			durationMs: 1000,
+			streamStatus: 'stopped-on-route',
+			toolCalls: [],
+			subAgentToolCalls: [],
+			spawnedAgents: [],
+			skillsLoaded: [],
+			askUserQuestions: [],
+			finalText: '',
+			fullText: '',
+			...overrides,
+		};
+	}
+
+	const judge = {
+		judge: async (input: JudgeInput): Promise<JudgeResult> =>
+			await Promise.resolve({
+				verdict: {
+					kind: input.mode === 'ask-user' ? 'clarify' : 'answer',
+					steer: 'workflow',
+					reason: 'Asks which trigger starts the workflow.',
+				},
+				cached: false,
+			}),
+	};
+
+	async function writeAndReadRouting(): Promise<DispatcherView> {
+		const dir = mkdtempSync(join(tmpdir(), 'eval-results-routing-contract-'));
+		const output = new RoutingEvalOutput(
+			dir,
+			judge,
+			{
+				runId: 'run-1',
+				variant: 'baseline',
+				model: 'anthropic/claude-opus-5-5',
+				trialsPerCase: 3,
+				stopOnRoute: true,
+				startedAt: '2026-10-01T10:00:00.000Z',
+			},
+			1,
+		);
+		await output.record(0, {
+			routingCase,
+			fileName: 'lt-route-agent-support-1a2b3c4d',
+			result: {
+				id: routingCase.id,
+				userMessage: routingCase.userMessage,
+				trials: [
+					trial(1, { toolCalls: [{ toolName: 'build-agent', args: {}, status: 'pending' }] }),
+					trial(2, {
+						toolCalls: [
+							{
+								toolName: 'ask-user',
+								args: { questions: [{ question: 'Which trigger?', options: ['Schedule'] }] },
+								status: 'pending',
+							},
+						],
+					}),
+					trial(3, { streamStatus: 'timed-out' }),
+				],
+			},
+			threadIds: ['discovery-thread-a', 'discovery-thread-b', 'discovery-thread-c'],
+			transcripts: [[], [], []],
+		});
+		await output.finish();
+		return jsonParse<DispatcherView>(readFileSync(output.evalResultsPath, 'utf8'));
+	}
+
+	it('serializes the fields the dispatcher reads for a routing case', async () => {
+		const report = await writeAndReadRouting();
+
+		// The dispatcher ingests `testCases[0]` of a single-case run.
+		expect(report.testCases).toHaveLength(1);
+		const tc = report.testCases[0];
+		expect(tc.totalRuns).toBe(3);
+		expect(tc.status).toBe('verified');
+		expect(tc.threadIds).toEqual([
+			'discovery-thread-a',
+			'discovery-thread-b',
+			'discovery-thread-c',
+		]);
+
+		// The same text the LangTracer push stores as the case's process expectation.
+		const expectation = routingExpectationText(routingCase.accepts);
+		expect(tc.buildExpectations).toEqual([
+			{ expectation, passCount: 1, evaluatedCount: 2, passAtK: 1, passHatK: 0.25 },
+		]);
+		expect(tc.buildExpectationResultsPerRun).toEqual([
+			[{ expectation, pass: true, reason: expect.stringContaining('Route agent') }],
+			[
+				{
+					expectation,
+					pass: false,
+					reason: expect.stringContaining('Route clarify:workflow'),
+					attribution: 'builder_issue',
+				},
+			],
+			[
+				{
+					expectation,
+					pass: false,
+					reason: expect.stringContaining('timed out'),
+					incomplete: true,
+					attribution: 'timeout',
+				},
+			],
+		]);
+
+		// Dispatcher rule: a strict majority of the evaluated runs. 1 of 2 fails.
+		const [unit] = tc.buildExpectations;
+		expect(unit.passCount * 2 > unit.evaluatedCount).toBe(false);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/cli-args.test.ts

@@ -0,0 +1,79 @@
+import { resolve } from 'node:path';
+
+import { isRoutingMode, parseCliArgs, parseSkillOverride } from '../cli-args';
+
+describe('parseCliArgs', () => {
+	it('accepts the dispatcher flag names as aliases', () => {
+		const args = parseCliArgs([
+			'--cases-dir',
+			'/tmp/cases',
+			'--filter',
+			'lt-route-agent-x-1a2b3c4d',
+			'--iterations',
+			'2',
+			'--timeout-ms',
+			'90000',
+			'--output-dir',
+			'/tmp/out',
+		]);
+
+		expect(args).toMatchObject({
+			casesDir: '/tmp/cases',
+			filter: 'lt-route-agent-x-1a2b3c4d',
+			trials: 2,
+			timeoutMs: 90_000,
+			outputDir: '/tmp/out',
+		});
+		expect(isRoutingMode(args)).toBe(true);
+		expect(parseCliArgs(['--trials', '4', '--timeout', '5000'])).toMatchObject({
+			trials: 4,
+			timeoutMs: 5000,
+		});
+	});
+
+	it('needs a case source for routing-only flags', () => {
+		expect(() => parseCliArgs(['--stop-on-route'])).toThrow(
+			'--stop-on-route works only in routing mode: pass --cases-dir <dir>, or --source langtracer --suite <slug|id>.',
+		);
+		expect(() => parseCliArgs(['--output-dir', '/tmp/out'])).toThrow(
+			/--output-dir works only in routing mode/,
+		);
+		expect(() => parseCliArgs(['--source', 'local'])).toThrow(
+			'--source local needs --cases-dir <dir>.',
+		);
+		expect(() => parseCliArgs(['--source', 'langtracer'])).toThrow(
+			'--source langtracer needs --suite <slug|id>.',
+		);
+		expect(
+			isRoutingMode(parseCliArgs(['--source', 'langtracer', '--suite', 'intent-routing'])),
+		).toBe(true);
+		expect(isRoutingMode(parseCliArgs([]))).toBe(false);
+	});
+
+	it('refuses unknown flags and invalid values', () => {
+		expect(() => parseCliArgs(['--iteration', '3'])).toThrow('Unknown argument: --iteration');
+		expect(() => parseCliArgs(['--iterations', '0'])).toThrow(/expected a positive integer/);
+		expect(() => parseCliArgs(['--filter'])).toThrow('Missing value for --filter');
+	});
+
+	it('collects repeated --skill-file values', () => {
+		const args = parseCliArgs([
+			'--skill-file',
+			'intent-recognition=skills/variant.md',
+			'--skill-file',
+			'planning=/abs/planning.md',
+		]);
+
+		expect(args.skillFiles).toEqual([
+			{ skillId: 'intent-recognition', path: resolve('skills/variant.md') },
+			{ skillId: 'planning', path: '/abs/planning.md' },
+		]);
+		expect(() => parseSkillOverride('intent-recognition')).toThrow(
+			'--skill-file expects <skillId>=<path>, got "intent-recognition"',
+		);
+	});
+
+	it('stops at --help', () => {
+		expect(parseCliArgs(['--help', '--bogus']).help).toBe(true);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/grade-resolve.test.ts

@@ -0,0 +1,225 @@
+import type { GradedCase, GradedTrial } from '../../routing/grade-report';
+import { computeScopeMetrics } from '../../routing/grade-report';
+import {
+	isSameDirectionClarify,
+	resolveTrial,
+	type RouteResolution,
+	staleStopCall,
+	strictTrialPasses,
+	trialPasses,
+} from '../../routing/grade-resolve';
+import type { Bucket, RoutingCase, TrialResult, TrialToolCall } from '../../routing/grade-types';
+
+function routingCase(bucket: Bucket, accepts: RoutingCase['accepts']): RoutingCase {
+	return {
+		id: `case-${bucket}`,
+		bucket,
+		userMessage: 'Do the thing.',
+		accepts,
+		policyDependent: false,
+	};
+}
+
+function call(toolName: string, args: Record<string, unknown> = {}): TrialToolCall {
+	return { toolName, args };
+}
+
+function trial(toolCalls: TrialToolCall[], overrides: Partial<TrialResult> = {}): TrialResult {
+	return {
+		trial: 1,
+		streamStatus: 'completed',
+		toolCalls,
+		spawnedAgents: [],
+		skillsLoaded: [],
+		askUserQuestions: [],
+		finalText: 'Here is what I found.',
+		...overrides,
+	};
+}
+
+function resolvedRoute(calls: TrialToolCall[]) {
+	const pending = resolveTrial(routingCase('workflow', ['workflow']), trial(calls));
+	return pending.kind === 'resolved'
+		? { kind: pending.kind, route: pending.resolution.route, rule: pending.resolution.rule }
+		: { kind: pending.kind, rule: pending.rule };
+}
+
+const clarify = (steer: RouteResolution['steer']): RouteResolution => ({
+	route: 'clarify',
+	steer,
+	rule: 6,
+	evidence: 'ask-user',
+});
+
+describe('resolveTrial', () => {
+	it('looks past data-table schema setup to the build that follows', () => {
+		expect(
+			resolvedRoute([
+				call('data-tables', { action: 'create' }),
+				call('build-workflow', { executionIntent: 'recurring' }),
+			]),
+		).toEqual({ kind: 'resolved', route: 'workflow', rule: 2 });
+	});
+
+	it('does not route data-table schema setup alone to one-off', () => {
+		expect(resolvedRoute([call('data-tables', { action: 'create' })])).toEqual({
+			kind: 'judge',
+			rule: 7,
+		});
+	});
+
+	it('routes a data-table row write to one-off', () => {
+		expect(
+			resolvedRoute([
+				call('data-tables', { action: 'create' }),
+				call('data-tables', { action: 'insert-rows' }),
+			]),
+		).toEqual({ kind: 'resolved', route: 'one-off', rule: 4 });
+	});
+
+	it('does not route a debugging-executions skill load alone to debug', () => {
+		expect(resolvedRoute([call('load_skill', { skillId: 'debugging-executions' })])).toEqual({
+			kind: 'judge',
+			rule: 7,
+		});
+	});
+
+	it.each(['list', 'get', 'debug'])('routes executions %s to debug', (action) => {
+		expect(
+			resolvedRoute([
+				call('load_skill', { skillId: 'debugging-executions' }),
+				call('executions', { action }),
+			]),
+		).toEqual({ kind: 'resolved', route: 'debug', rule: 5 });
+	});
+});
+
+describe('same-direction clarification', () => {
+	it.each([
+		['workflow', 'workflow', true],
+		['workflow', 'both', true],
+		['workflow', 'agent', false],
+		['workflow', 'none', false],
+		['agent', 'agent', true],
+		['agent', 'both', true],
+		['agent', 'workflow', false],
+		['agent', 'none', false],
+		['one-off', 'both', false],
+		['clarify', 'agent', false],
+	] as const)('bucket %s with steer %s: %s', (bucket, steer, expected) => {
+		expect(isSameDirectionClarify(bucket, clarify(steer))).toBe(expected);
+	});
+
+	it('does not count a non-clarify route', () => {
+		expect(
+			isSameDirectionClarify('workflow', {
+				route: 'workflow',
+				rule: 2,
+				evidence: 'build-workflow',
+			}),
+		).toBe(false);
+	});
+
+	it('passes the v2 score but not the strict score', () => {
+		const workflowCase = routingCase('workflow', ['workflow']);
+		expect(trialPasses(workflowCase, clarify('workflow'))).toBe(true);
+		expect(strictTrialPasses(workflowCase, clarify('workflow'))).toBe(false);
+
+		const agentCase = routingCase('agent', ['agent']);
+		expect(trialPasses(agentCase, clarify('both'))).toBe(true);
+		expect(strictTrialPasses(agentCase, clarify('both'))).toBe(false);
+		expect(trialPasses(agentCase, clarify('workflow'))).toBe(false);
+	});
+
+	it('keeps accept tokens in both scores', () => {
+		const clarifyCase = routingCase('clarify', ['clarify:open']);
+		expect(trialPasses(clarifyCase, clarify('none'))).toBe(true);
+		expect(strictTrialPasses(clarifyCase, clarify('none'))).toBe(true);
+		expect(trialPasses(clarifyCase, clarify('workflow'))).toBe(false);
+	});
+});
+
+describe('staleStopCall', () => {
+	it('flags a stopped trial whose calls no longer commit', () => {
+		const stopped = trial(
+			[call('data-tables', { action: 'list' }), call('data-tables', { action: 'create' })],
+			{ streamStatus: 'stopped-on-route' },
+		);
+		expect(staleStopCall(stopped)).toBe('data-tables create');
+	});
+
+	it('accepts a stopped trial that ends on a committing call', () => {
+		const stopped = trial([call('data-tables', { action: 'create' }), call('build-workflow')], {
+			streamStatus: 'stopped-on-route',
+		});
+		expect(staleStopCall(stopped)).toBeUndefined();
+	});
+
+	it('ignores trials that were not stopped', () => {
+		expect(staleStopCall(trial([call('data-tables', { action: 'create' })]))).toBeUndefined();
+	});
+});
+
+describe('computeScopeMetrics', () => {
+	function gradedTrial(
+		pass: boolean,
+		strictPass: boolean,
+		sameDirectionClarify: boolean,
+	): GradedTrial {
+		return {
+			trial: 1,
+			streamStatus: 'completed',
+			skillLoaded: false,
+			resolution: clarify('workflow'),
+			label: 'clarify:workflow',
+			pass,
+			strictPass,
+			sameDirectionClarify,
+			judged: true,
+		};
+	}
+
+	function gradedCase(bucket: Bucket, trials: GradedTrial[]): GradedCase {
+		const passedTrials = trials.filter((entry) => entry.pass).length;
+		const strictPassedTrials = trials.filter((entry) => entry.strictPass).length;
+		return {
+			id: `case-${bucket}`,
+			bucket,
+			accepts: [],
+			policyDependent: false,
+			trials,
+			passedTrials,
+			pass: passedTrials * 3 >= trials.length * 2,
+			strictPassedTrials,
+			strictPass: strictPassedTrials * 3 >= trials.length * 2,
+		};
+	}
+
+	it('reports strict scores and over-asking over the agent and workflow buckets', () => {
+		const metrics = computeScopeMetrics([
+			gradedCase('workflow', [
+				gradedTrial(true, false, true),
+				gradedTrial(true, false, true),
+				gradedTrial(true, true, false),
+			]),
+			gradedCase('agent', [
+				gradedTrial(true, true, false),
+				gradedTrial(true, true, false),
+				gradedTrial(false, false, false),
+			]),
+			gradedCase('clarify', [
+				gradedTrial(true, true, false),
+				gradedTrial(true, true, false),
+				gradedTrial(true, true, false),
+			]),
+		]);
+
+		expect(metrics.macroAccuracy).toBe(1);
+		expect(metrics.strictMacroAccuracy).toBeCloseTo(2 / 3);
+		expect(metrics.strictTrialPass).toMatchObject({ num: 6, den: 9 });
+		expect(metrics.overAsking).toMatchObject({ num: 2, den: 6 });
+		expect(metrics.buckets.workflow?.overAsking).toMatchObject({ num: 2, den: 3 });
+		expect(metrics.buckets.agent?.overAsking).toMatchObject({ num: 0, den: 3 });
+		expect(metrics.buckets.clarify?.overAsking).toBeUndefined();
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/route-rules.test.ts

@@ -0,0 +1,72 @@
+import {
+	isCommittingCall,
+	isCommittingDataTablesAction,
+	isMutatingDataTablesAction,
+} from '../../routing/route-rules';
+
+describe('isCommittingCall', () => {
+	it.each([
+		['build-agent', { operation: 'create' }],
+		['build-workflow', {}],
+		['create-tasks', {}],
+		['ask-user', { questions: [] }],
+		['nodes', { action: 'execute' }],
+		['executions', { action: 'run' }],
+		['executions', { action: 'debug' }],
+		['executions', { action: 'stop' }],
+		['data-tables', { action: 'insert-rows' }],
+		['data-tables', { action: 'update-rows' }],
+		['data-tables', { action: 'delete-rows' }],
+		['data-tables', { action: 'upsert-rows' }],
+		['data-tables', { action: 'delete' }],
+		['workflows', { action: 'delete' }],
+		['workflows', { action: 'publish' }],
+	])('treats %s %j as committing', (toolName, args) => {
+		expect(isCommittingCall(toolName, args)).toBe(true);
+	});
+
+	it.each([
+		['load_skill', { skillId: 'intent-recognition' }],
+		['load_skill', { skillId: 'debugging-executions' }],
+		['search_tools', { query: 'agent' }],
+		['n8n-docs', { query: 'credentials' }],
+		['research', { action: 'web-search' }],
+		['nodes', { action: 'search' }],
+		['executions', { action: 'list' }],
+		['executions', { action: 'get' }],
+		['data-tables', { action: 'list' }],
+		['data-tables', { action: 'query' }],
+		['data-tables', { action: 'schema' }],
+		['data-tables', { action: 'create' }],
+		['data-tables', { action: 'add-column' }],
+		['data-tables', { action: 'rename-column' }],
+		['data-tables', { action: 'delete-column' }],
+		['data-tables', {}],
+		['workflows', { action: 'list' }],
+		['workflows', { action: 'get-as-code' }],
+		['workflows', { action: 'validate' }],
+		['workflows', {}],
+		['credentials', { action: 'list' }],
+	])('treats %s %j as exploration', (toolName, args) => {
+		expect(isCommittingCall(toolName, args)).toBe(false);
+	});
+});
+
+describe('data-tables actions', () => {
+	it('counts schema setup as mutating but not committing', () => {
+		expect(isMutatingDataTablesAction('create')).toBe(true);
+		expect(isCommittingDataTablesAction('create')).toBe(false);
+	});
+
+	it('counts row writes as mutating and committing', () => {
+		expect(isMutatingDataTablesAction('insert-rows')).toBe(true);
+		expect(isCommittingDataTablesAction('insert-rows')).toBe(true);
+	});
+
+	it('counts reads and a missing action as neither', () => {
+		for (const action of ['list', 'get', 'query', 'schema', undefined]) {
+			expect(isMutatingDataTablesAction(action)).toBe(false);
+			expect(isCommittingDataTablesAction(action)).toBe(false);
+		}
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-case-dir.test.ts

@@ -0,0 +1,118 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { routingExpectationText } from '../../routing/expectation';
+import { toLangTracerCreateBody } from '../../routing/langtracer-cases';
+import { parseRoutingCase, type RoutingCase } from '../../routing/loader';
+import { isRoutingExportBody, loadRoutingCaseDir } from '../routing-case-dir';
+
+function routingCase(raw: Record<string, unknown> = {}): RoutingCase {
+	const parsed = parseRoutingCase({
+		id: 'route-agent-support',
+		bucket: 'agent',
+		userMessage: 'Set up AI support in our group.',
+		accepts: ['agent', 'clarify:agent'],
+		source: 'synthetic',
+		rationale: 'Ongoing conversational support role.',
+		...raw,
+	});
+	if (!parsed.success) throw new Error(parsed.issues.join('; '));
+	return parsed.data;
+}
+
+/** The body LangTracer exports (and the dispatcher writes) for a pushed routing case. */
+function exportBody(source: RoutingCase): Record<string, unknown> {
+	const body = toLangTracerCreateBody(source, { suiteId: 64, setKind: 'regression' });
+	return {
+		complexity: body.evalComplexity,
+		tags: body.evalTags,
+		conversation: body.conversation,
+		description: body.description,
+		processExpectations: [routingExpectationText(source.accepts)],
+	};
+}
+
+function caseDir(files: Record<string, unknown>): string {
+	const dir = mkdtempSync(join(tmpdir(), 'routing-case-dir-'));
+	for (const [name, body] of Object.entries(files)) {
+		writeFileSync(join(dir, name), typeof body === 'string' ? body : JSON.stringify(body));
+	}
+	return dir;
+}
+
+describe('loadRoutingCaseDir', () => {
+	it('reads authored case files and export bodies side by side', () => {
+		const dir = caseDir({
+			'route-debug-failed-run.json': routingCase({
+				id: 'route-debug-failed-run',
+				bucket: 'debug',
+				accepts: ['debug', 'clarify'],
+			}),
+			'route-agent-support.json': exportBody(routingCase()),
+			'results.json': { cases: [] },
+			'notes.json': 'not json',
+		});
+
+		const loaded = loadRoutingCaseDir(dir);
+
+		expect(loaded.map((c) => [c.routingCase.id, c.fileName])).toEqual([
+			['route-agent-support', 'route-agent-support'],
+			['route-debug-failed-run', 'route-debug-failed-run'],
+		]);
+		expect(loaded[0].routingCase).toMatchObject({
+			bucket: 'agent',
+			accepts: ['agent', 'clarify:agent'],
+			rationale: 'Ongoing conversational support role.',
+		});
+		expect(loaded[0].scenario.userMessage).toBe('Set up AI support in our group.');
+	});
+
+	it('uses the file name as the id of an export body without a name', () => {
+		const dir = caseDir({ 'lt-route-agent-support-1a2b3c4d.json': exportBody(routingCase()) });
+
+		const [loaded] = loadRoutingCaseDir(dir, { filter: 'lt-route-agent-support-1a2b3c4d' });
+
+		expect(loaded.routingCase.id).toBe('lt-route-agent-support-1a2b3c4d');
+		expect(loaded.scenario.id).toBe('lt-route-agent-support-1a2b3c4d');
+	});
+
+	it('uses the name of an export body when it carries one', () => {
+		const dir = caseDir({
+			'lt-case.json': { ...exportBody(routingCase()), name: 'route-agent-support' },
+		});
+
+		expect(loadRoutingCaseDir(dir, { filter: 'route-agent-support' })).toMatchObject([
+			{ routingCase: { id: 'route-agent-support' }, fileName: 'lt-case' },
+		]);
+	});
+
+	it('selects only the exact match when the filter names one case', () => {
+		const dir = caseDir({
+			'route-agent-support.json': exportBody(routingCase()),
+			'route-agent-support-2.json': exportBody(routingCase()),
+		});
+
+		expect(loadRoutingCaseDir(dir, { filter: 'route-agent-support' })).toHaveLength(1);
+		expect(loadRoutingCaseDir(dir, { filter: 'agent-support' })).toHaveLength(2);
+	});
+
+	it('reports every invalid case file in one error', () => {
+		const dir = caseDir({
+			'route-agent-bad.json': routingCase({ id: 'route-agent-other' }),
+			'lt-broken.json': { tags: ['routing', 'accepts:agent'], conversation: [] },
+		});
+
+		expect(() => loadRoutingCaseDir(dir)).toThrow(
+			/lt-broken\.json: no "bucket:" tag[\s\S]*route-agent-bad\.json: id "route-agent-other" must match the file name/,
+		);
+	});
+});
+
+describe('isRoutingExportBody', () => {
+	it('needs a routing tag', () => {
+		expect(isRoutingExportBody({ tags: ['routing'] })).toBe(true);
+		expect(isRoutingExportBody({ tags: ['agents'] })).toBe(false);
+		expect(isRoutingExportBody(routingCase())).toBe(false);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-context.test.ts

@@ -0,0 +1,94 @@
+import {
+	buildThreadArtifactsBlock,
+	buildThreadContextBlock,
+} from '../../../../../cli/src/modules/instance-ai/internal-messages';
+import { parseRoutingCase, toDiscoveryScenario } from '../../routing/loader';
+import {
+	buildRoutingTurnMessage,
+	createSeededMemory,
+	STUB_PROJECT_ID,
+	toResourceAttachment,
+} from '../routing-context';
+import type { RoutingSeed } from '../types';
+
+function seedOf(raw: Record<string, unknown>): RoutingSeed {
+	const parsed = parseRoutingCase({
+		id: 'route-answer-context',
+		bucket: 'answer',
+		userMessage: 'What does it do?',
+		accepts: ['answer'],
+		source: 'synthetic',
+		seed: { mode: 'inline', ...raw },
+	});
+	if (!parsed.success) throw new Error(parsed.issues.join('; '));
+	const { seed } = toDiscoveryScenario(parsed.data);
+	if (!seed) throw new Error('expected a seed');
+	return seed;
+}
+
+const seed = seedOf({
+	workflows: [{ id: 'wf-context-01', name: 'Order sync', nodes: [], connections: {} }],
+	agents: [
+		{
+			id: 'agent-context-01',
+			config: { name: 'Order bot', model: 'anthropic/claude-sonnet-4-5', instructions: 'Help.' },
+		},
+	],
+	messages: [
+		{ role: 'user', text: 'Build an order sync.' },
+		{ role: 'assistant', text: 'Done: "Order sync" is saved.' },
+	],
+});
+
+describe('toResourceAttachment', () => {
+	it('names the seeded workflow or Agent the way the editor hands it off', () => {
+		expect(toResourceAttachment({ workflow: 'wf-context-01' }, seed)).toEqual({
+			type: 'workflow',
+			id: 'wf-context-01',
+			name: 'Order sync',
+		});
+		expect(toResourceAttachment({ agent: 'agent-context-01' }, seed)).toEqual({
+			type: 'agent',
+			id: 'agent-context-01',
+			name: 'Order bot',
+			projectId: STUB_PROJECT_ID,
+		});
+	});
+
+	it('refuses an attachment the seed does not declare', () => {
+		expect(() => toResourceAttachment({ workflow: 'wf-missing-01' }, seed)).toThrow(
+			'is not a seeded workflow',
+		);
+	});
+});
+
+describe('buildRoutingTurnMessage', () => {
+	it('sends the user message unchanged without an attachment', () => {
+		expect(buildRoutingTurnMessage('Hello', undefined)).toBe('Hello');
+	});
+
+	it('prepends the thread-context block production builds for the hand-off', () => {
+		const attachment = toResourceAttachment({ workflow: 'wf-context-01' }, seed);
+		const message = buildRoutingTurnMessage('Change the schedule.', attachment);
+
+		const productionBlock = buildThreadContextBlock([
+			buildThreadArtifactsBlock(undefined, [attachment]),
+		]);
+		expect(message).toBe(`${productionBlock}\n\nChange the schedule.`);
+		expect(message.startsWith('<thread-context>\n<thread-artifacts>\n')).toBe(true);
+		expect(message).toContain('Workflow "Order sync" (id: `wf-context-01`)');
+	});
+});
+
+describe('createSeededMemory', () => {
+	it('holds the seeded messages as the thread history, in order', async () => {
+		const memory = await createSeededMemory('thread-1', 'eval-user', seed.messages);
+
+		const history = await memory.getMessages('thread-1', { resourceId: 'eval-user' });
+		expect(history.map((m) => ('role' in m ? m.role : m.type))).toEqual(['user', 'assistant']);
+		expect(history.every((m) => m.createdAt instanceof Date)).toBe(true);
+		expect(history[1]).toMatchObject({
+			content: [{ type: 'text', text: 'Done: "Order sync" is saved.' }],
+		});
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-eval-results.test.ts

@@ -0,0 +1,245 @@
+import { mkdtempSync, readFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import type { JudgeInput, JudgeResult } from '../../routing/judge';
+import { parseRoutingCase, type RoutingCase } from '../../routing/loader';
+import {
+	gradeRoutingCase,
+	RoutingEvalOutput,
+	toDispatcherTestCase,
+	type RoutingEvalMeta,
+} from '../routing-eval-results';
+import type { RoutingTrialRecord } from '../types';
+
+function routingCase(raw: Record<string, unknown> = {}): RoutingCase {
+	const parsed = parseRoutingCase({
+		id: 'route-agent-support',
+		bucket: 'agent',
+		userMessage: 'Set up AI support in our group.',
+		accepts: ['agent', 'clarify:agent'],
+		source: 'synthetic',
+		...raw,
+	});
+	if (!parsed.success) throw new Error(parsed.issues.join('; '));
+	return parsed.data;
+}
+
+function trial(n: number, overrides: Partial<RoutingTrialRecord> = {}): RoutingTrialRecord {
+	return {
+		trial: n,
+		durationMs: 1000,
+		streamStatus: 'stopped-on-route',
+		toolCalls: [],
+		subAgentToolCalls: [],
+		spawnedAgents: [],
+		skillsLoaded: [],
+		askUserQuestions: [],
+		finalText: '',
+		fullText: '',
+		...overrides,
+	};
+}
+
+const askUser = (question: string) => ({
+	toolName: 'ask-user',
+	args: { questions: [{ question, options: ['A', 'B'] }] },
+	status: 'pending' as const,
+});
+
+/** Steers by keyword; a final text of "THROW" makes the judge fail. */
+const judge = {
+	judge: async (input: JudgeInput): Promise<JudgeResult> => {
+		if (input.finalText === 'THROW') throw new Error('HTTP 529: overloaded');
+		const text = JSON.stringify(input.askUserQuestions);
+		const steer = text.includes('trigger')
+			? 'workflow'
+			: text.includes('persona')
+				? 'agent'
+				: 'none';
+		return await Promise.resolve({
+			verdict: { kind: input.mode === 'ask-user' ? 'clarify' : 'answer', steer, reason: 'test' },
+			cached: false,
+		});
+	},
+};
+
+const meta: RoutingEvalMeta = {
+	runId: 'run-1',
+	variant: 'baseline',
+	model: 'anthropic/claude-opus-5-5',
+	trialsPerCase: 5,
+	stopOnRoute: true,
+	startedAt: '2026-10-01T10:00:00.000Z',
+};
+
+const expectation = 'Routes to one of: agent, clarify:agent';
+
+async function gradedFixture() {
+	const source = routingCase();
+	return await gradeRoutingCase(
+		{
+			routingCase: source,
+			fileName: 'lt-route-agent-support-1a2b3c4d',
+			result: {
+				id: source.id,
+				userMessage: source.userMessage,
+				trials: [
+					trial(1, {
+						toolCalls: [{ toolName: 'build-agent', args: {}, status: 'pending' }],
+					}),
+					trial(2, { toolCalls: [askUser('Which trigger should start the workflow?')] }),
+					trial(3, { streamStatus: 'timed-out' }),
+					trial(4, { streamStatus: 'completed', finalText: 'THROW' }),
+					trial(5, { streamStatus: 'errored', runError: 'socket hang up' }),
+				],
+			},
+			threadIds: ['t-1', 't-2', 't-3', 't-4', 't-5'],
+			transcripts: [[], [], [], [], []],
+		},
+		judge,
+		meta,
+	);
+}
+
+describe('toDispatcherTestCase', () => {
+	it('turns each trial into one verdict on the routing expectation', async () => {
+		const testCase = toDispatcherTestCase(await gradedFixture());
+
+		expect(testCase.buildExpectationResultsPerRun).toEqual([
+			[
+				{
+					expectation,
+					pass: true,
+					reason: 'Route agent (rule 1, build-agent). Accepted.',
+				},
+			],
+			[
+				{
+					expectation,
+					pass: false,
+					reason:
+						'Route clarify:workflow (rule 6, ask-user). Judge: test. Not in the accepted routes: agent, clarify:agent.',
+					attribution: 'builder_issue',
+				},
+			],
+			[
+				{
+					expectation,
+					pass: false,
+					reason: 'Not graded: the run timed out before a committing call (timed-out).',
+					incomplete: true,
+					attribution: 'timeout',
+				},
+			],
+			[
+				{
+					expectation,
+					pass: false,
+					reason: 'Not graded: the judge failed (HTTP 529: overloaded).',
+					incomplete: true,
+					attribution: 'verification_gap',
+				},
+			],
+			[
+				{
+					expectation,
+					pass: false,
+					reason: 'Not graded: the run failed before a committing call (errored: socket hang up).',
+					incomplete: true,
+					attribution: 'framework_issue',
+				},
+			],
+		]);
+		expect(testCase).toMatchObject({
+			name: 'route-agent-support',
+			testCaseFile: 'lt-route-agent-support-1a2b3c4d',
+			status: 'verified',
+			totalRuns: 5,
+			buildExpectations: [
+				{ expectation, passCount: 1, evaluatedCount: 2, passAtK: 1, passHatK: 0.25 },
+			],
+			threadIds: ['t-1', 't-2', 't-3', 't-4', 't-5'],
+		});
+	});
+
+	it('passes a same-direction clarification under the v2 score', async () => {
+		const source = routingCase({
+			id: 'route-workflow-digest',
+			bucket: 'workflow',
+			accepts: ['workflow'],
+		});
+		const graded = await gradeRoutingCase(
+			{
+				routingCase: source,
+				result: {
+					id: source.id,
+					userMessage: source.userMessage,
+					trials: [trial(1, { toolCalls: [askUser('Which trigger should start it?')] })],
+				},
+				threadIds: ['t-1'],
+				transcripts: [[]],
+			},
+			judge,
+			meta,
+		);
+
+		const [[verdict]] = toDispatcherTestCase(graded).buildExpectationResultsPerRun;
+		expect(verdict).toEqual({
+			expectation: 'Routes to one of: workflow',
+			pass: true,
+			reason:
+				'Route clarify:workflow (rule 6, ask-user). Judge: test. Accepted as a same-direction clarification.',
+		});
+	});
+
+	it('marks a case with no graded trial as not verified', async () => {
+		const source = routingCase();
+		const graded = await gradeRoutingCase(
+			{
+				routingCase: source,
+				result: {
+					id: source.id,
+					userMessage: source.userMessage,
+					trials: [trial(1, { streamStatus: 'timed-out' })],
+				},
+				threadIds: ['t-1'],
+				transcripts: [[]],
+			},
+			judge,
+			meta,
+		);
+
+		expect(toDispatcherTestCase(graded)).toMatchObject({
+			status: 'notVerified',
+			buildExpectations: [{ passCount: 0, evaluatedCount: 0, passAtK: 0, passHatK: 0 }],
+		});
+	});
+});
+
+describe('RoutingEvalOutput', () => {
+	it('writes eval-results.json and the summary after each case', async () => {
+		const dir = mkdtempSync(join(tmpdir(), 'routing-eval-output-'));
+		const output = new RoutingEvalOutput(dir, judge, meta, 1);
+		const { input } = await gradedFixture();
+
+		await output.record(0, input);
+		const partial: unknown = JSON.parse(readFileSync(output.evalResultsPath, 'utf8'));
+		expect(partial).toMatchObject({
+			complete: false,
+			testCases: [{ name: 'route-agent-support' }],
+		});
+
+		const final = await output.finish();
+		expect(final.summary).toEqual({ testCases: 1, passed: 0, notVerified: 0 });
+		expect(JSON.parse(readFileSync(output.evalResultsPath, 'utf8'))).toMatchObject({
+			complete: true,
+			totalRuns: 5,
+			routing: { runId: 'run-1', judgeModel: expect.any(String) },
+		});
+		const summary = readFileSync(output.summaryPath, 'utf8');
+		expect(summary).toContain(
+			'| route-agent-support | agent | agent, clarify:agent | agent, clarify:workflow, none, none, none | 1/2 fail (3 not graded) |',
+		);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-langtracer.test.ts

@@ -0,0 +1,273 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { routingExpectationText } from '../../routing/expectation';
+import {
+	expectedBehaviorSentence,
+	exportedExpectations,
+	isSyntheticSource,
+	parsePassingIds,
+	routingCaseDiff,
+	routingCaseJsonFromExport,
+	routingCasesFromExport,
+	toLangTracerCreateBody,
+	toLangTracerUpdateBody,
+	unsupportedRoutingPushReason,
+	type RoutingLangTracerCreateBody,
+} from '../../routing/langtracer-cases';
+import {
+	langTracerConfigFromClaudeConfig,
+	resolveRoutingLangTracerConfig,
+} from '../../routing/langtracer-config';
+import { parseRoutingPushArgs } from '../../routing/langtracer-push';
+import { parseRoutingCase, type RoutingCase } from '../../routing/loader';
+
+function routingCase(raw: Record<string, unknown>): RoutingCase {
+	const parsed = parseRoutingCase({
+		id: 'route-agent-support',
+		bucket: 'agent',
+		agentShaped: true,
+		userMessage: 'Set up AI support in our group.',
+		accepts: ['agent', 'clarify:agent'],
+		policyDependent: false,
+		source: 'suite-62:985',
+		rationale: 'Ongoing conversational support role.',
+		...raw,
+	});
+	if (!parsed.success) throw new Error(parsed.issues.join('; '));
+	return parsed.data;
+}
+
+/** What the suite export returns for a pushed body: evalTags as `tags`, plus the stored content. */
+function exportedBody(body: RoutingLangTracerCreateBody): Record<string, unknown> {
+	return {
+		complexity: body.evalComplexity,
+		tags: body.evalTags,
+		conversation: body.conversation,
+		// A case with an expectation gets no synthesized happy-path scenario.
+		executionScenarios: [],
+		...(body.description ? { description: body.description } : {}),
+		...(body.seed ? { seed: body.seed } : {}),
+		processExpectations: body.processExpectations,
+	};
+}
+
+const seeded = routingCase({
+	id: 'route-debug-seeded',
+	bucket: 'debug',
+	agentShaped: false,
+	accepts: ['debug', 'clarify'],
+	source: 'langtracer:thread-1',
+	seed: {
+		mode: 'inline',
+		workflows: [{ id: 'wf-seeded-0001', name: 'Sync', nodes: [], connections: {} }],
+		messages: [{ role: 'user', text: 'Build a sync.' }],
+		priorRuns: [{ workflow: 'wf-seeded-0001', hints: 'Sync failed' }],
+	},
+	attach: { workflow: 'wf-seeded-0001' },
+	language: 'deu',
+});
+
+describe('toLangTracerCreateBody', () => {
+	it('encodes the routing labels as tags, in the spec order', () => {
+		const body = toLangTracerCreateBody(routingCase({ policyDependent: true }), {
+			suiteId: 64,
+			setKind: 'regression',
+		});
+
+		expect(body.evalTags).toEqual([
+			'routing',
+			'bucket:agent',
+			'accepts:agent',
+			'accepts:clarify:agent',
+			'policy-dependent',
+			'agent-shaped',
+			'source:suite-62:985',
+			'lang:eng',
+		]);
+		// LangTracer keeps only kebab-case case tags.
+		expect(body.tags).toEqual(['routing', 'policy-dependent', 'agent-shaped']);
+		expect(body).toMatchObject({
+			name: 'route-agent-support',
+			setKind: 'regression',
+			synthetic: true,
+			suiteId: 64,
+			userPrompt: 'Set up AI support in our group.',
+			description: 'Ongoing conversational support role.',
+			conversation: [{ role: 'user', text: 'Set up AI support in our group.' }],
+		});
+		expect(body.expectedBehavior).toBe(
+			'The Assistant should build an Agent or ask a clarifying question that points toward an Agent (accepted routes: agent, clarify:agent).',
+		);
+		expect(body.processExpectations).toEqual(['Routes to one of: agent, clarify:agent']);
+		expect(body).not.toHaveProperty('outcomeExpectations');
+	});
+
+	it('carries the seed without folders and the attachment on the turn', () => {
+		const body = toLangTracerCreateBody(seeded, { suiteId: 64, setKind: 'capability_gap' });
+		expect(body.synthetic).toBe(false);
+		expect(body.seed).not.toHaveProperty('folders');
+		expect(body.seed?.priorRuns).toEqual([{ workflow: 'wf-seeded-0001', hints: 'Sync failed' }]);
+		expect(body.conversation?.[0].attach).toEqual({ workflow: 'wf-seeded-0001' });
+		expect(body.evalTags).toContain('lang:deu');
+	});
+
+	it('patches every routing field and clears an absent seed', () => {
+		const body = toLangTracerCreateBody(routingCase({}), { suiteId: 64, setKind: 'regression' });
+		const patch = toLangTracerUpdateBody(body);
+		expect(patch).not.toHaveProperty('suiteId');
+		expect(patch).not.toHaveProperty('synthetic');
+		expect(patch).toMatchObject({
+			seed: null,
+			containsUserData: false,
+			setKind: 'regression',
+			processExpectations: [routingExpectationText(['agent', 'clarify:agent'])],
+			outcomeExpectations: [],
+		});
+	});
+});
+
+describe('routing cases from a LangTracer export', () => {
+	it('round-trips plain and seeded cases', () => {
+		const plain = routingCase({});
+		const files = Object.fromEntries(
+			[plain, seeded].map((c) => [
+				`${c.id}.json`,
+				exportedBody(toLangTracerCreateBody(c, { suiteId: 64, setKind: 'regression' })),
+			]),
+		);
+
+		const { cases, errors } = routingCasesFromExport(files);
+
+		expect(errors).toEqual([]);
+		expect(cases.map((c) => c.id)).toEqual(['route-agent-support', 'route-debug-seeded']);
+		expect(routingCaseDiff(plain, cases[0])).toEqual([]);
+		expect(routingCaseDiff(seeded, cases[1])).toEqual([]);
+		expect(cases[1].attach).toEqual({ workflow: 'wf-seeded-0001' });
+	});
+
+	it('ignores the stored expectation and scenarios when it rebuilds a case', () => {
+		const plain = routingCase({});
+		const exported = {
+			...exportedBody(toLangTracerCreateBody(plain, { suiteId: 64, setKind: 'regression' })),
+			processExpectations: ['Routes to one of: workflow'],
+			outcomeExpectations: ['An Agent was created.'],
+			executionScenarios: [{ name: 'happy-path', description: '' }],
+		};
+
+		const { cases, errors } = routingCasesFromExport({ [`${plain.id}.json`]: exported });
+
+		expect(errors).toEqual([]);
+		expect(cases[0].accepts).toEqual(['agent', 'clarify:agent']);
+		expect(routingCaseDiff(plain, cases[0])).toEqual([]);
+	});
+
+	it('reads the expectation lists of an export body, trimmed', () => {
+		expect(
+			exportedExpectations({ processExpectations: [' Routes to one of: answer ', ''] }),
+		).toEqual({ process: ['Routes to one of: answer'], outcome: [] });
+		expect(exportedExpectations(null)).toEqual({ process: [], outcome: [] });
+	});
+
+	it('names what is missing from a case that is not a routing case', () => {
+		expect(routingCaseJsonFromExport('x', { tags: ['other'] })).toEqual({
+			success: false,
+			issues: ['no "routing" tag, so it is not a routing case'],
+		});
+		const { errors } = routingCasesFromExport({
+			'route-a.json': { tags: ['routing', 'accepts:agent'], conversation: [] },
+		});
+		expect(errors).toEqual([
+			'route-a: no "bucket:" tag; no "source:" tag; conversation must be exactly one user turn',
+		]);
+	});
+
+	it('reports a changed field', () => {
+		const changed = routingCase({ accepts: ['agent'] });
+		expect(routingCaseDiff(routingCase({}), changed)).toEqual(['accepts']);
+	});
+});
+
+describe('push helpers', () => {
+	it('marks only the synthetic sources as synthetic', () => {
+		expect(['suite-62:1', 'agent-344:x', 'discovery:y', 'synthetic'].every(isSyntheticSource)).toBe(
+			true,
+		);
+		expect(['langtracer:t', 'michael-report', 'one-off-page'].some(isSyntheticSource)).toBe(false);
+	});
+
+	it('reads passing ids as an array or under a known key', () => {
+		expect([...parsePassingIds(['a', 'b'])]).toEqual(['a', 'b']);
+		expect([...parsePassingIds({ passingIds: ['a'] })]).toEqual(['a']);
+		expect(() => parsePassingIds({ other: [] })).toThrow('passing ids must be');
+	});
+
+	it('refuses a case with instance state', () => {
+		expect(
+			unsupportedRoutingPushReason(routingCase({ instanceState: { folderExploration: true } })),
+		).toContain('instanceState');
+		expect(unsupportedRoutingPushReason(routingCase({}))).toBeNull();
+	});
+
+	it('describes one accepted route in one sentence', () => {
+		expect(expectedBehaviorSentence(['answer'])).toBe(
+			'The Assistant should answer the question (accepted routes: answer).',
+		);
+	});
+});
+
+describe('parseRoutingPushArgs', () => {
+	const required = ['--suite', 'intent-routing', '--passing-ids', 'passing.json'];
+
+	it('requires a cases directory', () => {
+		expect(() => parseRoutingPushArgs(required)).toThrow('--cases-dir <dir> is required');
+	});
+
+	it('reads the cases directory and the selection', () => {
+		expect(
+			parseRoutingPushArgs([...required, '--cases-dir', 'cases', '--filter', 'agent', '--dry-run']),
+		).toEqual({
+			suite: 'intent-routing',
+			passingIdsFile: 'passing.json',
+			casesDir: 'cases',
+			selection: { filter: 'agent' },
+			dryRun: true,
+		});
+	});
+});
+
+describe('resolveRoutingLangTracerConfig', () => {
+	it('prefers the environment', () => {
+		expect(
+			resolveRoutingLangTracerConfig(
+				{ LANGTRACER_URL: 'https://lt.example', LANGTRACER_API_KEY: 'lt_env' },
+				'/does/not/exist.json',
+			),
+		).toEqual({ baseUrl: 'https://lt.example', apiKey: 'lt_env' });
+	});
+
+	it('falls back to the lang-tracer MCP entry of the Claude config', () => {
+		const dir = mkdtempSync(join(tmpdir(), 'routing-lt-config-'));
+		const file = join(dir, 'claude.json');
+		writeFileSync(
+			file,
+			JSON.stringify({
+				mcpServers: {
+					'lang-tracer': {
+						url: 'https://lt.example/api/mcp',
+						headers: { Authorization: 'Bearer lt_file' },
+					},
+				},
+			}),
+		);
+		expect(resolveRoutingLangTracerConfig({}, file)).toEqual({
+			baseUrl: 'https://lt.example/api',
+			apiKey: 'lt_file',
+		});
+		expect(langTracerConfigFromClaudeConfig({ mcpServers: {} })).toBeUndefined();
+		expect(() => resolveRoutingLangTracerConfig({}, join(dir, 'missing.json'))).toThrow(
+			'LANGTRACER_URL',
+		);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-loader.test.ts

@@ -0,0 +1,190 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import {
+	isRoutingCaseSelected,
+	loadRoutingCases,
+	parseRoutingCase,
+	toDiscoveryScenario,
+} from '../../routing/loader';
+
+const workflow = {
+	id: 'wf-seeded-01',
+	name: 'Daily digest',
+	nodes: [{ name: 'Start', type: 'n8n-nodes-base.manualTrigger' }],
+	connections: {},
+};
+
+const agent = {
+	id: 'agent-seeded-01',
+	config: { name: 'Helper', model: 'anthropic/claude-sonnet-4-5', instructions: 'Be helpful.' },
+};
+
+function baseCase(overrides: Record<string, unknown> = {}) {
+	return {
+		id: 'route-debug-seeded',
+		bucket: 'debug',
+		userMessage: 'It failed again.',
+		accepts: ['debug', 'clarify'],
+		source: 'synthetic',
+		...overrides,
+	};
+}
+
+function issuesOf(raw: unknown): string[] {
+	const parsed = parseRoutingCase(raw);
+	return parsed.success ? [] : parsed.issues;
+}
+
+describe('routing case schema', () => {
+	it('accepts seed, attach, and language, and hands seed and attach to the runner', () => {
+		const parsed = parseRoutingCase(
+			baseCase({
+				seed: {
+					mode: 'inline',
+					workflows: [workflow],
+					agents: [agent],
+					messages: [{ role: 'user', text: 'Build me a digest.' }],
+					priorRuns: [{ workflow: 'wf-seeded-01', hints: 'Start failed' }],
+				},
+				attach: { workflow: 'wf-seeded-01' },
+				language: 'deu',
+			}),
+		);
+		if (!parsed.success) throw new Error(parsed.issues.join('; '));
+
+		const scenario = toDiscoveryScenario(parsed.data);
+		expect(scenario.attach).toEqual({ workflow: 'wf-seeded-01' });
+		expect(scenario.seed?.workflows.map((w) => w.id)).toEqual(['wf-seeded-01']);
+		expect(scenario.seed?.priorRuns).toEqual([{ workflow: 'wf-seeded-01', hints: 'Start failed' }]);
+		// The `{role, text}` shorthand becomes a full stored message.
+		expect(scenario.seed?.messages[0]).toMatchObject({
+			type: 'llm',
+			role: 'user',
+			content: [{ type: 'text', text: 'Build me a digest.' }],
+		});
+		expect(parsed.data.language).toBe('deu');
+	});
+
+	it('keeps an unseeded case unchanged', () => {
+		const parsed = parseRoutingCase(baseCase());
+		if (!parsed.success) throw new Error(parsed.issues.join('; '));
+		expect(toDiscoveryScenario(parsed.data)).toEqual({
+			id: 'route-debug-seeded',
+			userMessage: 'It failed again.',
+		});
+	});
+
+	it('refuses an attachment that names no seeded resource', () => {
+		expect(issuesOf(baseCase({ attach: { workflow: 'wf-missing-01' } }))).toEqual([
+			'attach.workflow: must be the id of a workflow in `seed.workflows`',
+		]);
+		expect(
+			issuesOf(
+				baseCase({
+					seed: { mode: 'inline', workflows: [workflow] },
+					attach: { agent: 'agent-missing-01' },
+				}),
+			),
+		).toEqual(['attach.agent: must be the id of an Agent in `seed.agents`']);
+	});
+
+	it('refuses a prior run of a workflow the seed does not declare', () => {
+		expect(
+			issuesOf(
+				baseCase({
+					seed: {
+						mode: 'inline',
+						workflows: [workflow],
+						priorRuns: [{ workflow: 'wf-missing-01' }],
+					},
+				}),
+			),
+		).toEqual(['seed.priorRuns.0.workflow: must be the id of a workflow in `seed.workflows`']);
+	});
+
+	it('refuses seed slots the stub instance cannot serve', () => {
+		const issues = issuesOf(
+			baseCase({
+				seed: {
+					mode: 'inline',
+					workflows: [{ ...workflow, id: 'short' }],
+					folders: [{ id: 'folder-seeded-01', name: 'Clients' }],
+					projects: [{ name: 'Other' }],
+				},
+			}),
+		);
+		expect(issues).toEqual(
+			expect.arrayContaining([
+				expect.stringContaining('seed.folders'),
+				expect.stringContaining('seed.projects'),
+				expect.stringContaining('seed.workflows.0.id'),
+			]),
+		);
+		expect(issuesOf(baseCase({ seed: { mode: 'replay', threadId: 't-1' } }))).toEqual([
+			'seed.mode: routing cases support only `mode: "inline"` seeds',
+		]);
+		expect(issuesOf(baseCase({ seed: { mode: 'inline' } }))).toEqual([
+			'seed: an inline seed must carry messages, workflows, agents, or dataTables',
+		]);
+	});
+
+	it('accepts a file-name id only for an imported case', () => {
+		const raw = baseCase({ id: 'lt-route-debug-seeded-1a2b3c4d' });
+		expect(issuesOf(raw)).toEqual(['id: id must look like route-<bucket>-<slug>']);
+		expect(parseRoutingCase(raw, { imported: true }).success).toBe(true);
+		expect(parseRoutingCase(baseCase({ id: '../escape' }), { imported: true })).toEqual({
+			success: false,
+			issues: ['id: id must be safe as a file name'],
+		});
+	});
+
+	it('refuses a language that is not an ISO 639-3 code', () => {
+		expect(issuesOf(baseCase({ language: 'en' }))).toEqual([
+			'language: language must be an ISO 639-3 code, e.g. "eng"',
+		]);
+	});
+});
+
+describe('routing case selection', () => {
+	it('applies filter, exclude, and exclude-prefix to the id', () => {
+		expect(isRoutingCaseSelected('route-v2-agent-a', { excludePrefix: 'route-v2-' })).toBe(false);
+		expect(isRoutingCaseSelected('route-agent-v2-a', { excludePrefix: 'route-v2-' })).toBe(true);
+		expect(isRoutingCaseSelected('route-agent-a', { filter: 'agent,debug' })).toBe(true);
+		expect(isRoutingCaseSelected('route-answer-a', { filter: 'agent,debug' })).toBe(false);
+		expect(isRoutingCaseSelected('route-agent-a', { exclude: 'agent-a' })).toBe(false);
+	});
+
+	it('loads only the selected files from a directory', () => {
+		const dir = mkdtempSync(join(tmpdir(), 'routing-loader-'));
+		for (const id of ['route-debug-one', 'route-v2-debug-two']) {
+			writeFileSync(join(dir, `${id}.json`), JSON.stringify(baseCase({ id })));
+		}
+		writeFileSync(join(dir, 'results.json'), '{}');
+
+		expect(loadRoutingCases(dir).map((c) => c.routingCase.id)).toEqual([
+			'route-debug-one',
+			'route-v2-debug-two',
+		]);
+		expect(
+			loadRoutingCases(dir, { excludePrefix: 'route-v2-' }).map((c) => c.routingCase.id),
+		).toEqual(['route-debug-one']);
+		expect(loadRoutingCases(dir, 'two').map((c) => c.routingCase.id)).toEqual([
+			'route-v2-debug-two',
+		]);
+	});
+
+	it('selects only the exact match when the filter is a full id', () => {
+		const dir = mkdtempSync(join(tmpdir(), 'routing-loader-'));
+		for (const id of ['route-debug-one', 'route-debug-one-more']) {
+			writeFileSync(join(dir, `${id}.json`), JSON.stringify(baseCase({ id })));
+		}
+
+		expect(loadRoutingCases(dir, 'route-debug-one').map((c) => c.routingCase.id)).toEqual([
+			'route-debug-one',
+		]);
+		expect(loadRoutingCases(dir, 'debug-one')).toHaveLength(2);
+		expect(loadRoutingCases(dir, 'route-debug-one,more')).toHaveLength(2);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-trial.test.ts

@@ -0,0 +1,114 @@
+import type { InstanceAiEvent } from '@n8n/api-types';
+
+import { buildRoutingTrialRecord, ORCHESTRATOR_AGENT_ID } from '../routing-trial';
+
+const base = { runId: 'run-1', agentId: ORCHESTRATOR_AGENT_ID };
+
+function text(value: string, agentId = ORCHESTRATOR_AGENT_ID): InstanceAiEvent {
+	return { type: 'text-delta', ...base, agentId, payload: { text: value } };
+}
+
+function call(
+	toolCallId: string,
+	toolName: string,
+	args: Record<string, unknown>,
+	agentId = ORCHESTRATOR_AGENT_ID,
+): InstanceAiEvent {
+	return { type: 'tool-call', ...base, agentId, payload: { toolCallId, toolName, args } };
+}
+
+function result(toolCallId: string, value: unknown = {}): InstanceAiEvent {
+	return { type: 'tool-result', ...base, payload: { toolCallId, result: value } };
+}
+
+const skillContent = { type: 'content', value: [{ type: 'text', text: '# Skill' }] };
+
+describe('buildRoutingTrialRecord', () => {
+	it('records orchestrator calls, skills, and the text after the last call', () => {
+		const events: InstanceAiEvent[] = [
+			text('Let me check. '),
+			call('c1', 'load_skill', { name: 'intent-recognition' }),
+			result('c1', skillContent),
+			call('c3', 'load_skill', { skillId: 'no-such-skill' }),
+			result('c3', { success: false, error: 'Unknown skill' }),
+			call('c2', 'n8n-docs', { query: 'webhooks' }),
+			{ type: 'tool-error', ...base, payload: { toolCallId: 'c2', error: 'boom' } },
+			text('Webhooks '),
+			text('start workflows.'),
+		];
+
+		const record = buildRoutingTrialRecord({
+			trial: 1,
+			events,
+			durationMs: 10,
+			streamStatus: 'completed',
+		});
+
+		expect(record.toolCalls).toEqual([
+			{ toolName: 'load_skill', args: { name: 'intent-recognition' }, status: 'completed' },
+			{ toolName: 'load_skill', args: { skillId: 'no-such-skill' }, status: 'completed' },
+			{ toolName: 'n8n-docs', args: { query: 'webhooks' }, status: 'errored', error: 'boom' },
+		]);
+		expect(record.skillsLoaded).toEqual(['intent-recognition']);
+		expect(record.finalText).toBe('Webhooks start workflows.');
+		expect(record.fullText).toBe('Let me check.\n\nWebhooks start workflows.');
+	});
+
+	it('drops calls after the stop and keeps ask-user questions', () => {
+		const events: InstanceAiEvent[] = [
+			text('A few questions first.'),
+			call('c1', 'ask-user', {
+				introMessage: 'Before I build',
+				questions: [{ id: 'q1', question: 'Agent or workflow?', options: ['Agent', 'Workflow'] }],
+			}),
+			call('c2', 'build-agent', { operation: 'create' }),
+		];
+
+		const record = buildRoutingTrialRecord({
+			trial: 2,
+			events,
+			durationMs: 10,
+			streamStatus: 'stopped-on-route',
+			stop: { eventIndex: 2, toolName: 'ask-user', args: {} },
+		});
+
+		expect(record.toolCalls.map((c) => [c.toolName, c.status])).toEqual([['ask-user', 'pending']]);
+		expect(record.askUserQuestions).toEqual([
+			{
+				question: 'Agent or workflow?',
+				options: ['Agent', 'Workflow'],
+				introMessage: 'Before I build',
+			},
+		]);
+		expect(record.finalText).toBe('A few questions first.');
+		expect(record.stoppedOn?.toolName).toBe('ask-user');
+	});
+
+	it('keeps sub-agent calls apart and reports spawned roles', () => {
+		const events: InstanceAiEvent[] = [
+			call('c1', 'build-agent', { operation: 'create' }),
+			{
+				type: 'agent-spawned',
+				...base,
+				agentId: 'builder-1',
+				payload: { parentId: ORCHESTRATOR_AGENT_ID, role: 'agent-builder', tools: [] },
+			},
+			call('c2', 'write_config', {}, 'builder-1'),
+			text('sub-agent text', 'builder-1'),
+		];
+
+		const record = buildRoutingTrialRecord({
+			trial: 1,
+			events,
+			durationMs: 10,
+			streamStatus: 'completed',
+		});
+
+		expect(record.toolCalls.map((c) => c.toolName)).toEqual(['build-agent']);
+		expect(record.subAgentToolCalls).toEqual([
+			{ toolName: 'write_config', args: {}, status: 'pending', agentRole: 'agent-builder' },
+		]);
+		expect(record.spawnedAgents).toEqual(['agent-builder']);
+		expect(record.finalText).toBe('');
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/skill-overrides.test.ts

@@ -0,0 +1,82 @@
+import { loadRuntimeSkillSourceFromDirectory } from '@n8n/agents';
+import { TOP_LEVEL_ITEM_CEILING } from 'n8n-workflow';
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import {
+	INSTANCE_AI_SKILLS_DIR,
+	substituteSkillPlaceholders,
+} from '../../../src/skills/runtime-skills';
+import { buildSkillOverrideSource } from '../skill-overrides';
+
+/** The bundled skills with every skill present, independent of the agents module flag. */
+function bundledSource() {
+	return loadRuntimeSkillSourceFromDirectory(INSTANCE_AI_SKILLS_DIR, {
+		transformInstructions: substituteSkillPlaceholders,
+	});
+}
+
+function skillFile(name: string, body: string): string {
+	const dir = mkdtempSync(join(tmpdir(), 'skill-override-'));
+	const path = join(dir, 'SKILL.md');
+	writeFileSync(
+		path,
+		`---\nname: ${name}\ndescription: Variant of the skill for an eval run.\n---\n\n${body}\n`,
+	);
+	return path;
+}
+
+describe('buildSkillOverrideSource', () => {
+	it('replaces only the named skill and keeps its bundled location', async () => {
+		const base = bundledSource();
+		const path = skillFile(
+			'intent-recognition',
+			'Variant body. Ceiling: {{TOP_LEVEL_ITEM_CEILING_PLACEHOLDER}}.',
+		);
+
+		const built = buildSkillOverrideSource([{ skillId: 'intent-recognition', path }], base);
+		if (!built) throw new Error('expected an override source');
+		const { source, records } = built;
+
+		const original = base.registry.skills.find(({ id }) => id === 'intent-recognition');
+		const replaced = source.registry.skills.find(({ id }) => id === 'intent-recognition');
+		expect(replaced?.description).toBe('Variant of the skill for an eval run.');
+		expect(replaced?.path).toBe(original?.path);
+		expect(replaced?.directory).toBe(original?.directory);
+		expect(source.registry.skillsHash).not.toBe(base.registry.skillsHash);
+		expect(source.registry.skills.map(({ id }) => id)).toEqual(
+			base.registry.skills.map(({ id }) => id),
+		);
+
+		const loaded = await source.loadSkill('intent-recognition');
+		expect(loaded?.instructions).toContain(`Ceiling: ${String(TOP_LEVEL_ITEM_CEILING)}.`);
+		expect(await source.loadSkill('planning')).toEqual(await base.loadSkill('planning'));
+		expect(records['intent-recognition']).toEqual({ path, sha256: expect.any(String) });
+	});
+
+	it('returns nothing without overrides', () => {
+		expect(buildSkillOverrideSource([], bundledSource())).toBeUndefined();
+	});
+
+	it('refuses an unknown skill, a mismatched name, and a repeated skill', () => {
+		const base = bundledSource();
+		const path = skillFile('intent-recognition', 'Body.');
+
+		expect(() => buildSkillOverrideSource([{ skillId: 'no-such-skill', path }], base)).toThrow(
+			/"no-such-skill" is not in the runtime skill catalog/,
+		);
+		expect(() => buildSkillOverrideSource([{ skillId: 'planning', path }], base)).toThrow(
+			/must declare name "planning", got "intent-recognition"/,
+		);
+		expect(() =>
+			buildSkillOverrideSource(
+				[
+					{ skillId: 'intent-recognition', path },
+					{ skillId: 'intent-recognition', path },
+				],
+				base,
+			),
+		).toThrow(/"intent-recognition" is given more than once/);
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts

@@ -65,4 +65,14 @@ describe('resolveStreamStatus', () => {
 	it('never reads a cancelled run as a clean finish', () => {
 		expect(resolveStreamStatus(streamResult({ status: 'cancelled' }), false)).toBe('timed-out');
 	});
+
+	it('reports a run ended at its committing call as stopped-on-route', () => {
+		expect(resolveStreamStatus(streamResult({ status: 'cancelled' }), true, true)).toBe(
+			'stopped-on-route',
+		);
+	});
+
+	it('prefers stopped-on-route over a budget that expired during the stop', () => {
+		expect(resolveStreamStatus('timed-out', true, true)).toBe('stopped-on-route');
+	});
 });

packages/@n8n/instance-ai/evaluations/discovery/__tests__/stub-seed.test.ts

@@ -0,0 +1,178 @@
+// The stub instance with and without a seed: seeded reads return the seeded
+// state, and an unseeded stub keeps its empty answers.
+
+import { promises as fs } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { createStubServices } from '../../harness/stub-services';
+import { createStubAgentContextReader } from '../stub-agent-context';
+
+async function nodesJsonPath(): Promise<string> {
+	const dir = await fs.mkdtemp(path.join(tmpdir(), 'eval-stub-seed-'));
+	const file = path.join(dir, 'nodes.json');
+	await fs.writeFile(file, JSON.stringify([]), 'utf8');
+	return file;
+}
+
+const seed = {
+	workflows: [
+		{
+			id: 'wf-lead-enrich-01',
+			name: 'Lead enrichment',
+			nodes: [
+				{ name: 'New lead', type: 'n8n-nodes-base.webhook' },
+				{ name: 'Enrich lead', type: 'n8n-nodes-base.httpRequest', position: [220, 0] },
+			],
+			connections: {
+				'New lead': { main: [[{ node: 'Enrich lead', type: 'main', index: 0 }]] },
+			},
+		},
+		{ id: 'wf-other-0001', name: 'Weekly report', nodes: [], connections: {}, published: true },
+	],
+	dataTables: [
+		{ id: 'dt-leads-0001', name: 'Leads', columns: [{ name: 'email', type: 'string' as const }] },
+	],
+	priorRuns: [
+		{ workflow: 'wf-lead-enrich-01', hints: 'Enrich lead: 401 Unauthorized' },
+		{ workflow: 'wf-lead-enrich-01' },
+	],
+};
+
+describe('createStubServices without a seed', () => {
+	it('keeps the empty instance', async () => {
+		const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath() });
+
+		expect(await context.workflowService.list()).toEqual({
+			workflows: [],
+			total: 0,
+			totalInScope: 0,
+		});
+		expect((await context.workflowService.get('any-id')).nodes).toEqual([]);
+		expect(await context.executionService.list()).toEqual([]);
+		expect((await context.executionService.getStatus('seed-exec-1')).error).toBe(
+			'stub: execution disabled in eval',
+		);
+		expect(await context.dataTableService.list()).toEqual([]);
+	});
+});
+
+describe('createStubServices with a seed', () => {
+	it('returns the seeded workflows from list, get, and the JSON read', async () => {
+		const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed });
+
+		const all = await context.workflowService.list();
+		expect(all.workflows.map((w) => w.name)).toEqual(['Lead enrichment', 'Weekly report']);
+		expect(all.workflows[1].activeVersionId).not.toBeNull();
+
+		const filtered = await context.workflowService.list({ query: 'lead' });
+		expect(filtered.workflows.map((w) => w.id)).toEqual(['wf-lead-enrich-01']);
+		expect(filtered).toMatchObject({ total: 1, totalInScope: 2 });
+
+		const detail = await context.workflowService.get('wf-lead-enrich-01');
+		expect(detail.name).toBe('Lead enrichment');
+		expect(detail.nodes.map((n) => n.name)).toEqual(['New lead', 'Enrich lead']);
+
+		const json = await context.workflowService.getAsWorkflowJSON('wf-lead-enrich-01');
+		expect(json.nodes[0]).toMatchObject({ id: 'seed-node-0', typeVersion: 1, position: [0, 0] });
+		expect(json.nodes[1].position).toEqual([220, 0]);
+		expect(json.connections).toEqual(seed.workflows[0].connections);
+	});
+
+	it('keeps an edit to a seeded workflow', async () => {
+		const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed });
+		const edited = await context.workflowService.getAsWorkflowJSON('wf-lead-enrich-01');
+
+		await context.workflowService.updateFromWorkflowJSON('wf-lead-enrich-01', {
+			...edited,
+			name: 'Lead enrichment v2',
+		});
+
+		expect((await context.workflowService.get('wf-lead-enrich-01')).name).toBe(
+			'Lead enrichment v2',
+		);
+	});
+
+	it('reports each prior run as a failed execution with the hints as the error', async () => {
+		const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed });
+
+		const listed = await context.executionService.list({ workflowId: 'wf-lead-enrich-01' });
+		expect(listed.map((e) => e.id)).toEqual(['seed-exec-2', 'seed-exec-1']);
+		expect(listed.every((e) => e.status === 'error' && e.workflowName === 'Lead enrichment')).toBe(
+			true,
+		);
+		expect(Date.parse(listed[0].startedAt)).toBeGreaterThan(Date.parse(listed[1].startedAt));
+		expect(await context.executionService.list({ status: 'success' })).toEqual([]);
+		expect(await context.executionService.list({ workflowId: 'wf-other-0001' })).toEqual([]);
+
+		const status = await context.executionService.getStatus('seed-exec-1');
+		expect(status).toMatchObject({
+			status: 'error',
+			error: 'Enrich lead: 401 Unauthorized',
+			lastNodeExecuted: 'Enrich lead',
+		});
+
+		const debug = await context.executionService.getDebugInfo('seed-exec-1');
+		expect(debug.failedNode).toEqual({
+			name: 'Enrich lead',
+			type: 'n8n-nodes-base.httpRequest',
+			error: 'Enrich lead: 401 Unauthorized',
+		});
+		expect(debug.nodeTrace.map((n) => n.status)).toEqual(['error']);
+
+		const withoutHints = await context.executionService.getDebugInfo('seed-exec-2');
+		expect(withoutHints.error).toBe('The execution failed.');
+		expect(withoutHints.failedNode).toBeUndefined();
+	});
+
+	it('returns the seeded data tables and their columns', async () => {
+		const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed });
+
+		expect((await context.dataTableService.list()).map((t) => t.name)).toEqual(['Leads']);
+		expect(await context.dataTableService.getSchema('dt-leads-0001')).toEqual([
+			{ id: 'dt-leads-0001-col-0', name: 'email', type: 'string', index: 0 },
+		]);
+		expect(await context.dataTableService.getSchema('dt-unknown')).toEqual([]);
+	});
+});
+
+describe('createStubAgentContextReader', () => {
+	const reader = createStubAgentContextReader(
+		[
+			{
+				id: 'agent-support-01',
+				config: {
+					name: 'Support bot',
+					model: 'anthropic/claude-sonnet-4-5',
+					instructions: 'Answer order questions.',
+				},
+			},
+		],
+		'2026-09-30T00:00:00.000Z',
+	);
+
+	it('lists the seeded Agents and returns a config', async () => {
+		expect(await reader.lookup({ type: 'agents' })).toEqual({
+			agents: [
+				{
+					agentId: 'agent-support-01',
+					name: 'Support bot',
+					published: false,
+					updatedAt: '2026-09-30T00:00:00.000Z',
+				},
+			],
+		});
+		expect(await reader.lookup({ type: 'config', agentId: 'agent-support-01' })).toMatchObject({
+			configState: 'current-draft',
+			config: { model: 'anthropic/claude-sonnet-4-5' },
+		});
+		expect(await reader.lookup({ type: 'config-schema' })).toHaveProperty('configurableProperties');
+	});
+
+	it('fails for an unknown Agent and for lookups with no seeded data', async () => {
+		await expect(reader.lookup({ type: 'config', agentId: 'agent-missing' })).rejects.toThrow(
+			'Agent not found.',
+		);
+		await expect(reader.lookup({ type: 'capabilities' })).rejects.toThrow('not available');
+	});
+});

packages/@n8n/instance-ai/evaluations/discovery/cli-args.ts

@@ -0,0 +1,294 @@
+// ---------------------------------------------------------------------------
+// Arguments of the discovery CLI (cli.ts). Kept apart from the CLI so tests
+// can parse arguments without starting a run. Invalid arguments throw.
+// ---------------------------------------------------------------------------
+
+import { resolve } from 'node:path';
+
+export interface SkillOverride {
+	skillId: string;
+	/** Absolute path of the replacement SKILL.md. */
+	path: string;
+}
+
+/** Parses one `--skill-file` value: `<skillId>=<path>`. */
+export function parseSkillOverride(raw: string): SkillOverride {
+	const separator = raw.indexOf('=');
+	const skillId = separator === -1 ? '' : raw.slice(0, separator).trim();
+	const path = separator === -1 ? '' : raw.slice(separator + 1).trim();
+	if (!skillId || !path) {
+		throw new Error(`--skill-file expects <skillId>=<path>, got "${raw}"`);
+	}
+	return { skillId, path: resolve(path) };
+}
+
+export interface CliArgs {
+	help: boolean;
+	filter?: string;
+	verbose: boolean;
+	trials: number;
+	passThreshold: number;
+	timeoutMs: number;
+	/** Optional iteration cap; unset uses the SDK default. */
+	maxSteps?: number;
+	modelId: string;
+	/** Trials of one case that run at once. */
+	concurrency: number;
+	/** Cases that run at once. */
+	scenarioConcurrency: number;
+	nodesJsonPath?: string;
+	failOnZeroPass: boolean;
+	/** Routing mode: load routing cases (or LangTracer export bodies) from this directory. */
+	casesDir?: string;
+	stopOnRoute: boolean;
+	jsonOut?: string;
+	/** Routing mode: grade inline and write `eval-results.json` and friends here. */
+	outputDir?: string;
+	/** Routing mode with `--output-dir`: where the judge caches its verdicts. */
+	judgeCacheDir?: string;
+	variant: string;
+	validateOnly: boolean;
+	/** Routing mode: where the cases come from. Unset means the discovery scenarios. */
+	source?: 'local' | 'langtracer';
+	suite?: string;
+	/** Routing mode: drop cases whose id contains any comma-separated token. */
+	exclude?: string;
+	/** Routing mode: drop cases whose id starts with any comma-separated prefix. */
+	excludePrefix?: string;
+	/** Routing mode with `--validate-only`: compare the loaded cases with this directory. */
+	compareDir?: string;
+	/** Routing mode: write the loaded cases here, one `<id>.json` each, for the grader. */
+	saveCasesDir?: string;
+	/** Replacement SKILL.md files for the orchestrator's runtime skills. */
+	skillFiles: SkillOverride[];
+}
+
+export const DEFAULT_MODEL =
+	process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6';
+
+export const USAGE = `Usage: pnpm eval:discovery [options]
+
+Discovery mode (default): runs evaluations/data/discovery/ and reports pass rates.
+Routing mode: runs routing cases and records (or grades) the route of each trial.
+
+Common options:
+  --filter <text>              Cases whose id contains the text (comma-separated tokens).
+                               A single token that is exactly an id or file name
+                               selects only that case.
+  --trials, --iterations <n>   Trials per case (default 3).
+  --timeout, --timeout-ms <ms> Per-trial timeout (default 60000).
+  --model <provider/model>     Orchestrator model (default ${DEFAULT_MODEL}).
+  --concurrency <n>            Trials of one case that run at once (default 3).
+  --scenario-concurrency <n>   Cases that run at once (default 1).
+  --max-steps <n>              Iteration cap for the orchestrator.
+  --nodes-json <file>          Node catalog for the stub services.
+  --skill-file <id>=<path>     Use this SKILL.md for runtime skill <id> (repeatable).
+  --json-out <file>            Write the routing results JSON (both modes).
+  --verbose, -v                Print every trial's calls.
+
+Discovery mode only:
+  --pass-threshold <0..1>      Scenario pass threshold (default 0.67).
+  --fail-on-zero-pass          Exit non-zero only for scenarios with no passing trial.
+
+Routing mode (needs --cases-dir, or --source langtracer --suite):
+  --cases-dir <dir>            Routing case files (route-*.json) or LangTracer
+                               suite-export bodies (any *.json with a "routing" tag).
+  --source langtracer          Pull the cases from LangTracer instead.
+  --suite <slug|id>            The LangTracer suite for --source langtracer.
+  --stop-on-route              End each trial at its first committing call.
+  --output-dir <dir>           Grade inline and write eval-results.json (LangTracer
+                               dispatcher contract), routing-results.json, and
+                               routing-summary.md.
+  --judge-cache-dir <dir>      Judge verdict cache for --output-dir.
+  --variant <label>            Label written into the results (default baseline).
+  --exclude <text>             Drop cases whose id contains the text.
+  --exclude-prefix <prefix>    Drop cases whose id starts with the prefix.
+  --validate-only              Validate the cases and exit.
+  --compare-dir <dir>          With --validate-only: compare with the cases in <dir>.
+  --save-cases-dir <dir>       Write the loaded cases as <id>.json files.
+`;
+
+/** Flags that only routing mode reads. */
+const ROUTING_ONLY_FLAGS: Array<[keyof CliArgs, string]> = [
+	['stopOnRoute', '--stop-on-route'],
+	['outputDir', '--output-dir'],
+	['judgeCacheDir', '--judge-cache-dir'],
+	['validateOnly', '--validate-only'],
+	['exclude', '--exclude'],
+	['excludePrefix', '--exclude-prefix'],
+	['compareDir', '--compare-dir'],
+	['saveCasesDir', '--save-cases-dir'],
+	['suite', '--suite'],
+];
+
+function positiveInt(raw: string | undefined, flag: string): number {
+	const n = Number(raw);
+	if (!Number.isFinite(n) || n < 1 || !Number.isInteger(n)) {
+		throw new Error(`Invalid value for ${flag}: ${String(raw)} (expected a positive integer)`);
+	}
+	return n;
+}
+
+function numberInRange(raw: string | undefined, flag: string, lo: number, hi: number): number {
+	const n = Number(raw);
+	if (!Number.isFinite(n) || n < lo || n > hi) {
+		throw new Error(
+			`Invalid value for ${flag}: ${String(raw)} (expected number in [${lo}, ${hi}])`,
+		);
+	}
+	return n;
+}
+
+function value(raw: string | undefined, flag: string): string {
+	if (!raw || raw.startsWith('--')) throw new Error(`Missing value for ${flag}`);
+	return raw;
+}
+
+export function isRoutingMode(args: CliArgs): boolean {
+	return args.casesDir !== undefined || args.source !== undefined;
+}
+
+function validate(args: CliArgs): void {
+	if (args.source === 'langtracer') {
+		if (!args.suite) throw new Error('--source langtracer needs --suite <slug|id>.');
+		if (args.casesDir) throw new Error('Use either --cases-dir or --source langtracer, not both.');
+	}
+	if (args.source === 'local' && !args.casesDir) {
+		throw new Error('--source local needs --cases-dir <dir>.');
+	}
+	if (!isRoutingMode(args)) {
+		const given = ROUTING_ONLY_FLAGS.filter(([key]) => {
+			const setting = args[key];
+			return setting !== undefined && setting !== false;
+		}).map(([, flag]) => flag);
+		if (given.length > 0) {
+			const verb = given.length === 1 ? 'works' : 'work';
+			throw new Error(
+				`${given.join(', ')} ${verb} only in routing mode: pass --cases-dir <dir>, or --source langtracer --suite <slug|id>.`,
+			);
+		}
+	}
+	if (args.compareDir && !args.validateOnly) {
+		throw new Error('--compare-dir works only with --validate-only.');
+	}
+	if (args.judgeCacheDir && !args.outputDir) {
+		throw new Error('--judge-cache-dir works only with --output-dir.');
+	}
+}
+
+export function parseCliArgs(argv: string[]): CliArgs {
+	const args: CliArgs = {
+		help: false,
+		verbose: false,
+		trials: 3,
+		passThreshold: 2 / 3,
+		timeoutMs: 60_000,
+		modelId: DEFAULT_MODEL,
+		concurrency: 3,
+		scenarioConcurrency: 1,
+		failOnZeroPass: false,
+		stopOnRoute: false,
+		variant: 'baseline',
+		validateOnly: false,
+		skillFiles: [],
+	};
+
+	for (let i = 0; i < argv.length; i++) {
+		const arg = argv[i];
+		switch (arg) {
+			case '--help':
+			case '-h':
+				args.help = true;
+				return args;
+			case '--verbose':
+			case '-v':
+				args.verbose = true;
+				break;
+			case '--filter':
+				args.filter = value(argv[++i], arg);
+				break;
+			case '--trials':
+			case '--iterations':
+				args.trials = positiveInt(argv[++i], arg);
+				break;
+			case '--pass-threshold':
+				args.passThreshold = numberInRange(argv[++i], arg, 0, 1);
+				break;
+			case '--timeout':
+			case '--timeout-ms':
+				args.timeoutMs = positiveInt(argv[++i], arg);
+				break;
+			case '--max-steps':
+				args.maxSteps = positiveInt(argv[++i], arg);
+				break;
+			case '--model':
+				args.modelId = value(argv[++i], arg);
+				break;
+			case '--concurrency':
+				args.concurrency = positiveInt(argv[++i], arg);
+				break;
+			case '--scenario-concurrency':
+				args.scenarioConcurrency = positiveInt(argv[++i], arg);
+				break;
+			case '--nodes-json':
+				args.nodesJsonPath = value(argv[++i], arg);
+				break;
+			case '--fail-on-zero-pass':
+				args.failOnZeroPass = true;
+				break;
+			case '--cases-dir':
+				args.casesDir = value(argv[++i], arg);
+				break;
+			case '--stop-on-route':
+				args.stopOnRoute = true;
+				break;
+			case '--json-out':
+				args.jsonOut = value(argv[++i], arg);
+				break;
+			case '--output-dir':
+				args.outputDir = value(argv[++i], arg);
+				break;
+			case '--judge-cache-dir':
+				args.judgeCacheDir = value(argv[++i], arg);
+				break;
+			case '--variant':
+				args.variant = value(argv[++i], arg);
+				break;
+			case '--validate-only':
+				args.validateOnly = true;
+				break;
+			case '--source': {
+				const source = value(argv[++i], arg);
+				if (source !== 'local' && source !== 'langtracer') {
+					throw new Error(`Invalid value for --source: ${source} (expected local or langtracer)`);
+				}
+				args.source = source;
+				break;
+			}
+			case '--suite':
+				args.suite = value(argv[++i], arg);
+				break;
+			case '--exclude':
+				args.exclude = value(argv[++i], arg);
+				break;
+			case '--exclude-prefix':
+				args.excludePrefix = value(argv[++i], arg);
+				break;
+			case '--compare-dir':
+				args.compareDir = value(argv[++i], arg);
+				break;
+			case '--save-cases-dir':
+				args.saveCasesDir = value(argv[++i], arg);
+				break;
+			case '--skill-file':
+				args.skillFiles.push(parseSkillOverride(value(argv[++i], arg)));
+				break;
+			default:
+				// A misspelled flag would otherwise run with a default silently.
+				throw new Error(`Unknown argument: ${arg}. Run with --help for the options.`);
+		}
+	}
+
+	validate(args);
+	return args;
+}

packages/@n8n/instance-ai/evaluations/discovery/cli.ts

@@ -11,31 +11,65 @@
 // trials via the in-process runner, reports per-scenario pass-rates, exits
 // non-zero on any scenario below threshold, or on any scenario with zero passes
 // when --fail-on-zero-pass is set.
+//
+// Routing mode (cases live in LangTracer suite `intent-routing`):
+//   pnpm eval:discovery --cases-dir <dir> --stop-on-route \
+//     --json-out .data/routing/baseline.json --variant baseline --scenario-concurrency 4
+//   pnpm eval:discovery --cases-dir <dir> --filter <slug> --stop-on-route --output-dir <out>
+//   pnpm eval:discovery --source langtracer --suite intent-routing --validate-only
+//
+// `--cases-dir` reads routing case files and LangTracer suite-export bodies (see
+// routing-case-dir.ts). Without `--output-dir`, routing mode reports no pass
+// rates: it records each trial into the `--json-out` file for
+// evaluations/routing/grade.ts. With `--output-dir`, it grades each case inline
+// and writes `eval-results.json` for the LangTracer dispatcher (see
+// routing-eval-results.ts). `--source langtracer --suite <slug>` pulls the cases
+// from LangTracer instead. With `--validate-only`, `--compare-dir` checks the
+// loaded cases against another directory. `--save-cases-dir` writes the loaded
+// cases as `<id>.json`, so evaluations/routing/grade.ts can read them.
+//
+// `--skill-file <skillId>=<path>` runs the orchestrator with a different
+// SKILL.md for that runtime skill (see skill-overrides.ts).
 // ---------------------------------------------------------------------------
 
+import type { RuntimeSkillSource } from '@n8n/agents';
+import { nanoid } from 'nanoid';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { isRoutingMode, parseCliArgs, USAGE, type CliArgs } from './cli-args';
+import { loadRoutingCaseDir } from './routing-case-dir';
+import {
+	ROUTING_RESULTS_FILE,
+	RoutingEvalOutput,
+	type RoutingEvalMeta,
+} from './routing-eval-results';
+import {
+	RoutingResultsWriter,
+	toRoutingCaseResult,
+	type RoutingResultsMeta,
+} from './routing-results';
 import { runDiscoveryScenario, type DiscoveryRunResult } from './runner';
-import type { DiscoveryTestCase } from './types';
+import { buildSkillOverrideSource, type SkillOverrideRecord } from './skill-overrides';
+import type { DiscoveryScenario, DiscoveryStreamStatus } from './types';
 import { loadDiscoveryTestCasesWithFiles } from '../data/discovery';
+import { buildTranscriptFromEvents } from '../outcome/transcript-from-events';
+import { RoutingJudge, DEFAULT_JUDGE_CACHE_DIR } from '../routing/judge';
+import { routingCaseDiff, routingTags } from '../routing/langtracer-cases';
+import { loadRoutingCasesFromLangTracer } from '../routing/langtracer-source';
+import {
+	narrowToExactMatch,
+	type LoadedRoutingCase,
+	type RoutingCase,
+	type RoutingCaseSelection,
+} from '../routing/loader';
 
 // ---------------------------------------------------------------------------
 // Args
 // ---------------------------------------------------------------------------
 
-interface CliArgs {
-	filter?: string;
-	verbose: boolean;
-	trials: number;
-	passThreshold: number;
-	timeoutMs: number;
-	/** Optional iteration cap; unset uses the SDK default. */
-	maxSteps?: number;
-	modelId: string;
-	concurrency: number;
-	nodesJsonPath?: string;
-	failOnZeroPass: boolean;
-}
-
-const DEFAULT_MODEL = process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6';
+/** With routing results, how long a timed-out stream may take to wind down and report usage. */
+const SETTLE_GRACE_MS = 10_000;
 
 function previewArgs(args: Record<string, unknown>): string {
 	const entries = Object.entries(args).slice(0, 3);
@@ -57,80 +91,147 @@ function previewArgs(args: Record<string, unknown>): string {
 		.join(', ');
 }
 
-function requirePositiveInt(raw: string | undefined, flag: string): number {
-	const n = Number(raw);
-	if (!Number.isFinite(n) || n < 1 || !Number.isInteger(n)) {
-		console.error(`Invalid value for ${flag}: ${String(raw)} (expected a positive integer)`);
-		process.exit(1);
-	}
-	return n;
+// ---------------------------------------------------------------------------
+// Shared execution
+// ---------------------------------------------------------------------------
+
+interface CaseToRun {
+	scenario: DiscoveryScenario;
+	/** Case id. */
+	label: string;
+	/** Present in routing mode. */
+	routingCase?: RoutingCase;
+	/** Routing mode: the case file name without `.json`. */
+	fileName?: string;
 }
 
-function requireNumberInRange(
-	raw: string | undefined,
-	flag: string,
-	lo: number,
-	hi: number,
-): number {
-	const n = Number(raw);
-	if (!Number.isFinite(n) || n < lo || n > hi) {
-		console.error(`Invalid value for ${flag}: ${String(raw)} (expected number in [${lo}, ${hi}])`);
-		process.exit(1);
-	}
-	return n;
+/** The parsed arguments plus what main() builds from them once. */
+interface RunSettings {
+	args: CliArgs;
+	/** Built from `--skill-file`; absent runs the bundled skills. */
+	skills?: { source: RuntimeSkillSource; records: Record<string, SkillOverrideRecord> };
 }
 
-function parseArgs(argv: string[]): CliArgs {
-	const args: CliArgs = {
-		verbose: false,
-		trials: 3,
-		passThreshold: 2 / 3,
-		timeoutMs: 60_000,
-		modelId: DEFAULT_MODEL,
-		concurrency: 3,
-		failOnZeroPass: false,
-	};
+async function runTrials(
+	scenario: DiscoveryScenario,
+	{ args, skills }: RunSettings,
+): Promise<DiscoveryRunResult[]> {
+	const keepsResults = args.jsonOut !== undefined || args.outputDir !== undefined;
+	const trialResults: DiscoveryRunResult[] = [];
+	for (let i = 0; i < args.trials; i += args.concurrency) {
+		const batchSize = Math.min(args.concurrency, args.trials - i);
+		const batch = await Promise.all(
+			Array.from(
+				{ length: batchSize },
+				async () =>
+					await runDiscoveryScenario({
+						scenario,
+						modelId: args.modelId,
+						maxSteps: args.maxSteps,
+						timeoutMs: args.timeoutMs,
+						...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
+						...(args.stopOnRoute ? { stopOnRoute: true } : {}),
+						...(keepsResults ? { settleGraceMs: SETTLE_GRACE_MS } : {}),
+						...(skills ? { runtimeSkills: skills.source } : {}),
+					}),
+			),
+		);
+		trialResults.push(...batch);
+	}
+	return trialResults;
+}
 
-	for (let i = 0; i < argv.length; i++) {
-		const arg = argv[i];
-		switch (arg) {
-			case '--verbose':
-			case '-v':
-				args.verbose = true;
-				break;
-			case '--filter':
-				args.filter = argv[++i];
-				break;
-			case '--trials':
-				args.trials = requirePositiveInt(argv[++i], '--trials');
-				break;
-			case '--pass-threshold':
-				args.passThreshold = requireNumberInRange(argv[++i], '--pass-threshold', 0, 1);
-				break;
-			case '--timeout':
-				args.timeoutMs = requirePositiveInt(argv[++i], '--timeout');
-				break;
-			case '--max-steps':
-				args.maxSteps = requirePositiveInt(argv[++i], '--max-steps');
-				break;
-			case '--model':
-				args.modelId = argv[++i];
-				break;
-			case '--concurrency':
-				args.concurrency = requirePositiveInt(argv[++i], '--concurrency');
-				break;
-			case '--nodes-json':
-				args.nodesJsonPath = argv[++i];
-				break;
-			case '--fail-on-zero-pass':
-				args.failOnZeroPass = true;
-				break;
-			default:
-				break;
+/**
+ * Runs every case, `scenarioConcurrency` at a time. Sequential runs print the
+ * case label before its trials start; concurrent runs print each case's report
+ * in one piece when it finishes, so lines of different cases do not interleave.
+ */
+async function runCases(
+	cases: CaseToRun[],
+	settings: RunSettings,
+	report: (entry: CaseToRun, results: DiscoveryRunResult[]) => string[],
+	onCaseDone: (index: number, entry: CaseToRun, results: DiscoveryRunResult[]) => Promise<void>,
+): Promise<DiscoveryRunResult[][]> {
+	const { args } = settings;
+	const allResults = new Array<DiscoveryRunResult[]>(cases.length);
+	const sequential = args.scenarioConcurrency === 1;
+	let nextIndex = 0;
+
+	async function worker(): Promise<void> {
+		while (nextIndex < cases.length) {
+			const index = nextIndex++;
+			const entry = cases[index];
+			if (sequential) process.stdout.write(`▸ ${entry.label} ... `);
+			const results = await runTrials(entry.scenario, settings);
+			const lines = report(entry, results);
+			if (sequential) {
+				console.log(lines.join('\n'));
+			} else {
+				const [first = '', ...rest] = lines;
+				console.log([`▸ ${entry.label} ... ${first}`, ...rest].join('\n'));
+			}
+			allResults[index] = results;
+			await onCaseDone(index, entry, results);
 		}
 	}
 
-	return args;
+	const workerCount = Math.min(args.scenarioConcurrency, cases.length);
+	await Promise.all(Array.from({ length: workerCount }, async () => await worker()));
+	return allResults;
+}
+
+function verboseTrialLines(results: DiscoveryRunResult[]): string[] {
+	const lines: string[] = [];
+	for (const [idx, r] of results.entries()) {
+		const trial = `trial ${String(idx + 1)}`;
+		const duration = `${(r.durationMs / 1000).toFixed(1)}s`;
+		if (r.check) {
+			const icon = r.check.pass ? '  ✓' : '  ✗';
+			lines.push(`${icon} ${trial} (${duration}) — ${r.check.comment}`);
+		} else {
+			lines.push(`  • ${trial} (${duration}) — ${r.streamStatus}`);
+		}
+		if (r.runError) lines.push(`     run error: ${r.runError}`);
+		if (r.outcome.toolCalls.length > 0) {
+			lines.push('     tool calls:');
+			for (const tc of r.outcome.toolCalls) {
+				lines.push(`       • ${tc.toolName}(${previewArgs(tc.args)})`);
+			}
+		}
+		if (r.outcome.agentActivities.length > 0) {
+			lines.push('     spawned sub-agents:');
+			for (const a of r.outcome.agentActivities) {
+				lines.push(`       • role=${a.role}, tools=[${a.tools.join(', ')}]`);
+			}
+		}
+	}
+	return lines;
+}
+
+function resultsMeta({ args, skills }: RunSettings): RoutingResultsMeta {
+	const startedAt = new Date().toISOString();
+	return {
+		runId: `${args.variant}-${startedAt.replace(/[:.]/g, '-')}-${nanoid(4)}`,
+		variant: args.variant,
+		model: args.modelId,
+		...(skills ? { skillOverrides: skills.records } : {}),
+		trialsPerCase: args.trials,
+		stopOnRoute: args.stopOnRoute,
+		startedAt,
+	};
+}
+
+/** One writer per results file: `--json-out`, and `routing-results.json` under `--output-dir`. */
+function createResultsWriters(
+	{ args }: RunSettings,
+	meta: RoutingResultsMeta,
+	caseCount: number,
+): RoutingResultsWriter[] {
+	const paths = [
+		...(args.jsonOut ? [args.jsonOut] : []),
+		...(args.outputDir ? [join(args.outputDir, ROUTING_RESULTS_FILE)] : []),
+	];
+	return paths.map((path) => new RoutingResultsWriter(path, meta, caseCount));
 }
 
 // ---------------------------------------------------------------------------
@@ -138,14 +239,21 @@ function parseArgs(argv: string[]): CliArgs {
 // ---------------------------------------------------------------------------
 
 interface ScenarioAggregate {
-	scenario: DiscoveryTestCase;
+	scenario: DiscoveryScenario;
 	results: DiscoveryRunResult[];
 	passCount: number;
 	passRate: number;
 }
 
-async function runLocalMode(args: CliArgs): Promise<void> {
-	const cases = loadDiscoveryTestCasesWithFiles(args.filter);
+function passCountOf(results: DiscoveryRunResult[]): number {
+	return results.filter((r) => r.check?.pass === true).length;
+}
+
+async function runLocalMode(settings: RunSettings): Promise<void> {
+	const { args } = settings;
+	const cases: CaseToRun[] = loadDiscoveryTestCasesWithFiles(args.filter).map(
+		({ testCase, fileSlug }) => ({ scenario: testCase, label: fileSlug }),
+	);
 
 	if (cases.length === 0) {
 		console.log('No discovery scenarios found.');
@@ -156,65 +264,37 @@ async function runLocalMode(args: CliArgs): Promise<void> {
 		`Running ${String(cases.length)} discovery scenario(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
 	);
 
-	const aggregates: ScenarioAggregate[] = [];
+	const writers = createResultsWriters(settings, resultsMeta(settings), cases.length);
 
-	// Per scenario, run trials in parallel up to `concurrency`. Across scenarios, run sequentially
-	// — keeps the load profile predictable and per-scenario reports atomic.
-	for (const { testCase, fileSlug } of cases) {
-		process.stdout.write(`▸ ${fileSlug} ... `);
-		const trialResults: DiscoveryRunResult[] = [];
+	// Per scenario, run trials in parallel up to `concurrency`. Across scenarios, run
+	// `scenarioConcurrency` at a time (default 1: predictable load, atomic reports).
+	const allResults = await runCases(
+		cases,
+		settings,
+		(_entry, trialResults) => {
+			const passCount = passCountOf(trialResults);
+			const passRate = passCount / trialResults.length;
+			const status = passRate >= args.passThreshold ? '✓' : '✗';
+			return [
+				`${status} ${String(passCount)}/${String(trialResults.length)} passed (${(passRate * 100).toFixed(0)}%)`,
+				...(args.verbose ? verboseTrialLines(trialResults) : []),
+			];
+		},
+		async (index, entry, results) => {
+			const caseResult = toRoutingCaseResult(entry.scenario, undefined, results);
+			await Promise.all(writers.map(async (writer) => await writer.record(index, caseResult)));
+		},
+	);
+	await Promise.all(writers.map(async (writer) => await writer.finish()));
 
-		for (let i = 0; i < args.trials; i += args.concurrency) {
-			const batchSize = Math.min(args.concurrency, args.trials - i);
-			const batch = await Promise.all(
-				Array.from(
-					{ length: batchSize },
-					async () =>
-						await runDiscoveryScenario({
-							scenario: testCase,
-							modelId: args.modelId,
-							maxSteps: args.maxSteps,
-							timeoutMs: args.timeoutMs,
-							...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
-						}),
-				),
-			);
-			trialResults.push(...batch);
-		}
-
-		const passCount = trialResults.filter((r) => r.check.pass).length;
-		const passRate = passCount / trialResults.length;
-		const status = passRate >= args.passThreshold ? '✓' : '✗';
-		console.log(
-			`${status} ${String(passCount)}/${String(trialResults.length)} passed (${(passRate * 100).toFixed(0)}%)`,
-		);
-
-		if (args.verbose) {
-			for (const [idx, r] of trialResults.entries()) {
-				const icon = r.check.pass ? '  ✓' : '  ✗';
-				const trial = `trial ${String(idx + 1)}`;
-				console.log(`${icon} ${trial} (${(r.durationMs / 1000).toFixed(1)}s) — ${r.check.comment}`);
-				if (r.runError) console.log(`     run error: ${r.runError}`);
-				if (r.outcome.toolCalls.length > 0) {
-					console.log('     tool calls:');
-					for (const tc of r.outcome.toolCalls) {
-						const argsPreview = previewArgs(tc.args);
-						console.log(`       • ${tc.toolName}(${argsPreview})`);
-					}
-				}
-				if (r.outcome.agentActivities.length > 0) {
-					console.log('     spawned sub-agents:');
-					for (const a of r.outcome.agentActivities) {
-						console.log(`       • role=${a.role}, tools=[${a.tools.join(', ')}]`);
-					}
-				}
-			}
-		}
-
-		aggregates.push({ scenario: testCase, results: trialResults, passCount, passRate });
-	}
+	const aggregates: ScenarioAggregate[] = cases.map((entry, index) => {
+		const results = allResults[index];
+		const passCount = passCountOf(results);
+		return { scenario: entry.scenario, results, passCount, passRate: passCount / results.length };
+	});
 
 	printSummary(aggregates, args);
+	for (const writer of writers) console.log(`\nResults written to ${writer.filePath}`);
 
 	const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
 	const zeroPassScenarios = aggregates.filter((a) => a.passCount === 0);
@@ -248,29 +328,288 @@ function printSummary(aggregates: ScenarioAggregate[], args: CliArgs): void {
 		console.log('\nFailing scenarios:');
 		for (const a of failingScenarios) {
 			console.log(`  ✗ ${a.scenario.id} (${(a.passRate * 100).toFixed(0)}%)`);
-			const firstFailure = a.results.find((r) => !r.check.pass);
-			if (firstFailure) {
+			const firstFailure = a.results.find((r) => r.check?.pass !== true);
+			if (firstFailure?.check) {
 				console.log(`    ${firstFailure.check.comment}`);
 			}
 		}
 	}
 }
 
+// ---------------------------------------------------------------------------
+// Routing mode
+// ---------------------------------------------------------------------------
+
+function describeRoutingTrial(r: DiscoveryRunResult): string {
+	const duration = `${(r.durationMs / 1000).toFixed(1)}s`;
+	if (r.stop) return `stop@${r.stop.toolName} ${duration}`;
+	if (r.runError) return `${r.streamStatus} ${duration} (${r.runError.slice(0, 80)})`;
+	return `${r.streamStatus} ${duration}`;
+}
+
+function routingSelection(args: CliArgs): RoutingCaseSelection {
+	return {
+		...(args.filter !== undefined ? { filter: args.filter } : {}),
+		...(args.exclude !== undefined ? { exclude: args.exclude } : {}),
+		...(args.excludePrefix !== undefined ? { excludePrefix: args.excludePrefix } : {}),
+	};
+}
+
+async function loadRoutingCasesForArgs(
+	args: CliArgs,
+): Promise<{ loaded: LoadedRoutingCase[]; origin: string }> {
+	if (args.source === 'langtracer' && args.suite) {
+		const { suite, cases } = await loadRoutingCasesFromLangTracer(
+			args.suite,
+			routingSelection(args),
+		);
+		return {
+			loaded: narrowToExactMatch(cases, args.filter),
+			origin: `LangTracer suite "${suite.slug}" (#${String(suite.id)})`,
+		};
+	}
+	if (!args.casesDir) throw new Error('Routing mode needs --cases-dir <dir>.');
+	return {
+		loaded: loadRoutingCaseDir(args.casesDir, routingSelection(args)),
+		origin: args.casesDir,
+	};
+}
+
+/**
+ * Compares loaded cases with the files in `dir`: the same ids, and for each id
+ * the same routing fields and tags. Returns one line per difference.
+ */
+function compareWithDirectory(loaded: RoutingCase[], dir: string, args: CliArgs): string[] {
+	const local = new Map(
+		loadRoutingCaseDir(dir, routingSelection(args)).map(({ routingCase }) => [
+			routingCase.id,
+			routingCase,
+		]),
+	);
+	const differences: string[] = [];
+	const seen = new Set<string>();
+	for (const routingCase of loaded) {
+		seen.add(routingCase.id);
+		const counterpart = local.get(routingCase.id);
+		if (!counterpart) {
+			differences.push(`${routingCase.id}: not in ${dir}`);
+			continue;
+		}
+		const fields = routingCaseDiff(counterpart, routingCase);
+		if (fields.length > 0) differences.push(`${routingCase.id}: differs in ${fields.join(', ')}`);
+		const tags = routingTags(routingCase).join(' ');
+		if (tags !== routingTags(counterpart).join(' ')) {
+			differences.push(`${routingCase.id}: tags differ`);
+		}
+	}
+	for (const id of local.keys()) {
+		if (!seen.has(id)) differences.push(`${id}: missing from the loaded cases`);
+	}
+	return differences;
+}
+
+function countBy(values: string[]): string {
+	const counts = new Map<string, number>();
+	for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
+	return [...counts]
+		.sort(([a], [b]) => a.localeCompare(b))
+		.map(([key, count]) => `${key}=${String(count)}`)
+		.join(', ');
+}
+
+async function runRoutingMode(settings: RunSettings): Promise<void> {
+	const { args } = settings;
+	const { loaded, origin } = await loadRoutingCasesForArgs(args);
+	const cases: CaseToRun[] = loaded.map(({ routingCase, scenario, fileName }) => ({
+		scenario,
+		label: routingCase.id,
+		routingCase,
+		...(fileName ? { fileName } : {}),
+	}));
+	if (args.saveCasesDir) {
+		mkdirSync(args.saveCasesDir, { recursive: true });
+		for (const { routingCase } of loaded) {
+			const file = join(args.saveCasesDir, `${routingCase.id}.json`);
+			writeFileSync(file, `${JSON.stringify(routingCase, null, 2)}\n`);
+		}
+		console.log(`Wrote ${String(loaded.length)} routing case file(s) to ${args.saveCasesDir}.`);
+	}
+
+	if (args.validateOnly) {
+		const routingCases = loaded.map(({ routingCase }) => routingCase);
+		console.log(`${String(cases.length)} routing case(s) in ${origin} are valid.`);
+		if (routingCases.length > 0) {
+			console.log(`  buckets: ${countBy(routingCases.map((c) => c.bucket))}`);
+			const seeded = routingCases.filter((c) => c.seed !== undefined).length;
+			const attached = routingCases.filter((c) => c.attach !== undefined).length;
+			console.log(`  seeded: ${String(seeded)}, attached: ${String(attached)}`);
+		}
+		if (args.compareDir) {
+			const differences = compareWithDirectory(routingCases, args.compareDir, args);
+			if (differences.length === 0) {
+				console.log(`  matches ${args.compareDir}: same ids, fields, and tags`);
+			} else {
+				console.error(`  ${String(differences.length)} difference(s) from ${args.compareDir}:`);
+				for (const line of differences) console.error(`    - ${line}`);
+				process.exitCode = 1;
+			}
+		}
+		return;
+	}
+	if (cases.length === 0) {
+		console.log(`No routing cases found in ${origin}.`);
+		return;
+	}
+
+	if (!args.jsonOut && !args.outputDir) {
+		console.warn('Warning: no --json-out or --output-dir given; results will only be printed.');
+	}
+
+	console.log(
+		`Running ${String(cases.length)} routing case(s) × ${String(args.trials)} trial(s) (variant: ${args.variant}, model: ${args.modelId}, stop-on-route: ${String(args.stopOnRoute)}, scenario concurrency: ${String(args.scenarioConcurrency)}, trial concurrency: ${String(args.concurrency)}).\n`,
+	);
+
+	const meta = resultsMeta(settings);
+	const writers = createResultsWriters(settings, meta, cases.length);
+	const evalOutput = args.outputDir
+		? new RoutingEvalOutput(
+				args.outputDir,
+				new RoutingJudge({
+					cacheDir: args.judgeCacheDir ?? DEFAULT_JUDGE_CACHE_DIR,
+					apiKey: process.env.ANTHROPIC_API_KEY,
+				}),
+				meta satisfies RoutingEvalMeta,
+				cases.length,
+			)
+		: undefined;
+
+	const allResults = await runCases(
+		cases,
+		settings,
+		(entry, results) => [
+			`[${entry.routingCase?.bucket ?? '?'}] ${results.map(describeRoutingTrial).join(' | ')}`,
+			...(args.verbose ? verboseTrialLines(results) : []),
+		],
+		async (index, entry, results) => {
+			const caseResult = toRoutingCaseResult(entry.scenario, entry.routingCase, results);
+			await Promise.all(writers.map(async (writer) => await writer.record(index, caseResult)));
+			if (!evalOutput || !entry.routingCase) return;
+			const graded = await evalOutput.record(index, {
+				routingCase: entry.routingCase,
+				...(entry.fileName ? { fileName: entry.fileName } : {}),
+				result: caseResult,
+				threadIds: results.map((r) => r.threadId),
+				transcripts: results.map((r) =>
+					buildTranscriptFromEvents({
+						events: r.events,
+						openingMessage: entry.scenario.userMessage,
+					}),
+				),
+			});
+			const labels = graded.graded?.trials.map((trial) => trial.label).join(', ') ?? 'not graded';
+			console.log(`  ${entry.label}: graded ${labels}`);
+		},
+	);
+	await Promise.all(writers.map(async (writer) => await writer.finish()));
+	const evalResults = await evalOutput?.finish();
+
+	const trials = allResults.flat();
+	printRoutingSummary(trials);
+	for (const writer of writers) console.log(`\nResults written to ${writer.filePath}`);
+	if (evalOutput && evalResults) {
+		const { passed, testCases, notVerified } = evalResults.summary;
+		console.log(
+			`Graded: ${String(passed)}/${String(testCases)} case(s) passed, ${String(notVerified)} not verified.`,
+		);
+		console.log(`Eval results written to ${evalOutput.evalResultsPath}`);
+		console.log(`Summary written to ${evalOutput.summaryPath}`);
+	}
+
+	if (trials.length > 0 && trials.every((r) => r.streamStatus === 'errored')) {
+		process.exitCode = 1;
+	}
+}
+
+function printRoutingSummary(trials: DiscoveryRunResult[]): void {
+	console.log('\n=== Summary ===');
+	const byStatus = new Map<DiscoveryStreamStatus, number>();
+	const byStopTool = new Map<string, number>();
+	let inputTokens = 0;
+	let outputTokens = 0;
+	let costUsd = 0;
+	let withUsage = 0;
+	for (const r of trials) {
+		byStatus.set(r.streamStatus, (byStatus.get(r.streamStatus) ?? 0) + 1);
+		if (r.stop) byStopTool.set(r.stop.toolName, (byStopTool.get(r.stop.toolName) ?? 0) + 1);
+		if (r.usage) {
+			withUsage++;
+			inputTokens += r.usage.promptTokens;
+			outputTokens += r.usage.completionTokens;
+			costUsd += r.usage.costUsd;
+		}
+	}
+	const format = (counts: Map<string, number>) =>
+		[...counts].map(([key, count]) => `${key}=${String(count)}`).join(', ') || '∅';
+
+	console.log(`Trials: ${String(trials.length)} (${format(byStatus)})`);
+	console.log(`Stopped on: ${format(byStopTool)}`);
+	console.log(
+		`Total trial time: ${(trials.reduce((sum, r) => sum + r.durationMs, 0) / 1000).toFixed(1)}s`,
+	);
+	if (withUsage > 0) {
+		console.log(
+			`Orchestrator tokens (${String(withUsage)} trial(s) with usage): input=${String(inputTokens)}, output=${String(outputTokens)}, cost=$${costUsd.toFixed(2)}`,
+		);
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Main
 // ---------------------------------------------------------------------------
 
-async function main(): Promise<void> {
-	const args = parseArgs(process.argv.slice(2));
+function exitWithError(error: unknown): never {
+	console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+	process.exit(1);
+}
 
-	if (!process.env.ANTHROPIC_API_KEY) {
+async function main(): Promise<void> {
+	let args: CliArgs;
+	try {
+		args = parseCliArgs(process.argv.slice(2));
+	} catch (error) {
+		exitWithError(error);
+	}
+	if (args.help) {
+		console.log(USAGE);
+		return;
+	}
+
+	if (!args.validateOnly && !process.env.ANTHROPIC_API_KEY) {
 		console.error(
 			'Error: ANTHROPIC_API_KEY is required to run discovery evaluations (the runner calls Anthropic in-process).',
 		);
 		process.exit(1);
 	}
 
-	await runLocalMode(args);
+	const settings: RunSettings = { args };
+	let skills: RunSettings['skills'];
+	try {
+		skills = buildSkillOverrideSource(args.skillFiles);
+	} catch (error) {
+		exitWithError(error);
+	}
+	if (skills) {
+		settings.skills = skills;
+		for (const [skillId, record] of Object.entries(skills.records)) {
+			console.log(`Skill override: ${skillId} <- ${record.path}`);
+		}
+	}
+
+	if (isRoutingMode(args)) {
+		await runRoutingMode(settings);
+		return;
+	}
+	await runLocalMode(settings);
 }
 
 main()

packages/@n8n/instance-ai/evaluations/discovery/confirmation-policy.ts

@@ -27,7 +27,9 @@ export type ApprovalResponder = (
 	payload: Record<string, unknown>,
 ) => Record<string, unknown> | undefined;
 
-export function buildConfirmationPolicy(scenario: DiscoveryTestCase): ConfirmationPolicy {
+export function buildConfirmationPolicy(
+	scenario: Pick<DiscoveryTestCase, 'confirmations'>,
+): ConfirmationPolicy {
 	const policy: ConfirmationPolicy = new Map();
 	for (const [toolName, answer] of Object.entries(scenario.confirmations ?? {})) {
 		policy.set(toolName, typeof answer === 'string' ? { decision: answer } : answer);

packages/@n8n/instance-ai/evaluations/discovery/expected-tools-invoked.ts

@@ -242,6 +242,9 @@ function invalidTrialReason(trial: DiscoveryTrialFacts): string | undefined {
 		case 'errored':
 		case 'suspended':
 			return `Run did not complete (${trial.runError ? `${trial.streamStatus}: ${trial.runError}` : trial.streamStatus}).`;
+		// A deliberate stop at the first committing call settles the expectation as it stands.
+		case 'stopped-on-route':
+			return undefined;
 		case 'completed':
 			return trial.unmatchedConfirmations.length > 0
 				? `Scenario declared confirmation answers for [${trial.unmatchedConfirmations.join(', ')}] that no suspension asked for, so those decisions never ran.`

packages/@n8n/instance-ai/evaluations/discovery/routing-case-dir.ts

@@ -0,0 +1,117 @@
+// ---------------------------------------------------------------------------
+// Reads the routing cases in a `--cases-dir`.
+//
+// The directory can hold two formats:
+// - routing case files (`route-*.json`, see routing/loader.ts), and
+// - LangTracer suite-export case bodies, which have a `routing` entry in
+//   `tags`. The LangTracer dispatcher writes one such body for each run.
+//
+// An export body carries no case id unless it has a `name`, so its id is the
+// file name without `.json`. Other JSON files (results, notes) are skipped.
+// ---------------------------------------------------------------------------
+
+import { isRecord } from '@n8n/utils/is-record';
+import { readFileSync, readdirSync } from 'node:fs';
+import { basename, join, resolve } from 'node:path';
+
+import { ROUTING_TAG, routingCaseJsonFromExport } from '../routing/langtracer-cases';
+import {
+	isRoutingCaseSelected,
+	narrowToExactMatch,
+	parseRoutingCase,
+	toDiscoveryScenario,
+	type LoadedRoutingCase,
+	type RoutingCase,
+	type RoutingCaseSelection,
+} from '../routing/loader';
+
+const AUTHORED_FILE_PREFIX = 'route-';
+
+export function isRoutingExportBody(raw: unknown): raw is Record<string, unknown> {
+	return isRecord(raw) && Array.isArray(raw.tags) && raw.tags.includes(ROUTING_TAG);
+}
+
+/** The case id of an export body: its `name`, else the file name. */
+function exportBodyId(body: Record<string, unknown>, fileName: string): string {
+	return typeof body.name === 'string' && body.name.trim() ? body.name.trim() : fileName;
+}
+
+type FileOutcome = { routingCase: RoutingCase } | { error: string } | { skip: true };
+
+function readCaseFile(
+	filePath: string,
+	fileName: string,
+	selection: RoutingCaseSelection,
+): FileOutcome {
+	const authored = fileName.startsWith(AUTHORED_FILE_PREFIX);
+	let raw: unknown;
+	try {
+		raw = JSON.parse(readFileSync(filePath, 'utf-8'));
+	} catch (error) {
+		// Unreadable JSON is an error only for a selected case file; anything else is not a case.
+		if (!authored || !isRoutingCaseSelected(fileName, selection)) return { skip: true };
+		return { error: `${filePath}: ${error instanceof Error ? error.message : String(error)}` };
+	}
+
+	if (isRoutingExportBody(raw)) {
+		const id = exportBodyId(raw, fileName);
+		if (!isRoutingCaseSelected(id, selection) && !isRoutingCaseSelected(fileName, selection)) {
+			return { skip: true };
+		}
+		const rebuilt = routingCaseJsonFromExport(id, raw);
+		if (!rebuilt.success) return { error: `${filePath}: ${rebuilt.issues.join('; ')}` };
+		const parsed = parseRoutingCase(rebuilt.json, { imported: true });
+		if (!parsed.success) return { error: `${filePath}: ${parsed.issues.join('; ')}` };
+		return { routingCase: parsed.data };
+	}
+
+	if (!authored || !isRoutingCaseSelected(fileName, selection)) return { skip: true };
+	const parsed = parseRoutingCase(raw);
+	if (!parsed.success) return { error: `${filePath}: ${parsed.issues.join('; ')}` };
+	if (parsed.data.id !== fileName) {
+		return { error: `${filePath}: id "${parsed.data.id}" must match the file name "${fileName}"` };
+	}
+	return { routingCase: parsed.data };
+}
+
+/**
+ * Loads the selected cases of `dir`, sorted by file name. All invalid files
+ * are reported in one error, like `loadRoutingCases`.
+ */
+export function loadRoutingCaseDir(
+	dir: string,
+	selection: RoutingCaseSelection = {},
+): LoadedRoutingCase[] {
+	const root = resolve(dir);
+	const files = readdirSync(root)
+		.filter((file) => file.endsWith('.json'))
+		.sort();
+
+	const loaded: LoadedRoutingCase[] = [];
+	const errors: string[] = [];
+	const seenIds = new Map<string, string>();
+	for (const file of files) {
+		const fileName = basename(file, '.json');
+		const outcome = readCaseFile(join(root, file), fileName, selection);
+		if ('skip' in outcome) continue;
+		if ('error' in outcome) {
+			errors.push(outcome.error);
+			continue;
+		}
+		const { routingCase } = outcome;
+		const previous = seenIds.get(routingCase.id);
+		if (previous) {
+			errors.push(`${join(root, file)}: id "${routingCase.id}" is also used by ${previous}.json`);
+			continue;
+		}
+		seenIds.set(routingCase.id, fileName);
+		loaded.push({ routingCase, scenario: toDiscoveryScenario(routingCase), fileName });
+	}
+
+	if (errors.length > 0) {
+		throw new Error(
+			`Invalid routing case(s) in ${root}:\n${errors.map((e) => `  - ${e}`).join('\n')}`,
+		);
+	}
+	return narrowToExactMatch(loaded, selection.filter);
+}

packages/@n8n/instance-ai/evaluations/discovery/routing-context.ts

@@ -0,0 +1,89 @@
+// ---------------------------------------------------------------------------
+// What a routing case's seed and attachment add to the orchestrator's turn.
+//
+// Production (packages/cli/src/modules/instance-ai/instance-ai.service.ts)
+// sends an editor hand-off as a resource attachment and prepends a
+// `<thread-context>` block built by `internal-messages.ts`. The runner uses the
+// same builders, so the model reads the same block. Seeded prior messages go
+// into the agent's memory as thread history, the way a restored thread holds
+// them.
+// ---------------------------------------------------------------------------
+
+import { Memory, type AgentDbMessage, type BuiltMemory } from '@n8n/agents';
+import type { InstanceAiResourceAttachment } from '@n8n/api-types';
+import { jsonParse } from 'n8n-workflow';
+
+import type { RoutingAttachment, RoutingSeed } from './types';
+// Deep relative import, like the node-definition resolver in harness/stub-services.ts:
+// the eval must render the exact block production renders.
+import {
+	buildThreadArtifactsBlock,
+	buildThreadContextBlock,
+} from '../../../../cli/src/modules/instance-ai/internal-messages';
+
+/** The project the stub instance reports for every Agent. */
+export const STUB_PROJECT_ID = 'discovery-project';
+
+/**
+ * The resource attachment production receives when the user sends a message
+ * with this seeded workflow or Agent open. The loader already checked that the
+ * attachment names a seeded resource.
+ */
+export function toResourceAttachment(
+	attach: RoutingAttachment,
+	seed: RoutingSeed | undefined,
+): InstanceAiResourceAttachment {
+	if ('workflow' in attach) {
+		const workflow = seed?.workflows.find((w) => w.id === attach.workflow);
+		if (!workflow) throw new Error(`attach.workflow "${attach.workflow}" is not a seeded workflow`);
+		return { type: 'workflow', id: workflow.id, name: workflow.name };
+	}
+	const agent = seed?.agents.find((a) => a.id === attach.agent);
+	if (!agent) throw new Error(`attach.agent "${attach.agent}" is not a seeded Agent`);
+	return { type: 'agent', id: agent.id, name: agent.config.name, projectId: STUB_PROJECT_ID };
+}
+
+/**
+ * The text the orchestrator receives for the turn. With an attachment, it is
+ * the `<thread-context>` block with the `<thread-artifacts>` hand-off, a blank
+ * line, and the user's message, as production composes it. Without one, it is
+ * the user's message unchanged.
+ */
+export function buildRoutingTurnMessage(
+	userMessage: string,
+	attachment: InstanceAiResourceAttachment | undefined,
+): string {
+	if (!attachment) return userMessage;
+	const threadContextBlock = buildThreadContextBlock([
+		buildThreadArtifactsBlock(undefined, [attachment]),
+	]);
+	return [threadContextBlock, userMessage].filter(Boolean).join('\n\n');
+}
+
+/**
+ * An in-memory thread that already holds the seeded messages. The agent loads
+ * them as history when it streams with `persistence` for this thread.
+ */
+export async function createSeededMemory(
+	threadId: string,
+	resourceId: string,
+	messages: RoutingSeed['messages'],
+): Promise<BuiltMemory> {
+	const { memory } = new Memory().build();
+	await memory.saveThread({ id: threadId, resourceId });
+	await memory.saveMessages({ threadId, resourceId, messages: toDbMessages(messages) });
+	return memory;
+}
+
+/**
+ * Seed messages are the runtime's own stored shape, validated at load for
+ * their envelope only. The store keeps them verbatim, like production's thread
+ * restore; only `createdAt` becomes a `Date`.
+ */
+function toDbMessages(messages: RoutingSeed['messages']): AgentDbMessage[] {
+	return messages.map((message) => ({
+		...jsonParse<AgentDbMessage>(JSON.stringify(message)),
+		id: message.id,
+		createdAt: new Date(message.createdAt),
+	}));
+}

packages/@n8n/instance-ai/evaluations/discovery/routing-eval-results.ts

@@ -0,0 +1,387 @@
+// ---------------------------------------------------------------------------
+// `--output-dir` in routing mode: grades each case inline and writes
+//
+// - `eval-results.json`, in the contract the LangTracer dispatcher reads (see
+//   evaluations/__tests__/eval-results-dispatcher-contract.test.ts),
+// - `routing-results.json`, the `--json-out` format (written by the CLI), and
+// - `routing-summary.md`, a short summary for people.
+//
+// A routing case has one build expectation, `routingExpectationText(accepts)`,
+// the same text the LangTracer push stores, so LangTracer can match the two.
+// Each trial is one verdict on it. A trial that could not be graded (the judge
+// failed, or the run ended before a committing call because of a timeout or an
+// error) is `incomplete`: it counts neither as a pass nor as a failure.
+//
+// The files are rewritten after every graded case, so a run that is stopped
+// early still leaves the cases it finished.
+// ---------------------------------------------------------------------------
+
+import { mkdir, rename, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+import type { RoutingCaseResult, RoutingResultsFile } from './types';
+import type { EvalAttribution } from '../harness/attribution';
+import { routingExpectationText } from '../routing/expectation';
+import {
+	computeRunMetrics,
+	failureReason,
+	type GradedCase,
+	type GradedTrial,
+} from '../routing/grade-report';
+import { gradeRoutingResults } from '../routing/grade-run';
+import type { RoutingCase as GraderCase } from '../routing/grade-types';
+import { JUDGE_MODEL, type RoutingJudge } from '../routing/judge';
+import type { RoutingCase } from '../routing/loader';
+import { passAtK, passHatK } from '../run/aggregator';
+import type { BuildExpectationResult, CaseVerificationStatus, TranscriptTurn } from '../types';
+
+export const EVAL_RESULTS_FILE = 'eval-results.json';
+export const ROUTING_RESULTS_FILE = 'routing-results.json';
+export const ROUTING_SUMMARY_FILE = 'routing-summary.md';
+
+const JUDGE_CONCURRENCY = 4;
+const MAX_DETAIL_LENGTH = 300;
+
+export interface RoutingEvalCaseInput {
+	routingCase: RoutingCase;
+	/** The case file name without `.json`. */
+	fileName?: string;
+	result: RoutingCaseResult;
+	/** One per trial. */
+	threadIds: string[];
+	/** One per trial. */
+	transcripts: TranscriptTurn[][];
+}
+
+export interface RoutingEvalCase {
+	input: RoutingEvalCaseInput;
+	/** Absent when the grader left the case out (no trials, or a stale stop call). */
+	graded?: GradedCase;
+}
+
+export type RoutingEvalMeta = Pick<
+	RoutingResultsFile,
+	'runId' | 'variant' | 'model' | 'trialsPerCase' | 'stopOnRoute' | 'startedAt' | 'skillOverrides'
+>;
+
+export interface RoutingDispatcherTestCase {
+	name: string;
+	testCaseFile: string;
+	status: CaseVerificationStatus;
+	totalRuns: number;
+	buildExpectations: Array<{
+		expectation: string;
+		passCount: number;
+		evaluatedCount: number;
+		passAtK: number;
+		passHatK: number;
+	}>;
+	buildExpectationResultsPerRun: BuildExpectationResult[][];
+	threadIds: Array<string | null>;
+	transcriptPerRun: Array<TranscriptTurn[] | null>;
+}
+
+export interface RoutingEvalResultsFile {
+	timestamp: string;
+	duration: number;
+	totalRuns: number;
+	/** False while the run is still going. */
+	complete: boolean;
+	routing: RoutingEvalMeta & { judgeModel: string };
+	summary: {
+		testCases: number;
+		/** Cases that passed a strict majority of their graded trials, as LangTracer counts them. */
+		passed: number;
+		/** Cases with no graded trial. */
+		notVerified: number;
+	};
+	testCases: RoutingDispatcherTestCase[];
+}
+
+export function toGraderCase(routingCase: RoutingCase): GraderCase {
+	return {
+		id: routingCase.id,
+		bucket: routingCase.bucket,
+		userMessage: routingCase.userMessage,
+		accepts: routingCase.accepts,
+		policyDependent: routingCase.policyDependent ?? false,
+		...(routingCase.agentShaped !== undefined ? { agentShaped: routingCase.agentShaped } : {}),
+		source: routingCase.source,
+		...(routingCase.rationale !== undefined ? { rationale: routingCase.rationale } : {}),
+	};
+}
+
+function oneLine(text: string, max = MAX_DETAIL_LENGTH): string {
+	const flat = text.replace(/\s+/g, ' ').trim();
+	return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+
+/**
+ * Why a trial counts neither as a pass nor as a failure, as its attribution,
+ * or undefined when it is graded. A route decided by a committing call is
+ * graded even when the run failed later: the decision was already made.
+ */
+export function incompleteAttribution(trial: GradedTrial): EvalAttribution | undefined {
+	const { resolution } = trial;
+	if (resolution.judgeError !== undefined) return 'verification_gap';
+	const committed = resolution.rule !== undefined && resolution.rule <= 6;
+	if (committed) return undefined;
+	if (trial.streamStatus === 'timed-out') return 'timeout';
+	if (trial.runError !== undefined || trial.streamStatus === 'errored') return 'framework_issue';
+	return undefined;
+}
+
+/** Ends the text with a full stop unless it already ends a sentence. */
+function sentence(text: string): string {
+	return /[.!?]$/.test(text) ? text : `${text}.`;
+}
+
+function describeRoute(trial: GradedTrial): string {
+	const { resolution } = trial;
+	const rule = resolution.rule === undefined ? '' : `rule ${String(resolution.rule)}, `;
+	const judge = resolution.judgeReason
+		? ` Judge: ${sentence(oneLine(resolution.judgeReason))}`
+		: '';
+	return `Route ${trial.label} (${rule}${resolution.evidence}).${judge}`;
+}
+
+function incompleteReason(trial: GradedTrial, attribution: EvalAttribution): string {
+	const { resolution } = trial;
+	if (attribution === 'verification_gap') {
+		return `Not graded: the judge failed (${oneLine(resolution.judgeError ?? 'no verdict')}).`;
+	}
+	const detail = trial.runError
+		? `${trial.streamStatus}: ${oneLine(trial.runError)}`
+		: trial.streamStatus;
+	return attribution === 'timeout'
+		? `Not graded: the run timed out before a committing call (${detail}).`
+		: `Not graded: the run failed before a committing call (${detail}).`;
+}
+
+/** One trial as a verdict on the case's routing expectation. */
+export function trialVerdict(
+	accepts: RoutingCase['accepts'],
+	trial: GradedTrial,
+): BuildExpectationResult {
+	const expectation = routingExpectationText(accepts);
+	const attribution = incompleteAttribution(trial);
+	if (attribution) {
+		return {
+			expectation,
+			pass: false,
+			reason: incompleteReason(trial, attribution),
+			incomplete: true,
+			attribution,
+		};
+	}
+	if (trial.pass) {
+		const how = trial.strictPass ? 'Accepted.' : 'Accepted as a same-direction clarification.';
+		return { expectation, pass: true, reason: `${describeRoute(trial)} ${how}` };
+	}
+	return {
+		expectation,
+		pass: false,
+		reason: `${describeRoute(trial)} Not in the accepted routes: ${accepts.join(', ')}.`,
+		attribution: 'builder_issue',
+	};
+}
+
+function ungradedVerdict(accepts: RoutingCase['accepts']): BuildExpectationResult {
+	return {
+		expectation: routingExpectationText(accepts),
+		pass: false,
+		reason:
+			'Not graded: the grader left the case out (no trials, or a stop call that no longer commits).',
+		incomplete: true,
+		attribution: 'verification_gap',
+	};
+}
+
+export function toDispatcherTestCase({
+	input,
+	graded,
+}: RoutingEvalCase): RoutingDispatcherTestCase {
+	const { routingCase, result } = input;
+	const verdicts = graded
+		? graded.trials.map((trial) => trialVerdict(routingCase.accepts, trial))
+		: result.trials.map(() => ungradedVerdict(routingCase.accepts));
+	const evaluated = verdicts.filter((verdict) => !verdict.incomplete);
+	const passCount = evaluated.filter((verdict) => verdict.pass).length;
+	const n = evaluated.length;
+	return {
+		name: routingCase.id,
+		testCaseFile: input.fileName ?? routingCase.id,
+		status: n > 0 ? 'verified' : 'notVerified',
+		totalRuns: result.trials.length,
+		buildExpectations: [
+			{
+				expectation: routingExpectationText(routingCase.accepts),
+				passCount,
+				evaluatedCount: n,
+				// Terminal values, at k = the evaluated trials, as run/persist.ts writes them.
+				passAtK: passAtK(n, passCount, n),
+				passHatK: passHatK(n, passCount, n),
+			},
+		],
+		buildExpectationResultsPerRun: verdicts.map((verdict) => [verdict]),
+		threadIds: result.trials.map((_, index) => input.threadIds[index] ?? null),
+		transcriptPerRun: result.trials.map((_, index) => input.transcripts[index] ?? null),
+	};
+}
+
+function majorityPass(testCase: RoutingDispatcherTestCase): boolean {
+	const [unit] = testCase.buildExpectations;
+	return unit.evaluatedCount > 0 && unit.passCount * 2 > unit.evaluatedCount;
+}
+
+export function buildRoutingEvalResults(
+	cases: RoutingEvalCase[],
+	meta: RoutingEvalMeta,
+	complete: boolean,
+	now: Date = new Date(),
+): RoutingEvalResultsFile {
+	const testCases = cases.map(toDispatcherTestCase);
+	return {
+		timestamp: now.toISOString(),
+		duration: now.getTime() - Date.parse(meta.startedAt),
+		totalRuns: meta.trialsPerCase,
+		complete,
+		routing: { ...meta, judgeModel: JUDGE_MODEL },
+		summary: {
+			testCases: testCases.length,
+			passed: testCases.filter(majorityPass).length,
+			notVerified: testCases.filter((testCase) => testCase.status === 'notVerified').length,
+		},
+		testCases,
+	};
+}
+
+function pct(value: number | null): string {
+	return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
+}
+
+function cell(text: string): string {
+	return text.replace(/\|/g, '\\|');
+}
+
+export function renderRoutingSummary(cases: RoutingEvalCase[], meta: RoutingEvalMeta): string {
+	const graded = cases.flatMap(({ graded: gradedCase }) => (gradedCase ? [gradedCase] : []));
+	const { all } = computeRunMetrics(graded);
+	const lines = [
+		'# Routing eval summary',
+		'',
+		`Run \`${meta.runId}\`: variant ${meta.variant}, model ${meta.model}, ${String(cases.length)} case(s) × ${String(meta.trialsPerCase)} trial(s), stop on route: ${meta.stopOnRoute ? 'yes' : 'no'}.`,
+		'',
+		`Macro accuracy (v2): ${pct(all.macroAccuracy)}. Strict macro accuracy: ${pct(all.strictMacroAccuracy)}. Trial pass rate: ${pct(all.trialPass.rate)} (${String(all.trialPass.num)}/${String(all.trialPass.den)}). These grader metrics count every trial; the table counts graded trials only, as LangTracer does.`,
+		'',
+		'| Case | Bucket | Accepts | Routes | Passed |',
+		'|---|---|---|---|---|',
+	];
+	for (const entry of cases) {
+		const testCase = toDispatcherTestCase(entry);
+		const [unit] = testCase.buildExpectations;
+		const routes = entry.graded?.trials.map((trial) => trial.label).join(', ') ?? 'not graded';
+		const ungraded = testCase.totalRuns - unit.evaluatedCount;
+		const verdict =
+			testCase.status === 'notVerified' ? 'not verified' : majorityPass(testCase) ? 'pass' : 'fail';
+		const passed = `${String(unit.passCount)}/${String(unit.evaluatedCount)} ${verdict}${ungraded > 0 ? ` (${String(ungraded)} not graded)` : ''}`;
+		const { routingCase } = entry.input;
+		lines.push(
+			`| ${cell(routingCase.id)} | ${routingCase.bucket} | ${routingCase.accepts.join(', ')} | ${routes} | ${passed} |`,
+		);
+	}
+	const failing = graded.filter((gradedCase) => !gradedCase.pass);
+	if (failing.length > 0) {
+		lines.push('', '## Failing trials', '');
+		for (const gradedCase of failing) {
+			lines.push(`- ${gradedCase.id}: ${oneLine(failureReason(gradedCase))}`);
+		}
+	}
+	return `${lines.join('\n')}\n`;
+}
+
+/** Grades one finished case with the shared grader. */
+export async function gradeRoutingCase(
+	input: RoutingEvalCaseInput,
+	judge: Pick<RoutingJudge, 'judge'>,
+	meta: RoutingEvalMeta,
+): Promise<RoutingEvalCase> {
+	const run = await gradeRoutingResults(
+		{
+			runId: meta.runId,
+			variant: meta.variant,
+			model: meta.model,
+			startedAt: meta.startedAt,
+			cases: [{ id: input.result.id, trials: input.result.trials }],
+		},
+		new Map([[input.routingCase.id, toGraderCase(input.routingCase)]]),
+		judge,
+		JUDGE_CONCURRENCY,
+		'in-process',
+	);
+	return { input, graded: run.cases[0] };
+}
+
+async function writeAtomic(filePath: string, content: string): Promise<void> {
+	const tmpPath = `${filePath}.tmp`;
+	await writeFile(tmpPath, content, 'utf-8');
+	await rename(tmpPath, filePath);
+}
+
+export class RoutingEvalOutput {
+	readonly dir: string;
+
+	private readonly cases: Array<RoutingEvalCase | undefined>;
+
+	/** Serializes writes: cases that finish together must not race on the temp files. */
+	private pending: Promise<void> = Promise.resolve();
+
+	constructor(
+		dir: string,
+		private readonly judge: Pick<RoutingJudge, 'judge'>,
+		private readonly meta: RoutingEvalMeta,
+		caseCount: number,
+	) {
+		this.dir = resolve(dir);
+		this.cases = new Array<RoutingEvalCase | undefined>(caseCount);
+	}
+
+	get evalResultsPath(): string {
+		return join(this.dir, EVAL_RESULTS_FILE);
+	}
+
+	get summaryPath(): string {
+		return join(this.dir, ROUTING_SUMMARY_FILE);
+	}
+
+	/** Grades a finished case at its load-order position and rewrites the files. */
+	async record(index: number, input: RoutingEvalCaseInput): Promise<RoutingEvalCase> {
+		const graded = await gradeRoutingCase(input, this.judge, this.meta);
+		this.cases[index] = graded;
+		await this.enqueueWrite(false);
+		return graded;
+	}
+
+	async finish(): Promise<RoutingEvalResultsFile> {
+		await this.enqueueWrite(true);
+		return buildRoutingEvalResults(this.finishedCases(), this.meta, true);
+	}
+
+	private finishedCases(): RoutingEvalCase[] {
+		return this.cases.filter((entry): entry is RoutingEvalCase => entry !== undefined);
+	}
+
+	private async enqueueWrite(complete: boolean): Promise<void> {
+		// A failed write must not block later ones; the caller still sees its own failure.
+		this.pending = this.pending.catch(() => {}).then(async () => await this.write(complete));
+		await this.pending;
+	}
+
+	private async write(complete: boolean): Promise<void> {
+		const cases = this.finishedCases();
+		await mkdir(this.dir, { recursive: true });
+		const results = buildRoutingEvalResults(cases, this.meta, complete);
+		await writeAtomic(this.evalResultsPath, `${JSON.stringify(results, null, 2)}\n`);
+		await writeAtomic(this.summaryPath, renderRoutingSummary(cases, this.meta));
+	}
+}

packages/@n8n/instance-ai/evaluations/discovery/routing-results.ts

@@ -0,0 +1,90 @@
+// ---------------------------------------------------------------------------
+// Writes the `--json-out` results file. The file is rewritten after every
+// finished case, so an interrupted run still leaves the cases it finished.
+// ---------------------------------------------------------------------------
+
+import { mkdir, rename, writeFile } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+
+import { buildRoutingTrialRecord } from './routing-trial';
+import type { DiscoveryRunResult } from './runner';
+import type { RoutingCaseResult, RoutingResultsFile } from './types';
+import type { RoutingCase } from '../routing/loader';
+
+export type RoutingResultsMeta = Omit<RoutingResultsFile, 'finishedAt' | 'complete' | 'cases'>;
+
+export function toRoutingCaseResult(
+	scenario: { id: string; userMessage: string },
+	routingCase: RoutingCase | undefined,
+	results: DiscoveryRunResult[],
+): RoutingCaseResult {
+	return {
+		id: scenario.id,
+		...(routingCase
+			? {
+					bucket: routingCase.bucket,
+					accepts: routingCase.accepts,
+					policyDependent: routingCase.policyDependent ?? false,
+				}
+			: {}),
+		userMessage: scenario.userMessage,
+		trials: results.map((result, index) =>
+			buildRoutingTrialRecord({
+				trial: index + 1,
+				events: result.instanceEvents,
+				durationMs: result.durationMs,
+				streamStatus: result.streamStatus,
+				...(result.stop ? { stop: result.stop } : {}),
+				...(result.usage ? { usage: result.usage } : {}),
+				...(result.runError ? { runError: result.runError } : {}),
+			}),
+		),
+	};
+}
+
+export class RoutingResultsWriter {
+	readonly filePath: string;
+
+	private readonly cases: Array<RoutingCaseResult | undefined>;
+
+	/** Serializes writes: cases that finish together must not race on the temp file. */
+	private pending: Promise<void> = Promise.resolve();
+
+	constructor(
+		filePath: string,
+		private readonly meta: RoutingResultsMeta,
+		caseCount: number,
+	) {
+		this.filePath = resolve(filePath);
+		this.cases = new Array<RoutingCaseResult | undefined>(caseCount);
+	}
+
+	/** Records a finished case at its load-order position and rewrites the file. */
+	async record(index: number, result: RoutingCaseResult): Promise<void> {
+		this.cases[index] = result;
+		await this.enqueueWrite(false);
+	}
+
+	async finish(): Promise<void> {
+		await this.enqueueWrite(true);
+	}
+
+	private async enqueueWrite(complete: boolean): Promise<void> {
+		// A failed write must not block later ones; the caller still sees its own failure.
+		this.pending = this.pending.catch(() => {}).then(async () => await this.write(complete));
+		await this.pending;
+	}
+
+	private async write(complete: boolean): Promise<void> {
+		const file: RoutingResultsFile = {
+			...this.meta,
+			finishedAt: new Date().toISOString(),
+			complete,
+			cases: this.cases.filter((c): c is RoutingCaseResult => c !== undefined),
+		};
+		await mkdir(dirname(this.filePath), { recursive: true });
+		const tmpPath = `${this.filePath}.tmp`;
+		await writeFile(tmpPath, JSON.stringify(file, null, 2) + '\n', 'utf-8');
+		await rename(tmpPath, this.filePath);
+	}
+}

packages/@n8n/instance-ai/evaluations/discovery/routing-trial.ts

@@ -0,0 +1,201 @@
+// ---------------------------------------------------------------------------
+// Turns one trial's event stream into the routing results record.
+//
+// The route is read from the orchestrator's own calls, so sub-agent calls go
+// to a separate list. Under `--stop-on-route`, calls and text after the stop
+// are dropped: the grader sees the calls "until the stop". Tool results that
+// arrive later still settle the status of a recorded call.
+// ---------------------------------------------------------------------------
+
+import type { InstanceAiEvent } from '@n8n/api-types';
+import { isRecord } from '@n8n/utils/is-record';
+
+import type {
+	DiscoveryStreamStatus,
+	RoutingAskUserQuestion,
+	RoutingToolCall,
+	RoutingToolCallStatus,
+	RoutingTrialRecord,
+	RoutingSubAgentToolCall,
+} from './types';
+import type { RunTokenUsage } from '../../src/stream/usage-accumulator';
+import { DOMAIN_TOOL_IDS } from '../../src/tools/tool-ids';
+
+export const ORCHESTRATOR_AGENT_ID = 'n8n-instance-agent';
+
+const LOAD_SKILL_TOOL_NAME = 'load_skill';
+
+/** Keeps the results file small when a call carries generated code or file content. */
+const MAX_ARG_STRING_LENGTH = 4000;
+const MAX_ERROR_LENGTH = 500;
+
+export interface RouteStop {
+	/** Number of events published up to and including the committing call. */
+	eventIndex: number;
+	toolName: string;
+	args: Record<string, unknown>;
+}
+
+export interface RoutingTrialInput {
+	trial: number;
+	events: InstanceAiEvent[];
+	durationMs: number;
+	streamStatus: DiscoveryStreamStatus;
+	stop?: RouteStop;
+	usage?: RunTokenUsage;
+	runError?: string;
+}
+
+export function buildRoutingTrialRecord(input: RoutingTrialInput): RoutingTrialRecord {
+	const { events, stop } = input;
+	const beforeStop = stop ? events.slice(0, stop.eventIndex) : events;
+
+	const settled = new Map<string, { status: RoutingToolCallStatus; error?: string }>();
+	const results = new Map<string, unknown>();
+	const roles = new Map<string, string>();
+	const spawnedAgents: string[] = [];
+	const activatedSkills: string[] = [];
+
+	for (const event of events) {
+		if (event.type === 'tool-result') {
+			settled.set(event.payload.toolCallId, { status: 'completed' });
+			results.set(event.payload.toolCallId, event.payload.result);
+		} else if (event.type === 'tool-error' || event.type === 'tool-interrupted') {
+			settled.set(event.payload.toolCallId, {
+				status: 'errored',
+				error: truncate(event.payload.error, MAX_ERROR_LENGTH),
+			});
+		} else if (event.type === 'agent-spawned') {
+			roles.set(event.agentId, event.payload.role);
+			pushUnique(spawnedAgents, event.payload.role);
+		}
+		for (const skillId of activatedSkillIdsOf(event)) pushUnique(activatedSkills, skillId);
+	}
+
+	const toolCalls: RoutingToolCall[] = [];
+	const subAgentToolCalls: RoutingSubAgentToolCall[] = [];
+	const skillsLoaded: string[] = [];
+	const askUserQuestions: RoutingAskUserQuestion[] = [];
+	const textSegments: string[] = [];
+	let currentText = '';
+
+	for (const event of beforeStop) {
+		const fromOrchestrator = event.agentId === ORCHESTRATOR_AGENT_ID;
+		if (event.type === 'text-delta' && fromOrchestrator) {
+			currentText += event.payload.text;
+			continue;
+		}
+		if (event.type !== 'tool-call') continue;
+
+		const { toolCallId, toolName, args } = event.payload;
+		const outcome = settled.get(toolCallId) ?? { status: 'pending' as const };
+		const call: RoutingToolCall = { toolName, args: compactArgs(args), ...outcome };
+
+		if (!fromOrchestrator) {
+			subAgentToolCalls.push({ ...call, agentRole: roles.get(event.agentId) ?? event.agentId });
+			continue;
+		}
+
+		toolCalls.push(call);
+		if (currentText.trim()) textSegments.push(currentText);
+		currentText = '';
+
+		if (toolName === LOAD_SKILL_TOOL_NAME) {
+			const skillId = loadedSkillId(args, results.get(toolCallId));
+			if (skillId) pushUnique(skillsLoaded, skillId);
+		}
+		if (toolName === DOMAIN_TOOL_IDS.ASK_USER) askUserQuestions.push(...askUserQuestionsOf(args));
+	}
+
+	for (const skillId of activatedSkills) pushUnique(skillsLoaded, skillId);
+
+	const trailingText = currentText.trim() ? currentText : '';
+	const allSegments = trailingText ? [...textSegments, trailingText] : textSegments;
+
+	return {
+		trial: input.trial,
+		durationMs: input.durationMs,
+		streamStatus: input.streamStatus,
+		toolCalls,
+		subAgentToolCalls,
+		...(stop ? { stoppedOn: { toolName: stop.toolName, args: compactArgs(stop.args) } } : {}),
+		spawnedAgents,
+		skillsLoaded,
+		askUserQuestions,
+		finalText: (trailingText || textSegments.at(-1) || '').trim(),
+		fullText: allSegments.map((segment) => segment.trim()).join('\n\n'),
+		...(input.usage
+			? {
+					usage: {
+						inputTokens: input.usage.promptTokens,
+						outputTokens: input.usage.completionTokens,
+						costUsd: input.usage.costUsd,
+					},
+				}
+			: {}),
+		...(input.runError ? { runError: input.runError } : {}),
+	};
+}
+
+/**
+ * The skill a `load_skill` call loaded, or undefined when it failed or has no
+ * result yet. The call names the skill by `skillId` or `name`. A main-skill load
+ * returns text content; other loads return `{ success, skillId }`.
+ */
+function loadedSkillId(args: Record<string, unknown>, result: unknown): string | undefined {
+	if (!isRecord(result)) return undefined;
+	if (result.type !== 'content' && result.success !== true) return undefined;
+	if (typeof result.skillId === 'string') return result.skillId;
+	if (typeof args.skillId === 'string') return args.skillId;
+	return typeof args.name === 'string' ? args.name : undefined;
+}
+
+function askUserQuestionsOf(args: Record<string, unknown>): RoutingAskUserQuestion[] {
+	if (!Array.isArray(args.questions)) return [];
+	const introMessage = typeof args.introMessage === 'string' ? args.introMessage : undefined;
+	const questions: RoutingAskUserQuestion[] = [];
+	for (const entry of args.questions) {
+		if (!isRecord(entry) || typeof entry.question !== 'string') continue;
+		const options = Array.isArray(entry.options)
+			? entry.options.filter((option): option is string => typeof option === 'string')
+			: [];
+		questions.push({
+			question: entry.question,
+			options,
+			...(introMessage ? { introMessage } : {}),
+		});
+	}
+	return questions;
+}
+
+/** Programmatic skill activations, when an event reports them. */
+function activatedSkillIdsOf(event: InstanceAiEvent): string[] {
+	const payload: unknown = event.payload;
+	if (!isRecord(payload)) return [];
+	const sources = [payload.activatedSkillIds];
+	if (isRecord(payload.result)) sources.push(payload.result.activatedSkillIds);
+	return sources.flatMap((source) =>
+		Array.isArray(source) ? source.filter((id): id is string => typeof id === 'string') : [],
+	);
+}
+
+function compactArgs(args: Record<string, unknown>): Record<string, unknown> {
+	return Object.fromEntries(Object.entries(args).map(([key, value]) => [key, compact(value)]));
+}
+
+function compact(value: unknown): unknown {
+	if (typeof value === 'string') return truncate(value, MAX_ARG_STRING_LENGTH);
+	if (Array.isArray(value)) return value.map(compact);
+	if (isRecord(value)) return compactArgs(value);
+	return value;
+}
+
+function truncate(text: string, max: number): string {
+	return text.length > max
+		? `${text.slice(0, max)}… [truncated ${String(text.length - max)} chars]`
+		: text;
+}
+
+function pushUnique(list: string[], value: string): void {
+	if (!list.includes(value)) list.push(value);
+}

packages/@n8n/instance-ai/evaluations/discovery/runner.ts

@@ -14,8 +14,19 @@
 // timeout bounds the trial, not the run: at the budget the trial stops and fails,
 // and the abandoned stream is left to unwind on its own. Scenarios or --max-steps
 // can additionally opt into an iteration cap.
+//
+// Routing mode (`stopOnRoute`) aborts the run at the orchestrator's first
+// committing call (evaluations/routing/route-rules.ts). Routing scenarios have
+// no tool expectations, so no check runs; the routing grader reads the events.
+//
+// A routing scenario can carry a `seed` and an `attach`. The stub instance then
+// serves the seeded workflows, Agents, data tables, and failed prior runs; the
+// seeded messages become thread history; and the attached resource reaches the
+// orchestrator in the same `<thread-context>` block production sends. A case
+// that seeds Agents also gets a read-only `agent-context` tool over them.
 // ---------------------------------------------------------------------------
 
+import type { RuntimeSkillSource } from '@n8n/agents';
 import type { InstanceAiEvent, TaskList } from '@n8n/api-types';
 import { nanoid } from 'nanoid';
 
@@ -28,6 +39,7 @@ import {
 import { credentialAutoSetupResponder } from './credential-approval';
 import { evaluateDiscoveryTrial } from './expected-tools-invoked';
 import { resolveStreamStatus } from './stream-status';
+import { createStubAgentContextReader } from './stub-agent-context';
 import { createStubLocalMcpServer } from './stub-local-mcp';
 import {
 	createMcpConnectResponder,
@@ -37,15 +49,30 @@ import {
 	stubMcpServerConfigs,
 	type StubMcpRegistry,
 } from './stub-mcp-registry';
-import type { DiscoveryCheckResult, DiscoveryStreamStatus, DiscoveryTestCase } from './types';
+import {
+	buildRoutingTurnMessage,
+	createSeededMemory,
+	STUB_PROJECT_ID,
+	toResourceAttachment,
+} from './routing-context';
+import { ORCHESTRATOR_AGENT_ID, type RouteStop } from './routing-trial';
+import type {
+	DiscoveryCheckResult,
+	DiscoveryScenario,
+	DiscoveryStreamStatus,
+	RoutingSeed,
+} from './types';
 import { createInstanceAgent } from '../../src/agent/instance-agent';
 import type { InstanceAiEventBus } from '../../src/event-bus';
 import type { Logger } from '../../src/logger';
 import {
 	executeResumableStream,
 	normalizeStreamSource,
+	type ExecuteResumableStreamResult,
 } from '../../src/runtime/resumable-stream-executor';
 import { loadInstanceAiRuntimeSkillSource } from '../../src/skills/runtime-skills';
+import { ASK_USER_TOOL_ID } from '../../src/tools/tool-ids';
+import type { RunTokenUsage } from '../../src/stream/usage-accumulator';
 import type {
 	BuilderTurnStream,
 	InstanceAiContext,
@@ -61,14 +88,21 @@ import { createInMemoryEventBus, wrapEventBusWithObserver } from '../harness/in-
 import { createStubServices, defaultNodesJsonPath } from '../harness/stub-services';
 import { createStubWorkspace, stubWorkspaceRoot } from '../harness/stub-workspace';
 import { extractOutcomeFromEvents } from '../outcome/event-parser';
+import { isCommittingCall } from '../routing/route-rules';
 import type { CapturedEvent, EventOutcome } from '../types';
 
+/** How long a run stopped on its route may take to wind down and report usage. */
+const STOP_SETTLE_MS = 10_000;
+
+/** How long a stop at `ask-user` waits for the card to suspend before it aborts anyway. */
+const STOP_SUSPENSION_WAIT_MS = 5_000;
+
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
 
 export interface DiscoveryRunOptions {
-	scenario: DiscoveryTestCase;
+	scenario: DiscoveryScenario;
 	modelId: ModelConfig;
 	/** Defaults to `defaultNodesJsonPath()`. */
 	nodesJsonPath?: string;
@@ -76,16 +110,32 @@ export interface DiscoveryRunOptions {
 	maxSteps?: number;
 	/** Per-trial timeout in ms. */
 	timeoutMs?: number;
+	/** Abort the run at the orchestrator's first committing call. */
+	stopOnRoute?: boolean;
+	/** After a timeout, wait up to this long for the stream to wind down so its token
+	 *  usage is reported. Unset abandons a timed-out stream at once. */
+	settleGraceMs?: number;
+	/** Orchestrator skill catalog. Defaults to the bundled skills (see `--skill-file`). */
+	runtimeSkills?: RuntimeSkillSource;
 }
 
 export interface DiscoveryRunResult {
-	scenario: DiscoveryTestCase;
-	check: DiscoveryCheckResult;
+	scenario: DiscoveryScenario;
+	/** Absent when the scenario declares no tool expectations (routing cases). */
+	check?: DiscoveryCheckResult;
 	events: CapturedEvent[];
+	/** The same events with their schema types. */
+	instanceEvents: InstanceAiEvent[];
 	outcome: EventOutcome;
 	durationMs: number;
+	/** The in-process thread the trial ran on. */
+	threadId: string;
 	/** Final agent status — useful for diagnosing why noop / unexpected loops happened. */
 	streamStatus: DiscoveryStreamStatus;
+	/** The committing call that ended the run under `stopOnRoute`. */
+	stop?: RouteStop;
+	/** Orchestrator token usage, when the stream settled and reported it. */
+	usage?: RunTokenUsage;
 	/** Populated when the run errored before reaching the check. */
 	runError?: string;
 }
@@ -102,14 +152,20 @@ export async function runDiscoveryScenario(
 	const nodesJsonPath = options.nodesJsonPath ?? defaultNodesJsonPath();
 
 	const events: CapturedEvent[] = [];
+	const instanceEvents: InstanceAiEvent[] = [];
+	// Held in an object: the event observer sets it, and a plain `let` would stay
+	// narrowed to `undefined` for the reads below.
+	const route: { stop?: RouteStop; abortOnSuspensionOf?: string } = {};
 
 	let streamStatus: DiscoveryRunResult['streamStatus'] = 'completed';
 	let runError: string | undefined;
+	let usage: RunTokenUsage | undefined;
 
 	const confirmationPolicy = buildConfirmationPolicy(options.scenario);
 	const suspensions = new Map<string, SuspensionInfo>();
 
 	const abortController = new AbortController();
+	const threadId = 'discovery-thread-' + nanoid(6);
 	let mcpManager: StubMcpClientManager | undefined;
 	let timeoutHandle: NodeJS.Timeout | undefined;
 	const budgetExpired = new Promise<'timed-out'>((resolve) => {
@@ -118,21 +174,45 @@ export async function runDiscoveryScenario(
 			resolve('timed-out');
 		}, timeoutMs);
 	});
+	let stopSettleHandle: NodeJS.Timeout | undefined;
+	let stopSuspensionHandle: NodeJS.Timeout | undefined;
+	let abortForStop: () => void = () => {};
+	// Bounds the wind-down after a route stop, in case the aborted stream hangs.
+	const stopSettleExpired = new Promise<'timed-out'>((resolve) => {
+		abortForStop = () => {
+			if (abortController.signal.aborted) return;
+			abortController.abort();
+			stopSettleHandle = setTimeout(() => resolve('timed-out'), STOP_SETTLE_MS);
+		};
+	});
 
 	try {
-		const services = await createStubServices({ nodesJsonPath });
+		const seed = options.scenario.seed;
+		const services = await createStubServices({ nodesJsonPath, ...(seed ? { seed } : {}) });
 		const mcpState = options.scenario.instanceState?.mcp;
 		const mcpRegistry = mcpState ? createStubMcpRegistry(mcpState) : undefined;
+		const agentsEnabled = isAgentFeatureEnabled();
 		const context: InstanceAiContext = {
 			...applyInstanceState(services.context, options.scenario, mcpRegistry),
-			...(isAgentFeatureEnabled() ? { builderDelegate: createStubBuilderDelegate() } : {}),
+			...(agentsEnabled ? { builderDelegate: createStubBuilderDelegate(seed) } : {}),
+			// Only a case that seeds Agents gets `agent-context`, so every other case keeps its tool set.
+			...(agentsEnabled && seed?.agents.length
+				? { agentContextService: createStubAgentContextReader(seed.agents) }
+				: {}),
 			workspace: createStubWorkspace(),
 			workspaceRoot: stubWorkspaceRoot,
 		};
 
 		mcpManager = new StubMcpClientManager(createStubMcpToolRegistry(mcpState ?? {}));
-		const threadId = 'discovery-thread-' + nanoid(6);
 		const runId = 'discovery-run-' + nanoid(6);
+		// Only a seed with messages gets memory, so an unseeded run keeps its stateless path.
+		const memory = seed?.messages.length
+			? await createSeededMemory(threadId, context.userId, seed.messages)
+			: undefined;
+		const turnMessage = buildRoutingTurnMessage(
+			options.scenario.userMessage,
+			options.scenario.attach ? toResourceAttachment(options.scenario.attach, seed) : undefined,
+		);
 
 		const approvalResponders: ApprovalResponder[] = [
 			credentialAutoSetupResponder,
@@ -140,7 +220,30 @@ export async function runDiscoveryScenario(
 		];
 
 		const eventBus = wrapEventBusWithObserver(createInMemoryEventBus(), (event) => {
+			instanceEvents.push(event);
 			events.push(toCapturedEvent(event));
+			if (
+				options.stopOnRoute &&
+				!route.stop &&
+				event.type === 'tool-call' &&
+				event.agentId === ORCHESTRATOR_AGENT_ID &&
+				isCommittingCall(event.payload.toolName, event.payload.args)
+			) {
+				route.stop = {
+					eventIndex: instanceEvents.length,
+					toolName: event.payload.toolName,
+					args: event.payload.args,
+				};
+				if (event.payload.toolName === ASK_USER_TOOL_ID) {
+					// Abort once the card suspends: the suspension's finish chunk carries the
+					// turn's token usage, and an earlier abort drops it.
+					route.abortOnSuspensionOf = event.payload.toolCallId;
+					stopSuspensionHandle = setTimeout(abortForStop, STOP_SUSPENSION_WAIT_MS);
+				} else {
+					// Abort outside the publisher's call stack.
+					queueMicrotask(abortForStop);
+				}
+			}
 		});
 
 		// `OrchestrationContext` is required for the orchestrator to receive tools like
@@ -153,6 +256,7 @@ export async function runDiscoveryScenario(
 			threadId,
 			runId,
 			abortSignal: abortController.signal,
+			...(options.runtimeSkills ? { runtimeSkills: options.runtimeSkills } : {}),
 		});
 
 		const { agent } = await createInstanceAgent({
@@ -161,15 +265,18 @@ export async function runDiscoveryScenario(
 			orchestrationContext,
 			mcpServers: stubMcpServerConfigs(mcpState ?? {}),
 			mcpManager,
-			// No memory: discovery measures stateless first-step tool dispatch.
+			// No memory unless the case seeds prior messages: discovery measures
+			// first-step tool dispatch.
 			memoryConfig: {},
+			...(memory ? { memory } : {}),
 			thinkingEnabled: false,
 		});
 
 		const streamSource = normalizeStreamSource(
-			await agent.stream(options.scenario.userMessage, {
+			await agent.stream(turnMessage, {
 				maxIterations: maxSteps,
 				abortSignal: abortController.signal,
+				...(memory ? { persistence: { threadId, resourceId: context.userId } } : {}),
 				providerOptions: {
 					anthropic: { cacheControl: { type: 'ephemeral' as const } },
 				},
@@ -182,14 +289,17 @@ export async function runDiscoveryScenario(
 			context: {
 				threadId,
 				runId,
-				agentId: 'n8n-instance-agent',
+				agentId: ORCHESTRATOR_AGENT_ID,
 				eventBus,
 				signal: abortController.signal,
 				logger: silentLogger(),
 			},
 			control: {
 				mode: 'auto',
-				onSuspension: (suspension) => suspensions.set(suspension.requestId, suspension),
+				onSuspension: (suspension) => {
+					suspensions.set(suspension.requestId, suspension);
+					if (suspension.toolCallId === route.abortOnSuspensionOf) queueMicrotask(abortForStop);
+				},
 				waitForConfirmation: async (requestId: string): Promise<Record<string, unknown>> =>
 					await Promise.resolve(
 						resolveConfirmation(suspensions.get(requestId), confirmationPolicy, approvalResponders),
@@ -197,45 +307,84 @@ export async function runDiscoveryScenario(
 			},
 		});
 		void run.catch(() => {});
-		const result = await Promise.race([run, budgetExpired]);
+		const result = await Promise.race([run, budgetExpired, stopSettleExpired]);
 
-		streamStatus = resolveStreamStatus(result, abortController.signal.aborted);
+		const settled =
+			result === 'timed-out' && options.settleGraceMs
+				? await settleWithin(run, options.settleGraceMs)
+				: result;
+		if (settled !== 'timed-out') usage = settled.usage;
+
+		streamStatus = resolveStreamStatus(
+			result,
+			abortController.signal.aborted,
+			route.stop !== undefined,
+		);
 	} catch (error) {
-		runError = error instanceof Error ? error.message : String(error);
-		streamStatus = abortController.signal.aborted ? 'timed-out' : 'errored';
+		// A stopped run can reject as it unwinds; that is the stop, not a failure.
+		if (!route.stop) runError = error instanceof Error ? error.message : String(error);
+		streamStatus = route.stop
+			? 'stopped-on-route'
+			: abortController.signal.aborted
+				? 'timed-out'
+				: 'errored';
 	} finally {
 		clearTimeout(timeoutHandle);
+		clearTimeout(stopSettleHandle);
+		clearTimeout(stopSuspensionHandle);
 		await mcpManager?.disconnect();
 	}
 
 	const observedEvents = [...events];
 	const outcome = extractOutcomeFromEvents(observedEvents);
-	const check = evaluateDiscoveryTrial(options.scenario, outcome, {
-		streamStatus,
-		timeoutMs,
-		...(runError ? { runError } : {}),
-		unmatchedConfirmations: unmatchedConfirmations(confirmationPolicy, suspensions.values()),
-	});
+	const expectedToolInvocations = options.scenario.expectedToolInvocations;
+	const check = expectedToolInvocations
+		? evaluateDiscoveryTrial({ ...options.scenario, expectedToolInvocations }, outcome, {
+				streamStatus,
+				timeoutMs,
+				...(runError ? { runError } : {}),
+				unmatchedConfirmations: unmatchedConfirmations(confirmationPolicy, suspensions.values()),
+			})
+		: undefined;
 
 	return {
 		scenario: options.scenario,
-		check,
+		...(check ? { check } : {}),
 		// An abandoned stream can still publish, so hand back what the verdict saw.
-		events: [...events],
+		events: observedEvents,
+		instanceEvents: instanceEvents.slice(0, observedEvents.length),
 		outcome,
 		durationMs: Date.now() - started,
+		threadId,
 		streamStatus,
+		...(route.stop ? { stop: route.stop } : {}),
+		...(usage ? { usage } : {}),
 		...(runError ? { runError } : {}),
 	};
 }
 
+async function settleWithin(
+	run: Promise<ExecuteResumableStreamResult>,
+	graceMs: number,
+): Promise<ExecuteResumableStreamResult | 'timed-out'> {
+	let handle: NodeJS.Timeout | undefined;
+	const expired = new Promise<'timed-out'>((resolve) => {
+		handle = setTimeout(() => resolve('timed-out'), graceMs);
+	});
+	try {
+		return await Promise.race([run.catch((): 'timed-out' => 'timed-out'), expired]);
+	} finally {
+		clearTimeout(handle);
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Internal helpers
 // ---------------------------------------------------------------------------
 
 function applyInstanceState(
 	base: InstanceAiContext,
-	scenario: DiscoveryTestCase,
+	scenario: DiscoveryScenario,
 	mcpRegistry: StubMcpRegistry | undefined,
 ): InstanceAiContext {
 	const state = scenario.instanceState;
@@ -290,19 +439,28 @@ function completedBuilderTurn(): BuilderTurnStream {
 	};
 }
 
-function createStubBuilderDelegate(): InstanceAiBuilderDelegate {
+/** The builder's reads return the seeded Agents; a new Agent is "created" at once. */
+function createStubBuilderDelegate(seed?: RoutingSeed): InstanceAiBuilderDelegate {
+	const seededAgents = new Map((seed?.agents ?? []).map((agent) => [agent.id, agent]));
 	return {
 		createAgent: async (name) =>
 			await Promise.resolve({
 				agentId: 'discovery-agent',
-				projectId: 'discovery-project',
+				projectId: STUB_PROJECT_ID,
 				name,
 			}),
 		streamBuild: async () => await Promise.resolve(completedBuilderTurn()),
 		resumeBuild: async () => await Promise.resolve(completedBuilderTurn()),
 		findOpenSuspensions: async () => await Promise.resolve([]),
 		cancelOpenSuspension: async () => await Promise.resolve(),
-		resolveAgentName: async () => await Promise.resolve(undefined),
+		resolveAgentName: async (agentId) =>
+			await Promise.resolve(seededAgents.get(agentId)?.config.name),
+		readAgentArtifact: async (agentId) => {
+			const agent = seededAgents.get(agentId);
+			return await Promise.resolve(
+				agent ? { config: agent.config, skills: agent.skills ?? {}, configHash: null } : null,
+			);
+		},
 	};
 }
 
@@ -313,6 +471,7 @@ interface StubOrchestrationContextOptions {
 	threadId: string;
 	runId: string;
 	abortSignal: AbortSignal;
+	runtimeSkills?: RuntimeSkillSource;
 }
 
 function createStubOrchestrationContext(
@@ -329,11 +488,11 @@ function createStubOrchestrationContext(
 		threadId: opts.threadId,
 		runId: opts.runId,
 		userId: opts.context.userId,
-		orchestratorAgentId: 'n8n-instance-agent',
+		orchestratorAgentId: ORCHESTRATOR_AGENT_ID,
 		modelId: opts.modelId,
 		eventBus: opts.eventBus,
 		logger: silentLogger(),
-		runtimeSkills: loadInstanceAiRuntimeSkillSource(),
+		runtimeSkills: opts.runtimeSkills ?? loadInstanceAiRuntimeSkillSource(),
 		abortSignal: opts.abortSignal,
 		taskStorage,
 		// Surface the localMcpServer so Computer Use browser tools are available to the

packages/@n8n/instance-ai/evaluations/discovery/skill-overrides.ts

@@ -0,0 +1,115 @@
+// ---------------------------------------------------------------------------
+// `--skill-file <skillId>=<path>`: run the orchestrator with a different
+// SKILL.md for one or more runtime skills, without a change to src/.
+//
+// The override starts from the production source (same exclusions and
+// placeholder substitution as `loadInstanceAiRuntimeSkillSource`) and replaces
+// only the named entries. A replaced entry keeps the bundled location fields
+// and linked files, so the skill envelope the model sees differs only in the
+// skill content and the registry hash.
+//
+// The override reaches the orchestrator's catalog and `load_skill`. Sub-agents
+// that load their own skill source (for example `build-workflow`) still get the
+// bundled files.
+// ---------------------------------------------------------------------------
+
+import {
+	createRuntimeSkillRegistry,
+	filterRuntimeSkillSource,
+	formatSkillValidationErrors,
+	parseRuntimeSkillMarkdown,
+	type RuntimeSkill,
+	type RuntimeSkillSource,
+} from '@n8n/agents';
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+
+import type { SkillOverride } from './cli-args';
+import {
+	loadInstanceAiRuntimeSkillSource,
+	substituteSkillPlaceholders,
+} from '../../src/skills/runtime-skills';
+
+/** Recorded in the results file, so a run says which skill files it used. */
+export interface SkillOverrideRecord {
+	path: string;
+	sha256: string;
+}
+
+function replacementSkill(
+	source: RuntimeSkillSource,
+	override: SkillOverride,
+	content: string,
+): RuntimeSkill {
+	const entry = source.registry.skills.find(({ id }) => id === override.skillId);
+	if (!entry) {
+		const known = source.registry.skills.map(({ id }) => id).join(', ');
+		throw new Error(
+			`--skill-file: "${override.skillId}" is not in the runtime skill catalog (${known})`,
+		);
+	}
+	const parsed = parseRuntimeSkillMarkdown(content, {
+		...(entry.sourceName ? { sourceName: entry.sourceName } : {}),
+		...(entry.path ? { path: entry.path } : {}),
+		...(entry.sourcePath ? { sourcePath: entry.sourcePath } : {}),
+		...(entry.directory ? { directory: entry.directory } : {}),
+		...(entry.sourceDirectory ? { sourceDirectory: entry.sourceDirectory } : {}),
+		...(entry.category ? { category: entry.category } : {}),
+	});
+	if (!parsed.ok) {
+		throw new Error(
+			`--skill-file: ${override.path} is not a valid skill file: ${formatSkillValidationErrors(parsed.errors)}`,
+		);
+	}
+	if (parsed.skill.id !== entry.id) {
+		throw new Error(
+			`--skill-file: ${override.path} must declare name "${entry.id}", got "${parsed.skill.id}"`,
+		);
+	}
+	return {
+		...parsed.skill,
+		instructions: substituteSkillPlaceholders(parsed.skill.instructions),
+		linkedFiles: entry.linkedFiles,
+	};
+}
+
+/**
+ * The production runtime skill source with the given skills replaced, and a
+ * record of each replacement file. Returns `undefined` for no overrides.
+ */
+export function buildSkillOverrideSource(
+	overrides: SkillOverride[],
+	base: RuntimeSkillSource = loadInstanceAiRuntimeSkillSource(),
+): { source: RuntimeSkillSource; records: Record<string, SkillOverrideRecord> } | undefined {
+	if (overrides.length === 0) return undefined;
+
+	const replacements = new Map<string, RuntimeSkill>();
+	const records: Record<string, SkillOverrideRecord> = {};
+	for (const override of overrides) {
+		if (replacements.has(override.skillId)) {
+			throw new Error(`--skill-file: "${override.skillId}" is given more than once`);
+		}
+		const content = readFileSync(override.path, 'utf-8');
+		replacements.set(override.skillId, replacementSkill(base, override, content));
+		records[override.skillId] = {
+			path: override.path,
+			sha256: createHash('sha256').update(content).digest('hex'),
+		};
+	}
+
+	const entries = new Map(
+		createRuntimeSkillRegistry([...replacements.values()]).skills.map((entry) => [entry.id, entry]),
+	);
+	const skills = base.registry.skills.map((existing) => entries.get(existing.id) ?? existing);
+	// Filtering with no exclusions recomputes the registry hash from the new entries.
+	const source = filterRuntimeSkillSource(
+		{
+			...base,
+			registry: { ...base.registry, skills },
+			loadSkill: async (skillId, anchor) =>
+				replacements.get(skillId) ?? (await base.loadSkill(skillId, anchor)),
+		},
+		[],
+	);
+	return { source, records };
+}

packages/@n8n/instance-ai/evaluations/discovery/stream-status.ts

@@ -3,10 +3,14 @@ import type { FinishReason } from '@n8n/agents';
 import type { DiscoveryStreamStatus } from './types';
 import type { ExecuteResumableStreamResult } from '../../src/runtime/resumable-stream-executor';
 
+/** `stoppedOnRoute` wins over the abort it caused: `--stop-on-route` ends the
+ *  run on purpose, so the trial counts as a normal one. */
 export function resolveStreamStatus(
 	result: ExecuteResumableStreamResult | 'timed-out',
 	aborted: boolean,
+	stoppedOnRoute = false,
 ): DiscoveryStreamStatus {
+	if (stoppedOnRoute) return 'stopped-on-route';
 	if (result === 'timed-out' || aborted) return 'timed-out';
 	if (result.status === 'errored') return 'errored';
 	if (result.status === 'suspended') return 'suspended';

packages/@n8n/instance-ai/evaluations/discovery/stub-agent-context.ts

@@ -0,0 +1,103 @@
+// ---------------------------------------------------------------------------
+// Read-only `agent-context` reader over the Agents a routing case seeds.
+//
+// The stub instance has no Agents module, so it wires no `agent-context` tool.
+// A case that seeds Agents gets this reader instead, so a question about an
+// attached Agent can be answered from its config, as production can. The
+// shapes follow the production reader
+// (packages/cli/src/modules/agents/instance-ai-agent-context.adapter.ts).
+// The config schema is rendered by the production serializer. Instance-wide
+// lookups (channels, integrations, attachable workflows) have no seeded data
+// and return an error.
+// ---------------------------------------------------------------------------
+
+import { zodSchemaToJsonSchema } from '@n8n/ai-utilities/json-schema';
+import { AgentJsonConfigBaseSchema } from '@n8n/api-types';
+
+import type { RoutingSeed } from './types';
+// Deep relative import, like harness/stub-services.ts: the schema text must match production.
+import { jsonSchemaToCompactText } from '../../../../cli/src/modules/agents/json-config/schema-text-serializer';
+import type { InstanceAiAgentContextReader } from '../../src/types';
+
+type SeedAgent = RoutingSeed['agents'][number];
+
+const UNAVAILABLE = 'This lookup is not available in the eval instance.';
+
+export function createStubAgentContextReader(
+	agents: SeedAgent[],
+	updatedAt: string = new Date().toISOString(),
+): InstanceAiAgentContextReader {
+	const byId = new Map(agents.map((agent) => [agent.id, agent]));
+	return {
+		lookup: async (input) => {
+			await Promise.resolve();
+			if (input.type === 'agents') {
+				return {
+					agents: agents.map((agent) => ({
+						agentId: agent.id,
+						name: agent.config.name,
+						published: false,
+						updatedAt,
+					})),
+				};
+			}
+			if (input.type === 'config-schema') {
+				return {
+					configurableProperties: jsonSchemaToCompactText(
+						zodSchemaToJsonSchema(AgentJsonConfigBaseSchema),
+					),
+				};
+			}
+			if (
+				input.type === 'capabilities' ||
+				input.type === 'integrations' ||
+				input.type === 'attachable-workflows'
+			) {
+				throw new Error(UNAVAILABLE);
+			}
+
+			const agent = byId.get(input.agentId);
+			if (!agent) throw new Error('Agent not found.');
+			const skills = agent.skills ?? {};
+			switch (input.type) {
+				case 'config':
+					return {
+						agent: {
+							id: agent.id,
+							name: agent.config.name,
+							published: false,
+							draftVersionId: 'eval-version',
+							activeVersionId: null,
+							updatedAt,
+						},
+						configState: 'current-draft',
+						config: agent.config,
+						configHash: null,
+					};
+				case 'skills':
+					return {
+						skills: Object.entries(skills).map(([id, skill]) => ({
+							id,
+							name: skill.name,
+							description: skill.description,
+							attached: (agent.config.skills ?? []).some((ref) => ref.id === id),
+						})),
+					};
+				case 'skill': {
+					if (!Object.hasOwn(skills, input.skillId)) throw new Error('Skill not found.');
+					return { id: input.skillId, ...skills[input.skillId] };
+				}
+				case 'tasks':
+					return { tasks: [] };
+				case 'custom-tools':
+					return { tools: [] };
+				case 'custom-tool':
+					throw new Error('Custom tool not found.');
+				case 'sessions':
+					return { sessions: [], nextCursor: null };
+				case 'session':
+					return { notFound: true };
+			}
+		},
+	};
+}

packages/@n8n/instance-ai/evaluations/discovery/types.ts

@@ -10,6 +10,7 @@
 
 import type { DiscoveryMcpState } from './stub-mcp-registry';
 import type { ComputerUseState } from '../../src/types';
+import type { CaseSeed } from '../harness/schema';
 
 /**
  * Pass condition for tool invocations.
@@ -94,12 +95,30 @@ export interface DiscoveryTestCase {
 	timeoutMs?: number;
 }
 
+/** The inline seed a routing case restores into the stub instance before its turn. */
+export type RoutingSeed = Extract<CaseSeed, { mode: 'inline' }>;
+
+/** The seeded workflow or Agent the user had open when they sent the message. */
+export type RoutingAttachment = { workflow: string } | { agent: string };
+
+/** What the runner needs from a case. Routing cases have no tool expectations:
+ *  the routing grader reads the recorded calls afterwards. */
+export type DiscoveryScenario = Omit<DiscoveryTestCase, 'expectedToolInvocations'> &
+	Partial<Pick<DiscoveryTestCase, 'expectedToolInvocations'>> & {
+		/** Routing cases only: state the stub instance holds before the turn. */
+		seed?: RoutingSeed;
+		/** Routing cases only: sent with the message the way the editor hands it off. */
+		attach?: RoutingAttachment;
+	};
+
 export type DiscoveryStreamStatus =
 	| 'completed'
 	| 'errored'
 	| 'timed-out'
 	| 'suspended'
-	| 'step-exhausted';
+	| 'step-exhausted'
+	/** `--stop-on-route` ended the run at its first committing call. */
+	| 'stopped-on-route';
 
 export interface DiscoveryTrialFacts {
 	streamStatus: DiscoveryStreamStatus;
@@ -117,3 +136,78 @@ export interface DiscoveryCheckResult {
 	/** `spawn_sub_agent:<role>` markers for every spawned sub-agent. */
 	spawnedAgents: string[];
 }
+
+// ---------------------------------------------------------------------------
+// Routing results (`--json-out`), read by `evaluations/routing/grade.ts`.
+// ---------------------------------------------------------------------------
+
+export type RoutingToolCallStatus = 'completed' | 'errored' | 'pending';
+
+export interface RoutingToolCall {
+	toolName: string;
+	args: Record<string, unknown>;
+	status: RoutingToolCallStatus;
+	error?: string;
+}
+
+export interface RoutingSubAgentToolCall extends RoutingToolCall {
+	agentRole: string;
+}
+
+export interface RoutingAskUserQuestion {
+	question: string;
+	options: string[];
+	introMessage?: string;
+}
+
+export interface RoutingTrialUsage {
+	inputTokens: number;
+	outputTokens: number;
+	costUsd: number;
+}
+
+export interface RoutingTrialRecord {
+	trial: number;
+	durationMs: number;
+	streamStatus: DiscoveryStreamStatus;
+	/** Orchestrator calls in call order, up to and including the stop call. */
+	toolCalls: RoutingToolCall[];
+	/** Calls made by sub-agents. The route is read from `toolCalls` only. */
+	subAgentToolCalls: RoutingSubAgentToolCall[];
+	/** The committing call that ended the trial under `--stop-on-route`. */
+	stoppedOn?: { toolName: string; args: Record<string, unknown> };
+	spawnedAgents: string[];
+	skillsLoaded: string[];
+	askUserQuestions: RoutingAskUserQuestion[];
+	/** Orchestrator text after its last tool call, else its last non-empty text segment. */
+	finalText: string;
+	/** Every orchestrator text segment, in order, joined by blank lines. */
+	fullText: string;
+	/** Orchestrator tokens only; sub-agent usage is not included. */
+	usage?: RoutingTrialUsage;
+	runError?: string;
+}
+
+export interface RoutingCaseResult {
+	id: string;
+	bucket?: string;
+	accepts?: string[];
+	policyDependent?: boolean;
+	userMessage: string;
+	trials: RoutingTrialRecord[];
+}
+
+export interface RoutingResultsFile {
+	runId: string;
+	variant: string;
+	model: string;
+	/** `--skill-file` replacements by skill id; absent when the run used the bundled skills. */
+	skillOverrides?: Record<string, { path: string; sha256: string }>;
+	trialsPerCase: number;
+	stopOnRoute: boolean;
+	startedAt: string;
+	finishedAt: string;
+	/** False while the run is still going; the file is rewritten after each case. */
+	complete: boolean;
+	cases: RoutingCaseResult[];
+}

packages/@n8n/instance-ai/evaluations/harness/stub-seed.ts

@@ -0,0 +1,288 @@
+// ---------------------------------------------------------------------------
+// Seeded state for the in-process stub instance (`stub-services.ts`).
+//
+// The stub normally holds nothing: no workflows, no executions, no tables. A
+// routing case can seed workflows, data tables, and failed prior runs, so the
+// orchestrator's read tools find what a real instance would hold. This module
+// turns the authored seed into the shapes those services return. It keeps no
+// state of its own beyond what a service asks it to track.
+// ---------------------------------------------------------------------------
+
+import { isRecord } from '@n8n/utils/is-record';
+import type { WorkflowJSON } from '@n8n/workflow-sdk';
+import { jsonParse } from 'n8n-workflow';
+
+import type {
+	DataTableColumnInfo,
+	DataTableSummary,
+	ExecutionDebugInfo,
+	ExecutionResult,
+	ExecutionSummary,
+	WorkflowDetail,
+	WorkflowSummary,
+} from '../../src/types';
+
+/** The seed slots the stub instance can serve. Same field shapes as an inline case seed. */
+export interface StubInstanceSeed {
+	workflows?: Array<{
+		id: string;
+		name: string;
+		nodes: Array<Record<string, unknown>>;
+		connections: Record<string, unknown>;
+		published?: boolean;
+	}>;
+	dataTables?: Array<{
+		id: string;
+		name: string;
+		columns: Array<{ name: string; type: 'string' | 'number' | 'boolean' | 'date' }>;
+	}>;
+	priorRuns?: Array<{ workflow: string; hints?: string }>;
+}
+
+/** The error a prior run reports when the case gives no `hints`. */
+export const DEFAULT_PRIOR_RUN_ERROR = 'The execution failed.';
+
+/** Prior runs are spaced this far apart, the latest one this long before the stub starts. */
+const PRIOR_RUN_SPACING_MS = 60 * 60 * 1000;
+
+/** How long each prior run lasted. */
+const PRIOR_RUN_DURATION_MS = 2_000;
+
+/** The seed id a prior run execution carries, `seed-exec-1` for the first. */
+export function priorRunExecutionId(index: number): string {
+	return `seed-exec-${String(index + 1)}`;
+}
+
+/**
+ * Converts a seeded workflow to the WorkflowJSON the build and code tools read.
+ * A node without an id, a position, or a type version gets a stable default,
+ * so code generation and structure summaries do not fail on a sparse fixture.
+ */
+export function seedWorkflowToJSON(workflow: NonNullable<StubInstanceSeed['workflows']>[number]) {
+	const nodes = workflow.nodes.map((node, index) => {
+		if (typeof node.type !== 'string' || node.type.length === 0) {
+			throw new Error(`Seed workflow "${workflow.id}" node ${String(index)} has no "type"`);
+		}
+		return {
+			...node,
+			id:
+				typeof node.id === 'string' && node.id.length > 0 ? node.id : `seed-node-${String(index)}`,
+			typeVersion: typeof node.typeVersion === 'number' ? node.typeVersion : 1,
+			position: isPosition(node.position) ? node.position : [index * 220, 0],
+			parameters: isRecord(node.parameters) ? node.parameters : {},
+		};
+	});
+	// The fixture is plain JSON checked above; the round trip drops its source type.
+	return jsonParse<WorkflowJSON>(
+		JSON.stringify({
+			id: workflow.id,
+			name: workflow.name,
+			nodes,
+			connections: workflow.connections,
+		}),
+	);
+}
+
+function isPosition(value: unknown): value is [number, number] {
+	return (
+		Array.isArray(value) &&
+		value.length === 2 &&
+		typeof value[0] === 'number' &&
+		typeof value[1] === 'number'
+	);
+}
+
+export function seededWorkflowSummary(
+	json: WorkflowJSON,
+	opts: { versionId: string; activeVersionId: string | null; createdAt: string },
+): WorkflowSummary {
+	return {
+		id: json.id ?? '',
+		name: json.name,
+		versionId: opts.versionId,
+		activeVersionId: opts.activeVersionId,
+		isArchived: false,
+		createdAt: opts.createdAt,
+		updatedAt: opts.createdAt,
+	};
+}
+
+export function seededWorkflowDetail(
+	json: WorkflowJSON,
+	opts: { versionId: string; activeVersionId: string | null; createdAt: string },
+): WorkflowDetail {
+	return {
+		...seededWorkflowSummary(json, opts),
+		nodes: json.nodes.map((node) => ({ ...node, name: node.name ?? node.id })),
+		connections: json.connections,
+	};
+}
+
+/** Filters seeded workflows the way the real list does: by name query, node type, and status. */
+export function filterSeededWorkflows(
+	workflows: WorkflowJSON[],
+	options: { query?: string; status?: string; nodeTypes?: string[] } = {},
+): { inScope: WorkflowJSON[]; matching: WorkflowJSON[] } {
+	// Seeded workflows are never archived.
+	if (options.status === 'archived') return { inScope: [], matching: [] };
+	const nodeTypes = options.nodeTypes ?? [];
+	const inScope =
+		nodeTypes.length === 0
+			? workflows
+			: workflows.filter((json) => json.nodes.some((node) => nodeTypes.includes(node.type)));
+	const query = options.query?.trim().toLowerCase();
+	const matching = query
+		? inScope.filter((json) => json.name.toLowerCase().includes(query))
+		: inScope;
+	return { inScope, matching };
+}
+
+export function seededDataTableSummary(
+	table: NonNullable<StubInstanceSeed['dataTables']>[number],
+	createdAt: string,
+): DataTableSummary {
+	return {
+		id: table.id,
+		name: table.name,
+		columns: table.columns.map((column, index) => ({
+			id: `${table.id}-col-${String(index)}`,
+			name: column.name,
+			type: column.type,
+		})),
+		createdAt,
+		updatedAt: createdAt,
+	};
+}
+
+export function seededDataTableSchema(
+	table: NonNullable<StubInstanceSeed['dataTables']>[number],
+): DataTableColumnInfo[] {
+	return table.columns.map((column, index) => ({
+		id: `${table.id}-col-${String(index)}`,
+		name: column.name,
+		type: column.type,
+		index,
+	}));
+}
+
+/** One failed execution the stub reports for a seeded prior run. */
+export interface SeededExecution {
+	id: string;
+	workflowId: string;
+	workflowName: string;
+	error: string;
+	startedAt: string;
+	finishedAt: string;
+	/** The seeded node the hints name, when they name one. */
+	failedNode?: { name: string; type: string };
+}
+
+/**
+ * Builds one failed execution per prior run, in declared order, ending an
+ * hour before `now`. The hints are the error message. When the hints name a
+ * node of the workflow, that node is reported as the one that failed.
+ */
+export function buildSeededExecutions(
+	priorRuns: NonNullable<StubInstanceSeed['priorRuns']>,
+	workflowsById: Map<string, WorkflowJSON>,
+	now: number,
+): SeededExecution[] {
+	return priorRuns.map((priorRun, index) => {
+		const workflow = workflowsById.get(priorRun.workflow);
+		if (!workflow) {
+			throw new Error(`Prior run names seed workflow "${priorRun.workflow}", which is not seeded`);
+		}
+		const started = now - (priorRuns.length - index) * PRIOR_RUN_SPACING_MS;
+		const error = priorRun.hints ?? DEFAULT_PRIOR_RUN_ERROR;
+		const failedNode = findNamedNode(workflow, error);
+		return {
+			id: priorRunExecutionId(index),
+			workflowId: priorRun.workflow,
+			workflowName: workflow.name,
+			error,
+			startedAt: new Date(started).toISOString(),
+			finishedAt: new Date(started + PRIOR_RUN_DURATION_MS).toISOString(),
+			...(failedNode ? { failedNode } : {}),
+		};
+	});
+}
+
+/** The node whose name the text mentions; the longest name wins, so "Send email" beats "Send". */
+function findNamedNode(
+	workflow: WorkflowJSON,
+	text: string,
+): { name: string; type: string } | undefined {
+	const lower = text.toLowerCase();
+	let found: { name: string; type: string } | undefined;
+	for (const node of workflow.nodes) {
+		const name = node.name;
+		if (!name || !lower.includes(name.toLowerCase())) continue;
+		if (!found || name.length > found.name.length) found = { name, type: node.type };
+	}
+	return found;
+}
+
+/** Newest first, like the real execution list. */
+export function listSeededExecutions(
+	executions: SeededExecution[],
+	options: { workflowId?: string; status?: string; limit?: number } = {},
+	workflowVersionId: string,
+): ExecutionSummary[] {
+	// Every seeded run failed, so a status filter keeps all of them or none.
+	if (options.status !== undefined && options.status !== 'error') return [];
+	return executions
+		.filter((execution) => !options.workflowId || execution.workflowId === options.workflowId)
+		.reverse()
+		.slice(0, options.limit ?? executions.length)
+		.map((execution) => ({
+			id: execution.id,
+			workflowId: execution.workflowId,
+			workflowName: execution.workflowName,
+			status: 'error',
+			startedAt: execution.startedAt,
+			finishedAt: execution.finishedAt,
+			mode: 'trigger',
+			workflowVersionId,
+		}));
+}
+
+export function seededExecutionResult(
+	execution: SeededExecution,
+	workflowVersionId: string,
+): ExecutionResult {
+	return {
+		executionId: execution.id,
+		status: 'error',
+		error: execution.error,
+		startedAt: execution.startedAt,
+		finishedAt: execution.finishedAt,
+		workflowVersionId,
+		...(execution.failedNode
+			? {
+					lastNodeExecuted: execution.failedNode.name,
+					nodeErrors: [{ nodeName: execution.failedNode.name, message: execution.error }],
+				}
+			: {}),
+	};
+}
+
+export function seededExecutionDebugInfo(
+	execution: SeededExecution,
+	workflowVersionId: string,
+): ExecutionDebugInfo {
+	const { failedNode } = execution;
+	return {
+		...seededExecutionResult(execution, workflowVersionId),
+		...(failedNode ? { failedNode: { ...failedNode, error: execution.error } } : {}),
+		nodeTrace: failedNode
+			? [
+					{
+						...failedNode,
+						status: 'error',
+						startedAt: execution.startedAt,
+						finishedAt: execution.finishedAt,
+					},
+				]
+			: [],
+	};
+}

packages/@n8n/instance-ai/evaluations/harness/stub-services.ts

@@ -33,6 +33,19 @@ import {
 	listNodeDiscriminators,
 	resolveNodeTypeDefinition,
 } from '../../../../cli/src/modules/instance-ai/node-definition-resolver';
+import {
+	buildSeededExecutions,
+	filterSeededWorkflows,
+	listSeededExecutions,
+	seededDataTableSchema,
+	seededDataTableSummary,
+	seededExecutionDebugInfo,
+	seededExecutionResult,
+	seededWorkflowDetail,
+	seededWorkflowSummary,
+	seedWorkflowToJSON,
+	type StubInstanceSeed,
+} from './stub-seed';
 import type {
 	InstanceAiContext,
 	InstanceAiCredentialService,
@@ -70,6 +83,11 @@ export interface CreateStubServicesOptions {
 	nodesJsonPath: string;
 	/** Optional user id. */
 	userId?: string;
+	/**
+	 * State the stub instance holds before the turn: workflows its reads return,
+	 * data tables, and failed prior runs. Omitted keeps the empty instance.
+	 */
+	seed?: StubInstanceSeed;
 }
 
 export async function createStubServices(
@@ -83,6 +101,30 @@ export async function createStubServices(
 	// `live-current` — it cannot produce a stale live version.
 	let activeVersionId: string | null = null;
 
+	// Seeded workflows by id, updated in place when the agent saves one of them.
+	const seedCreatedAt = new Date().toISOString();
+	const seededWorkflows = new Map<string, WorkflowJSON>(
+		(options.seed?.workflows ?? []).map((workflow) => [workflow.id, seedWorkflowToJSON(workflow)]),
+	);
+	const publishedSeedIds = new Set(
+		(options.seed?.workflows ?? []).filter((w) => w.published).map((w) => w.id),
+	);
+	const seededVersion = (workflowId: string) => ({
+		versionId: EVAL_WORKFLOW_VERSION_ID,
+		activeVersionId:
+			publishedSeedIds.has(workflowId) || activeVersionId !== null
+				? EVAL_WORKFLOW_VERSION_ID
+				: null,
+		createdAt: seedCreatedAt,
+	});
+	const seededExecutions = buildSeededExecutions(
+		options.seed?.priorRuns ?? [],
+		seededWorkflows,
+		Date.now(),
+	);
+	const seededExecutionById = new Map(seededExecutions.map((e) => [e.id, e]));
+	const seededTables = new Map((options.seed?.dataTables ?? []).map((t) => [t.id, t]));
+
 	const workflowService: InstanceAiWorkflowService = {
 		async list(options) {
 			// A folder request gets a realistic miss so the transcript shows what the
@@ -100,20 +142,41 @@ export async function createStubServices(
 					},
 				};
 			}
-			return { workflows: [], total: 0, totalInScope: 0 };
+			const { inScope, matching } = filterSeededWorkflows([...seededWorkflows.values()], {
+				...(options?.query !== undefined ? { query: options.query } : {}),
+				...(options?.status !== undefined ? { status: options.status } : {}),
+				...(options?.nodeTypes !== undefined ? { nodeTypes: options.nodeTypes } : {}),
+			});
+			return {
+				workflows: matching
+					.slice(0, options?.limit ?? matching.length)
+					.map((json) => seededWorkflowSummary(json, seededVersion(json.id ?? ''))),
+				total: matching.length,
+				totalInScope: inScope.length,
+			};
 		},
 		async get(workflowId: string) {
+			const seeded = seededWorkflows.get(workflowId);
+			if (seeded) return seededWorkflowDetail(seeded, seededVersion(workflowId));
 			return { ...emptyWorkflowDetail(workflowId), activeVersionId };
 		},
 		async getAsWorkflowJSON(workflowId: string) {
+			const seeded = seededWorkflows.get(workflowId);
+			if (seeded) return seeded;
 			const latest = capturedWorkflows[capturedWorkflows.length - 1];
 			return latest ?? { id: workflowId, name: 'empty', nodes: [], connections: {} };
 		},
-		async getWorkflowHead() {
-			return { versionId: EVAL_WORKFLOW_VERSION_ID, activeVersionId, updatedAt: 0 };
+		async getWorkflowHead(workflowId: string) {
+			const head = seededWorkflows.has(workflowId) ? seededVersion(workflowId) : undefined;
+			return {
+				versionId: EVAL_WORKFLOW_VERSION_ID,
+				activeVersionId: head ? head.activeVersionId : activeVersionId,
+				updatedAt: 0,
+			};
 		},
 		async getWorkflowSnapshot(workflowId: string) {
-			const latest = capturedWorkflows[capturedWorkflows.length - 1];
+			const latest =
+				seededWorkflows.get(workflowId) ?? capturedWorkflows[capturedWorkflows.length - 1];
 			return {
 				json: latest ?? { id: workflowId, name: 'empty', nodes: [], connections: {} },
 				versionId: EVAL_WORKFLOW_VERSION_ID,
@@ -129,6 +192,8 @@ export async function createStubServices(
 		},
 		async updateFromWorkflowJSON(workflowId: string, json: WorkflowJSON) {
 			capturedWorkflows.push(json);
+			if (seededWorkflows.has(workflowId))
+				seededWorkflows.set(workflowId, { ...json, id: workflowId });
 			return {
 				...emptyWorkflowDetail(workflowId),
 				name: json.name,
@@ -147,7 +212,8 @@ export async function createStubServices(
 			activeVersionId = EVAL_WORKFLOW_VERSION_ID;
 			return { activeVersionId };
 		},
-		async unpublish() {
+		async unpublish(workflowId: string) {
+			publishedSeedIds.delete(workflowId);
 			activeVersionId = null;
 		},
 	};
@@ -229,8 +295,8 @@ export async function createStubServices(
 	};
 
 	const executionService: InstanceAiExecutionService = {
-		async list() {
-			return [];
+		async list(options) {
+			return listSeededExecutions(seededExecutions, options, EVAL_WORKFLOW_VERSION_ID);
 		},
 		// `verify-built-workflow` invokes `executionService.run()` after the
 		// eval has captured a built workflow JSON. The eval has no execution
@@ -262,16 +328,22 @@ export async function createStubServices(
 				finishedAt: new Date().toISOString(),
 			};
 		},
-		async getStatus() {
+		async getStatus(executionId: string) {
+			const seeded = seededExecutionById.get(executionId);
+			if (seeded) return seededExecutionResult(seeded, EVAL_WORKFLOW_VERSION_ID);
 			return stubExecutionResult('stub: execution disabled in eval');
 		},
-		async getResult() {
+		async getResult(executionId: string) {
+			const seeded = seededExecutionById.get(executionId);
+			if (seeded) return seededExecutionResult(seeded, EVAL_WORKFLOW_VERSION_ID);
 			return stubExecutionResult('stub: execution disabled in eval');
 		},
 		async stop() {
 			return { success: false, message: 'stub: execution disabled in eval' };
 		},
-		async getDebugInfo() {
+		async getDebugInfo(executionId: string) {
+			const seeded = seededExecutionById.get(executionId);
+			if (seeded) return seededExecutionDebugInfo(seeded, EVAL_WORKFLOW_VERSION_ID);
 			return {
 				...stubExecutionResult('stub: execution disabled in eval'),
 				nodeTrace: [],
@@ -301,7 +373,9 @@ export async function createStubServices(
 
 	const dataTableService: InstanceAiDataTableService = {
 		async list() {
-			return [];
+			return [...seededTables.values()].map((table) =>
+				seededDataTableSummary(table, seedCreatedAt),
+			);
 		},
 		async create(name: string) {
 			return {
@@ -313,8 +387,9 @@ export async function createStubServices(
 			};
 		},
 		async delete() {},
-		async getSchema() {
-			return [];
+		async getSchema(dataTableId: string) {
+			const seeded = seededTables.get(dataTableId);
+			return seeded ? seededDataTableSchema(seeded) : [];
 		},
 		async addColumn(_dataTableId: string, column) {
 			return { id: 'col-' + nanoid(), name: column.name, type: column.type, index: 0 };

packages/@n8n/instance-ai/evaluations/routing/expectation.ts

@@ -0,0 +1,15 @@
+// ---------------------------------------------------------------------------
+// The single build-expectation text a routing case carries.
+//
+// LangTracer matches a result's build expectation to the case's stored
+// expectation by exact (trimmed) text, so the push script and the
+// eval-results writer must produce the same string from the same accepts.
+// ---------------------------------------------------------------------------
+
+import type { RoutingCase } from './loader';
+
+const EXPECTATION_PREFIX = 'Routes to one of: ';
+
+export function routingExpectationText(accepts: RoutingCase['accepts']): string {
+	return `${EXPECTATION_PREFIX}${accepts.join(', ')}`;
+}

packages/@n8n/instance-ai/evaluations/routing/grade-report.ts

@@ -0,0 +1,819 @@
+// ---------------------------------------------------------------------------
+// Metrics and the markdown report for the routing grader (see the SPEC's
+// "Report" section and v2 addendum). Pure functions over graded runs; no I/O.
+//
+// `pass` is the v2 score. `strictPass` applies only the case's accept tokens.
+// ---------------------------------------------------------------------------
+
+import type { ResolutionRule, RouteResolution } from './grade-resolve';
+import { type AcceptToken, type Bucket, BUCKETS } from './grade-types';
+import type { JudgeStats } from './judge';
+
+export interface GradedTrial {
+	trial: number;
+	streamStatus: string;
+	runError?: string;
+	skillLoaded: boolean;
+	resolution: RouteResolution;
+	label: string;
+	pass: boolean;
+	strictPass: boolean;
+	/** A clarification toward the bucket's artifact (agent and workflow buckets only). */
+	sameDirectionClarify: boolean;
+	judged: boolean;
+	judgeCached?: boolean;
+}
+
+export interface GradedCase {
+	id: string;
+	bucket: Bucket;
+	accepts: AcceptToken[];
+	policyDependent: boolean;
+	trials: GradedTrial[];
+	passedTrials: number;
+	pass: boolean;
+	strictPassedTrials: number;
+	strictPass: boolean;
+}
+
+/** A case left out of the metrics because a trial stopped on a call that no longer commits. */
+export interface PendingRerunCase {
+	id: string;
+	bucket: Bucket;
+	trials: number;
+	staleTrials: Array<{ trial: number; stopCall: string }>;
+}
+
+export interface RunMeta {
+	file: string;
+	runId: string;
+	variant: string;
+	model: string;
+	startedAt?: string;
+	finishedAt?: string;
+}
+
+export interface GradedRun {
+	meta: RunMeta;
+	cases: GradedCase[];
+	/** Case ids in the results file that have no case file. */
+	missingCaseIds: string[];
+	/** Case ids in the results file with zero trials. */
+	emptyCaseIds: string[];
+	pendingRerun: PendingRerunCase[];
+}
+
+// ---------------------------------------------------------------------------
+// Metrics
+// ---------------------------------------------------------------------------
+
+export interface Rate {
+	num: number;
+	den: number;
+	rate: number | null;
+}
+
+function rate(num: number, den: number): Rate {
+	return { num, den, rate: den === 0 ? null : num / den };
+}
+
+export interface BucketMetrics {
+	cases: number;
+	casePass: Rate;
+	trialPass: Rate;
+	strictCasePass: Rate;
+	strictTrialPass: Rate;
+	/** Agent and workflow buckets only: share of trials that are a same-direction clarify. */
+	overAsking?: Rate;
+	skillLoad: Rate;
+	passWithSkill: Rate;
+	passWithoutSkill: Rate;
+}
+
+export interface ScopeMetrics {
+	cases: number;
+	trials: number;
+	/** Unweighted mean of per-bucket case pass rates, over buckets that have cases. */
+	macroAccuracy: number | null;
+	strictMacroAccuracy: number | null;
+	casePass: Rate;
+	trialPass: Rate;
+	strictCasePass: Rate;
+	strictTrialPass: Rate;
+	/** Share of agent-bucket and workflow-bucket trials that are a same-direction clarify. */
+	overAsking: Rate;
+	skillLoad: Rate;
+	passWithSkill: Rate;
+	passWithoutSkill: Rate;
+	buckets: Partial<Record<Bucket, BucketMetrics>>;
+}
+
+export interface RunMetrics {
+	all: ScopeMetrics;
+	excludingPolicyDependent: ScopeMetrics;
+}
+
+/** Buckets whose cases have a direction, so a clarification can be same-direction. */
+const DIRECTED_BUCKETS: ReadonlySet<Bucket> = new Set(['agent', 'workflow']);
+
+function trialStats(cases: readonly GradedCase[]) {
+	const trials = cases.flatMap((graded) => graded.trials);
+	const withSkill = trials.filter((trial) => trial.skillLoaded);
+	const withoutSkill = trials.filter((trial) => !trial.skillLoaded);
+	const passed = (list: GradedTrial[]) => list.filter((trial) => trial.pass).length;
+	return {
+		trials: trials.length,
+		trialPass: rate(passed(trials), trials.length),
+		strictTrialPass: rate(trials.filter((trial) => trial.strictPass).length, trials.length),
+		skillLoad: rate(withSkill.length, trials.length),
+		passWithSkill: rate(passed(withSkill), withSkill.length),
+		passWithoutSkill: rate(passed(withoutSkill), withoutSkill.length),
+	};
+}
+
+function overAskingRate(cases: readonly GradedCase[]): Rate {
+	const trials = cases
+		.filter((graded) => DIRECTED_BUCKETS.has(graded.bucket))
+		.flatMap((graded) => graded.trials);
+	return rate(trials.filter((trial) => trial.sameDirectionClarify).length, trials.length);
+}
+
+function mean(values: Array<number | null>): number | null {
+	const present = values.filter((value): value is number => value !== null);
+	return present.length === 0
+		? null
+		: present.reduce((sum, value) => sum + value, 0) / present.length;
+}
+
+export function computeScopeMetrics(cases: readonly GradedCase[]): ScopeMetrics {
+	const buckets: Partial<Record<Bucket, BucketMetrics>> = {};
+	for (const bucket of BUCKETS) {
+		const inBucket = cases.filter((graded) => graded.bucket === bucket);
+		if (inBucket.length === 0) continue;
+		const stats = trialStats(inBucket);
+		buckets[bucket] = {
+			cases: inBucket.length,
+			casePass: rate(inBucket.filter((graded) => graded.pass).length, inBucket.length),
+			trialPass: stats.trialPass,
+			strictCasePass: rate(inBucket.filter((graded) => graded.strictPass).length, inBucket.length),
+			strictTrialPass: stats.strictTrialPass,
+			overAsking: DIRECTED_BUCKETS.has(bucket) ? overAskingRate(inBucket) : undefined,
+			skillLoad: stats.skillLoad,
+			passWithSkill: stats.passWithSkill,
+			passWithoutSkill: stats.passWithoutSkill,
+		};
+	}
+	const bucketMetrics = Object.values(buckets);
+	const stats = trialStats(cases);
+	return {
+		cases: cases.length,
+		trials: stats.trials,
+		macroAccuracy: mean(bucketMetrics.map((metrics) => metrics.casePass.rate)),
+		strictMacroAccuracy: mean(bucketMetrics.map((metrics) => metrics.strictCasePass.rate)),
+		casePass: rate(cases.filter((graded) => graded.pass).length, cases.length),
+		trialPass: stats.trialPass,
+		strictCasePass: rate(cases.filter((graded) => graded.strictPass).length, cases.length),
+		strictTrialPass: stats.strictTrialPass,
+		overAsking: overAskingRate(cases),
+		skillLoad: stats.skillLoad,
+		passWithSkill: stats.passWithSkill,
+		passWithoutSkill: stats.passWithoutSkill,
+		buckets,
+	};
+}
+
+export function computeRunMetrics(cases: readonly GradedCase[]): RunMetrics {
+	return {
+		all: computeScopeMetrics(cases),
+		excludingPolicyDependent: computeScopeMetrics(
+			cases.filter((graded) => !graded.policyDependent),
+		),
+	};
+}
+
+/** Trial counts: bucket -> route label -> count. */
+export type Confusion = Partial<Record<Bucket, Record<string, number>>>;
+
+export function computeConfusion(cases: readonly GradedCase[]): Confusion {
+	const confusion: Confusion = {};
+	for (const graded of cases) {
+		const row = (confusion[graded.bucket] ??= {});
+		for (const trial of graded.trials) row[trial.label] = (row[trial.label] ?? 0) + 1;
+	}
+	return confusion;
+}
+
+const LABEL_ORDER = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'multi',
+	'clarify:agent',
+	'clarify:both',
+	'clarify:none',
+	'clarify:workflow',
+	'answer',
+	'decline',
+	'none',
+];
+
+function labelColumns(confusion: Confusion): string[] {
+	const present = new Set(Object.values(confusion).flatMap((row) => Object.keys(row)));
+	const known = LABEL_ORDER.filter((label) => present.has(label));
+	const extra = [...present].filter((label) => !LABEL_ORDER.includes(label)).sort();
+	return [...known, ...extra];
+}
+
+// ---------------------------------------------------------------------------
+// Compare
+// ---------------------------------------------------------------------------
+
+export interface FlippedCase {
+	id: string;
+	bucket: Bucket;
+	accepts: AcceptToken[];
+	direction: 'fixed' | 'broken';
+	before: string[];
+	after: string[];
+}
+
+export interface CompareSummary {
+	baseMeta: RunMeta;
+	sharedCases: number;
+	onlyInResults: string[];
+	onlyInCompare: string[];
+	/** Both runs restricted to shared cases, so deltas compare like with like. */
+	base: RunMetrics;
+	current: RunMetrics;
+	flipped: FlippedCase[];
+}
+
+export function computeCompare(current: GradedRun, base: GradedRun): CompareSummary {
+	const baseById = new Map(base.cases.map((graded) => [graded.id, graded]));
+	const currentIds = new Set(current.cases.map((graded) => graded.id));
+	const sharedCurrent = current.cases.filter((graded) => baseById.has(graded.id));
+	const sharedBase = base.cases.filter((graded) => currentIds.has(graded.id));
+
+	const flipped: FlippedCase[] = [];
+	for (const after of sharedCurrent) {
+		const before = baseById.get(after.id);
+		if (!before || before.pass === after.pass) continue;
+		flipped.push({
+			id: after.id,
+			bucket: after.bucket,
+			accepts: after.accepts,
+			direction: after.pass ? 'fixed' : 'broken',
+			before: before.trials.map((trial) => trial.label),
+			after: after.trials.map((trial) => trial.label),
+		});
+	}
+	flipped.sort((a, b) => a.direction.localeCompare(b.direction) || a.id.localeCompare(b.id));
+
+	return {
+		baseMeta: base.meta,
+		sharedCases: sharedCurrent.length,
+		onlyInResults: current.cases.filter((graded) => !baseById.has(graded.id)).map((c) => c.id),
+		onlyInCompare: base.cases.filter((graded) => !currentIds.has(graded.id)).map((c) => c.id),
+		base: computeRunMetrics(sharedBase),
+		current: computeRunMetrics(sharedCurrent),
+		flipped,
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Failure reasons
+// ---------------------------------------------------------------------------
+
+function oneLine(text: string, max: number): string {
+	const flat = text.replace(/\s+/g, ' ').trim();
+	return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat;
+}
+
+/** Groups the failing trials of a case into one line, most frequent route first. */
+export function failureReason(graded: GradedCase): string {
+	const groups = new Map<string, { count: number; detail: string }>();
+	for (const trial of graded.trials.filter((candidate) => !candidate.pass)) {
+		const { resolution } = trial;
+		let detail = resolution.evidence;
+		if (resolution.judgeError) detail = `judge error: ${oneLine(resolution.judgeError, 80)}`;
+		else if (resolution.judgeReason) detail = `judge: ${oneLine(resolution.judgeReason, 110)}`;
+		const key = `${trial.label}\u0000${resolution.evidence}`;
+		const group = groups.get(key);
+		if (group) group.count++;
+		else groups.set(key, { count: 1, detail });
+	}
+	return [...groups.entries()]
+		.sort((a, b) => b[1].count - a[1].count)
+		.map(([key, group]) => `${key.split('\u0000')[0]} x${group.count} (${group.detail})`)
+		.join('; ');
+}
+
+// ---------------------------------------------------------------------------
+// Markdown
+// ---------------------------------------------------------------------------
+
+function pct(value: number | null): string {
+	return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
+}
+
+function frac(value: Rate): string {
+	return value.rate === null ? 'n/a' : `${pct(value.rate)} (${value.num}/${value.den})`;
+}
+
+function pp(after: number | null, before: number | null): string {
+	if (after === null || before === null) return 'n/a';
+	const delta = (after - before) * 100;
+	return `${delta >= 0 ? '+' : ''}${delta.toFixed(1)} pp`;
+}
+
+function cell(text: string): string {
+	return text.replace(/\|/g, '\\|').replace(/\s*\n\s*/g, ' ');
+}
+
+function table(header: string[], rows: string[][]): string {
+	const lines = [
+		`| ${header.map(cell).join(' | ')} |`,
+		`| ${header.map(() => '---').join(' | ')} |`,
+		...rows.map((row) => `| ${row.map(cell).join(' | ')} |`),
+	];
+	return lines.join('\n');
+}
+
+function code(text: string): string {
+	return `\`${text}\``;
+}
+
+function acceptsText(accepts: readonly string[]): string {
+	return accepts.length === 0 ? '(none)' : accepts.join(', ');
+}
+
+function renderHeadline(metrics: RunMetrics): string {
+	const row = (name: string, scope: ScopeMetrics) => [
+		name,
+		String(scope.cases),
+		pct(scope.macroAccuracy),
+		frac(scope.casePass),
+		frac(scope.trialPass),
+		pct(scope.strictMacroAccuracy),
+		frac(scope.strictTrialPass),
+		frac(scope.overAsking),
+		frac(scope.skillLoad),
+	];
+	return table(
+		[
+			'Scope',
+			'Cases',
+			'Macro accuracy',
+			'Case pass rate',
+			'Trial accuracy',
+			'Strict macro',
+			'Strict trial accuracy',
+			'Over-asking',
+			'Skill-load rate',
+		],
+		[
+			row('All cases', metrics.all),
+			row('Excluding policyDependent', metrics.excludingPolicyDependent),
+		],
+	);
+}
+
+function renderBuckets(metrics: RunMetrics): string {
+	const rows = BUCKETS.flatMap((bucket) => {
+		const all = metrics.all.buckets[bucket];
+		if (!all) return [];
+		const excluded = metrics.excludingPolicyDependent.buckets[bucket];
+		return [
+			[
+				bucket,
+				String(all.cases),
+				frac(all.casePass),
+				frac(all.trialPass),
+				frac(all.strictCasePass),
+				frac(all.strictTrialPass),
+				all.overAsking ? frac(all.overAsking) : '-',
+				excluded ? frac(excluded.casePass) : 'n/a (0 cases)',
+			],
+		];
+	});
+	return table(
+		[
+			'Bucket',
+			'Cases',
+			'Case pass rate',
+			'Trial pass rate',
+			'Strict case pass rate',
+			'Strict trial pass rate',
+			'Over-asking',
+			'Case pass rate excl. policyDependent',
+		],
+		rows,
+	);
+}
+
+function renderSkill(metrics: RunMetrics): string {
+	const row = (name: string, scope: BucketMetrics | ScopeMetrics) => [
+		name,
+		String(scope.skillLoad.den),
+		frac(scope.skillLoad),
+		frac(scope.passWithSkill),
+		frac(scope.passWithoutSkill),
+	];
+	const rows = [
+		row('**All**', metrics.all),
+		...BUCKETS.flatMap((bucket) => {
+			const scope = metrics.all.buckets[bucket];
+			return scope ? [row(bucket, scope)] : [];
+		}),
+	];
+	return table(
+		['Bucket', 'Trials', 'Skill-load rate', 'Pass rate with skill', 'Pass rate without skill'],
+		rows,
+	);
+}
+
+function renderConfusion(confusion: Confusion): string {
+	const columns = labelColumns(confusion);
+	const rows = BUCKETS.flatMap((bucket) => {
+		const row = confusion[bucket];
+		if (!row) return [];
+		const total = Object.values(row).reduce((sum, count) => sum + count, 0);
+		return [[bucket, ...columns.map((label) => String(row[label] ?? 0)), String(total)]];
+	});
+	return table(['Bucket \\ Route', ...columns, 'Total'], rows);
+}
+
+function renderSameDirectionOnly(cases: readonly GradedCase[]): string {
+	const lenient = cases.filter((graded) => graded.pass && !graded.strictPass);
+	if (lenient.length === 0) return 'No case passes only because of a same-direction clarification.';
+	const rows = lenient.map((graded) => [
+		code(graded.id),
+		graded.bucket,
+		acceptsText(graded.accepts),
+		`${graded.strictPassedTrials} -> ${graded.passedTrials} of ${graded.trials.length}: ${graded.trials.map((trial) => trial.label).join(', ')}`,
+	]);
+	return table(['Case', 'Bucket', 'Accepts', 'Trials passed (strict -> v2): routes'], rows);
+}
+
+function renderPendingRerun(pending: readonly PendingRerunCase[]): string {
+	const rows = pending.map((entry) => [
+		code(entry.id),
+		entry.bucket,
+		`${entry.staleTrials.length} of ${entry.trials}`,
+		[...new Set(entry.staleTrials.map((stale) => stale.stopCall))].join(', '),
+	]);
+	return table(['Case', 'Bucket', 'Stale trials', 'Stopped on'], rows);
+}
+
+function renderFailing(cases: readonly GradedCase[]): string {
+	const failing = cases.filter((graded) => !graded.pass);
+	if (failing.length === 0) return 'No failing cases.';
+	const rows = failing.map((graded) => [
+		code(graded.id) + (graded.policyDependent ? ' (policyDependent)' : ''),
+		graded.bucket,
+		acceptsText(graded.accepts),
+		`${graded.passedTrials}/${graded.trials.length}: ${graded.trials.map((trial) => trial.label).join(', ')}`,
+		failureReason(graded),
+	]);
+	return table(['Case', 'Bucket', 'Accepts', 'Trials passed: routes', 'Reason'], rows);
+}
+
+function renderCompare(compare: CompareSummary, currentMeta: RunMeta): string {
+	const { base, current } = compare;
+	const lines: string[] = [
+		`Deltas are ${code(currentMeta.variant)} (\`--results\`) minus ${code(compare.baseMeta.variant)} (\`--compare\`, run ${code(compare.baseMeta.runId)}), over the ${compare.sharedCases} cases present in both runs.`,
+	];
+	if (compare.onlyInResults.length > 0 || compare.onlyInCompare.length > 0) {
+		lines.push(
+			`Cases only in \`--results\`: ${compare.onlyInResults.length}. Cases only in \`--compare\`: ${compare.onlyInCompare.length}. They are left out of the deltas.`,
+		);
+	}
+
+	const headlineRow = (scopeName: string, before: ScopeMetrics, after: ScopeMetrics) => [
+		[
+			scopeName,
+			'Macro accuracy',
+			pct(before.macroAccuracy),
+			pct(after.macroAccuracy),
+			pp(after.macroAccuracy, before.macroAccuracy),
+		],
+		[
+			scopeName,
+			'Case pass rate',
+			frac(before.casePass),
+			frac(after.casePass),
+			pp(after.casePass.rate, before.casePass.rate),
+		],
+		[
+			scopeName,
+			'Trial accuracy',
+			frac(before.trialPass),
+			frac(after.trialPass),
+			pp(after.trialPass.rate, before.trialPass.rate),
+		],
+		[
+			scopeName,
+			'Strict macro',
+			pct(before.strictMacroAccuracy),
+			pct(after.strictMacroAccuracy),
+			pp(after.strictMacroAccuracy, before.strictMacroAccuracy),
+		],
+		[
+			scopeName,
+			'Strict trial accuracy',
+			frac(before.strictTrialPass),
+			frac(after.strictTrialPass),
+			pp(after.strictTrialPass.rate, before.strictTrialPass.rate),
+		],
+		[
+			scopeName,
+			'Over-asking',
+			frac(before.overAsking),
+			frac(after.overAsking),
+			pp(after.overAsking.rate, before.overAsking.rate),
+		],
+		[
+			scopeName,
+			'Skill-load rate',
+			frac(before.skillLoad),
+			frac(after.skillLoad),
+			pp(after.skillLoad.rate, before.skillLoad.rate),
+		],
+	];
+	lines.push(
+		'',
+		table(
+			['Scope', 'Metric', compare.baseMeta.variant, currentMeta.variant, 'Delta'],
+			[
+				...headlineRow('All cases', base.all, current.all),
+				...headlineRow(
+					'Excl. policyDependent',
+					base.excludingPolicyDependent,
+					current.excludingPolicyDependent,
+				),
+			],
+		),
+	);
+
+	const bucketRows = BUCKETS.flatMap((bucket) => {
+		const before = base.all.buckets[bucket];
+		const after = current.all.buckets[bucket];
+		if (!before || !after) return [];
+		return [
+			[
+				bucket,
+				String(after.cases),
+				`${pct(before.casePass.rate)} -> ${pct(after.casePass.rate)}`,
+				pp(after.casePass.rate, before.casePass.rate),
+				`${pct(before.trialPass.rate)} -> ${pct(after.trialPass.rate)}`,
+				pp(after.trialPass.rate, before.trialPass.rate),
+				pp(after.strictCasePass.rate, before.strictCasePass.rate),
+				before.overAsking && after.overAsking
+					? pp(after.overAsking.rate, before.overAsking.rate)
+					: '-',
+				pp(after.skillLoad.rate, before.skillLoad.rate),
+			],
+		];
+	});
+	lines.push(
+		'',
+		'### Per-bucket deltas',
+		'',
+		table(
+			[
+				'Bucket',
+				'Cases',
+				'Case pass rate',
+				'Delta',
+				'Trial pass rate',
+				'Delta',
+				'Strict case pass delta',
+				'Over-asking delta',
+				'Skill-load delta',
+			],
+			bucketRows,
+		),
+		'',
+		'### Flipped cases',
+		'',
+	);
+
+	if (compare.flipped.length === 0) {
+		lines.push('No case changed between pass and fail.');
+	} else {
+		const fixed = compare.flipped.filter((flip) => flip.direction === 'fixed').length;
+		lines.push(
+			`${fixed} fixed (fail to pass), ${compare.flipped.length - fixed} broken (pass to fail).`,
+			'',
+			table(
+				[
+					'Case',
+					'Bucket',
+					'Change',
+					'Accepts',
+					`Routes (${compare.baseMeta.variant})`,
+					`Routes (${currentMeta.variant})`,
+				],
+				compare.flipped.map((flip) => [
+					code(flip.id),
+					flip.bucket,
+					flip.direction,
+					acceptsText(flip.accepts),
+					flip.before.join(', '),
+					flip.after.join(', '),
+				]),
+			),
+		);
+	}
+	return lines.join('\n');
+}
+
+export interface HealthSummary {
+	streamStatus: Record<string, number>;
+	rules: Record<string, number>;
+	runErrors: Array<{ caseId: string; trial: number; error: string }>;
+	unresolvedTrials: number;
+	judgedTrials: number;
+	judgeFailedTrials: number;
+	judge: JudgeStats;
+	warnings: string[];
+}
+
+export function computeHealth(
+	run: GradedRun,
+	judge: JudgeStats,
+	warnings: readonly string[],
+): HealthSummary {
+	const streamStatus: Record<string, number> = {};
+	const rules: Record<string, number> = {};
+	const runErrors: HealthSummary['runErrors'] = [];
+	let unresolvedTrials = 0;
+	let judgedTrials = 0;
+	let judgeFailedTrials = 0;
+	for (const graded of run.cases) {
+		for (const trial of graded.trials) {
+			streamStatus[trial.streamStatus] = (streamStatus[trial.streamStatus] ?? 0) + 1;
+			const rule: ResolutionRule | 'unresolved' = trial.resolution.rule ?? 'unresolved';
+			const ruleKey = rule === 'unresolved' ? rule : `rule ${rule}`;
+			rules[ruleKey] = (rules[ruleKey] ?? 0) + 1;
+			if (trial.runError) {
+				runErrors.push({ caseId: graded.id, trial: trial.trial, error: trial.runError });
+			}
+			if (trial.resolution.route === 'none') unresolvedTrials++;
+			if (trial.judged) judgedTrials++;
+			if (trial.resolution.judgeError) judgeFailedTrials++;
+		}
+	}
+	return {
+		streamStatus,
+		rules,
+		runErrors,
+		unresolvedTrials,
+		judgedTrials,
+		judgeFailedTrials,
+		judge,
+		warnings: [...warnings],
+	};
+}
+
+function renderHealth(run: GradedRun, health: HealthSummary): string {
+	const counts = (record: Record<string, number>) =>
+		Object.entries(record)
+			.sort((a, b) => a[0].localeCompare(b[0]))
+			.map(([key, count]) => `${key} ${count}`)
+			.join(', ') || 'none';
+	const lines = [
+		`- Stream status: ${counts(health.streamStatus)}.`,
+		`- Deciding rule: ${counts(health.rules)}.`,
+		`- Trials with no resolvable route (\`none\`): ${health.unresolvedTrials}.`,
+		`- Judge: ${health.judgedTrials} trials judged, ${health.judge.cacheHits} cache hits, ${health.judge.apiCalls} API requests, ${health.judgeFailedTrials} trials failed to judge, ${health.judge.inputTokens} input and ${health.judge.outputTokens} output tokens.`,
+	];
+	if (health.runErrors.length > 0) {
+		lines.push(`- Trials with a run error: ${health.runErrors.length}.`);
+		for (const runError of health.runErrors.slice(0, 10)) {
+			lines.push(
+				`  - ${code(runError.caseId)} trial ${runError.trial}: ${oneLine(runError.error, 160)}`,
+			);
+		}
+		if (health.runErrors.length > 10) lines.push(`  - and ${health.runErrors.length - 10} more.`);
+	}
+	if (run.missingCaseIds.length > 0) {
+		lines.push(
+			`- Case ids in the results with no case file (not graded): ${run.missingCaseIds.map(code).join(', ')}.`,
+		);
+	}
+	if (run.emptyCaseIds.length > 0) {
+		lines.push(`- Cases with no trials (not graded): ${run.emptyCaseIds.map(code).join(', ')}.`);
+	}
+	if (run.pendingRerun.length > 0) {
+		lines.push(
+			`- Cases pending a re-run (not graded): ${run.pendingRerun.length}. See "Pending re-run".`,
+		);
+	}
+	for (const warning of health.warnings) lines.push(`- Warning: ${warning}`);
+	return lines.join('\n');
+}
+
+function renderAllCases(cases: readonly GradedCase[]): string {
+	const rows = cases.map((graded) => [
+		code(graded.id),
+		graded.bucket,
+		acceptsText(graded.accepts),
+		graded.pass ? (graded.strictPass ? 'pass' : 'pass (strict fail)') : 'fail',
+		graded.trials
+			.map((trial) => `${trial.label} [${trial.resolution.evidence}]${trial.pass ? '' : ' (fail)'}`)
+			.join('; '),
+	]);
+	return table(['Case', 'Bucket', 'Accepts', 'Result', 'Trials'], rows);
+}
+
+export interface ReportInput {
+	run: GradedRun;
+	metrics: RunMetrics;
+	confusion: Confusion;
+	health: HealthSummary;
+	casesDir: string;
+	compare?: CompareSummary;
+}
+
+export function renderReport(input: ReportInput): string {
+	const { run, metrics, confusion, health, compare } = input;
+	const { meta } = run;
+	const sections: string[] = [
+		`# Routing eval report: ${meta.variant}`,
+		'',
+		`Run ${code(meta.runId)}, variant ${code(meta.variant)}, model ${code(meta.model)}. Started ${meta.startedAt ?? 'unknown'}, finished ${meta.finishedAt ?? 'unknown'}. ${metrics.all.cases} cases, ${metrics.all.trials} trials. Results: ${code(meta.file)}. Cases: ${code(input.casesDir)}.`,
+		'',
+		'## Headline',
+		'',
+		renderHeadline(metrics),
+		'',
+		'Macro accuracy is the unweighted mean of the per-bucket case pass rates. A case passes when at least 2 of 3 trials pass. Trial accuracy is the share of all trials that pass.',
+		'',
+		'A trial passes when its route matches an accept token of the case. In the agent bucket it also passes on `clarify:agent` or `clarify:both`, and in the workflow bucket on `clarify:workflow` or `clarify:both` (same-direction clarification). The strict scores use the accept tokens only. Over-asking is the share of agent-bucket and workflow-bucket trials that are a same-direction clarification.',
+	];
+	if (health.judgeFailedTrials > 0 || health.unresolvedTrials > 0) {
+		sections.push(
+			'',
+			`**Note:** trials with no resolvable route: ${health.unresolvedTrials} (judge failures: ${health.judgeFailedTrials}). They count as failures. See "Run health".`,
+		);
+	}
+	if (run.pendingRerun.length > 0) {
+		sections.push(
+			'',
+			`**Note:** ${run.pendingRerun.length} cases are not graded because a trial stopped on a call that no longer commits. See "Pending re-run".`,
+		);
+	}
+	if (compare) sections.push('', '## Compare', '', renderCompare(compare, meta));
+	sections.push(
+		'',
+		'## Per bucket',
+		'',
+		renderBuckets(metrics),
+		'',
+		'## Intent-recognition skill',
+		'',
+		`Share of trials with ${code('intent-recognition')} in ${code('skillsLoaded')}, and the trial pass rate with and without it.`,
+		'',
+		renderSkill(metrics),
+		'',
+		'## Confusion (trial counts)',
+		'',
+		'Rows are the case bucket. Columns are the resolved route; `clarify` is split by steer.',
+		'',
+		renderConfusion(confusion),
+		'',
+		'## Failing cases',
+		'',
+		renderFailing(run.cases),
+		'',
+		'## Pass only with same-direction clarification',
+		'',
+		renderSameDirectionOnly(run.cases),
+		'',
+		'## Run health',
+		'',
+		renderHealth(run, health),
+		'',
+	);
+	if (run.pendingRerun.length > 0) {
+		sections.push(
+			'## Pending re-run',
+			'',
+			'These cases are left out of every metric above. A trial stopped on a call that committed under the old rules but not under the current ones, so the run would have gone on. Re-run them to grade them.',
+			'',
+			renderPendingRerun(run.pendingRerun),
+			'',
+		);
+	}
+	sections.push(
+		'<details><summary>All cases</summary>',
+		'',
+		renderAllCases(run.cases),
+		'',
+		'</details>',
+		'',
+	);
+	return sections.join('\n');
+}

packages/@n8n/instance-ai/evaluations/routing/grade-resolve.ts

@@ -0,0 +1,281 @@
+// ---------------------------------------------------------------------------
+// Route resolution and accept tokens for the routing grader (see the SPEC's
+// "Route resolution" and "Accept tokens" sections, with the v2 addendum).
+//
+// The grader reads calls up to and including the first committing call. With
+// `--stop-on-route` the runner already ends there; applying the same window to
+// a run without the flag keeps the two gradable in the same way. Tool call
+// status is ignored: an errored `build-workflow` still shows the chosen route.
+//
+// Two scores: "strict" applies only the case's accept tokens. The default (v2)
+// score also passes a same-direction clarification in the agent and workflow
+// buckets.
+// ---------------------------------------------------------------------------
+
+import {
+	isCommittingCall,
+	isCommittingDataTablesAction,
+	isMutatingWorkflowsAction,
+} from './route-rules';
+import {
+	type AcceptToken,
+	type AskUserQuestion,
+	type Bucket,
+	parseAskUserQuestion,
+	type Route,
+	type RoutingCase,
+	type Steer,
+	type TrialResult,
+	type TrialToolCall,
+} from './grade-types';
+import type { JudgeInput, JudgeVerdict } from './judge';
+
+export const INTENT_SKILL_ID = 'intent-recognition';
+
+/** The runner writes this status when `--stop-on-route` ended the trial. */
+const STOPPED_ON_ROUTE = 'stopped-on-route';
+
+export type ResolutionRule = 1 | 2 | 3 | 4 | 5 | 6 | 7;
+
+export interface RouteResolution {
+	route: Route;
+	/** Set for `clarify`, and for judged text replies. */
+	steer?: Steer;
+	/** SPEC rule that decided the route; absent when nothing could be resolved. */
+	rule?: ResolutionRule;
+	/** The call or reply that decided the route, for reports. */
+	evidence: string;
+	judgeReason?: string;
+	judgeError?: string;
+}
+
+export type PendingResolution =
+	| { kind: 'resolved'; resolution: RouteResolution }
+	| { kind: 'judge'; rule: 6 | 7; evidence: string; input: JudgeInput };
+
+function actionOf(call: TrialToolCall): string | undefined {
+	return typeof call.args.action === 'string' ? call.args.action : undefined;
+}
+
+function skillIdOf(call: TrialToolCall): string | undefined {
+	const { skillId, name } = call.args;
+	if (typeof skillId === 'string') return skillId;
+	return typeof name === 'string' ? name : undefined;
+}
+
+function describeCall(call: TrialToolCall): string {
+	const action = actionOf(call);
+	return action ? `${call.toolName} ${action}` : call.toolName;
+}
+
+/** Calls up to and including the first committing call, or all calls when none commits. */
+export function routeWindow(calls: readonly TrialToolCall[]): TrialToolCall[] {
+	const stop = calls.findIndex((call) => isCommittingCall(call.toolName, call.args));
+	return stop === -1 ? [...calls] : calls.slice(0, stop + 1);
+}
+
+/**
+ * The call that ended a `--stop-on-route` trial when no recorded call commits
+ * under the current rules (for example `data-tables create`, which committed
+ * before v2). The run would have gone on, so the trial must be re-run to grade.
+ */
+export function staleStopCall(trial: TrialResult): string | undefined {
+	if (trial.streamStatus !== STOPPED_ON_ROUTE) return undefined;
+	if (trial.toolCalls.some((call) => isCommittingCall(call.toolName, call.args))) return undefined;
+	const last = trial.toolCalls.at(-1);
+	return last ? describeCall(last) : 'no recorded call';
+}
+
+function isOneOffAction(call: TrialToolCall): boolean {
+	const action = actionOf(call);
+	switch (call.toolName) {
+		case 'nodes':
+			return action === 'execute';
+		case 'executions':
+			return action === 'run' || action === 'stop';
+		case 'data-tables':
+			return isCommittingDataTablesAction(action);
+		case 'workflows':
+			return isMutatingWorkflowsAction(action);
+		default:
+			return false;
+	}
+}
+
+/** Loading the `debugging-executions` skill alone is not a debug route. */
+function isDebugSignal(call: TrialToolCall): boolean {
+	if (call.toolName !== 'executions') return false;
+	const action = actionOf(call);
+	return action === 'debug' || action === 'get' || action === 'list';
+}
+
+function askUserCard(
+	call: TrialToolCall,
+	trial: TrialResult,
+): { intro?: string; questions: AskUserQuestion[] } {
+	const fromArgs = Array.isArray(call.args.questions)
+		? call.args.questions
+				.map(parseAskUserQuestion)
+				.filter((question): question is AskUserQuestion => question !== undefined)
+		: [];
+	const intro = typeof call.args.introMessage === 'string' ? call.args.introMessage : undefined;
+	return { intro, questions: fromArgs.length > 0 ? fromArgs : trial.askUserQuestions };
+}
+
+/** Applies rules 1 to 7 in order. Rules 6 and 7 return a judge request. */
+export function resolveTrial(routingCase: RoutingCase, trial: TrialResult): PendingResolution {
+	const window = routeWindow(trial.toolCalls);
+	const resolved = (rule: ResolutionRule, route: Route, evidence: string): PendingResolution => ({
+		kind: 'resolved',
+		resolution: { route, rule, evidence },
+	});
+
+	if (
+		window.some((call) => call.toolName === 'build-agent' && call.args.operation !== 'exploring')
+	) {
+		return resolved(1, 'agent', 'build-agent');
+	}
+
+	const buildWorkflow = window.find((call) => call.toolName === 'build-workflow');
+	if (buildWorkflow) {
+		return buildWorkflow.args.executionIntent === 'one-off'
+			? resolved(2, 'one-off', 'build-workflow executionIntent=one-off')
+			: resolved(2, 'workflow', 'build-workflow');
+	}
+
+	if (window.some((call) => call.toolName === 'create-tasks')) {
+		return resolved(3, 'multi', 'create-tasks');
+	}
+
+	const oneOff = window.find(isOneOffAction);
+	if (oneOff) return resolved(4, 'one-off', describeCall(oneOff));
+
+	const debug = window.find(isDebugSignal);
+	if (debug) return resolved(5, 'debug', describeCall(debug));
+
+	const askUser = window.find((call) => call.toolName === 'ask-user');
+	if (askUser) {
+		const card = askUserCard(askUser, trial);
+		return {
+			kind: 'judge',
+			rule: 6,
+			evidence: 'ask-user',
+			input: {
+				mode: 'ask-user',
+				userMessage: routingCase.userMessage,
+				askUserIntro: card.intro,
+				askUserQuestions: card.questions,
+				finalText: trial.finalText,
+			},
+		};
+	}
+
+	if (trial.finalText.trim() === '') {
+		const status = trial.runError ? `${trial.streamStatus}: ${trial.runError}` : trial.streamStatus;
+		return {
+			kind: 'resolved',
+			resolution: {
+				route: 'none',
+				evidence: `no committing call and no final text (${status})`,
+			},
+		};
+	}
+
+	return {
+		kind: 'judge',
+		rule: 7,
+		evidence: 'final text',
+		input: {
+			mode: 'text',
+			userMessage: routingCase.userMessage,
+			askUserQuestions: [],
+			finalText: trial.finalText,
+		},
+	};
+}
+
+export function applyVerdict(
+	pending: Extract<PendingResolution, { kind: 'judge' }>,
+	verdict: JudgeVerdict,
+): RouteResolution {
+	// Rule 6 is clarify by construction; the judge only supplies the steer there.
+	const route: Route = pending.rule === 6 ? 'clarify' : verdict.kind;
+	return {
+		route,
+		steer: verdict.steer,
+		rule: pending.rule,
+		evidence: pending.evidence,
+		judgeReason: verdict.reason,
+	};
+}
+
+export function judgeFailure(
+	pending: Extract<PendingResolution, { kind: 'judge' }>,
+	error: unknown,
+): RouteResolution {
+	return {
+		route: 'none',
+		rule: pending.rule,
+		evidence: `${pending.evidence}, judge failed`,
+		judgeError: error instanceof Error ? error.message : String(error),
+	};
+}
+
+/** Short label for tables: the route, with the steer for `clarify`. */
+export function routeLabel(resolution: RouteResolution): string {
+	return resolution.route === 'clarify'
+		? `clarify:${resolution.steer ?? 'none'}`
+		: resolution.route;
+}
+
+export function acceptTokenMatches(token: AcceptToken, resolution: RouteResolution): boolean {
+	const { route, steer } = resolution;
+	switch (token) {
+		case 'clarify':
+			return route === 'clarify';
+		case 'clarify:agent':
+			return route === 'clarify' && (steer === 'agent' || steer === 'both');
+		case 'clarify:open':
+			return route === 'clarify' && steer !== undefined && steer !== 'workflow';
+		default:
+			return route === token;
+	}
+}
+
+/** The case's accept tokens only. */
+export function strictTrialPasses(routingCase: RoutingCase, resolution: RouteResolution): boolean {
+	return routingCase.accepts.some((token) => acceptTokenMatches(token, resolution));
+}
+
+/**
+ * A clarification that pushes toward the bucket's artifact: `clarify:agent` or
+ * `clarify:both` in the agent bucket, `clarify:workflow` or `clarify:both` in
+ * the workflow bucket. Other buckets have no direction.
+ */
+export function isSameDirectionClarify(bucket: Bucket, resolution: RouteResolution): boolean {
+	if (resolution.route !== 'clarify') return false;
+	const { steer } = resolution;
+	if (bucket === 'agent') return steer === 'agent' || steer === 'both';
+	if (bucket === 'workflow') return steer === 'workflow' || steer === 'both';
+	return false;
+}
+
+/** v2 score: the strict score, plus a same-direction clarification. */
+export function trialPasses(routingCase: RoutingCase, resolution: RouteResolution): boolean {
+	return (
+		strictTrialPasses(routingCase, resolution) ||
+		isSameDirectionClarify(routingCase.bucket, resolution)
+	);
+}
+
+/** SPEC: a case passes when at least 2 of 3 trials pass; other trial counts keep the 2/3 ratio. */
+export function casePasses(passedTrials: number, totalTrials: number): boolean {
+	return totalTrials > 0 && passedTrials * 3 >= totalTrials * 2;
+}
+
+export function intentSkillLoaded(trial: TrialResult): boolean {
+	if (trial.skillsLoaded.includes(INTENT_SKILL_ID)) return true;
+	return trial.toolCalls.some(
+		(call) => call.toolName === 'load_skill' && skillIdOf(call) === INTENT_SKILL_ID,
+	);
+}

packages/@n8n/instance-ai/evaluations/routing/grade-run.ts

@@ -0,0 +1,143 @@
+// ---------------------------------------------------------------------------
+// Grades a routing results set: resolves the route of each trial, asks the
+// judge where calls cannot decide it, and applies the accept tokens with the
+// v2 and strict scores. Shared by the grader CLI (grade.ts) and the discovery
+// CLI, which grades inline for `--output-dir`.
+// ---------------------------------------------------------------------------
+
+import {
+	applyVerdict,
+	casePasses,
+	intentSkillLoaded,
+	isSameDirectionClarify,
+	judgeFailure,
+	type PendingResolution,
+	resolveTrial,
+	routeLabel,
+	type RouteResolution,
+	staleStopCall,
+	strictTrialPasses,
+	trialPasses,
+} from './grade-resolve';
+import type { GradedCase, GradedRun, PendingRerunCase } from './grade-report';
+import type { RoutingCase, RoutingResults, TrialResult } from './grade-types';
+import { mapWithConcurrency, type RoutingJudge } from './judge';
+
+interface PendingTrial {
+	caseIndex: number;
+	trialIndex: number;
+	pending: PendingResolution;
+}
+
+export async function gradeRoutingResults(
+	results: RoutingResults,
+	cases: Map<string, RoutingCase>,
+	judge: Pick<RoutingJudge, 'judge'>,
+	concurrency: number,
+	/** Where the results came from, for the report. */
+	file: string,
+): Promise<GradedRun> {
+	const missingCaseIds: string[] = [];
+	const emptyCaseIds: string[] = [];
+	const pendingRerun: PendingRerunCase[] = [];
+	const graded: Array<{ routingCase: RoutingCase; trials: TrialResult[] }> = [];
+	for (const caseResult of results.cases) {
+		const routingCase = cases.get(caseResult.id);
+		if (!routingCase) {
+			missingCaseIds.push(caseResult.id);
+			continue;
+		}
+		if (caseResult.trials.length === 0) {
+			emptyCaseIds.push(caseResult.id);
+			continue;
+		}
+		const staleTrials = caseResult.trials.flatMap((trial) => {
+			const stopCall = staleStopCall(trial);
+			return stopCall === undefined ? [] : [{ trial: trial.trial, stopCall }];
+		});
+		if (staleTrials.length > 0) {
+			pendingRerun.push({
+				id: caseResult.id,
+				bucket: routingCase.bucket,
+				trials: caseResult.trials.length,
+				staleTrials,
+			});
+			continue;
+		}
+		graded.push({ routingCase, trials: caseResult.trials });
+	}
+
+	const pendingTrials: PendingTrial[] = graded.flatMap(({ routingCase, trials }, caseIndex) =>
+		trials.map((trial, trialIndex) => ({
+			caseIndex,
+			trialIndex,
+			pending: resolveTrial(routingCase, trial),
+		})),
+	);
+
+	const resolutions = await mapWithConcurrency(
+		pendingTrials,
+		concurrency,
+		async ({ pending }): Promise<{ resolution: RouteResolution; cached?: boolean }> => {
+			if (pending.kind === 'resolved') return { resolution: pending.resolution };
+			try {
+				const { verdict, cached } = await judge.judge(pending.input);
+				return { resolution: applyVerdict(pending, verdict), cached };
+			} catch (error) {
+				return { resolution: judgeFailure(pending, error) };
+			}
+		},
+	);
+
+	const gradedCases: GradedCase[] = graded.map(({ routingCase }) => ({
+		id: routingCase.id,
+		bucket: routingCase.bucket,
+		accepts: routingCase.accepts,
+		policyDependent: routingCase.policyDependent,
+		trials: [],
+		passedTrials: 0,
+		pass: false,
+		strictPassedTrials: 0,
+		strictPass: false,
+	}));
+	for (const [index, { caseIndex, trialIndex, pending }] of pendingTrials.entries()) {
+		const { routingCase, trials } = graded[caseIndex];
+		const trial = trials[trialIndex];
+		const { resolution, cached } = resolutions[index];
+		gradedCases[caseIndex].trials.push({
+			trial: trial.trial,
+			streamStatus: trial.streamStatus,
+			runError: trial.runError,
+			skillLoaded: intentSkillLoaded(trial),
+			resolution,
+			label: routeLabel(resolution),
+			pass: trialPasses(routingCase, resolution),
+			strictPass: strictTrialPasses(routingCase, resolution),
+			sameDirectionClarify: isSameDirectionClarify(routingCase.bucket, resolution),
+			judged: pending.kind === 'judge',
+			judgeCached: cached,
+		});
+	}
+	for (const gradedCase of gradedCases) {
+		const total = gradedCase.trials.length;
+		gradedCase.passedTrials = gradedCase.trials.filter((trial) => trial.pass).length;
+		gradedCase.pass = casePasses(gradedCase.passedTrials, total);
+		gradedCase.strictPassedTrials = gradedCase.trials.filter((trial) => trial.strictPass).length;
+		gradedCase.strictPass = casePasses(gradedCase.strictPassedTrials, total);
+	}
+
+	return {
+		meta: {
+			file,
+			runId: results.runId,
+			variant: results.variant,
+			model: results.model,
+			startedAt: results.startedAt,
+			finishedAt: results.finishedAt,
+		},
+		cases: gradedCases,
+		missingCaseIds,
+		emptyCaseIds,
+		pendingRerun,
+	};
+}

packages/@n8n/instance-ai/evaluations/routing/grade-types.ts

@@ -0,0 +1,277 @@
+// ---------------------------------------------------------------------------
+// Types and tolerant parsers for the routing grader's inputs: the results JSON
+// that the runner writes and the routing case files.
+//
+// The parsers accept missing optional fields so an older or partial results
+// file still grades. They reject only what the grader cannot work without.
+// ---------------------------------------------------------------------------
+
+import { readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+export const ROUTES = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'multi',
+	'clarify',
+	'answer',
+	'decline',
+	'none',
+] as const;
+export type Route = (typeof ROUTES)[number];
+
+export const BUCKETS = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'answer',
+	'clarify',
+	'multi',
+	'decline',
+] as const;
+export type Bucket = (typeof BUCKETS)[number];
+
+export const STEERS = ['agent', 'workflow', 'both', 'none'] as const;
+export type Steer = (typeof STEERS)[number];
+
+export const ACCEPT_TOKENS = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'multi',
+	'answer',
+	'decline',
+	'clarify',
+	'clarify:agent',
+	'clarify:open',
+] as const;
+export type AcceptToken = (typeof ACCEPT_TOKENS)[number];
+
+export interface RoutingCase {
+	id: string;
+	bucket: Bucket;
+	userMessage: string;
+	accepts: AcceptToken[];
+	policyDependent: boolean;
+	agentShaped?: boolean;
+	source?: string;
+	rationale?: string;
+}
+
+export interface TrialToolCall {
+	toolName: string;
+	args: Record<string, unknown>;
+	status?: string;
+}
+
+export interface AskUserQuestion {
+	question: string;
+	options: string[];
+}
+
+export interface TrialResult {
+	trial: number;
+	durationMs?: number;
+	streamStatus: string;
+	toolCalls: TrialToolCall[];
+	spawnedAgents: string[];
+	skillsLoaded: string[];
+	askUserQuestions: AskUserQuestion[];
+	finalText: string;
+	runError?: string;
+}
+
+export interface CaseResult {
+	id: string;
+	trials: TrialResult[];
+}
+
+export interface RoutingResults {
+	runId: string;
+	variant: string;
+	model: string;
+	startedAt?: string;
+	finishedAt?: string;
+	cases: CaseResult[];
+}
+
+// ---------------------------------------------------------------------------
+// Guards
+// ---------------------------------------------------------------------------
+
+export function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isOneOf<T extends string>(values: readonly T[], value: unknown): value is T {
+	return typeof value === 'string' && values.some((candidate) => candidate === value);
+}
+
+export const isRoute = (value: unknown): value is Route => isOneOf(ROUTES, value);
+export const isBucket = (value: unknown): value is Bucket => isOneOf(BUCKETS, value);
+export const isSteer = (value: unknown): value is Steer => isOneOf(STEERS, value);
+export const isAcceptToken = (value: unknown): value is AcceptToken =>
+	isOneOf(ACCEPT_TOKENS, value);
+
+function optionalString(value: unknown): string | undefined {
+	return typeof value === 'string' ? value : undefined;
+}
+
+function stringArray(value: unknown): string[] {
+	return Array.isArray(value)
+		? value.filter((item): item is string => typeof item === 'string')
+		: [];
+}
+
+// ---------------------------------------------------------------------------
+// Results
+// ---------------------------------------------------------------------------
+
+function parseToolCall(raw: unknown): TrialToolCall | undefined {
+	if (!isRecord(raw) || typeof raw.toolName !== 'string') return undefined;
+	return {
+		toolName: raw.toolName,
+		args: isRecord(raw.args) ? raw.args : {},
+		status: optionalString(raw.status),
+	};
+}
+
+export function parseAskUserQuestion(raw: unknown): AskUserQuestion | undefined {
+	if (!isRecord(raw) || typeof raw.question !== 'string') return undefined;
+	return { question: raw.question, options: stringArray(raw.options) };
+}
+
+function parseTrial(raw: unknown, index: number, where: string): TrialResult {
+	if (!isRecord(raw)) throw new Error(`${where}: trial ${index + 1} is not an object`);
+	const toolCalls = Array.isArray(raw.toolCalls)
+		? raw.toolCalls.map(parseToolCall).filter((call): call is TrialToolCall => call !== undefined)
+		: [];
+	const askUserQuestions = Array.isArray(raw.askUserQuestions)
+		? raw.askUserQuestions
+				.map(parseAskUserQuestion)
+				.filter((question): question is AskUserQuestion => question !== undefined)
+		: [];
+	return {
+		trial: typeof raw.trial === 'number' ? raw.trial : index + 1,
+		durationMs: typeof raw.durationMs === 'number' ? raw.durationMs : undefined,
+		streamStatus: optionalString(raw.streamStatus) ?? 'unknown',
+		toolCalls,
+		spawnedAgents: stringArray(raw.spawnedAgents),
+		skillsLoaded: stringArray(raw.skillsLoaded),
+		askUserQuestions,
+		finalText: optionalString(raw.finalText) ?? '',
+		runError: optionalString(raw.runError),
+	};
+}
+
+export function parseResults(raw: unknown, file: string): RoutingResults {
+	if (!isRecord(raw)) throw new Error(`${file}: results must be a JSON object`);
+	if (!Array.isArray(raw.cases)) throw new Error(`${file}: "cases" must be an array`);
+
+	// Merge entries that share an id, so a resumed run that appends trials still grades.
+	const byId = new Map<string, CaseResult>();
+	for (const [caseIndex, rawCase] of raw.cases.entries()) {
+		if (!isRecord(rawCase) || typeof rawCase.id !== 'string') {
+			throw new Error(`${file}: case ${caseIndex + 1} has no string "id"`);
+		}
+		const where = `${file}: case ${rawCase.id}`;
+		const trials = Array.isArray(rawCase.trials)
+			? rawCase.trials.map((trial, index) => parseTrial(trial, index, where))
+			: [];
+		const existing = byId.get(rawCase.id);
+		if (existing) existing.trials.push(...trials);
+		else byId.set(rawCase.id, { id: rawCase.id, trials });
+	}
+
+	return {
+		runId: optionalString(raw.runId) ?? 'unknown',
+		variant: optionalString(raw.variant) ?? 'unknown',
+		model: optionalString(raw.model) ?? 'unknown',
+		startedAt: optionalString(raw.startedAt),
+		finishedAt: optionalString(raw.finishedAt),
+		cases: [...byId.values()],
+	};
+}
+
+export function readResultsFile(file: string): RoutingResults {
+	const parsed: unknown = JSON.parse(readFileSync(file, 'utf8'));
+	return parseResults(parsed, file);
+}
+
+// ---------------------------------------------------------------------------
+// Cases
+// ---------------------------------------------------------------------------
+
+export interface LoadedCases {
+	byId: Map<string, RoutingCase>;
+	warnings: string[];
+}
+
+function parseCase(raw: unknown, file: string, warnings: string[]): RoutingCase | undefined {
+	if (!isRecord(raw)) {
+		warnings.push(`${file}: not a JSON object, skipped`);
+		return undefined;
+	}
+	if (typeof raw.id !== 'string' || typeof raw.userMessage !== 'string') {
+		warnings.push(`${file}: missing "id" or "userMessage", skipped`);
+		return undefined;
+	}
+	if (!isBucket(raw.bucket)) {
+		warnings.push(`${raw.id}: unknown bucket ${JSON.stringify(raw.bucket)}, skipped`);
+		return undefined;
+	}
+	const rawAccepts = stringArray(raw.accepts);
+	const accepts = rawAccepts.filter(isAcceptToken);
+	for (const token of rawAccepts) {
+		if (!isAcceptToken(token)) warnings.push(`${raw.id}: unknown accept token "${token}" ignored`);
+	}
+	if (accepts.length === 0) {
+		warnings.push(`${raw.id}: no valid accept tokens, so every trial fails`);
+	}
+	return {
+		id: raw.id,
+		bucket: raw.bucket,
+		userMessage: raw.userMessage,
+		accepts,
+		policyDependent: raw.policyDependent === true,
+		agentShaped: typeof raw.agentShaped === 'boolean' ? raw.agentShaped : undefined,
+		source: optionalString(raw.source),
+		rationale: optionalString(raw.rationale),
+	};
+}
+
+/** Reads every `*.json` file in the directory and indexes the cases by `id`. */
+export function loadRoutingCases(dir: string): LoadedCases {
+	const warnings: string[] = [];
+	const byId = new Map<string, RoutingCase>();
+	const files = readdirSync(dir)
+		.filter((name) => name.endsWith('.json'))
+		.sort();
+	for (const name of files) {
+		const file = join(dir, name);
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(readFileSync(file, 'utf8'));
+		} catch (error) {
+			warnings.push(
+				`${name}: invalid JSON (${error instanceof Error ? error.message : 'unknown'})`,
+			);
+			continue;
+		}
+		const routingCase = parseCase(parsed, name, warnings);
+		if (!routingCase) continue;
+		if (`${routingCase.id}.json` !== name) {
+			warnings.push(`${name}: file name does not match id "${routingCase.id}"`);
+		}
+		if (byId.has(routingCase.id)) {
+			warnings.push(`${name}: duplicate id "${routingCase.id}", later file ignored`);
+			continue;
+		}
+		byId.set(routingCase.id, routingCase);
+	}
+	return { byId, warnings };
+}

packages/@n8n/instance-ai/evaluations/routing/grade.ts

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+// ---------------------------------------------------------------------------
+// Routing eval grader.
+//
+// Usage (from packages/@n8n/instance-ai):
+//   pnpm tsx evaluations/routing/grade.ts --results <file> --cases-dir <dir> [--compare <file>] --out <report.md>
+//
+// Options:
+//   --cases-dir <dir>          Routing cases, one `<id>.json` each (required).
+//   --judge-cache-dir <dir>    Judge verdict cache (default evaluations/.data/routing-judge-cache).
+//   --judge-concurrency <n>    Parallel judge requests (default 4).
+//
+// Reads the results JSON, resolves the route of each trial (see route rules in
+// grade-resolve.ts), asks the judge only for ask-user cards and text-only
+// replies, applies the case's accept tokens, and writes a markdown report and
+// a JSON summary next to it (<report>.json). The judge needs ANTHROPIC_API_KEY
+// unless every verdict is already cached. Exits 2 when a judge call failed.
+//
+// A case with a trial that stopped on a call that no longer commits (the stop
+// rules changed after the run) is not graded. The report lists it under
+// "Pending re-run".
+// ---------------------------------------------------------------------------
+
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, parse, resolve } from 'node:path';
+
+import {
+	computeCompare,
+	computeConfusion,
+	computeHealth,
+	computeRunMetrics,
+	failureReason,
+	type GradedRun,
+	renderReport,
+} from './grade-report';
+import { gradeRoutingResults } from './grade-run';
+import { loadRoutingCases, readResultsFile, type RoutingCase } from './grade-types';
+import { DEFAULT_JUDGE_CACHE_DIR, JUDGE_MODEL, RoutingJudge } from './judge';
+
+interface CliArgs {
+	results: string;
+	compare?: string;
+	out: string;
+	casesDir: string;
+	judgeCacheDir: string;
+	judgeConcurrency: number;
+}
+
+function fail(message: string): never {
+	console.error(message);
+	process.exit(1);
+}
+
+function parseArgs(argv: string[]): CliArgs {
+	const values = new Map<string, string>();
+	for (let index = 0; index < argv.length; index++) {
+		const flag = argv[index];
+		if (flag === '--help' || flag === '-h') {
+			console.log(
+				'Usage: grade.ts --results <file> --cases-dir <dir> [--compare <file>] --out <report.md> [--judge-cache-dir <dir>] [--judge-concurrency <n>]',
+			);
+			process.exit(0);
+		}
+		if (!flag.startsWith('--')) fail(`Unexpected argument: ${flag}`);
+		const value = argv[index + 1];
+		if (value === undefined || value.startsWith('--')) fail(`Missing value for ${flag}`);
+		values.set(flag.slice(2), value);
+		index++;
+	}
+	const known = new Set([
+		'results',
+		'compare',
+		'out',
+		'cases-dir',
+		'judge-cache-dir',
+		'judge-concurrency',
+	]);
+	for (const key of values.keys()) if (!known.has(key)) fail(`Unknown option: --${key}`);
+
+	const results = values.get('results') ?? fail('--results is required');
+	const out = values.get('out') ?? fail('--out is required');
+	const casesDir = values.get('cases-dir') ?? fail('--cases-dir is required');
+	const concurrency = Number(values.get('judge-concurrency') ?? '4');
+	if (!Number.isInteger(concurrency) || concurrency < 1) {
+		fail('--judge-concurrency must be a positive integer');
+	}
+	const compare = values.get('compare');
+	return {
+		results: resolve(results),
+		compare: compare === undefined ? undefined : resolve(compare),
+		out: resolve(out),
+		casesDir: resolve(casesDir),
+		judgeCacheDir: resolve(values.get('judge-cache-dir') ?? DEFAULT_JUDGE_CACHE_DIR),
+		judgeConcurrency: concurrency,
+	};
+}
+
+async function gradeRun(
+	file: string,
+	cases: Map<string, RoutingCase>,
+	judge: RoutingJudge,
+	concurrency: number,
+): Promise<GradedRun> {
+	return await gradeRoutingResults(readResultsFile(file), cases, judge, concurrency, file);
+}
+
+function summaryPath(reportPath: string): string {
+	const { dir, name, ext } = parse(reportPath);
+	return join(dir, `${ext === '.md' ? name : `${name}${ext}`}.json`);
+}
+
+async function main() {
+	const args = parseArgs(process.argv.slice(2));
+	if (!existsSync(args.results)) fail(`Results file not found: ${args.results}`);
+	if (args.compare && !existsSync(args.compare)) fail(`Compare file not found: ${args.compare}`);
+	if (!existsSync(args.casesDir)) fail(`Cases directory not found: ${args.casesDir}`);
+
+	const { byId: cases, warnings } = loadRoutingCases(args.casesDir);
+	const judge = new RoutingJudge({
+		cacheDir: args.judgeCacheDir,
+		apiKey: process.env.ANTHROPIC_API_KEY,
+	});
+
+	const run = await gradeRun(args.results, cases, judge, args.judgeConcurrency);
+	const base = args.compare
+		? await gradeRun(args.compare, cases, judge, args.judgeConcurrency)
+		: undefined;
+
+	const metrics = computeRunMetrics(run.cases);
+	const confusion = computeConfusion(run.cases);
+	const health = computeHealth(run, judge.stats, warnings);
+	const compare = base ? computeCompare(run, base) : undefined;
+	const report = renderReport({
+		run,
+		metrics,
+		confusion,
+		health,
+		casesDir: args.casesDir,
+		compare,
+	});
+
+	const summary = {
+		generatedAt: new Date().toISOString(),
+		casesDir: args.casesDir,
+		judgeModel: JUDGE_MODEL,
+		results: run.meta,
+		metrics,
+		confusion,
+		health,
+		failingCases: run.cases
+			.filter((graded) => !graded.pass)
+			.map((graded) => ({
+				id: graded.id,
+				bucket: graded.bucket,
+				accepts: graded.accepts,
+				policyDependent: graded.policyDependent,
+				routes: graded.trials.map((trial) => trial.label),
+				reason: failureReason(graded),
+			})),
+		pendingRerun: run.pendingRerun,
+		cases: run.cases,
+		compare: compare && base ? { ...compare, baseCases: base.cases } : undefined,
+	};
+
+	mkdirSync(dirname(args.out), { recursive: true });
+	writeFileSync(args.out, report);
+	const jsonPath = summaryPath(args.out);
+	writeFileSync(jsonPath, `${JSON.stringify(summary, null, 2)}\n`);
+
+	const pct = (value: number | null) => (value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`);
+	console.log(
+		`${run.meta.variant}: macro ${pct(metrics.all.macroAccuracy)}, trial ${pct(metrics.all.trialPass.rate)}, ` +
+			`strict macro ${pct(metrics.all.strictMacroAccuracy)}, over-asking ${pct(metrics.all.overAsking.rate)}, ` +
+			`${metrics.all.cases} cases, ${metrics.all.trials} trials, ${run.pendingRerun.length} pending re-run ` +
+			`(judge: ${health.judgedTrials} judged, ${judge.stats.cacheHits} cached, ${health.judgeFailedTrials} failed)`,
+	);
+	if (compare) {
+		const before = compare.base.all.macroAccuracy;
+		const after = compare.current.all.macroAccuracy;
+		const delta =
+			before === null || after === null ? 'n/a' : `${((after - before) * 100).toFixed(1)} pp`;
+		console.log(
+			`vs ${compare.baseMeta.variant}: macro delta ${delta}, ${compare.flipped.length} flipped cases`,
+		);
+	}
+	console.log(`Report: ${args.out}`);
+	console.log(`Summary: ${jsonPath}`);
+	const judgeFailures = health.judgeFailedTrials + (base ? countJudgeFailures(base) : 0);
+	if (judgeFailures > 0) {
+		console.error(
+			`${judgeFailures} trials failed to judge. Fix the cause and re-run; cached verdicts are reused.`,
+		);
+		process.exitCode = 2;
+	}
+}
+
+function countJudgeFailures(run: GradedRun): number {
+	return run.cases
+		.flatMap((graded) => graded.trials)
+		.filter((trial) => trial.resolution.judgeError !== undefined).length;
+}
+
+main().catch((error: unknown) => {
+	console.error(error instanceof Error ? (error.stack ?? error.message) : String(error));
+	process.exit(1);
+});

packages/@n8n/instance-ai/evaluations/routing/judge.ts

@@ -0,0 +1,330 @@
+// ---------------------------------------------------------------------------
+// LLM judge for the routing grader.
+//
+// The grader calls the judge only when tool calls cannot decide the route: an
+// `ask-user` card (the judge gives the steer) or a reply with no committing
+// call (the judge gives the kind and the steer). Verdicts are cached on disk by
+// a hash of the full request, so a re-grade makes no API calls and a prompt
+// change starts a fresh cache.
+//
+// It calls the Anthropic Messages API with fetch instead of `createEvalAgent`
+// (src/utils/eval-agents.ts), because that factory always turns on adaptive
+// thinking and has no temperature setting, which would change the calibrated
+// temperature-0 forced-tool verdicts, and its `generate()` hides the HTTP status
+// and `retry-after` header that the retry loop reads.
+// ---------------------------------------------------------------------------
+
+import { createHash } from 'node:crypto';
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+
+import { type AskUserQuestion, isRecord, isSteer, type Steer } from './grade-types';
+
+export const JUDGE_MODEL = 'claude-sonnet-4-6';
+
+export const DEFAULT_JUDGE_CACHE_DIR = resolve(__dirname, '..', '.data', 'routing-judge-cache');
+
+const API_URL = 'https://api.anthropic.com/v1/messages';
+const API_VERSION = '2023-06-01';
+const TOOL_NAME = 'record_routing_judgment';
+const REQUEST_TIMEOUT_MS = 90_000;
+const DEFAULT_MAX_ATTEMPTS = 6;
+const MAX_BACKOFF_MS = 30_000;
+
+export const JUDGE_KINDS = ['answer', 'clarify', 'decline'] as const;
+export type JudgeKind = (typeof JUDGE_KINDS)[number];
+
+const isJudgeKind = (value: unknown): value is JudgeKind =>
+	typeof value === 'string' && JUDGE_KINDS.some((kind) => kind === value);
+
+export interface JudgeVerdict {
+	kind: JudgeKind;
+	steer: Steer;
+	reason: string;
+}
+
+export interface JudgeInput {
+	/** `ask-user` when the Assistant showed a question card, `text` when it only wrote text. */
+	mode: 'ask-user' | 'text';
+	userMessage: string;
+	askUserIntro?: string;
+	askUserQuestions: AskUserQuestion[];
+	finalText: string;
+}
+
+export interface JudgeResult {
+	verdict: JudgeVerdict;
+	cached: boolean;
+}
+
+export interface JudgeStats {
+	apiCalls: number;
+	cacheHits: number;
+	failures: number;
+	inputTokens: number;
+	outputTokens: number;
+}
+
+const SYSTEM_PROMPT = `You grade one reply from the n8n Assistant. n8n is a workflow automation product. The Assistant can build two different artifacts:
+
+- A workflow: a trigger and a fixed graph of steps (nodes) that runs the same way each time. A workflow can contain AI steps, including an "AI Agent" node. It is still a workflow.
+- An Agent: a standalone n8n Agent made with the Agent Builder. It has instructions, tools, and memory. It holds conversations or owns an ongoing role that needs judgment. It can also run scheduled tasks.
+
+The Assistant can also answer questions, do a one-time action, or ask the user for more detail.
+
+You get the user's message and the Assistant's reply. The reply is a structured question card from the ask-user tool (questions with options), free text, or both. Record your judgment with the ${TOOL_NAME} tool.
+
+kind:
+- answer: The reply deals with the request directly. It gives information, instructions, an explanation, or a result, and it does not need input from the user to continue. An answer that ends with an optional offer ("Do you want me to build this?") is still an answer.
+- clarify: The main purpose of the reply is to get a decision or a missing detail from the user before the Assistant continues. It asks one or more questions and stops.
+- decline: The reply refuses the request, or it says that it cannot help with it. It can suggest an alternative.
+
+steer is the artifact that the reply pushes the user toward:
+- agent: The reply recommends or assumes an Agent, or its questions only make sense for an Agent (persona, tone, what the Agent remembers, where it talks to people). It does not offer a workflow as a real option.
+- workflow: The reply recommends or assumes a workflow, or its questions are about workflow details (trigger, schedule, nodes, steps, field mappings). It does not offer an Agent as a real option. A workflow with an AI Agent node is a workflow.
+- both: The reply offers an Agent and a workflow as real options, and the user can pick either one. This applies also when the reply marks one of them as recommended.
+- none: The reply pushes toward neither artifact. It asks about the goal or the use case without favouring one, it asks only about details that fit both (which app holds the data, which channel), or it only answers or declines.
+
+Rules:
+- Judge what the reply does, not what the reply should do.
+- The words "agent", "assistant", or "bot" alone do not decide the steer. Decide from what the reply proposes to build.
+- Consider the questions, the options, and the text together.
+- reason: one sentence that quotes or names the part of the reply that decided the steer.`;
+
+const JUDGE_TOOL = {
+	name: TOOL_NAME,
+	description: 'Record the kind and the steer of the Assistant reply.',
+	input_schema: {
+		type: 'object',
+		properties: {
+			kind: {
+				type: 'string',
+				enum: [...JUDGE_KINDS],
+				description: 'What the reply does: answer, clarify, or decline.',
+			},
+			steer: {
+				type: 'string',
+				enum: ['agent', 'workflow', 'both', 'none'],
+				description: 'The artifact that the reply pushes the user toward.',
+			},
+			reason: { type: 'string', description: 'One sentence that justifies the steer.' },
+		},
+		required: ['kind', 'steer', 'reason'],
+		additionalProperties: false,
+	},
+} as const;
+
+function renderQuestions(input: JudgeInput): string {
+	const lines: string[] = [];
+	if (input.askUserIntro) lines.push(`Intro: ${input.askUserIntro}`);
+	for (const [index, question] of input.askUserQuestions.entries()) {
+		lines.push(`${index + 1}. ${question.question}`);
+		if (question.options.length === 0) lines.push('   (free-text answer, no options)');
+		for (const option of question.options) lines.push(`   - ${option}`);
+	}
+	return lines.length > 0 ? lines.join('\n') : '(the card has no readable questions)';
+}
+
+export function renderJudgePrompt(input: JudgeInput): string {
+	const parts = [`<user_message>\n${input.userMessage}\n</user_message>`];
+	if (input.mode === 'ask-user') {
+		parts.push(`<ask_user_card>\n${renderQuestions(input)}\n</ask_user_card>`);
+	}
+	parts.push(
+		`<assistant_text>\n${input.finalText.trim() || '(no text)'}\n</assistant_text>`,
+		input.mode === 'ask-user'
+			? 'The Assistant showed the ask-user card above and waited for the answer, so the kind is clarify. Decide the steer from the questions, the options, and the text.'
+			: 'The Assistant made no build or action call. Decide the kind and the steer from its text.',
+	);
+	return parts.join('\n\n');
+}
+
+function buildRequestBody(input: JudgeInput) {
+	return {
+		model: JUDGE_MODEL,
+		max_tokens: 1024,
+		temperature: 0,
+		system: SYSTEM_PROMPT,
+		tools: [JUDGE_TOOL],
+		tool_choice: { type: 'tool', name: TOOL_NAME },
+		messages: [{ role: 'user', content: renderJudgePrompt(input) }],
+	};
+}
+
+/** Cache key: a hash of the full request, so a prompt or model change starts a fresh cache. */
+export function judgeCacheKey(input: JudgeInput): string {
+	return createHash('sha256')
+		.update(JSON.stringify(buildRequestBody(input)))
+		.digest('hex');
+}
+
+function parseVerdict(raw: unknown): JudgeVerdict | undefined {
+	if (!isRecord(raw) || !isJudgeKind(raw.kind) || !isSteer(raw.steer)) return undefined;
+	return {
+		kind: raw.kind,
+		steer: raw.steer,
+		reason: typeof raw.reason === 'string' ? raw.reason : '',
+	};
+}
+
+function isRetryableStatus(status: number): boolean {
+	return status === 408 || status === 409 || status === 429 || status >= 500;
+}
+
+function backoffMs(attempt: number, retryAfter: string | null): number {
+	const seconds = retryAfter === null ? Number.NaN : Number(retryAfter);
+	if (Number.isFinite(seconds) && seconds >= 0) return Math.min(seconds * 1000, MAX_BACKOFF_MS);
+	const exponential = 1000 * 2 ** (attempt - 1);
+	return Math.min(exponential + Math.floor(Math.random() * 500), MAX_BACKOFF_MS);
+}
+
+export interface RoutingJudgeOptions {
+	cacheDir: string;
+	apiKey?: string;
+	maxAttempts?: number;
+}
+
+export class RoutingJudge {
+	readonly stats: JudgeStats = {
+		apiCalls: 0,
+		cacheHits: 0,
+		failures: 0,
+		inputTokens: 0,
+		outputTokens: 0,
+	};
+
+	/** Identical inputs in one run share one request. */
+	private readonly inFlight = new Map<string, Promise<JudgeResult>>();
+
+	constructor(private readonly options: RoutingJudgeOptions) {}
+
+	async judge(input: JudgeInput): Promise<JudgeResult> {
+		const body = buildRequestBody(input);
+		const key = judgeCacheKey(input);
+		const pending = this.inFlight.get(key);
+		if (pending) return await pending;
+
+		const promise = this.judgeUncached(key, body, input);
+		this.inFlight.set(key, promise);
+		try {
+			return await promise;
+		} catch (error) {
+			this.stats.failures++;
+			throw error;
+		}
+	}
+
+	private cachePath(key: string): string {
+		return join(this.options.cacheDir, `${key}.json`);
+	}
+
+	private async readCache(key: string): Promise<JudgeVerdict | undefined> {
+		try {
+			const parsed: unknown = JSON.parse(await readFile(this.cachePath(key), 'utf8'));
+			return isRecord(parsed) ? parseVerdict(parsed.verdict) : undefined;
+		} catch {
+			return undefined;
+		}
+	}
+
+	private async writeCache(key: string, input: JudgeInput, verdict: JudgeVerdict): Promise<void> {
+		await mkdir(this.options.cacheDir, { recursive: true });
+		const file = this.cachePath(key);
+		const temp = `${file}.${process.pid}.tmp`;
+		const entry = { model: JUDGE_MODEL, createdAt: new Date().toISOString(), input, verdict };
+		await writeFile(temp, `${JSON.stringify(entry, null, 2)}\n`);
+		await rename(temp, file);
+	}
+
+	private async judgeUncached(
+		key: string,
+		body: ReturnType<typeof buildRequestBody>,
+		input: JudgeInput,
+	): Promise<JudgeResult> {
+		const cached = await this.readCache(key);
+		if (cached) {
+			this.stats.cacheHits++;
+			return { verdict: cached, cached: true };
+		}
+		const verdict = await this.callApi(body);
+		await this.writeCache(key, input, verdict);
+		return { verdict, cached: false };
+	}
+
+	private async callApi(body: ReturnType<typeof buildRequestBody>): Promise<JudgeVerdict> {
+		const apiKey = this.options.apiKey;
+		if (!apiKey) {
+			throw new Error('ANTHROPIC_API_KEY is not set, and the verdict is not in the judge cache');
+		}
+		const maxAttempts = this.options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+		let lastError = 'no attempt made';
+
+		for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+			let response: Response;
+			try {
+				this.stats.apiCalls++;
+				response = await fetch(API_URL, {
+					method: 'POST',
+					headers: {
+						'content-type': 'application/json',
+						'x-api-key': apiKey,
+						'anthropic-version': API_VERSION,
+					},
+					body: JSON.stringify(body),
+					signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+				});
+			} catch (error) {
+				// Network failures and timeouts are transient.
+				lastError = error instanceof Error ? error.message : String(error);
+				if (attempt < maxAttempts) await sleep(backoffMs(attempt, null));
+				continue;
+			}
+
+			if (!response.ok) {
+				const text = (await response.text()).slice(0, 500);
+				lastError = `HTTP ${response.status}: ${text}`;
+				if (!isRetryableStatus(response.status)) break;
+				if (attempt < maxAttempts) {
+					await sleep(backoffMs(attempt, response.headers.get('retry-after')));
+				}
+				continue;
+			}
+
+			const payload: unknown = await response.json();
+			if (isRecord(payload) && isRecord(payload.usage)) {
+				const { input_tokens: inputTokens, output_tokens: outputTokens } = payload.usage;
+				if (typeof inputTokens === 'number') this.stats.inputTokens += inputTokens;
+				if (typeof outputTokens === 'number') this.stats.outputTokens += outputTokens;
+			}
+			const content = isRecord(payload) && Array.isArray(payload.content) ? payload.content : [];
+			const toolUse = content.find(
+				(block) => isRecord(block) && block.type === 'tool_use' && block.name === TOOL_NAME,
+			);
+			const verdict = isRecord(toolUse) ? parseVerdict(toolUse.input) : undefined;
+			if (verdict) return verdict;
+			// A malformed tool input is rare with forced tool use; one more sample usually fixes it.
+			lastError = `judge returned no valid ${TOOL_NAME} input`;
+		}
+
+		throw new Error(`Judge request failed: ${lastError}`);
+	}
+}
+
+/** Runs `worker` over `items` with at most `limit` in flight, keeping input order. */
+export async function mapWithConcurrency<T, R>(
+	items: readonly T[],
+	limit: number,
+	worker: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+	const results = new Array<R>(items.length);
+	let next = 0;
+	const runners = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
+		while (next < items.length) {
+			const index = next++;
+			results[index] = await worker(items[index], index);
+		}
+	});
+	await Promise.all(runners);
+	return results;
+}

packages/@n8n/instance-ai/evaluations/routing/langtracer-cases.ts

@@ -0,0 +1,370 @@
+// ---------------------------------------------------------------------------
+// Maps routing cases to LangTracer cases and back.
+//
+// A LangTracer case has no routing fields, so the routing labels travel as
+// tags (see the v2 addendum in the routing spec):
+//
+//   routing, bucket:<bucket>, accepts:<token> (one per token), policy-dependent,
+//   agent-shaped, source:<source>, lang:<ISO 639-3 code>
+//
+// The full list goes in `evalTags`, which the suite export returns as `tags`,
+// so the pull reads it from there. LangTracer's own case `tags` drop every tag
+// that is not kebab-case, so they get only the kebab-case subset, for filtering
+// in the LangTracer UI.
+//
+// `name` is the case id, `userPrompt` and the single conversation turn carry
+// `userMessage`, `description` carries `rationale`, and `seed` and
+// `conversation[0].attach` are LangTracer's own fields. `agent-shaped` is an
+// addition to the spec's list, so the pull keeps `agentShaped`.
+//
+// Each case also carries one process expectation, `routingExpectationText`.
+// LangTracer stores it as the case's only expectation and matches a run's
+// result to it by text. The pull ignores it: the `accepts:` tags are the
+// source of truth, and the text is derived from them.
+//
+// Pure, no network: the push CLI and the pull loader share it, and the tests
+// cover the round trip without a server.
+// ---------------------------------------------------------------------------
+
+import { isRecord } from '@n8n/utils/is-record';
+
+import { routingExpectationText } from './expectation';
+import {
+	DEFAULT_ROUTING_LANGUAGE,
+	parseRoutingCase,
+	type ROUTING_ACCEPT_TOKENS,
+	type RoutingCase,
+} from './loader';
+import type { LangTracerUpdateCaseBody } from '../langtracer/client';
+import type { LangTracerCreateCaseBody, PushableSeed } from '../langtracer/to-exported';
+
+export const ROUTING_TAG = 'routing';
+const POLICY_DEPENDENT_TAG = 'policy-dependent';
+const AGENT_SHAPED_TAG = 'agent-shaped';
+const BUCKET_PREFIX = 'bucket:';
+const ACCEPTS_PREFIX = 'accepts:';
+const SOURCE_PREFIX = 'source:';
+const LANG_PREFIX = 'lang:';
+
+/** Sources whose cases hold no user data (addendum: `synthetic: true` for these only). */
+const SYNTHETIC_SOURCE_PREFIXES = ['suite-62:', 'agent-344:', 'discovery:'];
+
+export type RoutingSetKind = 'regression' | 'capability_gap';
+
+/** Fields the routing push sends in addition to the n8n create-case contract. */
+interface RoutingCaseFields {
+	userPrompt: string;
+	expectedBehavior: string;
+	/** LangTracer's own case tags: the kebab-case subset of `evalTags`. */
+	tags: string[];
+	/** Exactly one entry: `routingExpectationText(accepts)`. */
+	processExpectations: string[];
+}
+
+export type RoutingLangTracerCreateBody = LangTracerCreateCaseBody & RoutingCaseFields;
+
+export type RoutingLangTracerUpdateBody = LangTracerUpdateCaseBody &
+	Partial<RoutingCaseFields> & {
+		setKind?: RoutingSetKind;
+		/** The patchable inverse of the create-only `synthetic` flag. */
+		containsUserData?: boolean;
+	};
+
+export function isSyntheticSource(source: string): boolean {
+	return source === 'synthetic' || SYNTHETIC_SOURCE_PREFIXES.some((p) => source.startsWith(p));
+}
+
+export function routingTags(routingCase: RoutingCase): string[] {
+	return [
+		ROUTING_TAG,
+		`${BUCKET_PREFIX}${routingCase.bucket}`,
+		...routingCase.accepts.map((token) => `${ACCEPTS_PREFIX}${token}`),
+		...(routingCase.policyDependent ? [POLICY_DEPENDENT_TAG] : []),
+		...(routingCase.agentShaped ? [AGENT_SHAPED_TAG] : []),
+		`${SOURCE_PREFIX}${routingCase.source}`,
+		`${LANG_PREFIX}${routingCase.language ?? DEFAULT_ROUTING_LANGUAGE}`,
+	];
+}
+
+/** LangTracer keeps a case tag only when it is kebab-case. */
+const CASE_TAG_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+
+/** The routing tags LangTracer stores as case tags. */
+export function langTracerCaseTags(tags: string[]): string[] {
+	return tags.filter((tag) => CASE_TAG_PATTERN.test(tag));
+}
+
+const ROUTE_PHRASES: Record<(typeof ROUTING_ACCEPT_TOKENS)[number], string> = {
+	agent: 'build an Agent',
+	workflow: 'build a workflow',
+	'one-off': 'do the task once now',
+	debug: 'inspect the executions',
+	multi: 'plan the work as several tasks',
+	answer: 'answer the question',
+	decline: 'decline the request',
+	clarify: 'ask a clarifying question',
+	'clarify:agent': 'ask a clarifying question that points toward an Agent',
+	'clarify:open': 'ask an open clarifying question that does not push a workflow only',
+};
+
+/** One sentence that names the accepted routes, for LangTracer's `expectedBehavior`. */
+export function expectedBehaviorSentence(accepts: RoutingCase['accepts']): string {
+	const phrases = accepts.map((token) => ROUTE_PHRASES[token]);
+	const list =
+		phrases.length <= 1
+			? (phrases[0] ?? '')
+			: `${phrases.slice(0, -1).join(', ')}${phrases.length > 2 ? ',' : ''} or ${phrases[phrases.length - 1]}`;
+	return `The Assistant should ${list} (accepted routes: ${accepts.join(', ')}).`;
+}
+
+/** Why a routing case cannot be stored in LangTracer, or null when it can. */
+export function unsupportedRoutingPushReason(routingCase: RoutingCase): string | null {
+	if (routingCase.instanceState !== undefined) {
+		return 'sets instanceState, which a LangTracer case cannot store';
+	}
+	return null;
+}
+
+function pushableSeedOf(routingCase: RoutingCase): PushableSeed | undefined {
+	const seed = routingCase.seed;
+	if (seed?.mode !== 'inline') return undefined;
+	const { folders: _folders, ...rest } = seed;
+	return rest;
+}
+
+export function toLangTracerCreateBody(
+	routingCase: RoutingCase,
+	opts: { suiteId: number; setKind: RoutingSetKind },
+): RoutingLangTracerCreateBody {
+	const tags = routingTags(routingCase);
+	const seed = pushableSeedOf(routingCase);
+	return {
+		name: routingCase.id,
+		setKind: opts.setKind,
+		synthetic: isSyntheticSource(routingCase.source),
+		suiteId: opts.suiteId,
+		...(routingCase.rationale !== undefined ? { description: routingCase.rationale } : {}),
+		userPrompt: routingCase.userMessage,
+		expectedBehavior: expectedBehaviorSentence(routingCase.accepts),
+		conversation: [
+			{
+				role: 'user',
+				text: routingCase.userMessage,
+				...(routingCase.attach ? { attach: routingCase.attach } : {}),
+			},
+		],
+		// A routing case is one message; the build fields are required by the contract only.
+		evalComplexity: 'simple',
+		evalTags: tags,
+		tags: langTracerCaseTags(tags),
+		processExpectations: [routingExpectationText(routingCase.accepts)],
+		...(seed ? { seed } : {}),
+	};
+}
+
+/** The PATCH for an existing case: every routing field, so a re-push converges. */
+export function toLangTracerUpdateBody(
+	body: RoutingLangTracerCreateBody,
+): RoutingLangTracerUpdateBody {
+	const { suiteId: _suiteId, synthetic, seed, ...rest } = body;
+	// An absent seed is sent as null and the outcome list as empty: a partial
+	// PATCH would keep a stored seed or a second expectation alive.
+	return { ...rest, outcomeExpectations: [], seed: seed ?? null, containsUserData: !synthetic };
+}
+
+/** The expectation lists of one suite-export body, trimmed like LangTracer stores them. */
+export function exportedExpectations(raw: unknown): { process: string[]; outcome: string[] } {
+	const list = (value: unknown) =>
+		stringList(value)
+			.map((text) => text.trim())
+			.filter((text) => text.length > 0);
+	return isRecord(raw)
+		? { process: list(raw.processExpectations), outcome: list(raw.outcomeExpectations) }
+		: { process: [], outcome: [] };
+}
+
+// ---------------------------------------------------------------------------
+// LangTracer export → routing case
+// ---------------------------------------------------------------------------
+
+function stringList(value: unknown): string[] {
+	return Array.isArray(value) ? value.filter((v): v is string => typeof v === 'string') : [];
+}
+
+function tagValues(tags: string[], prefix: string): string[] {
+	return tags.filter((tag) => tag.startsWith(prefix)).map((tag) => tag.slice(prefix.length));
+}
+
+/**
+ * Rebuilds the routing-case JSON from one suite-export body (`<name>.json`).
+ * Returns the raw case for `parseRoutingCase`, or the reasons it cannot be
+ * rebuilt. The export carries the case tags as `tags`. Its expectations and
+ * scenarios are not read: `accepts` comes from the tags only.
+ */
+export function routingCaseJsonFromExport(
+	name: string,
+	raw: unknown,
+): { success: true; json: Record<string, unknown> } | { success: false; issues: string[] } {
+	if (!isRecord(raw)) return { success: false, issues: ['export body is not an object'] };
+	const tags = stringList(raw.tags);
+	if (!tags.includes(ROUTING_TAG)) {
+		return { success: false, issues: [`no "${ROUTING_TAG}" tag, so it is not a routing case`] };
+	}
+
+	const issues: string[] = [];
+	const single = (prefix: string): string | undefined => {
+		const values = tagValues(tags, prefix);
+		if (values.length > 1) issues.push(`more than one "${prefix}" tag`);
+		return values[0];
+	};
+	const bucket = single(BUCKET_PREFIX);
+	const source = single(SOURCE_PREFIX);
+	const language = single(LANG_PREFIX);
+	if (bucket === undefined) issues.push(`no "${BUCKET_PREFIX}" tag`);
+	if (source === undefined) issues.push(`no "${SOURCE_PREFIX}" tag`);
+
+	const conversation = Array.isArray(raw.conversation) ? raw.conversation : [];
+	const [turn] = conversation;
+	if (conversation.length !== 1 || !isRecord(turn) || turn.role !== 'user') {
+		issues.push('conversation must be exactly one user turn');
+	}
+	if (issues.length > 0) return { success: false, issues };
+
+	const opening = isRecord(turn) ? turn : {};
+	return {
+		success: true,
+		json: {
+			id: name,
+			bucket,
+			agentShaped: tags.includes(AGENT_SHAPED_TAG),
+			userMessage: opening.text,
+			accepts: tagValues(tags, ACCEPTS_PREFIX),
+			policyDependent: tags.includes(POLICY_DEPENDENT_TAG),
+			source,
+			...(typeof raw.description === 'string' && raw.description.length > 0
+				? { rationale: raw.description }
+				: {}),
+			...(raw.seed !== undefined && raw.seed !== null ? { seed: raw.seed } : {}),
+			...(opening.attach !== undefined ? { attach: opening.attach } : {}),
+			...(language !== undefined && language !== DEFAULT_ROUTING_LANGUAGE ? { language } : {}),
+		},
+	};
+}
+
+/** Rebuilds and validates every routing case of a suite export, reporting all failures at once. */
+export function routingCasesFromExport(files: Record<string, unknown>): {
+	cases: RoutingCase[];
+	errors: string[];
+} {
+	const cases: RoutingCase[] = [];
+	const errors: string[] = [];
+	for (const [fileName, raw] of Object.entries(files)) {
+		const name = fileName.replace(/\.json$/i, '');
+		const rebuilt = routingCaseJsonFromExport(name, raw);
+		if (!rebuilt.success) {
+			errors.push(`${name}: ${rebuilt.issues.join('; ')}`);
+			continue;
+		}
+		const parsed = parseRoutingCase(rebuilt.json);
+		if (!parsed.success) {
+			errors.push(`${name}: ${parsed.issues.join('; ')}`);
+			continue;
+		}
+		cases.push(parsed.data);
+	}
+	cases.sort((a, b) => a.id.localeCompare(b.id));
+	return { cases, errors };
+}
+
+// ---------------------------------------------------------------------------
+// Comparison
+// ---------------------------------------------------------------------------
+
+const COMPARED_FIELDS = [
+	'bucket',
+	'agentShaped',
+	'userMessage',
+	'accepts',
+	'policyDependent',
+	'source',
+	'rationale',
+	'instanceState',
+	'seed',
+	'attach',
+	'language',
+] as const satisfies ReadonlyArray<keyof RoutingCase>;
+
+/** Folds the defaults both sides may omit, so only a real difference counts. */
+function comparable(routingCase: RoutingCase): Record<string, unknown> {
+	return {
+		...routingCase,
+		agentShaped: routingCase.agentShaped ?? false,
+		policyDependent: routingCase.policyDependent ?? false,
+		language: routingCase.language ?? DEFAULT_ROUTING_LANGUAGE,
+		seed: comparableSeed(routingCase.seed),
+	};
+}
+
+/**
+ * Empty seed slots and message ids are dropped: the loader defaults every slot
+ * to `[]`, and `{role, text}` shorthand mints a new message id on each parse.
+ */
+function comparableSeed(seed: RoutingCase['seed']): unknown {
+	if (!seed) return undefined;
+	const out: Record<string, unknown> = {};
+	for (const [key, slot] of Object.entries(seed)) {
+		if (slot === undefined || (Array.isArray(slot) && slot.length === 0)) continue;
+		out[key] =
+			key === 'messages' && Array.isArray(slot)
+				? slot.map((message: unknown) => {
+						if (!isRecord(message)) return message;
+						const { id: _id, ...rest } = message;
+						return rest;
+					})
+				: slot;
+	}
+	return out;
+}
+
+/** The routing fields that differ between two versions of one case. */
+export function routingCaseDiff(a: RoutingCase, b: RoutingCase): string[] {
+	const left = comparable(a);
+	const right = comparable(b);
+	return COMPARED_FIELDS.filter((field) => canonical(left[field]) !== canonical(right[field]));
+}
+
+function canonical(value: unknown): string {
+	return value === undefined ? 'undefined' : JSON.stringify(sortKeysDeep(value));
+}
+
+function sortKeysDeep(value: unknown): unknown {
+	if (Array.isArray(value)) return value.map(sortKeysDeep);
+	if (isRecord(value)) {
+		const sorted: Record<string, unknown> = {};
+		for (const key of Object.keys(value).sort()) sorted[key] = sortKeysDeep(value[key]);
+		return sorted;
+	}
+	return value;
+}
+
+// ---------------------------------------------------------------------------
+// Passing ids (setKind)
+// ---------------------------------------------------------------------------
+
+/**
+ * Reads the grader's list of case ids that pass on the baseline: a JSON array
+ * of ids, or an object with one array under `ids`, `passingIds`, or `passing`.
+ */
+export function parsePassingIds(raw: unknown): Set<string> {
+	const list = Array.isArray(raw)
+		? raw
+		: isRecord(raw)
+			? (raw.ids ?? raw.passingIds ?? raw.passing)
+			: undefined;
+	if (!Array.isArray(list) || !list.every((id): id is string => typeof id === 'string')) {
+		throw new Error(
+			'passing ids must be a JSON array of case ids, or an object with that array under "ids", "passingIds", or "passing"',
+		);
+	}
+	return new Set(list);
+}

packages/@n8n/instance-ai/evaluations/routing/langtracer-config.ts

@@ -0,0 +1,54 @@
+// ---------------------------------------------------------------------------
+// LangTracer connection for the routing eval tools.
+//
+// Env first (LANGTRACER_URL + LANGTRACER_API_KEY, like the other eval CLIs).
+// When they are not set, the lang-tracer MCP server entry in ~/.claude.json
+// supplies them: its URL without the trailing `/mcp`, and the bearer token of
+// its Authorization header. The key is never printed.
+// ---------------------------------------------------------------------------
+
+import { isRecord } from '@n8n/utils/is-record';
+import { jsonParse } from 'n8n-workflow';
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+import { resolveLangTracerConfig, type LangTracerConfig } from '../langtracer/config';
+
+const MCP_SERVER_NAME = 'lang-tracer';
+
+export function defaultClaudeConfigPath(): string {
+	return join(homedir(), '.claude.json');
+}
+
+/** Reads the lang-tracer MCP entry of a Claude config file, or undefined when it has none. */
+export function langTracerConfigFromClaudeConfig(raw: unknown): LangTracerConfig | undefined {
+	if (!isRecord(raw) || !isRecord(raw.mcpServers)) return undefined;
+	const server = raw.mcpServers[MCP_SERVER_NAME];
+	if (!isRecord(server) || typeof server.url !== 'string' || !isRecord(server.headers)) {
+		return undefined;
+	}
+	const authorization = server.headers.Authorization;
+	if (typeof authorization !== 'string') return undefined;
+	const apiKey = authorization.replace(/^Bearer\s+/i, '').trim();
+	const baseUrl = server.url.trim().replace(/\/mcp\/?$/, '');
+	if (!apiKey || !baseUrl) return undefined;
+	return { baseUrl, apiKey };
+}
+
+export function resolveRoutingLangTracerConfig(
+	env: NodeJS.ProcessEnv = process.env,
+	claudeConfigPath: string = defaultClaudeConfigPath(),
+): LangTracerConfig {
+	if (env.LANGTRACER_URL?.trim() && env.LANGTRACER_API_KEY?.trim()) {
+		return resolveLangTracerConfig(env);
+	}
+	if (existsSync(claudeConfigPath)) {
+		const fromFile = langTracerConfigFromClaudeConfig(
+			jsonParse<unknown>(readFileSync(claudeConfigPath, 'utf-8'), { fallbackValue: undefined }),
+		);
+		if (fromFile) return fromFile;
+	}
+	// Throws with the message that names the missing variables.
+	return resolveLangTracerConfig(env);
+}

packages/@n8n/instance-ai/evaluations/routing/langtracer-push.ts

@@ -0,0 +1,349 @@
+#!/usr/bin/env node
+// ---------------------------------------------------------------------------
+// Push routing cases (one `<id>.json` per case) into a LangTracer suite.
+// Create-or-update by case name, so a re-push converges and leaves unchanged
+// cases alone. The mapping is in langtracer-cases.ts.
+//
+//   pnpm tsx evaluations/routing/langtracer-push.ts --suite intent-routing \
+//     --cases-dir <dir> --passing-ids <file> --dry-run
+//
+// Credentials: LANGTRACER_URL and LANGTRACER_API_KEY, else the lang-tracer MCP
+// entry in ~/.claude.json (see langtracer-config.ts).
+// ---------------------------------------------------------------------------
+
+import { isRecord } from '@n8n/utils/is-record';
+import { jsonParse } from 'n8n-workflow';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { z } from 'zod';
+
+import {
+	exportedExpectations,
+	parsePassingIds,
+	routingCaseDiff,
+	routingCasesFromExport,
+	routingTags,
+	toLangTracerCreateBody,
+	toLangTracerUpdateBody,
+	unsupportedRoutingPushReason,
+	type RoutingLangTracerCreateBody,
+	type RoutingSetKind,
+} from './langtracer-cases';
+import { resolveRoutingLangTracerConfig } from './langtracer-config';
+import { loadRoutingCases, type RoutingCase, type RoutingCaseSelection } from './loader';
+import { findLangTracerSuite, LangTracerClient } from '../langtracer/client';
+import type { LangTracerConfig } from '../langtracer/config';
+
+const HELP = `Push routing cases into a LangTracer suite (create or update by case name).
+
+Usage:
+  tsx evaluations/routing/langtracer-push.ts --suite <slug|id> --cases-dir <dir> --passing-ids <file> [options]
+
+Options:
+  --suite <slug|id>         Target suite (required; it must exist)
+  --cases-dir <dir>         Directory with one routing case per <id>.json (required)
+  --passing-ids <file>      JSON list of ids that pass on the baseline: they get
+                            setKind "regression", every other case "capability_gap" (required)
+  --filter <csv>            Keep ids that contain any token
+  --exclude <csv>           Drop ids that contain any token
+  --exclude-prefix <csv>    Drop ids that start with any prefix (e.g. route-v2-)
+  --dry-run                 Print the plan and the payloads without writing
+  -h, --help                Show this help`;
+
+export interface RoutingPushArgs {
+	suite: string;
+	passingIdsFile: string;
+	casesDir: string;
+	selection: RoutingCaseSelection;
+	dryRun: boolean;
+}
+
+export function parseRoutingPushArgs(argv: string[]): RoutingPushArgs | 'help' {
+	const args: RoutingPushArgs = {
+		suite: '',
+		passingIdsFile: '',
+		casesDir: '',
+		selection: {},
+		dryRun: false,
+	};
+	const value = (i: number, flag: string): string => {
+		const next = argv[i + 1];
+		if (next === undefined || next.startsWith('--')) throw new Error(`Missing value for ${flag}`);
+		return next;
+	};
+	for (let i = 0; i < argv.length; i++) {
+		const arg = argv[i];
+		switch (arg) {
+			case '--suite':
+				args.suite = value(i++, arg);
+				break;
+			case '--passing-ids':
+				args.passingIdsFile = value(i++, arg);
+				break;
+			case '--cases-dir':
+				args.casesDir = value(i++, arg);
+				break;
+			case '--filter':
+				args.selection.filter = value(i++, arg);
+				break;
+			case '--exclude':
+				args.selection.exclude = value(i++, arg);
+				break;
+			case '--exclude-prefix':
+				args.selection.excludePrefix = value(i++, arg);
+				break;
+			case '--dry-run':
+				args.dryRun = true;
+				break;
+			case '-h':
+			case '--help':
+				return 'help';
+			default:
+				throw new Error(`Unknown argument: ${arg} (use --help)`);
+		}
+	}
+	if (!args.suite) throw new Error('--suite <slug|id> is required');
+	if (!args.casesDir) throw new Error('--cases-dir <dir> is required');
+	if (!args.passingIdsFile) throw new Error('--passing-ids <file> is required');
+	return args;
+}
+
+// The client's suite read keeps only id + name. The push also needs the stored
+// setKind, flags, and prompt fields to tell an unchanged case from a changed one.
+const suiteRowSchema = z.object({
+	id: z.number(),
+	name: z.string(),
+	setKind: z.string(),
+	containsUserData: z.boolean().optional(),
+	userPrompt: z.string().nullable().optional(),
+	expectedBehavior: z.string().nullable().optional(),
+	tags: z.array(z.string()).nullable().optional(),
+	processExpectations: z.array(z.string()).nullable().optional(),
+	outcomeExpectations: z.array(z.string()).nullable().optional(),
+});
+type SuiteRow = z.infer<typeof suiteRowSchema>;
+
+async function fetchSuiteRows(config: LangTracerConfig, suiteId: number): Promise<SuiteRow[]> {
+	const path = `/api/v1/suites/${String(suiteId)}`;
+	const response = await fetch(new URL(path, config.baseUrl), {
+		headers: { Authorization: `Bearer ${config.apiKey}` },
+	});
+	if (!response.ok) throw new Error(`lang-tracer GET ${path} returned ${String(response.status)}`);
+	const body: unknown = await response.json();
+	const data = isRecord(body) && 'data' in body ? body.data : body;
+	return z.object({ cases: z.array(suiteRowSchema) }).parse(data).cases;
+}
+
+/** Which stored row fields differ from the body the push would send. */
+function rowDiff(row: SuiteRow, body: RoutingLangTracerCreateBody): string[] {
+	const diff: string[] = [];
+	if (row.setKind !== body.setKind) diff.push('setKind');
+	if (row.containsUserData !== !body.synthetic) diff.push('containsUserData');
+	if ((row.userPrompt ?? '') !== body.userPrompt) diff.push('userPrompt');
+	if ((row.expectedBehavior ?? '') !== body.expectedBehavior) diff.push('expectedBehavior');
+	if ((row.tags ?? []).join(' ') !== body.tags.join(' ')) diff.push('tags');
+	diff.push(
+		...expectationDiff(
+			{ process: row.processExpectations ?? [], outcome: row.outcomeExpectations ?? [] },
+			body,
+		),
+	);
+	return diff;
+}
+
+/** A routing case stores exactly the body's process expectation and no outcome expectation. */
+function expectationDiff(
+	stored: { process: string[]; outcome: string[] },
+	body: RoutingLangTracerCreateBody,
+): string[] {
+	const diff: string[] = [];
+	if (stored.process.join('\n') !== body.processExpectations.join('\n')) {
+		diff.push('processExpectations');
+	}
+	if (stored.outcome.length > 0) diff.push('outcomeExpectations');
+	return diff;
+}
+
+interface PlannedCase {
+	routingCase: RoutingCase;
+	setKind: RoutingSetKind;
+	body: RoutingLangTracerCreateBody;
+}
+
+async function main(): Promise<void> {
+	const parsed = parseRoutingPushArgs(process.argv.slice(2));
+	if (parsed === 'help') {
+		console.log(HELP);
+		return;
+	}
+	const args = parsed;
+
+	const passing = parsePassingIds(
+		jsonParse<unknown>(readFileSync(resolve(args.passingIdsFile), 'utf-8')),
+	);
+	const loaded = loadRoutingCases(args.casesDir, args.selection).map((c) => c.routingCase);
+	const unknownPassing = [...passing].filter((id) => !loaded.some((c) => c.id === id));
+
+	const config = resolveRoutingLangTracerConfig();
+	const client = new LangTracerClient(config);
+	const suite = findLangTracerSuite(await client.listSuites(), args.suite);
+	if (!suite) throw new Error(`LangTracer suite "${args.suite}" not found; create it first.`);
+
+	const skipped: Array<{ id: string; reason: string }> = [];
+	const planned: PlannedCase[] = [];
+	for (const routingCase of loaded) {
+		const reason = unsupportedRoutingPushReason(routingCase);
+		if (reason) {
+			skipped.push({ id: routingCase.id, reason });
+			continue;
+		}
+		const setKind: RoutingSetKind = passing.has(routingCase.id) ? 'regression' : 'capability_gap';
+		planned.push({
+			routingCase,
+			setKind,
+			body: toLangTracerCreateBody(routingCase, { suiteId: suite.id, setKind }),
+		});
+	}
+
+	const [rows, exported] = await Promise.all([
+		fetchSuiteRows(config, suite.id),
+		client.exportSuite(suite.id),
+	]);
+	const rowsByName = new Map(rows.map((row) => [row.name, row]));
+	const stored = new Map(
+		routingCasesFromExport(exported.files).cases.map((routingCase) => [
+			routingCase.id,
+			routingCase,
+		]),
+	);
+
+	const toCreate: PlannedCase[] = [];
+	const toUpdate: Array<PlannedCase & { id: number; changed: string[] }> = [];
+	const unchanged: PlannedCase[] = [];
+	for (const item of planned) {
+		const row = rowsByName.get(item.routingCase.id);
+		if (!row) {
+			toCreate.push(item);
+			continue;
+		}
+		const storedCase = stored.get(item.routingCase.id);
+		const changed = [
+			...rowDiff(row, item.body),
+			...(storedCase ? routingCaseDiff(storedCase, item.routingCase) : ['export']),
+		];
+		if (changed.length > 0) toUpdate.push({ ...item, id: row.id, changed });
+		else unchanged.push(item);
+	}
+
+	const count = (items: PlannedCase[], pred: (item: PlannedCase) => boolean) =>
+		items.filter(pred).length;
+	console.log(
+		`Suite "${suite.slug}" (#${String(suite.id)}), ${args.dryRun ? 'DRY RUN' : 'push'}: ${String(planned.length)} case(s) from ${args.casesDir}`,
+	);
+	console.log(
+		`  setKind: regression=${String(count(planned, (c) => c.setKind === 'regression'))}, capability_gap=${String(count(planned, (c) => c.setKind === 'capability_gap'))}`,
+	);
+	console.log(
+		`  synthetic: true=${String(count(planned, (c) => c.body.synthetic))}, false=${String(count(planned, (c) => !c.body.synthetic))}`,
+	);
+	console.log(`  create: ${String(toCreate.length)}`);
+	console.log(`  update: ${String(toUpdate.length)}`);
+	for (const item of toUpdate)
+		console.log(`    ~ ${item.routingCase.id}: ${item.changed.join(', ')}`);
+	console.log(`  unchanged: ${String(unchanged.length)}`);
+	for (const s of skipped) console.log(`  skipped ${s.id}: ${s.reason}`);
+	if (unknownPassing.length > 0) {
+		console.warn(
+			`  warning: ${String(unknownPassing.length)} passing id(s) are not among the selected cases: ${unknownPassing.join(', ')}`,
+		);
+	}
+	const remote = [...rowsByName.keys()].filter((name) => !loaded.some((c) => c.id === name));
+	if (remote.length > 0) {
+		console.log(`  in the suite but not selected (left as they are): ${String(remote.length)}`);
+	}
+
+	if (args.dryRun) {
+		const [created] = toCreate;
+		const [updated] = toUpdate;
+		if (created) {
+			console.log(`\nSample create payload (${created.routingCase.id}):`);
+			console.log(JSON.stringify(created.body, null, 2));
+		}
+		if (updated) {
+			console.log(`\nSample update payload (${updated.routingCase.id}, #${String(updated.id)}):`);
+			console.log(JSON.stringify(toLangTracerUpdateBody(updated.body), null, 2));
+		}
+		return;
+	}
+
+	for (const item of toCreate) {
+		const res = await client.createCase(item.body);
+		console.log(
+			`  + ${res.alreadyExisted ? 'found' : 'created'} ${item.routingCase.id} (#${String(res.case.id)})`,
+		);
+	}
+	for (const item of toUpdate) {
+		const res = await client.updateCase(item.id, toLangTracerUpdateBody(item.body));
+		console.log(
+			`  ~ updated ${item.routingCase.id} (#${String(item.id)}, rev ${String(res.revision)})`,
+		);
+	}
+
+	await verifyRoundTrip(config, client, suite.id, planned);
+	console.log(
+		`\nDone: ${String(toCreate.length)} created, ${String(toUpdate.length)} updated, ${String(unchanged.length)} unchanged, ${String(skipped.length)} skipped.`,
+	);
+}
+
+/**
+ * Re-reads the suite and checks that every pushed case rebuilds to the local
+ * case with the same tags, setKind, user-data flag, and expectation, in both
+ * the stored row and the export the dispatcher sends. A server that ignores a
+ * key still answers 200, so only a read-back proves the write.
+ */
+async function verifyRoundTrip(
+	config: LangTracerConfig,
+	client: LangTracerClient,
+	suiteId: number,
+	planned: PlannedCase[],
+): Promise<void> {
+	const [rows, exported] = await Promise.all([
+		fetchSuiteRows(config, suiteId),
+		client.exportSuite(suiteId),
+	]);
+	const rowsByName = new Map(rows.map((row) => [row.name, row]));
+	const { cases, errors } = routingCasesFromExport(exported.files);
+	const stored = new Map(cases.map((routingCase) => [routingCase.id, routingCase]));
+	const problems = [...errors];
+	for (const item of planned) {
+		const id = item.routingCase.id;
+		const row = rowsByName.get(id);
+		const storedCase = stored.get(id);
+		if (!row || !storedCase) {
+			problems.push(`${id}: missing from the suite`);
+			continue;
+		}
+		const changed = [...rowDiff(row, item.body), ...routingCaseDiff(storedCase, item.routingCase)];
+		if (routingTags(storedCase).join(' ') !== item.body.evalTags.join(' ')) {
+			changed.push('export tags');
+		}
+		const exportedDiff = expectationDiff(
+			exportedExpectations(exported.files[`${id}.json`]),
+			item.body,
+		);
+		changed.push(...exportedDiff.map((field) => `export ${field}`));
+		if (changed.length > 0) problems.push(`${id}: ${[...new Set(changed)].join(', ')}`);
+	}
+	if (problems.length > 0) {
+		for (const problem of problems) console.error(`  ! ${problem}`);
+		throw new Error(`${String(problems.length)} case(s) did not round-trip.`);
+	}
+	console.log(`  verified ${String(planned.length)} case(s) round-trip intact`);
+}
+
+if (require.main === module) {
+	main().catch((error: unknown) => {
+		console.error(error instanceof Error ? error.message : String(error));
+		process.exit(1);
+	});
+}

packages/@n8n/instance-ai/evaluations/routing/langtracer-source.ts

@@ -0,0 +1,58 @@
+// ---------------------------------------------------------------------------
+// Pulls routing cases from a LangTracer suite (`--source langtracer`).
+// ---------------------------------------------------------------------------
+
+import { routingCasesFromExport } from './langtracer-cases';
+import { resolveRoutingLangTracerConfig } from './langtracer-config';
+import {
+	isRoutingCaseSelected,
+	toDiscoveryScenario,
+	type LoadedRoutingCase,
+	type RoutingCaseSelection,
+} from './loader';
+import { findLangTracerSuite, LangTracerClient } from '../langtracer/client';
+
+export interface LangTracerRoutingSuite {
+	suite: { id: number; slug: string };
+	cases: LoadedRoutingCase[];
+}
+
+/**
+ * Exports the suite and rebuilds every case from its tags. Any case that does
+ * not rebuild fails the whole load with every fault named, like the disk loader.
+ */
+export async function loadRoutingCasesFromLangTracer(
+	suiteRef: string,
+	selection: RoutingCaseSelection = {},
+	client: LangTracerClient = new LangTracerClient(resolveRoutingLangTracerConfig()),
+): Promise<LangTracerRoutingSuite> {
+	const suites = await client.listSuites();
+	const suite = findLangTracerSuite(suites, suiteRef);
+	if (!suite) {
+		const known = suites
+			.map((s) => s.slug)
+			.sort()
+			.join(', ');
+		throw new Error(`LangTracer suite "${suiteRef}" not found. Available: ${known || '(none)'}.`);
+	}
+
+	const exported = await client.exportSuite(suite.id);
+	const selected = Object.fromEntries(
+		Object.entries(exported.files).filter(([fileName]) =>
+			isRoutingCaseSelected(fileName.replace(/\.json$/i, ''), selection),
+		),
+	);
+	const { cases, errors } = routingCasesFromExport(selected);
+	if (errors.length > 0) {
+		throw new Error(
+			`Invalid routing case(s) in LangTracer suite "${suite.slug}":\n${errors.map((e) => `  - ${e}`).join('\n')}`,
+		);
+	}
+	return {
+		suite: { id: suite.id, slug: suite.slug },
+		cases: cases.map((routingCase) => ({
+			routingCase,
+			scenario: toDiscoveryScenario(routingCase),
+		})),
+	};
+}

packages/@n8n/instance-ai/evaluations/routing/loader.ts

@@ -0,0 +1,327 @@
+// ---------------------------------------------------------------------------
+// Loader for routing eval cases (one JSON file per case, `<id>.json`).
+//
+// Routing cases carry no tool expectations: the grader resolves the route
+// from the recorded calls afterwards. The loader validates the authoring
+// format and maps each case to the scenario shape the discovery runner needs.
+// ---------------------------------------------------------------------------
+
+import { readFileSync, readdirSync } from 'fs';
+import { basename, join, resolve } from 'path';
+import { z } from 'zod';
+
+import { discoveryTestCaseSchema } from '../data/discovery';
+import type { DiscoveryScenario, RoutingAttachment, RoutingSeed } from '../discovery/types';
+import { CaseSeedSchema } from '../harness/schema';
+
+export const ROUTING_BUCKETS = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'answer',
+	'clarify',
+	'multi',
+	'decline',
+] as const;
+
+export const ROUTING_ACCEPT_TOKENS = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'multi',
+	'answer',
+	'decline',
+	'clarify',
+	'clarify:agent',
+	'clarify:open',
+] as const;
+
+export const DEFAULT_ROUTING_LANGUAGE = 'eng';
+
+/** The seed id rule LangTracer and the n8n restore path apply to every artifact. */
+const MIN_SEED_ID_LENGTH = 8;
+
+/**
+ * The inline seed shape of the main eval harness (and LangTracer's
+ * `create_test_case`). The in-process stub instance can serve workflows,
+ * Agents, data tables, prior messages, and prior runs. It has no folders or
+ * projects, so a seed that declares them is refused instead of run without them.
+ */
+const routingSeedSchema = CaseSeedSchema.superRefine((seed, ctx) => {
+	if (seed.mode !== 'inline') {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['mode'],
+			message: 'routing cases support only `mode: "inline"` seeds',
+		});
+		return;
+	}
+	if (seed.folders.length > 0) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['folders'],
+			message: 'the in-process stub instance has no folders, so routing cases cannot seed them',
+		});
+	}
+	if (seed.projects.length > 0) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['projects'],
+			message: 'the in-process stub instance has one project, so routing cases cannot seed more',
+		});
+	}
+	for (const [index, workflow] of seed.workflows.entries()) {
+		if (workflow.id.length < MIN_SEED_ID_LENGTH) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ['workflows', index, 'id'],
+				message: `seed workflow ids need at least ${String(MIN_SEED_ID_LENGTH)} characters`,
+			});
+		}
+	}
+	const empty =
+		seed.messages.length === 0 &&
+		seed.workflows.length === 0 &&
+		seed.agents.length === 0 &&
+		seed.dataTables.length === 0;
+	if (empty) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			message: 'an inline seed must carry messages, workflows, agents, or dataTables',
+		});
+	}
+});
+
+const routingAttachmentSchema = z.union([
+	z.object({ workflow: z.string().min(1) }).strict(),
+	z.object({ agent: z.string().min(1) }).strict(),
+]);
+
+/** Authored case ids, which name the case file. */
+const AUTHORED_ID_PATTERN = /^route-[a-z0-9-]+$/;
+
+/**
+ * Ids of imported cases. A LangTracer export body has no id, so the id is its
+ * file name, which only has to be safe as a file name (the dispatcher writes
+ * `lt-<case>-<run>.json`, for example).
+ */
+const IMPORTED_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+function routingCaseObject(id: z.ZodString) {
+	return z
+		.object({
+			id,
+			bucket: z.enum(ROUTING_BUCKETS),
+			agentShaped: z.boolean().optional(),
+			userMessage: z.string().min(1),
+			accepts: z.array(z.enum(ROUTING_ACCEPT_TOKENS)).min(1),
+			policyDependent: z.boolean().optional(),
+			source: z.string().min(1),
+			rationale: z.string().min(1).optional(),
+			instanceState: discoveryTestCaseSchema.shape.instanceState,
+			seed: routingSeedSchema.optional(),
+			/** The workflow or Agent the user had open when they sent the message. */
+			attach: routingAttachmentSchema.optional(),
+			/** ISO 639-3 code of `userMessage`. Absent means English. */
+			language: z
+				.string()
+				.regex(/^[a-z]{3}$/, 'language must be an ISO 639-3 code, e.g. "eng"')
+				.optional(),
+		})
+		.strict();
+}
+
+/** Every attachment and prior run must name a seeded resource. */
+function checkSeedReferences(
+	routingCase: z.infer<ReturnType<typeof routingCaseObject>>,
+	ctx: z.RefinementCtx,
+): void {
+	const seed = inlineSeedOf(routingCase.seed);
+	const workflowIds = new Set(seed?.workflows.map((workflow) => workflow.id));
+	const agentIds = new Set(seed?.agents.map((agent) => agent.id));
+	const { attach } = routingCase;
+	if (attach && 'workflow' in attach && !workflowIds.has(attach.workflow)) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['attach', 'workflow'],
+			message: 'must be the id of a workflow in `seed.workflows`',
+		});
+	}
+	if (attach && 'agent' in attach && !agentIds.has(attach.agent)) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['attach', 'agent'],
+			message: 'must be the id of an Agent in `seed.agents`',
+		});
+	}
+	for (const [index, priorRun] of (seed?.priorRuns ?? []).entries()) {
+		if (!workflowIds.has(priorRun.workflow)) {
+			ctx.addIssue({
+				code: z.ZodIssueCode.custom,
+				path: ['seed', 'priorRuns', index, 'workflow'],
+				message: 'must be the id of a workflow in `seed.workflows`',
+			});
+		}
+	}
+}
+
+export const routingCaseSchema = routingCaseObject(
+	z.string().regex(AUTHORED_ID_PATTERN, 'id must look like route-<bucket>-<slug>'),
+).superRefine(checkSeedReferences);
+
+const importedRoutingCaseSchema = routingCaseObject(
+	z.string().regex(IMPORTED_ID_PATTERN, 'id must be safe as a file name'),
+).superRefine(checkSeedReferences);
+
+export type RoutingCase = z.infer<typeof routingCaseSchema>;
+
+export interface LoadedRoutingCase {
+	routingCase: RoutingCase;
+	scenario: DiscoveryScenario;
+	/** The case file name without `.json`, when the case came from a file. */
+	fileName?: string;
+}
+
+/** Narrows a parsed seed to its inline arm; the schema refuses every other arm. */
+function inlineSeedOf(seed: RoutingCase['seed']): RoutingSeed | undefined {
+	return seed?.mode === 'inline' ? seed : undefined;
+}
+
+export function toDiscoveryScenario(routingCase: RoutingCase): DiscoveryScenario {
+	const seed = inlineSeedOf(routingCase.seed);
+	const attach: RoutingAttachment | undefined = routingCase.attach;
+	return {
+		id: routingCase.id,
+		userMessage: routingCase.userMessage,
+		...(routingCase.instanceState ? { instanceState: routingCase.instanceState } : {}),
+		...(routingCase.rationale ? { rationale: routingCase.rationale } : {}),
+		...(seed ? { seed } : {}),
+		...(attach ? { attach } : {}),
+	};
+}
+
+/**
+ * Validates one parsed case body. Returns the issues as `path: message` strings.
+ * `imported` accepts any file-name-safe id instead of `route-<bucket>-<slug>`.
+ */
+export function parseRoutingCase(
+	raw: unknown,
+	options: { imported?: boolean } = {},
+): { success: true; data: RoutingCase } | { success: false; issues: string[] } {
+	const schema = options.imported ? importedRoutingCaseSchema : routingCaseSchema;
+	const parsed = schema.safeParse(raw);
+	if (parsed.success) return { success: true, data: parsed.data };
+	return {
+		success: false,
+		issues: parsed.error.issues.map(
+			(issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`,
+		),
+	};
+}
+
+function parseRoutingCaseFile(filePath: string): RoutingCase {
+	let raw: unknown;
+	try {
+		raw = JSON.parse(readFileSync(filePath, 'utf-8'));
+	} catch (error) {
+		throw new Error(`${filePath}: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	const parsed = parseRoutingCase(raw);
+	if (!parsed.success) throw new Error(`${filePath}: ${parsed.issues.join('; ')}`);
+	const slug = basename(filePath, '.json');
+	if (parsed.data.id !== slug) {
+		throw new Error(`${filePath}: id "${parsed.data.id}" must match the file name "${slug}"`);
+	}
+	return parsed.data;
+}
+
+function csvTokens(value: string | undefined): string[] {
+	return (value ?? '')
+		.split(',')
+		.map((token) => token.trim().toLowerCase())
+		.filter((token) => token.length > 0);
+}
+
+export interface RoutingCaseSelection {
+	/**
+	 * Keep ids that contain any comma-separated token. A single token that is
+	 * exactly the id or file name of a case selects only that case.
+	 */
+	filter?: string;
+	/** Drop ids that contain any comma-separated token. */
+	exclude?: string;
+	/** Drop ids that start with any comma-separated prefix. */
+	excludePrefix?: string;
+}
+
+/** Applies `filter`, `exclude`, and `excludePrefix` to one case id or file name. */
+export function isRoutingCaseSelected(name: string, selection: RoutingCaseSelection = {}): boolean {
+	const lower = name.toLowerCase();
+	const include = csvTokens(selection.filter);
+	if (include.length > 0 && !include.some((token) => lower.includes(token))) return false;
+	if (csvTokens(selection.exclude).some((token) => lower.includes(token))) return false;
+	if (csvTokens(selection.excludePrefix).some((prefix) => lower.startsWith(prefix))) return false;
+	return true;
+}
+
+/**
+ * Keeps only the cases whose id or file name is exactly `filter`, when there
+ * are any. So `--filter route-agent-x` does not also run `route-agent-x-2`.
+ */
+export function narrowToExactMatch(
+	cases: LoadedRoutingCase[],
+	filter: string | undefined,
+): LoadedRoutingCase[] {
+	const tokens = csvTokens(filter);
+	if (tokens.length !== 1) return cases;
+	const [token] = tokens;
+	const exact = cases.filter(
+		({ routingCase, fileName }) =>
+			routingCase.id.toLowerCase() === token || fileName?.toLowerCase() === token,
+	);
+	return exact.length > 0 ? exact : cases;
+}
+
+/**
+ * Loads every `route-*.json` case in `dir` (sorted by file name), so results
+ * or notes kept next to the cases are not read as cases. A string `selection`
+ * is a `filter`. All invalid files are reported in one error, so an author can
+ * fix them in one pass.
+ */
+export function loadRoutingCases(
+	dir: string,
+	selection?: string | RoutingCaseSelection,
+): LoadedRoutingCase[] {
+	const select: RoutingCaseSelection =
+		typeof selection === 'string' ? { filter: selection } : (selection ?? {});
+	const root = resolve(dir);
+	const files = readdirSync(root)
+		.filter(
+			(f) => f.startsWith('route-') && f.endsWith('.json') && isRoutingCaseSelected(f, select),
+		)
+		.sort();
+
+	const loaded: LoadedRoutingCase[] = [];
+	const errors: string[] = [];
+	for (const file of files) {
+		try {
+			const routingCase = parseRoutingCaseFile(join(root, file));
+			loaded.push({
+				routingCase,
+				scenario: toDiscoveryScenario(routingCase),
+				fileName: basename(file, '.json'),
+			});
+		} catch (error) {
+			errors.push(error instanceof Error ? error.message : String(error));
+		}
+	}
+
+	if (errors.length > 0) {
+		throw new Error(
+			`Invalid routing case(s) in ${root}:\n${errors.map((e) => `  - ${e}`).join('\n')}`,
+		);
+	}
+	return narrowToExactMatch(loaded, select.filter);
+}

packages/@n8n/instance-ai/evaluations/routing/route-rules.ts

@@ -0,0 +1,82 @@
+// ---------------------------------------------------------------------------
+// Committing calls for the routing eval.
+//
+// A committing call is the first call that picks a route for the user's
+// request. `--stop-on-route` ends the trial there. Every other call (skill and
+// tool loading, docs, research, reads, and data-table schema setup) is
+// exploration, so the run goes on.
+// ---------------------------------------------------------------------------
+
+import { DOMAIN_TOOL_IDS, ORCHESTRATION_TOOL_IDS } from '../../src/tools/tool-ids';
+
+const DATA_TABLES_READ_ACTIONS: ReadonlySet<string> = new Set(['list', 'get', 'query', 'schema']);
+
+/**
+ * Schema-only setup. The Assistant often creates a table before it builds a
+ * workflow, so these calls do not pick a route. Row writes and table deletes do.
+ */
+const DATA_TABLES_SETUP_ACTIONS: ReadonlySet<string> = new Set([
+	'create',
+	'add-column',
+	'rename-column',
+	'delete-column',
+]);
+
+const WORKFLOWS_READ_ACTIONS: ReadonlySet<string> = new Set([
+	'list',
+	'get',
+	'get-as-code',
+	'node-usage',
+	'list-versions',
+	'validate',
+]);
+
+const EXECUTIONS_COMMITTING_ACTIONS: ReadonlySet<string> = new Set(['run', 'debug', 'stop']);
+
+function actionOf(args: Record<string, unknown>): string | undefined {
+	return typeof args.action === 'string' ? args.action : undefined;
+}
+
+/**
+ * Any data-tables action that is not a read, including schema setup.
+ * A call without an action fails schema validation, so it commits to nothing.
+ */
+export function isMutatingDataTablesAction(action: string | undefined): boolean {
+	return action !== undefined && !DATA_TABLES_READ_ACTIONS.has(action);
+}
+
+/** A mutating data-tables action that is not schema setup: row writes, table deletes, and unknown actions. */
+export function isCommittingDataTablesAction(action: string | undefined): boolean {
+	return (
+		action !== undefined &&
+		!DATA_TABLES_READ_ACTIONS.has(action) &&
+		!DATA_TABLES_SETUP_ACTIONS.has(action)
+	);
+}
+
+export function isMutatingWorkflowsAction(action: string | undefined): boolean {
+	return action !== undefined && !WORKFLOWS_READ_ACTIONS.has(action);
+}
+
+export function isCommittingCall(toolName: string, args: Record<string, unknown>): boolean {
+	const action = actionOf(args);
+	switch (toolName) {
+		case ORCHESTRATION_TOOL_IDS.BUILD_AGENT:
+			// `exploring` only reads an existing Agent, so it does not commit to a route.
+			return args.operation !== 'exploring';
+		case DOMAIN_TOOL_IDS.BUILD_WORKFLOW:
+		case ORCHESTRATION_TOOL_IDS.CREATE_TASKS:
+		case DOMAIN_TOOL_IDS.ASK_USER:
+			return true;
+		case DOMAIN_TOOL_IDS.NODES:
+			return action === 'execute';
+		case DOMAIN_TOOL_IDS.EXECUTIONS:
+			return action !== undefined && EXECUTIONS_COMMITTING_ACTIONS.has(action);
+		case DOMAIN_TOOL_IDS.DATA_TABLES:
+			return isCommittingDataTablesAction(action);
+		case DOMAIN_TOOL_IDS.WORKFLOWS:
+			return isMutatingWorkflowsAction(action);
+		default:
+			return false;
+	}
+}