From df9c6d1689c9969c8c8efcbd9f3fa8fbf114b98b Mon Sep 17 00:00:00 2001 From: Oleg Ivaniv Date: Thu, 1 Oct 2026 11:39:16 +0200 Subject: [PATCH] test(core): Add routing mode to Instance AI discovery evals (no-changelog) Add a routing mode to the discovery eval harness. It runs one user message through the real orchestrator, stops at the first action that commits to a route, and grades the route against the accepted routes of the case. - Load routing cases from a directory or from a LangTracer suite. Cases can seed workflows, Agents, prior messages and failed runs, and attach a workflow or Agent to the message. - Resolve the route from tool calls. A small judge classifies text-only replies and the steer of clarifying questions. - Write routing results, a markdown summary, and an eval-results.json in the LangTracer dispatcher contract. - Add a grader with per-bucket reports and run comparison, and a push script for LangTracer suites. - Add a manual workflow that runs a routing suite. --- .github/WORKFLOWS.md | 1 + .github/workflows/test-evals-routing.yml | 101 +++ .../@n8n/instance-ai/evaluations/README.md | 112 +++ .../eval-results-dispatcher-contract.test.ts | 140 +++ .../discovery/__tests__/cli-args.test.ts | 79 ++ .../discovery/__tests__/grade-resolve.test.ts | 225 +++++ .../discovery/__tests__/route-rules.test.ts | 72 ++ .../__tests__/routing-case-dir.test.ts | 118 +++ .../__tests__/routing-context.test.ts | 94 ++ .../__tests__/routing-eval-results.test.ts | 245 ++++++ .../__tests__/routing-langtracer.test.ts | 273 ++++++ .../__tests__/routing-loader.test.ts | 190 ++++ .../discovery/__tests__/routing-trial.test.ts | 114 +++ .../__tests__/skill-overrides.test.ts | 82 ++ .../discovery/__tests__/stream-status.test.ts | 10 + .../discovery/__tests__/stub-seed.test.ts | 178 ++++ .../evaluations/discovery/cli-args.ts | 294 +++++++ .../instance-ai/evaluations/discovery/cli.ts | 631 ++++++++++---- .../discovery/confirmation-policy.ts | 4 +- .../discovery/expected-tools-invoked.ts | 3 + .../evaluations/discovery/routing-case-dir.ts | 117 +++ .../evaluations/discovery/routing-context.ts | 89 ++ .../discovery/routing-eval-results.ts | 387 +++++++++ .../evaluations/discovery/routing-results.ts | 90 ++ .../evaluations/discovery/routing-trial.ts | 201 +++++ .../evaluations/discovery/runner.ts | 217 ++++- .../evaluations/discovery/skill-overrides.ts | 115 +++ .../evaluations/discovery/stream-status.ts | 4 + .../discovery/stub-agent-context.ts | 103 +++ .../evaluations/discovery/types.ts | 96 +- .../evaluations/harness/stub-seed.ts | 288 ++++++ .../evaluations/harness/stub-services.ts | 101 ++- .../evaluations/routing/expectation.ts | 15 + .../evaluations/routing/grade-report.ts | 819 ++++++++++++++++++ .../evaluations/routing/grade-resolve.ts | 281 ++++++ .../evaluations/routing/grade-run.ts | 143 +++ .../evaluations/routing/grade-types.ts | 277 ++++++ .../instance-ai/evaluations/routing/grade.ts | 206 +++++ .../instance-ai/evaluations/routing/judge.ts | 330 +++++++ .../evaluations/routing/langtracer-cases.ts | 370 ++++++++ .../evaluations/routing/langtracer-config.ts | 54 ++ .../evaluations/routing/langtracer-push.ts | 349 ++++++++ .../evaluations/routing/langtracer-source.ts | 58 ++ .../instance-ai/evaluations/routing/loader.ts | 327 +++++++ .../evaluations/routing/route-rules.ts | 82 ++ 45 files changed, 7895 insertions(+), 190 deletions(-) create mode 100644 .github/workflows/test-evals-routing.yml create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/cli-args.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/grade-resolve.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/route-rules.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-case-dir.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-context.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-eval-results.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-langtracer.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-loader.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-trial.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/skill-overrides.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/__tests__/stub-seed.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/cli-args.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/routing-case-dir.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/routing-context.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/routing-eval-results.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/routing-results.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/routing-trial.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/skill-overrides.ts create mode 100644 packages/@n8n/instance-ai/evaluations/discovery/stub-agent-context.ts create mode 100644 packages/@n8n/instance-ai/evaluations/harness/stub-seed.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/expectation.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/grade-report.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/grade-resolve.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/grade-run.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/grade-types.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/grade.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/judge.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/langtracer-cases.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/langtracer-config.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/langtracer-push.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/langtracer-source.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/loader.ts create mode 100644 packages/@n8n/instance-ai/evaluations/routing/route-rules.ts diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index 9ceed1a884a..45f309e029a 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -421,6 +421,7 @@ Run-workflow dropdown, which only a user with write access can pick. | `util-codespace-preview.yml`| Wake, re-serve or delete a PR preview instance by hand | | `util-data-tooling.yml` | SQLite/PostgreSQL export/import validation (manual) | | `util-probe-registry.yml` | Diagnose slow npm metadata fetches (temporary) | +| `test-evals-routing.yml` | Instance AI routing evals on a LangTracer suite (`eval:discovery --stop-on-route`) | --- diff --git a/.github/workflows/test-evals-routing.yml b/.github/workflows/test-evals-routing.yml new file mode 100644 index 00000000000..0510c77a418 --- /dev/null +++ b/.github/workflows/test-evals-routing.yml @@ -0,0 +1,101 @@ +name: 'Test: Instance AI Routing Evals' +run-name: "Test: Instance AI Routing Evals (branch=${{ inputs.branch || 'master' }}, suite=${{ inputs.suite || 'intent-routing' }}, model=${{ inputs.model || 'anthropic/claude-opus-5-5' }}, iterations=${{ inputs.iterations || '3' }}, filter=${{ inputs.filter || 'all' }})" + +# Manual only. LangTracer's dispatcher runs cases with eval:instance-ai, so the +# routing suite has no scheduled lane until the dispatcher can run eval:discovery. +on: + workflow_dispatch: + inputs: + branch: + description: 'GitHub branch to test' + required: false + default: 'master' + suite: + description: 'LangTracer suite slug or id with the routing cases' + required: false + default: 'intent-routing' + model: + description: 'Orchestrator model (provider/model)' + required: false + default: 'anthropic/claude-opus-5-5' + iterations: + description: 'Trials per case' + required: false + default: '3' + filter: + description: 'Comma-separated case-id tokens; an exact id runs only that case. Empty = all cases.' + required: false + default: '' + +permissions: + contents: read + +jobs: + run-routing-evals: + name: 'Run Routing Evals' + runs-on: blacksmith-4vcpu-ubuntu-2204 + timeout-minutes: 90 + env: + NODE_OPTIONS: '--max-old-space-size=7168' + OUTPUT_DIR: ${{ github.workspace }}/routing-eval-output + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.branch || github.ref }} + fetch-depth: 1 + + - name: Setup Environment + uses: ./.github/actions/setup-nodejs + + - name: Export Node Types + run: | + ./packages/cli/bin/n8n export:nodes --output ./packages/@n8n/ai-workflow-builder.ee/evaluations/nodes.json + + - name: Run Routing Evals + working-directory: packages/@n8n/instance-ai + env: + ANTHROPIC_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} + N8N_ENABLED_MODULES: agents,instance-ai + LANGTRACER_URL: ${{ secrets.EVALS_LANGTRACER_URL }} + LANGTRACER_API_KEY: ${{ secrets.LANGTRACER_API_KEY }} + SUITE: ${{ inputs.suite || 'intent-routing' }} + MODEL: ${{ inputs.model || 'anthropic/claude-opus-5-5' }} + ITERATIONS: ${{ inputs.iterations || '3' }} + FILTER: ${{ inputs.filter }} + run: | + ARGS=( + --source langtracer + --suite "$SUITE" + --stop-on-route + --trials "$ITERATIONS" + --model "$MODEL" + --scenario-concurrency 4 + --output-dir "$OUTPUT_DIR" + # The cases let evaluations/routing/grade.ts re-grade or compare the artifact. + --save-cases-dir "$OUTPUT_DIR/cases" + ) + if [ -n "$FILTER" ]; then + ARGS+=(--filter "$FILTER") + fi + pnpm eval:discovery "${ARGS[@]}" + + - name: Write job summary + if: ${{ always() }} + run: | + SUMMARY_FILE="$OUTPUT_DIR/routing-summary.md" + if [ -f "$SUMMARY_FILE" ]; then + cat "$SUMMARY_FILE" >> "$GITHUB_STEP_SUMMARY" + else + echo "No routing summary was written. Check the 'Run Routing Evals' step log." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload Results + if: ${{ always() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: instance-ai-routing-eval-results + path: routing-eval-output/ + if-no-files-found: warn + retention-days: 14 diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 0a678f2a78f..e18026cc1cf 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -21,6 +21,7 @@ Sections: - [Regression detection](#regression-detection) - [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows) - [Running discovery evals](#discovery-evals) +- [Running routing evals](#routing-evals) - [Running pairwise evals](#pairwise-evals) - [How the e2e harness works](#how-the-e2e-harness-works) - [How the workflow-build harness works](#how-the-workflow-build-harness-works) @@ -564,6 +565,117 @@ For data-table routing, look for `load_skill(skillId="data-table-manager")` and `data-tables(action="list")`, and verify there are no planning, workflow-builder, or spawned-agent entries in the spawned-agent section. +## Routing evals + +Routing evals measure how often the Assistant takes an acceptable path for a +prompt: build an Agent, build a workflow, do the task once, ask a question, and +so on. They use the discovery harness in routing mode. With `--stop-on-route`, +a trial stops at the first call that commits to a route, so each trial costs +one routing decision, not a full build. The grader then gives each trial a +route and checks it against the case's accept tokens. + +Each case has a `bucket` (its main expected route) and one or more `accepts` +tokens. A trial passes when its route matches one of the tokens. A case passes +when at least 2 of 3 trials pass. `routing/grade-resolve.ts` has the exact +rules and their order. + +| Route | The trial | Accept tokens | +|-------|-----------|---------------| +| `agent` | calls `build-agent` | `agent` | +| `workflow` | calls `build-workflow` | `workflow` | +| `one-off` | calls `build-workflow` with `executionIntent: "one-off"`, executes a node, runs or stops an execution, writes rows, or changes a workflow | `one-off` | +| `multi` | calls `create-tasks` | `multi` | +| `debug` | reads executions (`get`, `list`, or `debug`) | `debug` | +| `clarify` | calls `ask-user`, or replies with a question | `clarify` (any direction), `clarify:agent` (points to an Agent, or to both), `clarify:open` (does not push a workflow only) | +| `answer` | replies and commits to nothing | `answer` | +| `decline` | refuses the request | `decline` | + +An LLM judge reads `ask-user` cards and text-only replies to tell `clarify`, +`answer`, and `decline` apart, and to find the direction of a question. An +`agent` or `workflow` case also passes when the Assistant asks a question that +points the same way. The report shows this score next to the older "strict" +score, which does not count those questions. + +### Cases in LangTracer + +The cases live in the LangTracer suite `intent-routing`, not in this +repository. A LangTracer case has no routing fields, so the routing labels are +tags in `evalTags`: `routing`, `bucket:`, one `accepts:` for each +token, `policy-dependent`, `agent-shaped`, `source:`, and `lang:`. +The case name is the case id. The prompt is the single conversation turn. + +Each case also has one process expectation, `Routes to one of: `. +LangTracer uses this text to match a run's result to the case. The runner +ignores it and reads the accept tokens from the tags. `setKind` is +`regression` when the case passes on the baseline model, else +`capability_gap`. + +### Run locally + +```bash +# From packages/@n8n/instance-ai/ +ANTHROPIC_API_KEY=... N8N_ENABLED_MODULES=agents,instance-ai \ + pnpm eval:discovery --source langtracer --suite intent-routing --stop-on-route \ + --trials 3 --model anthropic/claude-opus-5-5 --scenario-concurrency 4 \ + --output-dir .data/routing/baseline --save-cases-dir .data/routing/cases +``` + +- `--source langtracer --suite intent-routing` pulls the cases from + LangTracer. Set `LANGTRACER_URL` and `LANGTRACER_API_KEY`. If they are not + set, the `lang-tracer` MCP entry in `~/.claude.json` supplies them. +- `--cases-dir ` reads the cases from a local directory instead. It reads + routing case files (`route-*.json`) and LangTracer suite-export bodies. +- `--output-dir ` grades each case when it finishes. It writes + `routing-results.json` (every trial), `routing-summary.md` (a short summary), + and `eval-results.json` (the format LangTracer reads). +- `--save-cases-dir ` writes the loaded cases, one `.json` file for + each case. The grader needs them. +- `--skill-file =` runs the orchestrator with a different + `SKILL.md` for one runtime skill, for example + `--skill-file intent-recognition=./SKILL.md`. Repeat the flag for more + skills. Sub-agents still use the bundled skills. +- `--filter ` runs only the case ids that contain one of the tokens. + +### Grade and compare + +```bash +pnpm tsx evaluations/routing/grade.ts --results .data/routing/baseline/routing-results.json \ + --cases-dir .data/routing/cases --out .data/routing/baseline/report.md \ + --compare .data/routing/previous/routing-results.json +``` + +The grader writes a markdown report and a JSON summary next to it. The report +shows the macro accuracy (the mean of the case pass rates of all buckets), the +score of each bucket, the skill-load rate, a confusion table, and the failing +cases. `--compare` is optional. It adds the change for each bucket and the +cases that flipped. The judge needs `ANTHROPIC_API_KEY` unless its cache +already has every verdict. + +### Push cases + +```bash +pnpm tsx evaluations/routing/langtracer-push.ts --suite intent-routing \ + --cases-dir --passing-ids --dry-run +``` + +The push creates or updates each case by name and leaves unchanged cases as +they are. `--passing-ids` is a JSON list of the case ids that pass on the +baseline. Those cases get `setKind: regression`, all others +`capability_gap`. Remove `--dry-run` to write. After a write, the script reads +the suite back and fails when a case does not match the local file. + +### CI + +Dispatch **Test: Instance AI Routing Evals** +(`.github/workflows/test-evals-routing.yml`) to run a suite on a branch. It has +no pull-request or schedule trigger. The job writes the summary to the run page +and uploads the output directory as an artifact. + +LangTracer's dispatcher runs every case with `eval:instance-ai`, which builds +and verifies a workflow. Routing cases need `eval:discovery`. Do not schedule +the `intent-routing` suite in LangTracer until the dispatcher can run +`eval:discovery`. + ## Pairwise evals Pairwise evals score a built workflow against the dataset's `dos` / `donts` diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts index 53b772c41d7..35f450c7278 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts @@ -5,7 +5,12 @@ import { tmpdir } from 'os'; import { join } from 'path'; import type { CheckOutcome } from '../binaryChecks/types'; +import { RoutingEvalOutput } from '../discovery/routing-eval-results'; +import type { RoutingTrialRecord } from '../discovery/types'; import { AGENT_ARTIFACT_CASE_CAP_BYTES } from '../harness/artifacts/agent-artifact'; +import { routingExpectationText } from '../routing/expectation'; +import type { JudgeInput, JudgeResult } from '../routing/judge'; +import { parseRoutingCase } from '../routing/loader'; import { aggregateResults } from '../run/aggregator'; import { writeEvalResults } from '../run/persist'; import type { @@ -139,15 +144,19 @@ interface DispatcherView { agentArtifactPerRun: Array | null>; totalRuns: number; workflowChecksPerRun: Array | null>; + status?: string; buildExpectations: Array<{ expectation: string; passCount: number; evaluatedCount: number; + passAtK?: number; + passHatK?: number; }>; buildExpectationResultsPerRun: Array | null>; buildCostUsdPerRun?: Array; @@ -415,3 +424,134 @@ describe('eval-results.json — dispatcher contract', () => { expect(tc.buildTurnsPerRun).toEqual([5, null]); }); }); + +// The discovery CLI's routing mode (`--output-dir`) writes the same file for a +// LangTracer routing case. The dispatcher matches each expectation to the +// case's stored process expectation by trimmed text, and passes it on a strict +// majority of the evaluated (not `incomplete`) runs. +describe('eval-results.json — dispatcher contract (routing)', () => { + const parsed = parseRoutingCase({ + id: 'route-agent-support', + bucket: 'agent', + userMessage: 'Set up AI support in our group.', + accepts: ['agent', 'clarify:agent'], + source: 'synthetic', + }); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + const routingCase = parsed.data; + + function trial(n: number, overrides: Partial): RoutingTrialRecord { + return { + trial: n, + durationMs: 1000, + streamStatus: 'stopped-on-route', + toolCalls: [], + subAgentToolCalls: [], + spawnedAgents: [], + skillsLoaded: [], + askUserQuestions: [], + finalText: '', + fullText: '', + ...overrides, + }; + } + + const judge = { + judge: async (input: JudgeInput): Promise => + await Promise.resolve({ + verdict: { + kind: input.mode === 'ask-user' ? 'clarify' : 'answer', + steer: 'workflow', + reason: 'Asks which trigger starts the workflow.', + }, + cached: false, + }), + }; + + async function writeAndReadRouting(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'eval-results-routing-contract-')); + const output = new RoutingEvalOutput( + dir, + judge, + { + runId: 'run-1', + variant: 'baseline', + model: 'anthropic/claude-opus-5-5', + trialsPerCase: 3, + stopOnRoute: true, + startedAt: '2026-10-01T10:00:00.000Z', + }, + 1, + ); + await output.record(0, { + routingCase, + fileName: 'lt-route-agent-support-1a2b3c4d', + result: { + id: routingCase.id, + userMessage: routingCase.userMessage, + trials: [ + trial(1, { toolCalls: [{ toolName: 'build-agent', args: {}, status: 'pending' }] }), + trial(2, { + toolCalls: [ + { + toolName: 'ask-user', + args: { questions: [{ question: 'Which trigger?', options: ['Schedule'] }] }, + status: 'pending', + }, + ], + }), + trial(3, { streamStatus: 'timed-out' }), + ], + }, + threadIds: ['discovery-thread-a', 'discovery-thread-b', 'discovery-thread-c'], + transcripts: [[], [], []], + }); + await output.finish(); + return jsonParse(readFileSync(output.evalResultsPath, 'utf8')); + } + + it('serializes the fields the dispatcher reads for a routing case', async () => { + const report = await writeAndReadRouting(); + + // The dispatcher ingests `testCases[0]` of a single-case run. + expect(report.testCases).toHaveLength(1); + const tc = report.testCases[0]; + expect(tc.totalRuns).toBe(3); + expect(tc.status).toBe('verified'); + expect(tc.threadIds).toEqual([ + 'discovery-thread-a', + 'discovery-thread-b', + 'discovery-thread-c', + ]); + + // The same text the LangTracer push stores as the case's process expectation. + const expectation = routingExpectationText(routingCase.accepts); + expect(tc.buildExpectations).toEqual([ + { expectation, passCount: 1, evaluatedCount: 2, passAtK: 1, passHatK: 0.25 }, + ]); + expect(tc.buildExpectationResultsPerRun).toEqual([ + [{ expectation, pass: true, reason: expect.stringContaining('Route agent') }], + [ + { + expectation, + pass: false, + reason: expect.stringContaining('Route clarify:workflow'), + attribution: 'builder_issue', + }, + ], + [ + { + expectation, + pass: false, + reason: expect.stringContaining('timed out'), + incomplete: true, + attribution: 'timeout', + }, + ], + ]); + + // Dispatcher rule: a strict majority of the evaluated runs. 1 of 2 fails. + const [unit] = tc.buildExpectations; + expect(unit.passCount * 2 > unit.evaluatedCount).toBe(false); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/cli-args.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/cli-args.test.ts new file mode 100644 index 00000000000..c20f103ea95 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/cli-args.test.ts @@ -0,0 +1,79 @@ +import { resolve } from 'node:path'; + +import { isRoutingMode, parseCliArgs, parseSkillOverride } from '../cli-args'; + +describe('parseCliArgs', () => { + it('accepts the dispatcher flag names as aliases', () => { + const args = parseCliArgs([ + '--cases-dir', + '/tmp/cases', + '--filter', + 'lt-route-agent-x-1a2b3c4d', + '--iterations', + '2', + '--timeout-ms', + '90000', + '--output-dir', + '/tmp/out', + ]); + + expect(args).toMatchObject({ + casesDir: '/tmp/cases', + filter: 'lt-route-agent-x-1a2b3c4d', + trials: 2, + timeoutMs: 90_000, + outputDir: '/tmp/out', + }); + expect(isRoutingMode(args)).toBe(true); + expect(parseCliArgs(['--trials', '4', '--timeout', '5000'])).toMatchObject({ + trials: 4, + timeoutMs: 5000, + }); + }); + + it('needs a case source for routing-only flags', () => { + expect(() => parseCliArgs(['--stop-on-route'])).toThrow( + '--stop-on-route works only in routing mode: pass --cases-dir , or --source langtracer --suite .', + ); + expect(() => parseCliArgs(['--output-dir', '/tmp/out'])).toThrow( + /--output-dir works only in routing mode/, + ); + expect(() => parseCliArgs(['--source', 'local'])).toThrow( + '--source local needs --cases-dir .', + ); + expect(() => parseCliArgs(['--source', 'langtracer'])).toThrow( + '--source langtracer needs --suite .', + ); + expect( + isRoutingMode(parseCliArgs(['--source', 'langtracer', '--suite', 'intent-routing'])), + ).toBe(true); + expect(isRoutingMode(parseCliArgs([]))).toBe(false); + }); + + it('refuses unknown flags and invalid values', () => { + expect(() => parseCliArgs(['--iteration', '3'])).toThrow('Unknown argument: --iteration'); + expect(() => parseCliArgs(['--iterations', '0'])).toThrow(/expected a positive integer/); + expect(() => parseCliArgs(['--filter'])).toThrow('Missing value for --filter'); + }); + + it('collects repeated --skill-file values', () => { + const args = parseCliArgs([ + '--skill-file', + 'intent-recognition=skills/variant.md', + '--skill-file', + 'planning=/abs/planning.md', + ]); + + expect(args.skillFiles).toEqual([ + { skillId: 'intent-recognition', path: resolve('skills/variant.md') }, + { skillId: 'planning', path: '/abs/planning.md' }, + ]); + expect(() => parseSkillOverride('intent-recognition')).toThrow( + '--skill-file expects =, got "intent-recognition"', + ); + }); + + it('stops at --help', () => { + expect(parseCliArgs(['--help', '--bogus']).help).toBe(true); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/grade-resolve.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/grade-resolve.test.ts new file mode 100644 index 00000000000..9c52e8be5a6 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/grade-resolve.test.ts @@ -0,0 +1,225 @@ +import type { GradedCase, GradedTrial } from '../../routing/grade-report'; +import { computeScopeMetrics } from '../../routing/grade-report'; +import { + isSameDirectionClarify, + resolveTrial, + type RouteResolution, + staleStopCall, + strictTrialPasses, + trialPasses, +} from '../../routing/grade-resolve'; +import type { Bucket, RoutingCase, TrialResult, TrialToolCall } from '../../routing/grade-types'; + +function routingCase(bucket: Bucket, accepts: RoutingCase['accepts']): RoutingCase { + return { + id: `case-${bucket}`, + bucket, + userMessage: 'Do the thing.', + accepts, + policyDependent: false, + }; +} + +function call(toolName: string, args: Record = {}): TrialToolCall { + return { toolName, args }; +} + +function trial(toolCalls: TrialToolCall[], overrides: Partial = {}): TrialResult { + return { + trial: 1, + streamStatus: 'completed', + toolCalls, + spawnedAgents: [], + skillsLoaded: [], + askUserQuestions: [], + finalText: 'Here is what I found.', + ...overrides, + }; +} + +function resolvedRoute(calls: TrialToolCall[]) { + const pending = resolveTrial(routingCase('workflow', ['workflow']), trial(calls)); + return pending.kind === 'resolved' + ? { kind: pending.kind, route: pending.resolution.route, rule: pending.resolution.rule } + : { kind: pending.kind, rule: pending.rule }; +} + +const clarify = (steer: RouteResolution['steer']): RouteResolution => ({ + route: 'clarify', + steer, + rule: 6, + evidence: 'ask-user', +}); + +describe('resolveTrial', () => { + it('looks past data-table schema setup to the build that follows', () => { + expect( + resolvedRoute([ + call('data-tables', { action: 'create' }), + call('build-workflow', { executionIntent: 'recurring' }), + ]), + ).toEqual({ kind: 'resolved', route: 'workflow', rule: 2 }); + }); + + it('does not route data-table schema setup alone to one-off', () => { + expect(resolvedRoute([call('data-tables', { action: 'create' })])).toEqual({ + kind: 'judge', + rule: 7, + }); + }); + + it('routes a data-table row write to one-off', () => { + expect( + resolvedRoute([ + call('data-tables', { action: 'create' }), + call('data-tables', { action: 'insert-rows' }), + ]), + ).toEqual({ kind: 'resolved', route: 'one-off', rule: 4 }); + }); + + it('does not route a debugging-executions skill load alone to debug', () => { + expect(resolvedRoute([call('load_skill', { skillId: 'debugging-executions' })])).toEqual({ + kind: 'judge', + rule: 7, + }); + }); + + it.each(['list', 'get', 'debug'])('routes executions %s to debug', (action) => { + expect( + resolvedRoute([ + call('load_skill', { skillId: 'debugging-executions' }), + call('executions', { action }), + ]), + ).toEqual({ kind: 'resolved', route: 'debug', rule: 5 }); + }); +}); + +describe('same-direction clarification', () => { + it.each([ + ['workflow', 'workflow', true], + ['workflow', 'both', true], + ['workflow', 'agent', false], + ['workflow', 'none', false], + ['agent', 'agent', true], + ['agent', 'both', true], + ['agent', 'workflow', false], + ['agent', 'none', false], + ['one-off', 'both', false], + ['clarify', 'agent', false], + ] as const)('bucket %s with steer %s: %s', (bucket, steer, expected) => { + expect(isSameDirectionClarify(bucket, clarify(steer))).toBe(expected); + }); + + it('does not count a non-clarify route', () => { + expect( + isSameDirectionClarify('workflow', { + route: 'workflow', + rule: 2, + evidence: 'build-workflow', + }), + ).toBe(false); + }); + + it('passes the v2 score but not the strict score', () => { + const workflowCase = routingCase('workflow', ['workflow']); + expect(trialPasses(workflowCase, clarify('workflow'))).toBe(true); + expect(strictTrialPasses(workflowCase, clarify('workflow'))).toBe(false); + + const agentCase = routingCase('agent', ['agent']); + expect(trialPasses(agentCase, clarify('both'))).toBe(true); + expect(strictTrialPasses(agentCase, clarify('both'))).toBe(false); + expect(trialPasses(agentCase, clarify('workflow'))).toBe(false); + }); + + it('keeps accept tokens in both scores', () => { + const clarifyCase = routingCase('clarify', ['clarify:open']); + expect(trialPasses(clarifyCase, clarify('none'))).toBe(true); + expect(strictTrialPasses(clarifyCase, clarify('none'))).toBe(true); + expect(trialPasses(clarifyCase, clarify('workflow'))).toBe(false); + }); +}); + +describe('staleStopCall', () => { + it('flags a stopped trial whose calls no longer commit', () => { + const stopped = trial( + [call('data-tables', { action: 'list' }), call('data-tables', { action: 'create' })], + { streamStatus: 'stopped-on-route' }, + ); + expect(staleStopCall(stopped)).toBe('data-tables create'); + }); + + it('accepts a stopped trial that ends on a committing call', () => { + const stopped = trial([call('data-tables', { action: 'create' }), call('build-workflow')], { + streamStatus: 'stopped-on-route', + }); + expect(staleStopCall(stopped)).toBeUndefined(); + }); + + it('ignores trials that were not stopped', () => { + expect(staleStopCall(trial([call('data-tables', { action: 'create' })]))).toBeUndefined(); + }); +}); + +describe('computeScopeMetrics', () => { + function gradedTrial( + pass: boolean, + strictPass: boolean, + sameDirectionClarify: boolean, + ): GradedTrial { + return { + trial: 1, + streamStatus: 'completed', + skillLoaded: false, + resolution: clarify('workflow'), + label: 'clarify:workflow', + pass, + strictPass, + sameDirectionClarify, + judged: true, + }; + } + + function gradedCase(bucket: Bucket, trials: GradedTrial[]): GradedCase { + const passedTrials = trials.filter((entry) => entry.pass).length; + const strictPassedTrials = trials.filter((entry) => entry.strictPass).length; + return { + id: `case-${bucket}`, + bucket, + accepts: [], + policyDependent: false, + trials, + passedTrials, + pass: passedTrials * 3 >= trials.length * 2, + strictPassedTrials, + strictPass: strictPassedTrials * 3 >= trials.length * 2, + }; + } + + it('reports strict scores and over-asking over the agent and workflow buckets', () => { + const metrics = computeScopeMetrics([ + gradedCase('workflow', [ + gradedTrial(true, false, true), + gradedTrial(true, false, true), + gradedTrial(true, true, false), + ]), + gradedCase('agent', [ + gradedTrial(true, true, false), + gradedTrial(true, true, false), + gradedTrial(false, false, false), + ]), + gradedCase('clarify', [ + gradedTrial(true, true, false), + gradedTrial(true, true, false), + gradedTrial(true, true, false), + ]), + ]); + + expect(metrics.macroAccuracy).toBe(1); + expect(metrics.strictMacroAccuracy).toBeCloseTo(2 / 3); + expect(metrics.strictTrialPass).toMatchObject({ num: 6, den: 9 }); + expect(metrics.overAsking).toMatchObject({ num: 2, den: 6 }); + expect(metrics.buckets.workflow?.overAsking).toMatchObject({ num: 2, den: 3 }); + expect(metrics.buckets.agent?.overAsking).toMatchObject({ num: 0, den: 3 }); + expect(metrics.buckets.clarify?.overAsking).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/route-rules.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/route-rules.test.ts new file mode 100644 index 00000000000..fcdc9696a0e --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/route-rules.test.ts @@ -0,0 +1,72 @@ +import { + isCommittingCall, + isCommittingDataTablesAction, + isMutatingDataTablesAction, +} from '../../routing/route-rules'; + +describe('isCommittingCall', () => { + it.each([ + ['build-agent', { operation: 'create' }], + ['build-workflow', {}], + ['create-tasks', {}], + ['ask-user', { questions: [] }], + ['nodes', { action: 'execute' }], + ['executions', { action: 'run' }], + ['executions', { action: 'debug' }], + ['executions', { action: 'stop' }], + ['data-tables', { action: 'insert-rows' }], + ['data-tables', { action: 'update-rows' }], + ['data-tables', { action: 'delete-rows' }], + ['data-tables', { action: 'upsert-rows' }], + ['data-tables', { action: 'delete' }], + ['workflows', { action: 'delete' }], + ['workflows', { action: 'publish' }], + ])('treats %s %j as committing', (toolName, args) => { + expect(isCommittingCall(toolName, args)).toBe(true); + }); + + it.each([ + ['load_skill', { skillId: 'intent-recognition' }], + ['load_skill', { skillId: 'debugging-executions' }], + ['search_tools', { query: 'agent' }], + ['n8n-docs', { query: 'credentials' }], + ['research', { action: 'web-search' }], + ['nodes', { action: 'search' }], + ['executions', { action: 'list' }], + ['executions', { action: 'get' }], + ['data-tables', { action: 'list' }], + ['data-tables', { action: 'query' }], + ['data-tables', { action: 'schema' }], + ['data-tables', { action: 'create' }], + ['data-tables', { action: 'add-column' }], + ['data-tables', { action: 'rename-column' }], + ['data-tables', { action: 'delete-column' }], + ['data-tables', {}], + ['workflows', { action: 'list' }], + ['workflows', { action: 'get-as-code' }], + ['workflows', { action: 'validate' }], + ['workflows', {}], + ['credentials', { action: 'list' }], + ])('treats %s %j as exploration', (toolName, args) => { + expect(isCommittingCall(toolName, args)).toBe(false); + }); +}); + +describe('data-tables actions', () => { + it('counts schema setup as mutating but not committing', () => { + expect(isMutatingDataTablesAction('create')).toBe(true); + expect(isCommittingDataTablesAction('create')).toBe(false); + }); + + it('counts row writes as mutating and committing', () => { + expect(isMutatingDataTablesAction('insert-rows')).toBe(true); + expect(isCommittingDataTablesAction('insert-rows')).toBe(true); + }); + + it('counts reads and a missing action as neither', () => { + for (const action of ['list', 'get', 'query', 'schema', undefined]) { + expect(isMutatingDataTablesAction(action)).toBe(false); + expect(isCommittingDataTablesAction(action)).toBe(false); + } + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-case-dir.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-case-dir.test.ts new file mode 100644 index 00000000000..4cfd058708a --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-case-dir.test.ts @@ -0,0 +1,118 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { routingExpectationText } from '../../routing/expectation'; +import { toLangTracerCreateBody } from '../../routing/langtracer-cases'; +import { parseRoutingCase, type RoutingCase } from '../../routing/loader'; +import { isRoutingExportBody, loadRoutingCaseDir } from '../routing-case-dir'; + +function routingCase(raw: Record = {}): RoutingCase { + const parsed = parseRoutingCase({ + id: 'route-agent-support', + bucket: 'agent', + userMessage: 'Set up AI support in our group.', + accepts: ['agent', 'clarify:agent'], + source: 'synthetic', + rationale: 'Ongoing conversational support role.', + ...raw, + }); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + return parsed.data; +} + +/** The body LangTracer exports (and the dispatcher writes) for a pushed routing case. */ +function exportBody(source: RoutingCase): Record { + const body = toLangTracerCreateBody(source, { suiteId: 64, setKind: 'regression' }); + return { + complexity: body.evalComplexity, + tags: body.evalTags, + conversation: body.conversation, + description: body.description, + processExpectations: [routingExpectationText(source.accepts)], + }; +} + +function caseDir(files: Record): string { + const dir = mkdtempSync(join(tmpdir(), 'routing-case-dir-')); + for (const [name, body] of Object.entries(files)) { + writeFileSync(join(dir, name), typeof body === 'string' ? body : JSON.stringify(body)); + } + return dir; +} + +describe('loadRoutingCaseDir', () => { + it('reads authored case files and export bodies side by side', () => { + const dir = caseDir({ + 'route-debug-failed-run.json': routingCase({ + id: 'route-debug-failed-run', + bucket: 'debug', + accepts: ['debug', 'clarify'], + }), + 'route-agent-support.json': exportBody(routingCase()), + 'results.json': { cases: [] }, + 'notes.json': 'not json', + }); + + const loaded = loadRoutingCaseDir(dir); + + expect(loaded.map((c) => [c.routingCase.id, c.fileName])).toEqual([ + ['route-agent-support', 'route-agent-support'], + ['route-debug-failed-run', 'route-debug-failed-run'], + ]); + expect(loaded[0].routingCase).toMatchObject({ + bucket: 'agent', + accepts: ['agent', 'clarify:agent'], + rationale: 'Ongoing conversational support role.', + }); + expect(loaded[0].scenario.userMessage).toBe('Set up AI support in our group.'); + }); + + it('uses the file name as the id of an export body without a name', () => { + const dir = caseDir({ 'lt-route-agent-support-1a2b3c4d.json': exportBody(routingCase()) }); + + const [loaded] = loadRoutingCaseDir(dir, { filter: 'lt-route-agent-support-1a2b3c4d' }); + + expect(loaded.routingCase.id).toBe('lt-route-agent-support-1a2b3c4d'); + expect(loaded.scenario.id).toBe('lt-route-agent-support-1a2b3c4d'); + }); + + it('uses the name of an export body when it carries one', () => { + const dir = caseDir({ + 'lt-case.json': { ...exportBody(routingCase()), name: 'route-agent-support' }, + }); + + expect(loadRoutingCaseDir(dir, { filter: 'route-agent-support' })).toMatchObject([ + { routingCase: { id: 'route-agent-support' }, fileName: 'lt-case' }, + ]); + }); + + it('selects only the exact match when the filter names one case', () => { + const dir = caseDir({ + 'route-agent-support.json': exportBody(routingCase()), + 'route-agent-support-2.json': exportBody(routingCase()), + }); + + expect(loadRoutingCaseDir(dir, { filter: 'route-agent-support' })).toHaveLength(1); + expect(loadRoutingCaseDir(dir, { filter: 'agent-support' })).toHaveLength(2); + }); + + it('reports every invalid case file in one error', () => { + const dir = caseDir({ + 'route-agent-bad.json': routingCase({ id: 'route-agent-other' }), + 'lt-broken.json': { tags: ['routing', 'accepts:agent'], conversation: [] }, + }); + + expect(() => loadRoutingCaseDir(dir)).toThrow( + /lt-broken\.json: no "bucket:" tag[\s\S]*route-agent-bad\.json: id "route-agent-other" must match the file name/, + ); + }); +}); + +describe('isRoutingExportBody', () => { + it('needs a routing tag', () => { + expect(isRoutingExportBody({ tags: ['routing'] })).toBe(true); + expect(isRoutingExportBody({ tags: ['agents'] })).toBe(false); + expect(isRoutingExportBody(routingCase())).toBe(false); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-context.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-context.test.ts new file mode 100644 index 00000000000..75c1d8cabfb --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-context.test.ts @@ -0,0 +1,94 @@ +import { + buildThreadArtifactsBlock, + buildThreadContextBlock, +} from '../../../../../cli/src/modules/instance-ai/internal-messages'; +import { parseRoutingCase, toDiscoveryScenario } from '../../routing/loader'; +import { + buildRoutingTurnMessage, + createSeededMemory, + STUB_PROJECT_ID, + toResourceAttachment, +} from '../routing-context'; +import type { RoutingSeed } from '../types'; + +function seedOf(raw: Record): RoutingSeed { + const parsed = parseRoutingCase({ + id: 'route-answer-context', + bucket: 'answer', + userMessage: 'What does it do?', + accepts: ['answer'], + source: 'synthetic', + seed: { mode: 'inline', ...raw }, + }); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + const { seed } = toDiscoveryScenario(parsed.data); + if (!seed) throw new Error('expected a seed'); + return seed; +} + +const seed = seedOf({ + workflows: [{ id: 'wf-context-01', name: 'Order sync', nodes: [], connections: {} }], + agents: [ + { + id: 'agent-context-01', + config: { name: 'Order bot', model: 'anthropic/claude-sonnet-4-5', instructions: 'Help.' }, + }, + ], + messages: [ + { role: 'user', text: 'Build an order sync.' }, + { role: 'assistant', text: 'Done: "Order sync" is saved.' }, + ], +}); + +describe('toResourceAttachment', () => { + it('names the seeded workflow or Agent the way the editor hands it off', () => { + expect(toResourceAttachment({ workflow: 'wf-context-01' }, seed)).toEqual({ + type: 'workflow', + id: 'wf-context-01', + name: 'Order sync', + }); + expect(toResourceAttachment({ agent: 'agent-context-01' }, seed)).toEqual({ + type: 'agent', + id: 'agent-context-01', + name: 'Order bot', + projectId: STUB_PROJECT_ID, + }); + }); + + it('refuses an attachment the seed does not declare', () => { + expect(() => toResourceAttachment({ workflow: 'wf-missing-01' }, seed)).toThrow( + 'is not a seeded workflow', + ); + }); +}); + +describe('buildRoutingTurnMessage', () => { + it('sends the user message unchanged without an attachment', () => { + expect(buildRoutingTurnMessage('Hello', undefined)).toBe('Hello'); + }); + + it('prepends the thread-context block production builds for the hand-off', () => { + const attachment = toResourceAttachment({ workflow: 'wf-context-01' }, seed); + const message = buildRoutingTurnMessage('Change the schedule.', attachment); + + const productionBlock = buildThreadContextBlock([ + buildThreadArtifactsBlock(undefined, [attachment]), + ]); + expect(message).toBe(`${productionBlock}\n\nChange the schedule.`); + expect(message.startsWith('\n\n')).toBe(true); + expect(message).toContain('Workflow "Order sync" (id: `wf-context-01`)'); + }); +}); + +describe('createSeededMemory', () => { + it('holds the seeded messages as the thread history, in order', async () => { + const memory = await createSeededMemory('thread-1', 'eval-user', seed.messages); + + const history = await memory.getMessages('thread-1', { resourceId: 'eval-user' }); + expect(history.map((m) => ('role' in m ? m.role : m.type))).toEqual(['user', 'assistant']); + expect(history.every((m) => m.createdAt instanceof Date)).toBe(true); + expect(history[1]).toMatchObject({ + content: [{ type: 'text', text: 'Done: "Order sync" is saved.' }], + }); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-eval-results.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-eval-results.test.ts new file mode 100644 index 00000000000..187147a1ae4 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-eval-results.test.ts @@ -0,0 +1,245 @@ +import { mkdtempSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import type { JudgeInput, JudgeResult } from '../../routing/judge'; +import { parseRoutingCase, type RoutingCase } from '../../routing/loader'; +import { + gradeRoutingCase, + RoutingEvalOutput, + toDispatcherTestCase, + type RoutingEvalMeta, +} from '../routing-eval-results'; +import type { RoutingTrialRecord } from '../types'; + +function routingCase(raw: Record = {}): RoutingCase { + const parsed = parseRoutingCase({ + id: 'route-agent-support', + bucket: 'agent', + userMessage: 'Set up AI support in our group.', + accepts: ['agent', 'clarify:agent'], + source: 'synthetic', + ...raw, + }); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + return parsed.data; +} + +function trial(n: number, overrides: Partial = {}): RoutingTrialRecord { + return { + trial: n, + durationMs: 1000, + streamStatus: 'stopped-on-route', + toolCalls: [], + subAgentToolCalls: [], + spawnedAgents: [], + skillsLoaded: [], + askUserQuestions: [], + finalText: '', + fullText: '', + ...overrides, + }; +} + +const askUser = (question: string) => ({ + toolName: 'ask-user', + args: { questions: [{ question, options: ['A', 'B'] }] }, + status: 'pending' as const, +}); + +/** Steers by keyword; a final text of "THROW" makes the judge fail. */ +const judge = { + judge: async (input: JudgeInput): Promise => { + if (input.finalText === 'THROW') throw new Error('HTTP 529: overloaded'); + const text = JSON.stringify(input.askUserQuestions); + const steer = text.includes('trigger') + ? 'workflow' + : text.includes('persona') + ? 'agent' + : 'none'; + return await Promise.resolve({ + verdict: { kind: input.mode === 'ask-user' ? 'clarify' : 'answer', steer, reason: 'test' }, + cached: false, + }); + }, +}; + +const meta: RoutingEvalMeta = { + runId: 'run-1', + variant: 'baseline', + model: 'anthropic/claude-opus-5-5', + trialsPerCase: 5, + stopOnRoute: true, + startedAt: '2026-10-01T10:00:00.000Z', +}; + +const expectation = 'Routes to one of: agent, clarify:agent'; + +async function gradedFixture() { + const source = routingCase(); + return await gradeRoutingCase( + { + routingCase: source, + fileName: 'lt-route-agent-support-1a2b3c4d', + result: { + id: source.id, + userMessage: source.userMessage, + trials: [ + trial(1, { + toolCalls: [{ toolName: 'build-agent', args: {}, status: 'pending' }], + }), + trial(2, { toolCalls: [askUser('Which trigger should start the workflow?')] }), + trial(3, { streamStatus: 'timed-out' }), + trial(4, { streamStatus: 'completed', finalText: 'THROW' }), + trial(5, { streamStatus: 'errored', runError: 'socket hang up' }), + ], + }, + threadIds: ['t-1', 't-2', 't-3', 't-4', 't-5'], + transcripts: [[], [], [], [], []], + }, + judge, + meta, + ); +} + +describe('toDispatcherTestCase', () => { + it('turns each trial into one verdict on the routing expectation', async () => { + const testCase = toDispatcherTestCase(await gradedFixture()); + + expect(testCase.buildExpectationResultsPerRun).toEqual([ + [ + { + expectation, + pass: true, + reason: 'Route agent (rule 1, build-agent). Accepted.', + }, + ], + [ + { + expectation, + pass: false, + reason: + 'Route clarify:workflow (rule 6, ask-user). Judge: test. Not in the accepted routes: agent, clarify:agent.', + attribution: 'builder_issue', + }, + ], + [ + { + expectation, + pass: false, + reason: 'Not graded: the run timed out before a committing call (timed-out).', + incomplete: true, + attribution: 'timeout', + }, + ], + [ + { + expectation, + pass: false, + reason: 'Not graded: the judge failed (HTTP 529: overloaded).', + incomplete: true, + attribution: 'verification_gap', + }, + ], + [ + { + expectation, + pass: false, + reason: 'Not graded: the run failed before a committing call (errored: socket hang up).', + incomplete: true, + attribution: 'framework_issue', + }, + ], + ]); + expect(testCase).toMatchObject({ + name: 'route-agent-support', + testCaseFile: 'lt-route-agent-support-1a2b3c4d', + status: 'verified', + totalRuns: 5, + buildExpectations: [ + { expectation, passCount: 1, evaluatedCount: 2, passAtK: 1, passHatK: 0.25 }, + ], + threadIds: ['t-1', 't-2', 't-3', 't-4', 't-5'], + }); + }); + + it('passes a same-direction clarification under the v2 score', async () => { + const source = routingCase({ + id: 'route-workflow-digest', + bucket: 'workflow', + accepts: ['workflow'], + }); + const graded = await gradeRoutingCase( + { + routingCase: source, + result: { + id: source.id, + userMessage: source.userMessage, + trials: [trial(1, { toolCalls: [askUser('Which trigger should start it?')] })], + }, + threadIds: ['t-1'], + transcripts: [[]], + }, + judge, + meta, + ); + + const [[verdict]] = toDispatcherTestCase(graded).buildExpectationResultsPerRun; + expect(verdict).toEqual({ + expectation: 'Routes to one of: workflow', + pass: true, + reason: + 'Route clarify:workflow (rule 6, ask-user). Judge: test. Accepted as a same-direction clarification.', + }); + }); + + it('marks a case with no graded trial as not verified', async () => { + const source = routingCase(); + const graded = await gradeRoutingCase( + { + routingCase: source, + result: { + id: source.id, + userMessage: source.userMessage, + trials: [trial(1, { streamStatus: 'timed-out' })], + }, + threadIds: ['t-1'], + transcripts: [[]], + }, + judge, + meta, + ); + + expect(toDispatcherTestCase(graded)).toMatchObject({ + status: 'notVerified', + buildExpectations: [{ passCount: 0, evaluatedCount: 0, passAtK: 0, passHatK: 0 }], + }); + }); +}); + +describe('RoutingEvalOutput', () => { + it('writes eval-results.json and the summary after each case', async () => { + const dir = mkdtempSync(join(tmpdir(), 'routing-eval-output-')); + const output = new RoutingEvalOutput(dir, judge, meta, 1); + const { input } = await gradedFixture(); + + await output.record(0, input); + const partial: unknown = JSON.parse(readFileSync(output.evalResultsPath, 'utf8')); + expect(partial).toMatchObject({ + complete: false, + testCases: [{ name: 'route-agent-support' }], + }); + + const final = await output.finish(); + expect(final.summary).toEqual({ testCases: 1, passed: 0, notVerified: 0 }); + expect(JSON.parse(readFileSync(output.evalResultsPath, 'utf8'))).toMatchObject({ + complete: true, + totalRuns: 5, + routing: { runId: 'run-1', judgeModel: expect.any(String) }, + }); + const summary = readFileSync(output.summaryPath, 'utf8'); + expect(summary).toContain( + '| route-agent-support | agent | agent, clarify:agent | agent, clarify:workflow, none, none, none | 1/2 fail (3 not graded) |', + ); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-langtracer.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-langtracer.test.ts new file mode 100644 index 00000000000..1044a4f010d --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-langtracer.test.ts @@ -0,0 +1,273 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { routingExpectationText } from '../../routing/expectation'; +import { + expectedBehaviorSentence, + exportedExpectations, + isSyntheticSource, + parsePassingIds, + routingCaseDiff, + routingCaseJsonFromExport, + routingCasesFromExport, + toLangTracerCreateBody, + toLangTracerUpdateBody, + unsupportedRoutingPushReason, + type RoutingLangTracerCreateBody, +} from '../../routing/langtracer-cases'; +import { + langTracerConfigFromClaudeConfig, + resolveRoutingLangTracerConfig, +} from '../../routing/langtracer-config'; +import { parseRoutingPushArgs } from '../../routing/langtracer-push'; +import { parseRoutingCase, type RoutingCase } from '../../routing/loader'; + +function routingCase(raw: Record): RoutingCase { + const parsed = parseRoutingCase({ + id: 'route-agent-support', + bucket: 'agent', + agentShaped: true, + userMessage: 'Set up AI support in our group.', + accepts: ['agent', 'clarify:agent'], + policyDependent: false, + source: 'suite-62:985', + rationale: 'Ongoing conversational support role.', + ...raw, + }); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + return parsed.data; +} + +/** What the suite export returns for a pushed body: evalTags as `tags`, plus the stored content. */ +function exportedBody(body: RoutingLangTracerCreateBody): Record { + return { + complexity: body.evalComplexity, + tags: body.evalTags, + conversation: body.conversation, + // A case with an expectation gets no synthesized happy-path scenario. + executionScenarios: [], + ...(body.description ? { description: body.description } : {}), + ...(body.seed ? { seed: body.seed } : {}), + processExpectations: body.processExpectations, + }; +} + +const seeded = routingCase({ + id: 'route-debug-seeded', + bucket: 'debug', + agentShaped: false, + accepts: ['debug', 'clarify'], + source: 'langtracer:thread-1', + seed: { + mode: 'inline', + workflows: [{ id: 'wf-seeded-0001', name: 'Sync', nodes: [], connections: {} }], + messages: [{ role: 'user', text: 'Build a sync.' }], + priorRuns: [{ workflow: 'wf-seeded-0001', hints: 'Sync failed' }], + }, + attach: { workflow: 'wf-seeded-0001' }, + language: 'deu', +}); + +describe('toLangTracerCreateBody', () => { + it('encodes the routing labels as tags, in the spec order', () => { + const body = toLangTracerCreateBody(routingCase({ policyDependent: true }), { + suiteId: 64, + setKind: 'regression', + }); + + expect(body.evalTags).toEqual([ + 'routing', + 'bucket:agent', + 'accepts:agent', + 'accepts:clarify:agent', + 'policy-dependent', + 'agent-shaped', + 'source:suite-62:985', + 'lang:eng', + ]); + // LangTracer keeps only kebab-case case tags. + expect(body.tags).toEqual(['routing', 'policy-dependent', 'agent-shaped']); + expect(body).toMatchObject({ + name: 'route-agent-support', + setKind: 'regression', + synthetic: true, + suiteId: 64, + userPrompt: 'Set up AI support in our group.', + description: 'Ongoing conversational support role.', + conversation: [{ role: 'user', text: 'Set up AI support in our group.' }], + }); + expect(body.expectedBehavior).toBe( + 'The Assistant should build an Agent or ask a clarifying question that points toward an Agent (accepted routes: agent, clarify:agent).', + ); + expect(body.processExpectations).toEqual(['Routes to one of: agent, clarify:agent']); + expect(body).not.toHaveProperty('outcomeExpectations'); + }); + + it('carries the seed without folders and the attachment on the turn', () => { + const body = toLangTracerCreateBody(seeded, { suiteId: 64, setKind: 'capability_gap' }); + expect(body.synthetic).toBe(false); + expect(body.seed).not.toHaveProperty('folders'); + expect(body.seed?.priorRuns).toEqual([{ workflow: 'wf-seeded-0001', hints: 'Sync failed' }]); + expect(body.conversation?.[0].attach).toEqual({ workflow: 'wf-seeded-0001' }); + expect(body.evalTags).toContain('lang:deu'); + }); + + it('patches every routing field and clears an absent seed', () => { + const body = toLangTracerCreateBody(routingCase({}), { suiteId: 64, setKind: 'regression' }); + const patch = toLangTracerUpdateBody(body); + expect(patch).not.toHaveProperty('suiteId'); + expect(patch).not.toHaveProperty('synthetic'); + expect(patch).toMatchObject({ + seed: null, + containsUserData: false, + setKind: 'regression', + processExpectations: [routingExpectationText(['agent', 'clarify:agent'])], + outcomeExpectations: [], + }); + }); +}); + +describe('routing cases from a LangTracer export', () => { + it('round-trips plain and seeded cases', () => { + const plain = routingCase({}); + const files = Object.fromEntries( + [plain, seeded].map((c) => [ + `${c.id}.json`, + exportedBody(toLangTracerCreateBody(c, { suiteId: 64, setKind: 'regression' })), + ]), + ); + + const { cases, errors } = routingCasesFromExport(files); + + expect(errors).toEqual([]); + expect(cases.map((c) => c.id)).toEqual(['route-agent-support', 'route-debug-seeded']); + expect(routingCaseDiff(plain, cases[0])).toEqual([]); + expect(routingCaseDiff(seeded, cases[1])).toEqual([]); + expect(cases[1].attach).toEqual({ workflow: 'wf-seeded-0001' }); + }); + + it('ignores the stored expectation and scenarios when it rebuilds a case', () => { + const plain = routingCase({}); + const exported = { + ...exportedBody(toLangTracerCreateBody(plain, { suiteId: 64, setKind: 'regression' })), + processExpectations: ['Routes to one of: workflow'], + outcomeExpectations: ['An Agent was created.'], + executionScenarios: [{ name: 'happy-path', description: '' }], + }; + + const { cases, errors } = routingCasesFromExport({ [`${plain.id}.json`]: exported }); + + expect(errors).toEqual([]); + expect(cases[0].accepts).toEqual(['agent', 'clarify:agent']); + expect(routingCaseDiff(plain, cases[0])).toEqual([]); + }); + + it('reads the expectation lists of an export body, trimmed', () => { + expect( + exportedExpectations({ processExpectations: [' Routes to one of: answer ', ''] }), + ).toEqual({ process: ['Routes to one of: answer'], outcome: [] }); + expect(exportedExpectations(null)).toEqual({ process: [], outcome: [] }); + }); + + it('names what is missing from a case that is not a routing case', () => { + expect(routingCaseJsonFromExport('x', { tags: ['other'] })).toEqual({ + success: false, + issues: ['no "routing" tag, so it is not a routing case'], + }); + const { errors } = routingCasesFromExport({ + 'route-a.json': { tags: ['routing', 'accepts:agent'], conversation: [] }, + }); + expect(errors).toEqual([ + 'route-a: no "bucket:" tag; no "source:" tag; conversation must be exactly one user turn', + ]); + }); + + it('reports a changed field', () => { + const changed = routingCase({ accepts: ['agent'] }); + expect(routingCaseDiff(routingCase({}), changed)).toEqual(['accepts']); + }); +}); + +describe('push helpers', () => { + it('marks only the synthetic sources as synthetic', () => { + expect(['suite-62:1', 'agent-344:x', 'discovery:y', 'synthetic'].every(isSyntheticSource)).toBe( + true, + ); + expect(['langtracer:t', 'michael-report', 'one-off-page'].some(isSyntheticSource)).toBe(false); + }); + + it('reads passing ids as an array or under a known key', () => { + expect([...parsePassingIds(['a', 'b'])]).toEqual(['a', 'b']); + expect([...parsePassingIds({ passingIds: ['a'] })]).toEqual(['a']); + expect(() => parsePassingIds({ other: [] })).toThrow('passing ids must be'); + }); + + it('refuses a case with instance state', () => { + expect( + unsupportedRoutingPushReason(routingCase({ instanceState: { folderExploration: true } })), + ).toContain('instanceState'); + expect(unsupportedRoutingPushReason(routingCase({}))).toBeNull(); + }); + + it('describes one accepted route in one sentence', () => { + expect(expectedBehaviorSentence(['answer'])).toBe( + 'The Assistant should answer the question (accepted routes: answer).', + ); + }); +}); + +describe('parseRoutingPushArgs', () => { + const required = ['--suite', 'intent-routing', '--passing-ids', 'passing.json']; + + it('requires a cases directory', () => { + expect(() => parseRoutingPushArgs(required)).toThrow('--cases-dir is required'); + }); + + it('reads the cases directory and the selection', () => { + expect( + parseRoutingPushArgs([...required, '--cases-dir', 'cases', '--filter', 'agent', '--dry-run']), + ).toEqual({ + suite: 'intent-routing', + passingIdsFile: 'passing.json', + casesDir: 'cases', + selection: { filter: 'agent' }, + dryRun: true, + }); + }); +}); + +describe('resolveRoutingLangTracerConfig', () => { + it('prefers the environment', () => { + expect( + resolveRoutingLangTracerConfig( + { LANGTRACER_URL: 'https://lt.example', LANGTRACER_API_KEY: 'lt_env' }, + '/does/not/exist.json', + ), + ).toEqual({ baseUrl: 'https://lt.example', apiKey: 'lt_env' }); + }); + + it('falls back to the lang-tracer MCP entry of the Claude config', () => { + const dir = mkdtempSync(join(tmpdir(), 'routing-lt-config-')); + const file = join(dir, 'claude.json'); + writeFileSync( + file, + JSON.stringify({ + mcpServers: { + 'lang-tracer': { + url: 'https://lt.example/api/mcp', + headers: { Authorization: 'Bearer lt_file' }, + }, + }, + }), + ); + expect(resolveRoutingLangTracerConfig({}, file)).toEqual({ + baseUrl: 'https://lt.example/api', + apiKey: 'lt_file', + }); + expect(langTracerConfigFromClaudeConfig({ mcpServers: {} })).toBeUndefined(); + expect(() => resolveRoutingLangTracerConfig({}, join(dir, 'missing.json'))).toThrow( + 'LANGTRACER_URL', + ); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-loader.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-loader.test.ts new file mode 100644 index 00000000000..4195a5758a5 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-loader.test.ts @@ -0,0 +1,190 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + isRoutingCaseSelected, + loadRoutingCases, + parseRoutingCase, + toDiscoveryScenario, +} from '../../routing/loader'; + +const workflow = { + id: 'wf-seeded-01', + name: 'Daily digest', + nodes: [{ name: 'Start', type: 'n8n-nodes-base.manualTrigger' }], + connections: {}, +}; + +const agent = { + id: 'agent-seeded-01', + config: { name: 'Helper', model: 'anthropic/claude-sonnet-4-5', instructions: 'Be helpful.' }, +}; + +function baseCase(overrides: Record = {}) { + return { + id: 'route-debug-seeded', + bucket: 'debug', + userMessage: 'It failed again.', + accepts: ['debug', 'clarify'], + source: 'synthetic', + ...overrides, + }; +} + +function issuesOf(raw: unknown): string[] { + const parsed = parseRoutingCase(raw); + return parsed.success ? [] : parsed.issues; +} + +describe('routing case schema', () => { + it('accepts seed, attach, and language, and hands seed and attach to the runner', () => { + const parsed = parseRoutingCase( + baseCase({ + seed: { + mode: 'inline', + workflows: [workflow], + agents: [agent], + messages: [{ role: 'user', text: 'Build me a digest.' }], + priorRuns: [{ workflow: 'wf-seeded-01', hints: 'Start failed' }], + }, + attach: { workflow: 'wf-seeded-01' }, + language: 'deu', + }), + ); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + + const scenario = toDiscoveryScenario(parsed.data); + expect(scenario.attach).toEqual({ workflow: 'wf-seeded-01' }); + expect(scenario.seed?.workflows.map((w) => w.id)).toEqual(['wf-seeded-01']); + expect(scenario.seed?.priorRuns).toEqual([{ workflow: 'wf-seeded-01', hints: 'Start failed' }]); + // The `{role, text}` shorthand becomes a full stored message. + expect(scenario.seed?.messages[0]).toMatchObject({ + type: 'llm', + role: 'user', + content: [{ type: 'text', text: 'Build me a digest.' }], + }); + expect(parsed.data.language).toBe('deu'); + }); + + it('keeps an unseeded case unchanged', () => { + const parsed = parseRoutingCase(baseCase()); + if (!parsed.success) throw new Error(parsed.issues.join('; ')); + expect(toDiscoveryScenario(parsed.data)).toEqual({ + id: 'route-debug-seeded', + userMessage: 'It failed again.', + }); + }); + + it('refuses an attachment that names no seeded resource', () => { + expect(issuesOf(baseCase({ attach: { workflow: 'wf-missing-01' } }))).toEqual([ + 'attach.workflow: must be the id of a workflow in `seed.workflows`', + ]); + expect( + issuesOf( + baseCase({ + seed: { mode: 'inline', workflows: [workflow] }, + attach: { agent: 'agent-missing-01' }, + }), + ), + ).toEqual(['attach.agent: must be the id of an Agent in `seed.agents`']); + }); + + it('refuses a prior run of a workflow the seed does not declare', () => { + expect( + issuesOf( + baseCase({ + seed: { + mode: 'inline', + workflows: [workflow], + priorRuns: [{ workflow: 'wf-missing-01' }], + }, + }), + ), + ).toEqual(['seed.priorRuns.0.workflow: must be the id of a workflow in `seed.workflows`']); + }); + + it('refuses seed slots the stub instance cannot serve', () => { + const issues = issuesOf( + baseCase({ + seed: { + mode: 'inline', + workflows: [{ ...workflow, id: 'short' }], + folders: [{ id: 'folder-seeded-01', name: 'Clients' }], + projects: [{ name: 'Other' }], + }, + }), + ); + expect(issues).toEqual( + expect.arrayContaining([ + expect.stringContaining('seed.folders'), + expect.stringContaining('seed.projects'), + expect.stringContaining('seed.workflows.0.id'), + ]), + ); + expect(issuesOf(baseCase({ seed: { mode: 'replay', threadId: 't-1' } }))).toEqual([ + 'seed.mode: routing cases support only `mode: "inline"` seeds', + ]); + expect(issuesOf(baseCase({ seed: { mode: 'inline' } }))).toEqual([ + 'seed: an inline seed must carry messages, workflows, agents, or dataTables', + ]); + }); + + it('accepts a file-name id only for an imported case', () => { + const raw = baseCase({ id: 'lt-route-debug-seeded-1a2b3c4d' }); + expect(issuesOf(raw)).toEqual(['id: id must look like route--']); + expect(parseRoutingCase(raw, { imported: true }).success).toBe(true); + expect(parseRoutingCase(baseCase({ id: '../escape' }), { imported: true })).toEqual({ + success: false, + issues: ['id: id must be safe as a file name'], + }); + }); + + it('refuses a language that is not an ISO 639-3 code', () => { + expect(issuesOf(baseCase({ language: 'en' }))).toEqual([ + 'language: language must be an ISO 639-3 code, e.g. "eng"', + ]); + }); +}); + +describe('routing case selection', () => { + it('applies filter, exclude, and exclude-prefix to the id', () => { + expect(isRoutingCaseSelected('route-v2-agent-a', { excludePrefix: 'route-v2-' })).toBe(false); + expect(isRoutingCaseSelected('route-agent-v2-a', { excludePrefix: 'route-v2-' })).toBe(true); + expect(isRoutingCaseSelected('route-agent-a', { filter: 'agent,debug' })).toBe(true); + expect(isRoutingCaseSelected('route-answer-a', { filter: 'agent,debug' })).toBe(false); + expect(isRoutingCaseSelected('route-agent-a', { exclude: 'agent-a' })).toBe(false); + }); + + it('loads only the selected files from a directory', () => { + const dir = mkdtempSync(join(tmpdir(), 'routing-loader-')); + for (const id of ['route-debug-one', 'route-v2-debug-two']) { + writeFileSync(join(dir, `${id}.json`), JSON.stringify(baseCase({ id }))); + } + writeFileSync(join(dir, 'results.json'), '{}'); + + expect(loadRoutingCases(dir).map((c) => c.routingCase.id)).toEqual([ + 'route-debug-one', + 'route-v2-debug-two', + ]); + expect( + loadRoutingCases(dir, { excludePrefix: 'route-v2-' }).map((c) => c.routingCase.id), + ).toEqual(['route-debug-one']); + expect(loadRoutingCases(dir, 'two').map((c) => c.routingCase.id)).toEqual([ + 'route-v2-debug-two', + ]); + }); + + it('selects only the exact match when the filter is a full id', () => { + const dir = mkdtempSync(join(tmpdir(), 'routing-loader-')); + for (const id of ['route-debug-one', 'route-debug-one-more']) { + writeFileSync(join(dir, `${id}.json`), JSON.stringify(baseCase({ id }))); + } + + expect(loadRoutingCases(dir, 'route-debug-one').map((c) => c.routingCase.id)).toEqual([ + 'route-debug-one', + ]); + expect(loadRoutingCases(dir, 'debug-one')).toHaveLength(2); + expect(loadRoutingCases(dir, 'route-debug-one,more')).toHaveLength(2); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-trial.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-trial.test.ts new file mode 100644 index 00000000000..b23004b6bc2 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/routing-trial.test.ts @@ -0,0 +1,114 @@ +import type { InstanceAiEvent } from '@n8n/api-types'; + +import { buildRoutingTrialRecord, ORCHESTRATOR_AGENT_ID } from '../routing-trial'; + +const base = { runId: 'run-1', agentId: ORCHESTRATOR_AGENT_ID }; + +function text(value: string, agentId = ORCHESTRATOR_AGENT_ID): InstanceAiEvent { + return { type: 'text-delta', ...base, agentId, payload: { text: value } }; +} + +function call( + toolCallId: string, + toolName: string, + args: Record, + agentId = ORCHESTRATOR_AGENT_ID, +): InstanceAiEvent { + return { type: 'tool-call', ...base, agentId, payload: { toolCallId, toolName, args } }; +} + +function result(toolCallId: string, value: unknown = {}): InstanceAiEvent { + return { type: 'tool-result', ...base, payload: { toolCallId, result: value } }; +} + +const skillContent = { type: 'content', value: [{ type: 'text', text: '# Skill' }] }; + +describe('buildRoutingTrialRecord', () => { + it('records orchestrator calls, skills, and the text after the last call', () => { + const events: InstanceAiEvent[] = [ + text('Let me check. '), + call('c1', 'load_skill', { name: 'intent-recognition' }), + result('c1', skillContent), + call('c3', 'load_skill', { skillId: 'no-such-skill' }), + result('c3', { success: false, error: 'Unknown skill' }), + call('c2', 'n8n-docs', { query: 'webhooks' }), + { type: 'tool-error', ...base, payload: { toolCallId: 'c2', error: 'boom' } }, + text('Webhooks '), + text('start workflows.'), + ]; + + const record = buildRoutingTrialRecord({ + trial: 1, + events, + durationMs: 10, + streamStatus: 'completed', + }); + + expect(record.toolCalls).toEqual([ + { toolName: 'load_skill', args: { name: 'intent-recognition' }, status: 'completed' }, + { toolName: 'load_skill', args: { skillId: 'no-such-skill' }, status: 'completed' }, + { toolName: 'n8n-docs', args: { query: 'webhooks' }, status: 'errored', error: 'boom' }, + ]); + expect(record.skillsLoaded).toEqual(['intent-recognition']); + expect(record.finalText).toBe('Webhooks start workflows.'); + expect(record.fullText).toBe('Let me check.\n\nWebhooks start workflows.'); + }); + + it('drops calls after the stop and keeps ask-user questions', () => { + const events: InstanceAiEvent[] = [ + text('A few questions first.'), + call('c1', 'ask-user', { + introMessage: 'Before I build', + questions: [{ id: 'q1', question: 'Agent or workflow?', options: ['Agent', 'Workflow'] }], + }), + call('c2', 'build-agent', { operation: 'create' }), + ]; + + const record = buildRoutingTrialRecord({ + trial: 2, + events, + durationMs: 10, + streamStatus: 'stopped-on-route', + stop: { eventIndex: 2, toolName: 'ask-user', args: {} }, + }); + + expect(record.toolCalls.map((c) => [c.toolName, c.status])).toEqual([['ask-user', 'pending']]); + expect(record.askUserQuestions).toEqual([ + { + question: 'Agent or workflow?', + options: ['Agent', 'Workflow'], + introMessage: 'Before I build', + }, + ]); + expect(record.finalText).toBe('A few questions first.'); + expect(record.stoppedOn?.toolName).toBe('ask-user'); + }); + + it('keeps sub-agent calls apart and reports spawned roles', () => { + const events: InstanceAiEvent[] = [ + call('c1', 'build-agent', { operation: 'create' }), + { + type: 'agent-spawned', + ...base, + agentId: 'builder-1', + payload: { parentId: ORCHESTRATOR_AGENT_ID, role: 'agent-builder', tools: [] }, + }, + call('c2', 'write_config', {}, 'builder-1'), + text('sub-agent text', 'builder-1'), + ]; + + const record = buildRoutingTrialRecord({ + trial: 1, + events, + durationMs: 10, + streamStatus: 'completed', + }); + + expect(record.toolCalls.map((c) => c.toolName)).toEqual(['build-agent']); + expect(record.subAgentToolCalls).toEqual([ + { toolName: 'write_config', args: {}, status: 'pending', agentRole: 'agent-builder' }, + ]); + expect(record.spawnedAgents).toEqual(['agent-builder']); + expect(record.finalText).toBe(''); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/skill-overrides.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/skill-overrides.test.ts new file mode 100644 index 00000000000..a85d3a72c2b --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/skill-overrides.test.ts @@ -0,0 +1,82 @@ +import { loadRuntimeSkillSourceFromDirectory } from '@n8n/agents'; +import { TOP_LEVEL_ITEM_CEILING } from 'n8n-workflow'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + INSTANCE_AI_SKILLS_DIR, + substituteSkillPlaceholders, +} from '../../../src/skills/runtime-skills'; +import { buildSkillOverrideSource } from '../skill-overrides'; + +/** The bundled skills with every skill present, independent of the agents module flag. */ +function bundledSource() { + return loadRuntimeSkillSourceFromDirectory(INSTANCE_AI_SKILLS_DIR, { + transformInstructions: substituteSkillPlaceholders, + }); +} + +function skillFile(name: string, body: string): string { + const dir = mkdtempSync(join(tmpdir(), 'skill-override-')); + const path = join(dir, 'SKILL.md'); + writeFileSync( + path, + `---\nname: ${name}\ndescription: Variant of the skill for an eval run.\n---\n\n${body}\n`, + ); + return path; +} + +describe('buildSkillOverrideSource', () => { + it('replaces only the named skill and keeps its bundled location', async () => { + const base = bundledSource(); + const path = skillFile( + 'intent-recognition', + 'Variant body. Ceiling: {{TOP_LEVEL_ITEM_CEILING_PLACEHOLDER}}.', + ); + + const built = buildSkillOverrideSource([{ skillId: 'intent-recognition', path }], base); + if (!built) throw new Error('expected an override source'); + const { source, records } = built; + + const original = base.registry.skills.find(({ id }) => id === 'intent-recognition'); + const replaced = source.registry.skills.find(({ id }) => id === 'intent-recognition'); + expect(replaced?.description).toBe('Variant of the skill for an eval run.'); + expect(replaced?.path).toBe(original?.path); + expect(replaced?.directory).toBe(original?.directory); + expect(source.registry.skillsHash).not.toBe(base.registry.skillsHash); + expect(source.registry.skills.map(({ id }) => id)).toEqual( + base.registry.skills.map(({ id }) => id), + ); + + const loaded = await source.loadSkill('intent-recognition'); + expect(loaded?.instructions).toContain(`Ceiling: ${String(TOP_LEVEL_ITEM_CEILING)}.`); + expect(await source.loadSkill('planning')).toEqual(await base.loadSkill('planning')); + expect(records['intent-recognition']).toEqual({ path, sha256: expect.any(String) }); + }); + + it('returns nothing without overrides', () => { + expect(buildSkillOverrideSource([], bundledSource())).toBeUndefined(); + }); + + it('refuses an unknown skill, a mismatched name, and a repeated skill', () => { + const base = bundledSource(); + const path = skillFile('intent-recognition', 'Body.'); + + expect(() => buildSkillOverrideSource([{ skillId: 'no-such-skill', path }], base)).toThrow( + /"no-such-skill" is not in the runtime skill catalog/, + ); + expect(() => buildSkillOverrideSource([{ skillId: 'planning', path }], base)).toThrow( + /must declare name "planning", got "intent-recognition"/, + ); + expect(() => + buildSkillOverrideSource( + [ + { skillId: 'intent-recognition', path }, + { skillId: 'intent-recognition', path }, + ], + base, + ), + ).toThrow(/"intent-recognition" is given more than once/); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts index 08e9b9a5fe3..73b545388d1 100644 --- a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts @@ -65,4 +65,14 @@ describe('resolveStreamStatus', () => { it('never reads a cancelled run as a clean finish', () => { expect(resolveStreamStatus(streamResult({ status: 'cancelled' }), false)).toBe('timed-out'); }); + + it('reports a run ended at its committing call as stopped-on-route', () => { + expect(resolveStreamStatus(streamResult({ status: 'cancelled' }), true, true)).toBe( + 'stopped-on-route', + ); + }); + + it('prefers stopped-on-route over a budget that expired during the stop', () => { + expect(resolveStreamStatus('timed-out', true, true)).toBe('stopped-on-route'); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stub-seed.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stub-seed.test.ts new file mode 100644 index 00000000000..54c795ef388 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stub-seed.test.ts @@ -0,0 +1,178 @@ +// The stub instance with and without a seed: seeded reads return the seeded +// state, and an unseeded stub keeps its empty answers. + +import { promises as fs } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { createStubServices } from '../../harness/stub-services'; +import { createStubAgentContextReader } from '../stub-agent-context'; + +async function nodesJsonPath(): Promise { + const dir = await fs.mkdtemp(path.join(tmpdir(), 'eval-stub-seed-')); + const file = path.join(dir, 'nodes.json'); + await fs.writeFile(file, JSON.stringify([]), 'utf8'); + return file; +} + +const seed = { + workflows: [ + { + id: 'wf-lead-enrich-01', + name: 'Lead enrichment', + nodes: [ + { name: 'New lead', type: 'n8n-nodes-base.webhook' }, + { name: 'Enrich lead', type: 'n8n-nodes-base.httpRequest', position: [220, 0] }, + ], + connections: { + 'New lead': { main: [[{ node: 'Enrich lead', type: 'main', index: 0 }]] }, + }, + }, + { id: 'wf-other-0001', name: 'Weekly report', nodes: [], connections: {}, published: true }, + ], + dataTables: [ + { id: 'dt-leads-0001', name: 'Leads', columns: [{ name: 'email', type: 'string' as const }] }, + ], + priorRuns: [ + { workflow: 'wf-lead-enrich-01', hints: 'Enrich lead: 401 Unauthorized' }, + { workflow: 'wf-lead-enrich-01' }, + ], +}; + +describe('createStubServices without a seed', () => { + it('keeps the empty instance', async () => { + const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath() }); + + expect(await context.workflowService.list()).toEqual({ + workflows: [], + total: 0, + totalInScope: 0, + }); + expect((await context.workflowService.get('any-id')).nodes).toEqual([]); + expect(await context.executionService.list()).toEqual([]); + expect((await context.executionService.getStatus('seed-exec-1')).error).toBe( + 'stub: execution disabled in eval', + ); + expect(await context.dataTableService.list()).toEqual([]); + }); +}); + +describe('createStubServices with a seed', () => { + it('returns the seeded workflows from list, get, and the JSON read', async () => { + const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed }); + + const all = await context.workflowService.list(); + expect(all.workflows.map((w) => w.name)).toEqual(['Lead enrichment', 'Weekly report']); + expect(all.workflows[1].activeVersionId).not.toBeNull(); + + const filtered = await context.workflowService.list({ query: 'lead' }); + expect(filtered.workflows.map((w) => w.id)).toEqual(['wf-lead-enrich-01']); + expect(filtered).toMatchObject({ total: 1, totalInScope: 2 }); + + const detail = await context.workflowService.get('wf-lead-enrich-01'); + expect(detail.name).toBe('Lead enrichment'); + expect(detail.nodes.map((n) => n.name)).toEqual(['New lead', 'Enrich lead']); + + const json = await context.workflowService.getAsWorkflowJSON('wf-lead-enrich-01'); + expect(json.nodes[0]).toMatchObject({ id: 'seed-node-0', typeVersion: 1, position: [0, 0] }); + expect(json.nodes[1].position).toEqual([220, 0]); + expect(json.connections).toEqual(seed.workflows[0].connections); + }); + + it('keeps an edit to a seeded workflow', async () => { + const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed }); + const edited = await context.workflowService.getAsWorkflowJSON('wf-lead-enrich-01'); + + await context.workflowService.updateFromWorkflowJSON('wf-lead-enrich-01', { + ...edited, + name: 'Lead enrichment v2', + }); + + expect((await context.workflowService.get('wf-lead-enrich-01')).name).toBe( + 'Lead enrichment v2', + ); + }); + + it('reports each prior run as a failed execution with the hints as the error', async () => { + const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed }); + + const listed = await context.executionService.list({ workflowId: 'wf-lead-enrich-01' }); + expect(listed.map((e) => e.id)).toEqual(['seed-exec-2', 'seed-exec-1']); + expect(listed.every((e) => e.status === 'error' && e.workflowName === 'Lead enrichment')).toBe( + true, + ); + expect(Date.parse(listed[0].startedAt)).toBeGreaterThan(Date.parse(listed[1].startedAt)); + expect(await context.executionService.list({ status: 'success' })).toEqual([]); + expect(await context.executionService.list({ workflowId: 'wf-other-0001' })).toEqual([]); + + const status = await context.executionService.getStatus('seed-exec-1'); + expect(status).toMatchObject({ + status: 'error', + error: 'Enrich lead: 401 Unauthorized', + lastNodeExecuted: 'Enrich lead', + }); + + const debug = await context.executionService.getDebugInfo('seed-exec-1'); + expect(debug.failedNode).toEqual({ + name: 'Enrich lead', + type: 'n8n-nodes-base.httpRequest', + error: 'Enrich lead: 401 Unauthorized', + }); + expect(debug.nodeTrace.map((n) => n.status)).toEqual(['error']); + + const withoutHints = await context.executionService.getDebugInfo('seed-exec-2'); + expect(withoutHints.error).toBe('The execution failed.'); + expect(withoutHints.failedNode).toBeUndefined(); + }); + + it('returns the seeded data tables and their columns', async () => { + const { context } = await createStubServices({ nodesJsonPath: await nodesJsonPath(), seed }); + + expect((await context.dataTableService.list()).map((t) => t.name)).toEqual(['Leads']); + expect(await context.dataTableService.getSchema('dt-leads-0001')).toEqual([ + { id: 'dt-leads-0001-col-0', name: 'email', type: 'string', index: 0 }, + ]); + expect(await context.dataTableService.getSchema('dt-unknown')).toEqual([]); + }); +}); + +describe('createStubAgentContextReader', () => { + const reader = createStubAgentContextReader( + [ + { + id: 'agent-support-01', + config: { + name: 'Support bot', + model: 'anthropic/claude-sonnet-4-5', + instructions: 'Answer order questions.', + }, + }, + ], + '2026-09-30T00:00:00.000Z', + ); + + it('lists the seeded Agents and returns a config', async () => { + expect(await reader.lookup({ type: 'agents' })).toEqual({ + agents: [ + { + agentId: 'agent-support-01', + name: 'Support bot', + published: false, + updatedAt: '2026-09-30T00:00:00.000Z', + }, + ], + }); + expect(await reader.lookup({ type: 'config', agentId: 'agent-support-01' })).toMatchObject({ + configState: 'current-draft', + config: { model: 'anthropic/claude-sonnet-4-5' }, + }); + expect(await reader.lookup({ type: 'config-schema' })).toHaveProperty('configurableProperties'); + }); + + it('fails for an unknown Agent and for lookups with no seeded data', async () => { + await expect(reader.lookup({ type: 'config', agentId: 'agent-missing' })).rejects.toThrow( + 'Agent not found.', + ); + await expect(reader.lookup({ type: 'capabilities' })).rejects.toThrow('not available'); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/cli-args.ts b/packages/@n8n/instance-ai/evaluations/discovery/cli-args.ts new file mode 100644 index 00000000000..f66202e85ff --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/discovery/cli-args.ts @@ -0,0 +1,294 @@ +// --------------------------------------------------------------------------- +// Arguments of the discovery CLI (cli.ts). Kept apart from the CLI so tests +// can parse arguments without starting a run. Invalid arguments throw. +// --------------------------------------------------------------------------- + +import { resolve } from 'node:path'; + +export interface SkillOverride { + skillId: string; + /** Absolute path of the replacement SKILL.md. */ + path: string; +} + +/** Parses one `--skill-file` value: `=`. */ +export function parseSkillOverride(raw: string): SkillOverride { + const separator = raw.indexOf('='); + const skillId = separator === -1 ? '' : raw.slice(0, separator).trim(); + const path = separator === -1 ? '' : raw.slice(separator + 1).trim(); + if (!skillId || !path) { + throw new Error(`--skill-file expects =, got "${raw}"`); + } + return { skillId, path: resolve(path) }; +} + +export interface CliArgs { + help: boolean; + filter?: string; + verbose: boolean; + trials: number; + passThreshold: number; + timeoutMs: number; + /** Optional iteration cap; unset uses the SDK default. */ + maxSteps?: number; + modelId: string; + /** Trials of one case that run at once. */ + concurrency: number; + /** Cases that run at once. */ + scenarioConcurrency: number; + nodesJsonPath?: string; + failOnZeroPass: boolean; + /** Routing mode: load routing cases (or LangTracer export bodies) from this directory. */ + casesDir?: string; + stopOnRoute: boolean; + jsonOut?: string; + /** Routing mode: grade inline and write `eval-results.json` and friends here. */ + outputDir?: string; + /** Routing mode with `--output-dir`: where the judge caches its verdicts. */ + judgeCacheDir?: string; + variant: string; + validateOnly: boolean; + /** Routing mode: where the cases come from. Unset means the discovery scenarios. */ + source?: 'local' | 'langtracer'; + suite?: string; + /** Routing mode: drop cases whose id contains any comma-separated token. */ + exclude?: string; + /** Routing mode: drop cases whose id starts with any comma-separated prefix. */ + excludePrefix?: string; + /** Routing mode with `--validate-only`: compare the loaded cases with this directory. */ + compareDir?: string; + /** Routing mode: write the loaded cases here, one `.json` each, for the grader. */ + saveCasesDir?: string; + /** Replacement SKILL.md files for the orchestrator's runtime skills. */ + skillFiles: SkillOverride[]; +} + +export const DEFAULT_MODEL = + process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6'; + +export const USAGE = `Usage: pnpm eval:discovery [options] + +Discovery mode (default): runs evaluations/data/discovery/ and reports pass rates. +Routing mode: runs routing cases and records (or grades) the route of each trial. + +Common options: + --filter Cases whose id contains the text (comma-separated tokens). + A single token that is exactly an id or file name + selects only that case. + --trials, --iterations Trials per case (default 3). + --timeout, --timeout-ms Per-trial timeout (default 60000). + --model Orchestrator model (default ${DEFAULT_MODEL}). + --concurrency Trials of one case that run at once (default 3). + --scenario-concurrency Cases that run at once (default 1). + --max-steps Iteration cap for the orchestrator. + --nodes-json Node catalog for the stub services. + --skill-file = Use this SKILL.md for runtime skill (repeatable). + --json-out Write the routing results JSON (both modes). + --verbose, -v Print every trial's calls. + +Discovery mode only: + --pass-threshold <0..1> Scenario pass threshold (default 0.67). + --fail-on-zero-pass Exit non-zero only for scenarios with no passing trial. + +Routing mode (needs --cases-dir, or --source langtracer --suite): + --cases-dir Routing case files (route-*.json) or LangTracer + suite-export bodies (any *.json with a "routing" tag). + --source langtracer Pull the cases from LangTracer instead. + --suite The LangTracer suite for --source langtracer. + --stop-on-route End each trial at its first committing call. + --output-dir Grade inline and write eval-results.json (LangTracer + dispatcher contract), routing-results.json, and + routing-summary.md. + --judge-cache-dir Judge verdict cache for --output-dir. + --variant