diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 2c741e853d7..6144b599e36 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -147,7 +147,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai --iterations 3 | `--tier` | — | Filter to test cases whose `datasets` array contains this value (e.g. `--tier pr` for the PR-time set). Combines with `--filter`/`--exclude`. | | `--source` | `disk` | Where test cases come from. `disk` (default) reads `data/workflows/`; `langtracer` pulls a suite from LangTracer's REST API — see [Sourcing from LangTracer](#sourcing-test-cases-from-langtracer) | | `--suite` | — | LangTracer suite slug (or numeric id) to pull when `--source langtracer` (required in that mode) | -| `--build-via-mcp` | `false` | Build each workflow by driving the lane's MCP server with `claude -p`, then verify it on that same lane — see [Building via MCP (`--build-via-mcp`)](#building-via-mcp---build-via-mcp). Works across multiple `--base-url` lanes; mutually exclusive with `--prebuilt-workflows` | +| `--build-via-mcp` | `false` | Build each workflow by driving the lane's MCP server with `claude -p`, then verify it on that same lane — see [Building via MCP (`--build-via-mcp`)](#building-via-mcp---build-via-mcp). Works across multiple `--base-url` lanes; requires `LANGSMITH_API_KEY`; mutually exclusive with `--prebuilt-workflows` | | `--mcp-server` | `n8n-local` | MCP server name for the staged `claude` config + tool allowlist (`--build-via-mcp`) | | `ANTHROPIC_MODEL` (env) | `claude-opus-4-8` | Anthropic model for the `claude` MCP build (`--build-via-mcp`); distinct from the verifier model. Not a flag: it rides `claude`'s native env var, and the CLI pins the default when unset so builds never float with claude-code's bundled default | | `--build-cwd` | — | Working directory for the `claude` build subprocess (`--build-via-mcp`); loads that project's Claude config/skills | @@ -405,11 +405,14 @@ How it differs from the manifest flow: `claude` prints, so a build that times out or never emits the trailer can leave its workflow behind on the lane even though cleanup is on. -**Prerequisites**: the `claude` CLI installed and authenticated (the build -subprocess reads `ANTHROPIC_API_KEY`; set `ANTHROPIC_MODEL` to pick the build -model, defaulting to `claude-opus-4-8` when unset); each lane reachable and -seeded with the E2E owner. The MCP module is on by default, so no server-side -config is needed beyond a running instance. +**Prerequisites**: `LANGSMITH_API_KEY` set — MCP builds only run on the +LangSmith path, whose lane allocator caps builds at 4 per lane globally (the +keyless direct loop parallelizes iterations, which would multiply concurrent +`claude` sessions by the iteration count). Plus the `claude` CLI installed and +authenticated (the build subprocess reads `ANTHROPIC_API_KEY`; set +`ANTHROPIC_MODEL` to pick the build model, defaulting to `claude-opus-4-8` when +unset); each lane reachable and seeded with the E2E owner. The MCP module is on +by default, so no server-side config is needed beyond a running instance. Local run against a pool of container lanes (reuses `scripts/run-eval-lanes.sh`, which starts + seeds the lanes and forwards everything after `--`): diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts index 98af34b7c45..b0ff5444589 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts @@ -178,10 +178,21 @@ describe('parseCliArgs --source langtracer dataset isolation', () => { }); describe('parseCliArgs --build-via-mcp', () => { + beforeEach(() => { + // --build-via-mcp is LangSmith-only; give every test in this suite a key + // so the requirement doesn't drown out what each test actually asserts. + vi.stubEnv('LANGSMITH_API_KEY', 'test-key'); + }); + afterEach(() => { vi.unstubAllEnvs(); }); + it('requires LANGSMITH_API_KEY (the direct loop does not support MCP builds)', () => { + vi.stubEnv('LANGSMITH_API_KEY', ''); + expect(() => parseCliArgs(['--build-via-mcp'])).toThrow(/--build-via-mcp requires LangSmith/); + }); + it('defaults to disabled with sensible build knobs', () => { vi.stubEnv('ANTHROPIC_MODEL', ''); const args = parseCliArgs([]); diff --git a/packages/@n8n/instance-ai/evaluations/cli/args.ts b/packages/@n8n/instance-ai/evaluations/cli/args.ts index 90ccd5a3161..6771c8c7697 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/args.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/args.ts @@ -180,6 +180,16 @@ export function parseCliArgs(argv: string[]): CliArgs { '--delete-prebuilt-workflows applies to --prebuilt-workflows. --build-via-mcp already cleans up the workflows it builds unless --keep-workflows is set.', ); } + // MCP builds are LangSmith-only: the keyless direct loop parallelizes + // iterations without the lane allocator, so its 4-per-lane build cap applies + // PER ITERATION — concurrent `claude` sessions would scale with + // lanes × iterations × 4 and flood the shared Anthropic budget. Fail fast + // instead of teaching the direct loop (tech debt slated for removal) MCP builds. + if (validated.buildViaMcp && !process.env.LANGSMITH_API_KEY) { + throw new Error( + '--build-via-mcp requires LangSmith experiment tracking — set LANGSMITH_API_KEY. The no-LangSmith direct loop does not support MCP builds.', + ); + } // Build knobs without --build-via-mcp would parse fine and then be silently // ignored — the run would look like it honored them. Fail loudly instead. if (!validated.buildViaMcp && raw.buildOnlyFlags.length > 0) { diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index 7ca497fb017..fea5951b6a2 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -1250,16 +1250,8 @@ async function runDirectLoop(config: RunConfig): Promise<{ evaluation: MultiRunEvaluation; slugByTestCase: Map; }> { - const { - args, - lanes, - logger, - testCasesWithFiles, - prebuiltManifest, - cleanupBuiltWorkflows, - mcpBuildLogDir, - mcpBuildSpend, - } = config; + const { args, lanes, logger, testCasesWithFiles, prebuiltManifest, cleanupBuiltWorkflows } = + config; const slugByTestCase = new Map( testCasesWithFiles.map(({ testCase, fileSlug }) => [testCase, fileSlug]), @@ -1283,7 +1275,11 @@ async function runDirectLoop(config: RunConfig): Promise<{ const indexed = testCasesWithFiles.map((tc, origIdx) => ({ tc, origIdx })); const buckets = partitionRoundRobin(indexed, lanes.length); - // Iterations are independent — run them in parallel. + // Iterations are independent — run them in parallel. NOTE: this makes the + // 4-per-lane build cap apply per iteration (lanes × iterations × 4 builds at + // peak), which is why --build-via-mcp never reaches this loop: parseCliArgs + // requires LangSmith for that mode, whose allocator caps builds per lane + // globally. const allRunResults: WorkflowTestCaseResult[][] = await Promise.all( Array.from({ length: args.iterations }, async (_unused, iter) => { if (args.iterations > 1) { @@ -1302,23 +1298,6 @@ async function runDirectLoop(config: RunConfig): Promise<{ tc.fileSlug, iter, ); - // --build-via-mcp: build this case on THIS lane via `claude`, - // then verify it here. The static lane partitioning already - // keeps build + verify on the same lane (required — the - // workflow only exists on the lane that built it). - const buildOverride = args.buildViaMcp - ? async () => - await buildWorkflowViaMcpOnLane({ - lane, - conversation: tc.testCase.conversation, - slug: tc.fileSlug, - iteration: iter, - args, - logDir: mcpBuildLogDir ?? process.cwd(), - logger, - buildSpend: mcpBuildSpend, - }) - : undefined; const result = await runWorkflowTestCase({ client: lane.client, baseUrl: lane.baseUrl, @@ -1331,11 +1310,8 @@ async function runDirectLoop(config: RunConfig): Promise<{ keepWorkflows: args.keepWorkflows, laneTag, prebuiltWorkflowId, - buildOverride, pinAiRoots: args.pinAiRoots, }); - // Prebuilt only: MCP builds are registered for cleanup inside - // buildWorkflowViaMcpOnLane as soon as `claude` reports the id. if ( prebuiltWorkflowId !== undefined && cleanupBuiltWorkflows && diff --git a/packages/@n8n/instance-ai/evaluations/harness/runner.ts b/packages/@n8n/instance-ai/evaluations/harness/runner.ts index eb94a25f1a1..3465bd6719a 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/runner.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/runner.ts @@ -176,12 +176,6 @@ interface WorkflowTestCaseConfig { /** When set, skip the orchestrator build and verify this existing workflow * instead. The harness leaves it in place — caller owns its lifecycle. */ prebuiltWorkflowId?: string; - /** When set, obtain the BuildResult from this callback instead of building via - * the orchestrator or fetching `prebuiltWorkflowId`. Used by `--build-via-mcp` - * to build the workflow on this lane via `claude`, then verify it here. Treated - * like prebuilt (no transcript) for downstream judging. Never combined with - * `prebuiltWorkflowId`. */ - buildOverride?: () => Promise; /** AI root nodes (Agent, Chain) to keep pinned — opt-out from the default-on * wire-server interception path. Omit (or pass empty) to intercept every * interceptable AI root the workflow contains. Server-side gated by the @@ -207,31 +201,27 @@ export async function runWorkflowTestCase( n8nBaseUrl: config.baseUrl, }; - // A build override (--build-via-mcp) and prebuilt lookup both yield a - // workflow with no build transcript, so they share the prebuilt judging path. - const isPrebuilt = config.prebuiltWorkflowId !== undefined || config.buildOverride !== undefined; - const build = config.buildOverride - ? await config.buildOverride() - : config.prebuiltWorkflowId - ? await fetchPrebuiltBuild(client, config.prebuiltWorkflowId, logger) - : await buildWorkflow({ - client, - conversation: testCase.conversation, - messageBudget: testCase.messageBudget, - credentials: testCase.credentials, - seedFile: testCase.seedFile, - priorConversation: testCase.priorConversation, - seedThread: testCase.seedThread, - createdCredentialIds: config.createdCredentialIds, - timeoutMs, - preRunWorkflowIds: config.preRunWorkflowIds, - claimedWorkflowIds: config.claimedWorkflowIds, - logger, - laneTag: config.laneTag, - }); + const isPrebuilt = config.prebuiltWorkflowId !== undefined; + const build = config.prebuiltWorkflowId + ? await fetchPrebuiltBuild(client, config.prebuiltWorkflowId, logger) + : await buildWorkflow({ + client, + conversation: testCase.conversation, + messageBudget: testCase.messageBudget, + credentials: testCase.credentials, + seedFile: testCase.seedFile, + priorConversation: testCase.priorConversation, + seedThread: testCase.seedThread, + createdCredentialIds: config.createdCredentialIds, + timeoutMs, + preRunWorkflowIds: config.preRunWorkflowIds, + claimedWorkflowIds: config.claimedWorkflowIds, + logger, + laneTag: config.laneTag, + }); if (isPrebuilt && build.success && !build.workflowChecks) { - // No transcript in prebuilt/MCP mode, but the authored conversation still + // No transcript in prebuilt mode, but the authored conversation still // carries the user's request — feed it so prompt-aware checks (e.g. // fulfills_user_request) grade against real intent instead of "". build.workflowChecks = await runWorkflowChecks({ diff --git a/packages/cli/src/modules/mcp/evaluations/README.md b/packages/cli/src/modules/mcp/evaluations/README.md index ef53463322d..8813025b9ae 100644 --- a/packages/cli/src/modules/mcp/evaluations/README.md +++ b/packages/cli/src/modules/mcp/evaluations/README.md @@ -11,8 +11,8 @@ troubleshooting, use the [Instance AI workflow evaluation README](../../../../.. - **Fused — recommended.** `eval:instance-ai --build-via-mcp` builds each workflow through the MCP server and verifies it in the **same run**, across one - or more lanes. One command, one LangSmith experiment — and it's what CI - (`ci-mcp-evals.yml`) runs. See + or more lanes. One command, one LangSmith experiment (`LANGSMITH_API_KEY` + required) — and it's what CI (`ci-mcp-evals.yml`) runs. See [Building via MCP](../../../../../@n8n/instance-ai/evaluations/README.md#building-via-mcp---build-via-mcp). - **Two-phase — decoupled (documented below).** `eval:build-mcp-manifest` writes a manifest of workflow IDs, then `eval:instance-ai --prebuilt-workflows` scores