diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index 06b6c874423..58a9b77b78e 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -55,6 +55,7 @@ import { type BuildResult, } from '../harness/runner'; import { syncDataset, type DatasetExampleInputs } from '../langsmith/dataset-sync'; +import { seedMcpRegistry } from '../mcp-registry/seeder'; import { snapshotWorkflowIds } from '../outcome/workflow-discovery'; import { writeWorkflowReport } from '../report/workflow-report'; import type { @@ -212,6 +213,14 @@ async function main(): Promise { const seedResult = await seedCredentials(client, undefined, logger); logger.info(`Seeded ${String(seedResult.credentialIds.length)} credential(s)${tag}`); + logger.info(`Seeding MCP registry...${tag}`); + const mcpSeedResult = await seedMcpRegistry(client, logger); + if (mcpSeedResult.seeded) { + logger.info(`Seeded ${String(mcpSeedResult.count)} MCP registry server(s)${tag}`); + } else { + logger.info(`Skipped MCP registry seed (test endpoint unavailable)${tag}`); + } + const preRunWorkflowIds = await snapshotWorkflowIds(client); const claimedWorkflowIds = new Set(); return { client, baseUrl, preRunWorkflowIds, claimedWorkflowIds, seedResult }; diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index eeff07627f7..cd4f14a6c38 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -435,6 +435,19 @@ export class N8nClient { return { id: result.data.id }; } + /** + * Seed the MCP registry with the test fixture (Notion + Linear mock servers) + * and trigger a synthetic node-type reload. Requires the server to be running + * with `E2E_TESTS=true` so the test controller is mounted. + * POST /rest/mcp-registry/test/seed body: none + */ + async seedMcpRegistry(): Promise<{ count: number }> { + const result = (await this.fetch('/rest/mcp-registry/test/seed', { + method: 'POST', + })) as { data: { ok: boolean; count: number } }; + return { count: result.data.count }; + } + /** * Delete a credential by ID. * DELETE /rest/credentials/:id diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-with-notion-mcp.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-with-notion-mcp.json new file mode 100644 index 00000000000..456ea970662 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-with-notion-mcp.json @@ -0,0 +1,13 @@ +{ + "prompt": "Build a workflow with a chat-triggered AI agent that can search my Notion workspace to answer questions about ongoing projects. The agent should use the Notion connection from n8n's built-in service connections (the one optimized for agent tool use), not the standard Notion action node. Configure all nodes as completely as possible and don't ask me for credentials, I'll set them up later.", + "complexity": "medium", + "tags": ["build", "agent", "ai-tool", "mcp-registry", "notion"], + "scenarios": [ + { + "name": "happy-path", + "description": "User asks about a project; agent uses the Notion MCP registry node to find the answer", + "dataSetup": "User asks 'what's the status of the Phoenix project?'. The Notion MCP server responds to a notion-search call with one matching page summarized as: title 'Phoenix', status 'in progress', owner 'Mira', last updated '3 days ago'.", + "successCriteria": "The workflow contains a chat trigger, an Agent node, and a Notion MCP registry node (the node provided by the @n8n/mcp-registry package — its node type is 'notion' and its credential type is 'notionMcpOAuth2Api', NOT the native 'n8n-nodes-base.notion' node with 'notionApi' credentials). The Notion MCP node is wired to the Agent as an ai_tool connection. When executed, the Agent's final response references 'Phoenix', 'in progress', and 'Mira'." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/index.ts b/packages/@n8n/instance-ai/evaluations/index.ts index 99b02cd00fc..51ba24312bc 100644 --- a/packages/@n8n/instance-ai/evaluations/index.ts +++ b/packages/@n8n/instance-ai/evaluations/index.ts @@ -17,6 +17,10 @@ export type { WorkflowTestCaseWithFile } from './data/workflows'; export { seedCredentials, cleanupCredentials } from './credentials/seeder'; export type { SeedResult } from './credentials/seeder'; +// -- MCP Registry -- +export { seedMcpRegistry } from './mcp-registry/seeder'; +export type { McpRegistrySeedResult } from './mcp-registry/seeder'; + // -- Runner (all-in-one) -- export { runWorkflowTestCase, runWithConcurrency } from './harness/runner'; diff --git a/packages/@n8n/instance-ai/evaluations/mcp-registry/seeder.ts b/packages/@n8n/instance-ai/evaluations/mcp-registry/seeder.ts new file mode 100644 index 00000000000..e6c13cc5e85 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/mcp-registry/seeder.ts @@ -0,0 +1,46 @@ +// --------------------------------------------------------------------------- +// MCP registry seeding for evaluation runs +// +// The MCP registry catalog drives the synthetic registry nodes (Notion MCP, +// Linear MCP, etc.) that workflow-builder evals depend on. In production the +// catalog is fetched from the n8n.io Strapi API on the leader instance — but +// the fire-and-forget boot fetch swallows errors, so an eval run can otherwise +// silently start with an empty catalog and no MCP nodes available to the +// builder agent. +// +// This seeder calls the test endpoint to deterministically upsert the Notion + +// Linear mock entries and trigger a node-type reload. Best-effort and +// idempotent — multiple calls are no-ops. +// +// Requires the n8n server to be running with E2E_TESTS=true so the test +// controller is mounted. CI containers already set this; local dev does not by +// default. +// --------------------------------------------------------------------------- + +import type { N8nClient } from '../clients/n8n-client'; +import type { EvalLogger } from '../harness/logger'; + +export interface McpRegistrySeedResult { + seeded: boolean; + count: number; +} + +/** + * Seed the MCP registry into the n8n instance for evaluation runs. + * + * Non-fatal on failure — the most common cause is `E2E_TESTS=true` not being + * set on the server, which is informative rather than a run-stopper. + */ +export async function seedMcpRegistry( + client: N8nClient, + logger?: EvalLogger, +): Promise { + try { + const { count } = await client.seedMcpRegistry(); + return { seeded: true, count }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger?.verbose(` Skipping MCP registry seed: ${message}`); + return { seeded: false, count: 0 }; + } +} diff --git a/packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts b/packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts index ad55e1c241d..30e797082fb 100644 --- a/packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts +++ b/packages/cli/src/modules/mcp-registry/__tests__/node-description-transform.test.ts @@ -213,6 +213,9 @@ describe('serverToNodeDescription', () => { expect(description).toMatchInlineSnapshot(` { + "builderHint": { + "searchHint": "Agent-optimised Notion integration. When wiring an ai_tool to an AI Agent for Notion, use THIS node, not the native action node — this variant exposes Notion's tools in the shape AI Agents expect and ships pre-configured connection details.", + }, "codex": { "alias": [ "MCP", diff --git a/packages/cli/src/modules/mcp-registry/node-description-transform.ts b/packages/cli/src/modules/mcp-registry/node-description-transform.ts index 2b3e76e13e1..cfc2e0a2645 100644 --- a/packages/cli/src/modules/mcp-registry/node-description-transform.ts +++ b/packages/cli/src/modules/mcp-registry/node-description-transform.ts @@ -203,6 +203,10 @@ export function serverToNodeDescription( remote.transport, remote.endpointUrl, ); + description.builderHint = { + ...description.builderHint, + searchHint: `Agent-optimised ${server.title} integration. When wiring an ai_tool to an AI Agent for ${server.title}, use THIS node, not the native action node — this variant exposes ${server.title}'s tools in the shape AI Agents expect and ships pre-configured connection details.`, + }; return description; }