diff --git a/packages/@n8n/agents/src/sdk/__tests__/inline-sub-agent-tools.test.ts b/packages/@n8n/agents/src/sdk/__tests__/inline-sub-agent-tools.test.ts index e4b8a1a6f9c..8e48cbcb426 100644 --- a/packages/@n8n/agents/src/sdk/__tests__/inline-sub-agent-tools.test.ts +++ b/packages/@n8n/agents/src/sdk/__tests__/inline-sub-agent-tools.test.ts @@ -351,3 +351,45 @@ describe('inline sub-agent tool filtering', () => { expect(providerToolNames(runtimeConfigs[1])).toEqual(['anthropic.web_search_20250305']); }); }); + +describe('inline sub-agent transport inheritance', () => { + beforeEach(() => { + runtimeConfigs.length = 0; + runtimeGenerateResults.length = 0; + }); + + it('passes the parent modelFetch to inline child runtimes', async () => { + const parentFetch = vi.fn(); + const agent = new Agent('parent'); + agent.modelFetch(parentFetch as never); + const runInline = (agent as unknown as AgentWithInlineRunner).createInlineSubAgentRunner({ + deferredTools: [], + modelConfig: 'openai/gpt-4o-mini', + tools: [makeTool('lookup')], + }); + + await runInline({ + subAgentId: INLINE_SUB_AGENT_ID, + taskName: 'research', + goal: 'Find the answer', + taskPath: ['research'], + } as unknown as DelegateSubAgentRequest); + + expect(runtimeConfigs).toHaveLength(1); + expect(runtimeConfigs[0].modelFetch).toBe(parentFetch); + }); + + it('leaves modelFetch unset on child runtimes when the parent has none', async () => { + const runInline = createInlineRunner({}); + + await runInline({ + subAgentId: INLINE_SUB_AGENT_ID, + taskName: 'research', + goal: 'Find the answer', + taskPath: ['research'], + } as unknown as DelegateSubAgentRequest); + + expect(runtimeConfigs).toHaveLength(1); + expect('modelFetch' in runtimeConfigs[0]).toBe(false); + }); +}); diff --git a/packages/@n8n/agents/src/sdk/agent.ts b/packages/@n8n/agents/src/sdk/agent.ts index 882ac8789d3..7de0b00c68d 100644 --- a/packages/@n8n/agents/src/sdk/agent.ts +++ b/packages/@n8n/agents/src/sdk/agent.ts @@ -1116,6 +1116,9 @@ export class Agent implements BuiltAgent, AgentBuilder { const childRuntime = new AgentRuntime({ name: `${this.name}:${request.taskName}`, model: childModelConfig, + // Inherit the parent's transport so children can't bypass a + // guarded/instrumented fetch via the ambient default. + ...(this.modelFetchValue !== undefined ? { modelFetch: this.modelFetchValue } : {}), instructions: 'You are a focused subagent working on a specific delegated task. Complete the delegated task independently and return a concise, self-contained summary to your parent agent.', tools: tools.length > 0 ? tools : undefined, diff --git a/packages/@n8n/api-types/src/index.ts b/packages/@n8n/api-types/src/index.ts index 07168dda48b..48f6c8f7d73 100644 --- a/packages/@n8n/api-types/src/index.ts +++ b/packages/@n8n/api-types/src/index.ts @@ -337,6 +337,7 @@ export { GATEWAY_CONFIRMATION_REQUIRED_PREFIX, InstanceAiSendMessageRequest, InstanceAiEvalExecutionRequest, + InstanceAiEvalAgentExecutionRequest, InstanceAiEvalCredentialAllowlistRequest, INSTANCE_AI_MEMORY_TASK_WAIT_TIMEOUT_MS, AI_GATEWAY_MANAGED_TAG, @@ -466,6 +467,11 @@ export type { InstanceAiEvalMockedCredential, InstanceAiEvalRewrittenCredential, InstanceAiEvalExecutionResult, + InstanceAiEvalAgentToolCallRecord, + InstanceAiEvalAgentModelTurnRecord, + InstanceAiEvalAgentScenarioSeed, + InstanceAiEvalAgentSkippedFeature, + InstanceAiEvalAgentExecutionResult, InstanceAiEvalSeedWorkflow, InstanceAiEvalSeedDataTable, } from './schemas/instance-ai.schema'; diff --git a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts index 5ef3ba5a9e2..1b38a06ad1f 100644 --- a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts +++ b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts @@ -1609,6 +1609,80 @@ export class InstanceAiEvalExecutionRequest extends Z.class({ pinNodes: z.array(z.string().min(1)).max(50).optional(), }) {} +// --------------------------------------------------------------------------- +// Eval agent execution — run a built first-class Agent for one scenario turn. +// Tool-side HTTP is mocked at the wire (same layer as workflow eval); the +// agent's own model call runs for real and is recorded, not mocked. +// --------------------------------------------------------------------------- + +export interface InstanceAiEvalAgentToolCallRecord { + /** Sanitized tool name — matches what the model called. */ + tool: string; + /** Where the tool executes. 'other' covers built-ins (skills, todos, environment). */ + kind: 'node' | 'workflow' | 'custom' | 'mcp' | 'other'; + input?: unknown; + output?: unknown; + /** Tool-level failure. Unlike workflow node errors, this does NOT flip run `success` — agents may recover. */ + error?: string; + /** True when at least one outbound HTTP request behind this call was served by the mock layer. */ + mocked: boolean; + interceptedRequests: InstanceAiEvalInterceptedRequest[]; + /** True when the call required approval and eval auto-approved it. */ + autoApproved?: boolean; +} + +/** One recorded (passthrough) call to the agent's real model provider. Bodies are redacted and truncated. */ +export interface InstanceAiEvalAgentModelTurnRecord { + url: string; + provider?: string; + status?: number; + durationMs?: number; + streamed: boolean; + requestBody?: unknown; + responseBody?: unknown; + error?: string; +} + +/** Phase-1 output for agent scenarios: the opening user message plays the role trigger pin data plays for workflows. */ +export interface InstanceAiEvalAgentScenarioSeed { + openingMessage: string; + globalContext: string; + /** Per-tool data hints, keyed by sanitized tool name. */ + toolHints: Record; + warnings: string[]; +} + +/** A config feature the eval runtime pruned before the run (not yet mockable). */ +export interface InstanceAiEvalAgentSkippedFeature { + feature: string; + reason: string; +} + +export interface InstanceAiEvalAgentExecutionResult { + runId: string; + /** The run completed without framework/model errors. Tool-level errors live on toolCalls[].error. */ + success: boolean; + errors: string[]; + /** The agent's final assistant text for the turn. */ + finalText: string; + model?: string; + finishReason?: string; + toolCalls: InstanceAiEvalAgentToolCallRecord[]; + modelTurns: InstanceAiEvalAgentModelTurnRecord[]; + usage?: { inputTokens?: number; outputTokens?: number }; + seed: InstanceAiEvalAgentScenarioSeed; + skippedFeatures: InstanceAiEvalAgentSkippedFeature[]; + mockedCredentials: InstanceAiEvalMockedCredential[]; +} + +export class InstanceAiEvalAgentExecutionRequest extends Z.class({ + /** Project the agent lives in (agent routes are project-scoped). */ + projectId: z.string().min(1), + scenarioHints: z.string().max(2000).optional(), + /** Overall run budget. Server default applies when omitted. */ + timeoutMs: z.number().int().min(30_000).max(900_000).optional(), +}) {} + export class InstanceAiEvalCredentialAllowlistRequest extends Z.class({ threadId: z.string().uuid(), /** diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/agent-report-rendering.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/agent-report-rendering.test.ts new file mode 100644 index 00000000000..e8ca518b3b8 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/agent-report-rendering.test.ts @@ -0,0 +1,137 @@ +import type { InstanceAiEvalAgentExecutionResult } from '@n8n/api-types'; +import { describe, expect, it } from 'vitest'; + +import { parseTargetOutput } from '../cli/reshape'; +import { generateWorkflowReport } from '../report/workflow-report'; +import type { WorkflowTestCase, WorkflowTestCaseResult } from '../types'; + +const agentRun: InstanceAiEvalAgentExecutionResult = { + runId: 'run-1', + success: true, + errors: [], + finalText: 'Commented on existing issue #482.', + model: 'openai/gpt-4o-mini', + finishReason: 'stop', + toolCalls: [ + { + tool: 'github_comment_on_issue', + kind: 'node', + input: { issueNumber: 482, body: 'Duplicate report' }, + output: { ok: true }, + mocked: true, + interceptedRequests: [ + { + url: 'https://api.github.com/repos/acme/widgets/issues/482/comments', + method: 'POST', + nodeType: 'n8n-nodes-base.githubTool', + requestBody: { body: 'Duplicate report' }, + mockResponse: { id: 9001 }, + }, + ], + }, + ], + modelTurns: [ + { + url: 'https://api.openai.com/v1/responses', + provider: 'openai', + status: 200, + durationMs: 3400, + streamed: true, + responseBody: 'data: {"type":"response.completed"}', + }, + ], + usage: { inputTokens: 1200, outputTokens: 90 }, + seed: { + openingMessage: 'A user reports CSV export truncates rows over 10k.', + globalContext: 'Repo acme/widgets, open issue 482', + toolHints: { github_comment_on_issue: 'Comment posts succeed' }, + warnings: [], + }, + skippedFeatures: [{ feature: 'memory', reason: 'not mockable yet.' }], + mockedCredentials: [], +}; + +const testCase = { + description: 'Agent triage case', + conversation: [{ role: 'user', text: 'Create an agent...' }], + complexity: 'medium', + tags: ['agent'], + executionScenarios: [ + { + name: 'comments-on-duplicate', + description: 'Duplicate exists', + dataSetup: 'Search returns issue 482.', + successCriteria: 'Commented on 482, no new issue.', + }, + ], +} as unknown as WorkflowTestCase; + +describe('agent run rendering in the HTML report', () => { + it('renders the built agent and the agent scenario trace', () => { + const result: WorkflowTestCaseResult = { + testCase, + fileSlug: 'agent-github-triage', + workflowBuildSuccess: true, + agentId: 'agent-1', + agentArtifactContext: '## Agent config\n{"name":"Bug Triage"}', + executionScenarioResults: [ + { + scenario: testCase.executionScenarios![0], + success: true, + agentEvalResult: agentRun, + agentId: 'agent-1', + score: 1, + reasoning: 'The agent commented on issue 482 and created nothing.', + }, + ], + }; + + const html = generateWorkflowReport([result]); + + expect(html).toContain('Built agent (agent-1)'); + expect(html).toContain('Bug Triage'); + expect(html).toContain('Scenario seed'); + expect(html).toContain('A user reports CSV export truncates rows over 10k.'); + expect(html).toContain('github_comment_on_issue'); + expect(html).toContain('https://api.github.com/repos/acme/widgets/issues/482/comments'); + expect(html).toContain('Mock returned'); + expect(html).toContain('Model turns (real, recorded)'); + expect(html).toContain('Commented on existing issue #482.'); + expect(html).toContain('Agent features disabled for this run'); + }); +}); + +describe('agent fields in LangSmith run outputs', () => { + it('round-trips agentEvalResult, agentId, and agentContext', () => { + const output = parseTargetOutput({ + buildSuccess: true, + passed: true, + score: 1, + reasoning: 'ok', + agentId: 'agent-1', + agentContext: '## Agent config', + agentEvalResult: agentRun, + execDurationMs: 100, + nodeCount: 0, + }); + + expect(output?.agentId).toBe('agent-1'); + expect(output?.agentContext).toBe('## Agent config'); + expect(output?.agentEvalResult?.toolCalls[0].tool).toBe('github_comment_on_issue'); + }); + + it('drops a malformed agentEvalResult instead of crashing the row', () => { + const output = parseTargetOutput({ + buildSuccess: true, + passed: true, + score: 1, + reasoning: 'ok', + agentEvalResult: { nope: true }, + execDurationMs: 100, + nodeCount: 0, + }); + + expect(output).toBeDefined(); + expect(output?.agentEvalResult).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/agent-scenario-artifact.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/agent-scenario-artifact.test.ts new file mode 100644 index 00000000000..b9a9d7cf77b --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/agent-scenario-artifact.test.ts @@ -0,0 +1,104 @@ +import type { InstanceAiEvalAgentExecutionResult } from '@n8n/api-types'; +import { describe, expect, it } from 'vitest'; + +import { buildAgentVerificationArtifact } from '../harness/runner'; + +// Structurally an ExecutionScenario; the schema-inferred type is error-typed +// under type-aware lint (deep ZodEffects instantiation — see harness/schema.ts). +const scenario = { + name: 'announces-release', + description: 'Happy path announcement', + dataSetup: 'Slack chat.postMessage succeeds.', + successCriteria: 'One Slack message announcing v2.31 was posted.', +}; + +const agentRun: InstanceAiEvalAgentExecutionResult = { + runId: 'run-1', + success: true, + errors: [], + finalText: 'Posted the announcement to #alerts.', + model: 'openai/gpt-4o-mini', + finishReason: 'stop', + toolCalls: [ + { + tool: 'Send_Slack_Message', + kind: 'node', + input: { channel: 'alerts', message: 'v2.31 shipped!' }, + output: { ok: true }, + mocked: true, + interceptedRequests: [ + { + url: 'https://slack.com/api/chat.postMessage', + method: 'POST', + nodeType: 'n8n-nodes-base.slackTool', + requestBody: { channel: 'alerts', text: 'v2.31 shipped!' }, + mockResponse: { ok: true, ts: '1700000000.0001' }, + }, + ], + }, + ], + modelTurns: [ + { url: 'https://api.openai.com/v1/responses', provider: 'openai', status: 200, streamed: true }, + ], + usage: { inputTokens: 1200, outputTokens: 80 }, + seed: { + openingMessage: 'Hey, v2.31 shipped — announce it in #alerts please.', + globalContext: 'Platform team, release v2.31', + toolHints: { Send_Slack_Message: 'Posting succeeds' }, + warnings: [], + }, + skippedFeatures: [{ feature: 'memory', reason: 'not mockable yet.' }], + mockedCredentials: [], +}; + +describe('buildAgentVerificationArtifact', () => { + it('renders the agent context as the stable block and the run as the scenario block', () => { + const artifact = buildAgentVerificationArtifact(scenario, '## Agent config\n{...}', agentRun); + + expect(artifact.workflowContext).toContain('Agent under test'); + expect(artifact.workflowContext).toContain('## Agent config'); + + const ctx = artifact.scenarioContext; + expect(ctx).toContain('announces-release'); + expect(ctx).toContain('Slack chat.postMessage succeeds.'); + expect(ctx).toContain('Hey, v2.31 shipped'); + expect(ctx).toContain('Send_Slack_Message (node)'); + expect(ctx).toContain('https://slack.com/api/chat.postMessage'); + expect(ctx).toContain('1700000000.0001'); + expect(ctx).toContain('Posted the announcement to #alerts.'); + expect(ctx).toContain('HARNESS LIMITATION: agent feature "memory"'); + }); + + it('flags errored tool calls and mock-generation failures', () => { + const failedRun: InstanceAiEvalAgentExecutionResult = { + ...agentRun, + success: false, + finishReason: 'error', + errors: ['Model run error: rate limited'], + finalText: '', + toolCalls: [ + { + tool: 'Send_Slack_Message', + kind: 'node', + error: 'Tool call failed (not reported in the run result — see interceptedRequests)', + mocked: true, + interceptedRequests: [ + { + url: 'https://slack.com/api/chat.postMessage', + method: 'POST', + nodeType: 'n8n-nodes-base.slackTool', + mockResponse: { _evalMockError: true, message: 'mock generation timed out' }, + }, + ], + }, + ], + }; + + const ctx = buildAgentVerificationArtifact(scenario, '(config)', failedRun).scenarioContext; + expect(ctx).toContain('FAILED'); + expect(ctx).toContain('Model run error: rate limited'); + expect(ctx).toContain('ERRORED'); + expect(ctx).toContain('MOCK ISSUE: tool "Send_Slack_Message"'); + expect(ctx).toContain('(no final text)'); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts index 430973bf64b..8fb5f7439a0 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts @@ -66,4 +66,34 @@ describe('cleanupBuild', () => { expect(mocks.deleteDataTable).toHaveBeenCalledWith('project-1', 'DT1'); expect(mocks.deleteThread).toHaveBeenCalledWith('T1'); }); + + it('deletes the built agent of an agent-anchored build', async () => { + const { client, mocks } = makeClient({ deleteAgent: vi.fn().mockResolvedValue(undefined) }); + const build = { ...makeBuild(), artifactRefs: [{ type: 'agent' as const, id: 'agent-1' }] }; + + await expect(cleanupBuild(client, build, silentLogger)).resolves.toBe(true); + + expect(mocks.deleteAgent).toHaveBeenCalledWith('project-1', 'agent-1'); + }); + + it('reports not clean when the agent deletion fails, so the caller can retry', async () => { + const { client, mocks } = makeClient({ + deleteAgent: vi.fn().mockRejectedValue(new Error('HTTP 502')), + }); + const build = { ...makeBuild(), artifactRefs: [{ type: 'agent' as const, id: 'agent-1' }] }; + + await expect(cleanupBuild(client, build, silentLogger)).resolves.toBe(false); + + // The remaining artifacts are still cleaned up. + expect(mocks.deleteDataTable).toHaveBeenCalledWith('project-1', 'DT1'); + expect(mocks.deleteThread).toHaveBeenCalledWith('T1'); + }); + + it('never calls deleteAgent for a build without an agent ref', async () => { + const { client, mocks } = makeClient({ deleteAgent: vi.fn() }); + + await expect(cleanupBuild(client, makeBuild(), silentLogger)).resolves.toBe(true); + + expect(mocks.deleteAgent).not.toHaveBeenCalled(); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index 236f3b7c296..ab0cf5b99bc 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -70,14 +70,18 @@ import { import { abortedWorkflowTestCaseResult, buildWorkflow, + executeAgentScenario, executeScenario, cleanupBuild, + fetchAgentScenarioContext, + findAgentArtifactRef, runWorkflowChecks, effectiveTimeoutMs, runWorkflowTestCase, runWithConcurrency, workflowExpectedForCase, type BuildResult, + warnAgentSeedDataTablesIgnored, } from '../harness/runner'; import { classifyScenarioExecutionError, @@ -632,6 +636,14 @@ async function runWithLangSmith(config: RunConfig): Promise<{ buildTrace?: BuildResult['buildTrace']; timeoutMs: number; }) => Promise>>; + tracedExecuteAgent: (execArgs: { + agentId: string; + scenario: ExecutionScenario; + agentContext: string; + buildTrace?: BuildResult['buildTrace']; + timeoutMs: number; + testCaseName?: string; + }) => Promise>>; } const laneStates: LaneState[] = lanes.map((lane, idx) => { @@ -694,6 +706,32 @@ async function runWithLangSmith(config: RunConfig): Promise<{ metadata: { lane: laneNum }, }, ), + tracedExecuteAgent: traceable( + async (execArgs: { + agentId: string; + scenario: ExecutionScenario; + agentContext: string; + buildTrace?: BuildResult['buildTrace']; + timeoutMs: number; + testCaseName?: string; + }) => + await executeAgentScenario( + lane.client, + execArgs.agentId, + execArgs.scenario, + execArgs.agentContext, + logger, + execArgs.timeoutMs, + execArgs.testCaseName, + execArgs.buildTrace, + ), + { + name: 'agent_scenario_execution', + run_type: 'chain', + client: lsClient, + metadata: { lane: laneNum }, + }, + ), }; }); @@ -880,6 +918,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{ buildDurations.set(key, buildDurationMs); stashTranscript(build); stashBuildExpectations(key, fileSlug, build, false); + stashAgentContext(key, lane.runner.client, build); stashRunDebug(lane.runner.client, build); logger.info( `[lane ${String(lane.laneNum)}] built ${fileSlug} (iteration ${String(iteration)}) thread=${build.threadId ?? 'none'} success=${String(build.success)}`, @@ -910,6 +949,16 @@ async function runWithLangSmith(config: RunConfig): Promise<{ } } + // Agent config + skills, fetched once per build and shared by every + // scenario row of the case (the agent analog of the cached workflow JSON). + const agentContextByKey = new Map>(); + + function stashAgentContext(key: string, client: N8nClient, build: BuildResult): void { + const agentRef = findAgentArtifactRef(build.artifactRefs); + if (!agentRef) return; + agentContextByKey.set(key, fetchAgentScenarioContext(client, agentRef, logger)); + } + function stashRunDebug(client: N8nClient, build: BuildResult): void { if (!build.threadId) return; runDebugByThreadId.set(build.threadId, captureThreadRunDebug(client, build.threadId, logger)); @@ -977,6 +1026,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{ const { build, lane } = await cached; // Run-debug capture reads the thread — let it settle before deletion. if (build.threadId) await runDebugByThreadId.get(build.threadId)?.catch(() => {}); + // cleanupBuild also deletes any built agent (agent-anchored builds). const clean = await cleanupBuild(lane.runner.client, build, logger); if (!clean) { // Leave the entry in buildCache — the end-of-run pass retries it. @@ -1035,13 +1085,17 @@ async function runWithLangSmith(config: RunConfig): Promise<{ lane: builtOnLane, buildDurationMs, } = await getOrBuild(iteration, inputs.testCaseFile); + const cacheKey = `${String(iteration)}:${inputs.testCaseFile}`; + + // Agent-anchored build: scenarios target the agent and a missing workflow is + // not a build failure (helper workflows are its tools) — mirrors the direct loop. + const agentRef = findAgentArtifactRef(build.artifactRefs); + const agentRunnable = agentRef !== undefined && build.transcript !== undefined; // Stashed at build time with a `.catch` attached, so awaiting never rejects. // Awaited only after each branch's own work is done, keeping the judge off // the scenario critical path while persisting verdicts to run outputs. - const verdictsPromise = buildExpectationsByKey.get( - `${String(iteration)}:${inputs.testCaseFile}`, - ); + const verdictsPromise = buildExpectationsByKey.get(cacheKey); const attachExpectations = async (output: TargetOutput): Promise => { const verdicts = await verdictsPromise; return verdicts && verdicts.length > 0 ? { ...output, expectationResults: verdicts } : output; @@ -1052,12 +1106,14 @@ async function runWithLangSmith(config: RunConfig): Promise<{ // so LangSmith pass metrics stay truthful for scenario-less cases. Checked before // the workflowId guard because answer-only cases legitimately end without a // workflow (workflowExpectedForCase). - if (inputs.scenarioName === BUILD_ONLY_SCENARIO_NAME && build.success) { + if (inputs.scenarioName === BUILD_ONLY_SCENARIO_NAME && (build.success || agentRunnable)) { const verdicts = await verdictsPromise; const outcome = sentinelOutcomeFromVerdicts(verdicts); return { buildSuccess: true, workflowId: build.workflowId, + ...(agentRef ? { agentId: agentRef.id } : {}), + ...(agentRef ? { agentContext: await agentContextByKey.get(cacheKey) } : {}), passed: outcome.passed, score: outcome.score, reasoning: outcome.reasoning, @@ -1075,7 +1131,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{ }; } - if (!build.success || !build.workflowId) { + if (!agentRunnable && (!build.success || !build.workflowId)) { return await attachExpectations({ buildSuccess: false, passed: false, @@ -1096,6 +1152,92 @@ async function runWithLangSmith(config: RunConfig): Promise<{ }); } + // Agent scenario path — real model, mocked tool HTTP. Mirrors the + // workflow branch below (retry loop, framework_issue guard, output shape). + if (agentRunnable && agentRef) { + // Dataset rows don't carry seedDataTables — check the authored scenario. + warnAgentSeedDataTablesIgnored( + logger, + scenario.name, + testCaseByFileSlug + .get(inputs.testCaseFile) + ?.executionScenarios?.find((s) => s.name === scenario.name)?.seedDataTables, + ); + const agentExecStart = Date.now(); + const agentContext = + (await agentContextByKey.get(cacheKey)) ?? '(agent configuration could not be fetched)'; + let agentResult; + for (let attempt = 1; ; attempt++) { + try { + agentResult = await builtOnLane.tracedExecuteAgent({ + agentId: agentRef.id, + scenario, + agentContext, + buildTrace: build.buildTrace, + timeoutMs: effectiveTimeoutMs( + testCaseByFileSlug.get(inputs.testCaseFile)?.complexity, + args.timeoutMs, + ), + testCaseName: inputs.testCaseFile, + }); + break; + } catch (error: unknown) { + const errorMessage = extractErrorMessage(error); + if (shouldRetryScenarioExecution(errorMessage, attempt)) { + logger.warn( + ` [${scenario.name}] agent execution attempt ${attempt}/${MAX_EXEC_ATTEMPTS} failed (${errorMessage}); retrying`, + ); + await new Promise((resolve) => setTimeout(resolve, 500 * attempt)); + continue; + } + logger.error(` ERROR [${scenario.name}]: ${errorMessage}`); + return await attachExpectations({ + buildSuccess: true, + agentId: agentRef.id, + agentContext, + passed: false, + score: 0, + reasoning: `Agent scenario execution error: ${errorMessage}`, + failureCategory: 'framework_issue', + execErrors: [errorMessage], + buildDurationMs, + execDurationMs: Date.now() - agentExecStart, + nodeCount: 0, + threadId: build.threadId, + buildTrace: build.buildTrace, + planRejections: build.proxyDecisionStats?.rejection ?? 0, + }); + } + } + + const agentFailureCategory = agentResult.success ? undefined : agentResult.failureCategory; + const agentRootCause = agentResult.success ? undefined : agentResult.rootCause; + return await attachExpectations({ + buildSuccess: true, + agentId: agentRef.id, + agentContext, + agentEvalResult: agentResult.agentEvalResult, + passed: agentResult.success, + score: agentResult.score, + reasoning: agentResult.reasoning, + failureCategory: agentFailureCategory, + rootCause: agentRootCause, + ...(agentResult.incomplete ? { incomplete: true } : {}), + execErrors: agentResult.agentEvalResult?.errors ?? [], + buildDurationMs, + execDurationMs: Date.now() - agentExecStart, + nodeCount: 0, + threadId: build.threadId, + buildTrace: build.buildTrace, + planRejections: build.proxyDecisionStats?.rejection ?? 0, + }); + } + if (!build.workflowId) { + // agentRunnable without an agentRef is unreachable; this narrows for TS + // and guards the workflow path below. + throw new Error(`No runnable artifact for scenario ${scenario.name}`); + } + const execStart = Date.now(); const nodeCount = build.workflowJsons[0]?.nodes.length ?? 0; let result; @@ -1954,14 +2096,18 @@ export function writeEvalResults( passHatK: terminalRate(sa.passHatK), runs: sa.runs.map((sr, runIndex) => ({ workflowId: sr.workflowId ?? tc.runs[runIndex]?.workflowId ?? null, + ...((sr.agentId ?? tc.runs[runIndex]?.agentId) + ? { agentId: sr.agentId ?? tc.runs[runIndex]?.agentId } + : {}), passed: sr.success, ...(sr.incomplete ? { incomplete: true } : {}), score: sr.score, reasoning: sr.reasoning, failureCategory: sr.failureCategory, rootCause: sr.rootCause, - execErrors: sr.evalResult?.errors ?? [], + execErrors: sr.evalResult?.errors ?? sr.agentEvalResult?.errors ?? [], evalResult: sr.evalResult, + ...(sr.agentEvalResult ? { agentEvalResult: sr.agentEvalResult } : {}), })), })), })), diff --git a/packages/@n8n/instance-ai/evaluations/cli/reshape.ts b/packages/@n8n/instance-ai/evaluations/cli/reshape.ts index 91170028765..6960b825edb 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/reshape.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/reshape.ts @@ -7,7 +7,11 @@ // unit-testable on its own (index.ts runs main() at import time). // --------------------------------------------------------------------------- -import type { InstanceAiEvalExecutionResult, InstanceAiRunDebugResponse } from '@n8n/api-types'; +import type { + InstanceAiEvalAgentExecutionResult, + InstanceAiEvalExecutionResult, + InstanceAiRunDebugResponse, +} from '@n8n/api-types'; import type { Run } from 'langsmith/schemas'; import { z } from 'zod'; @@ -50,6 +54,8 @@ const targetOutputSchema = z.object({ reasoning: z.string().default(''), workflowId: z.string().optional(), scenarioWorkflowId: z.string().optional(), + /** Set when the scenario ran against a built first-class Agent instead of a workflow. */ + agentId: z.string().optional(), failureCategory: z.string().optional(), rootCause: z.string().optional(), /** Verifier returned no verdict — run is excluded from scoring but stays visible. */ @@ -58,6 +64,9 @@ const targetOutputSchema = z.object({ expectationResults: expectationResultsSchema.optional().catch(undefined), execErrors: z.array(z.string()).default([]), evalResult: z.unknown().optional(), + agentEvalResult: z.unknown().optional(), + /** Rendered agent config + skills — attached to every agent row, deduped on reshape (first write wins). */ + agentContext: z.string().optional(), /** Only set on the scenario that initiated the build. */ buildDurationMs: z.number().optional(), execDurationMs: z.number().default(0), @@ -73,9 +82,10 @@ const targetOutputSchema = z.object({ export type TargetOutput = Omit< z.infer, - 'evalResult' | 'workflowJson' | 'buildTrace' + 'evalResult' | 'agentEvalResult' | 'workflowJson' | 'buildTrace' > & { evalResult?: InstanceAiEvalExecutionResult; + agentEvalResult?: InstanceAiEvalAgentExecutionResult; workflowJson?: WorkflowResponse; buildTrace?: BuildTrace; }; @@ -95,6 +105,16 @@ function isEvalResult(v: unknown): v is InstanceAiEvalExecutionResult { ); } +function isAgentEvalResult(v: unknown): v is InstanceAiEvalAgentExecutionResult { + if (!isPlainObject(v)) return false; + return ( + typeof v.runId === 'string' && + Array.isArray(v.toolCalls) && + Array.isArray(v.modelTurns) && + isPlainObject(v.seed) + ); +} + function isWorkflowResponse(v: unknown): v is WorkflowResponse { if (!isPlainObject(v)) return false; return ( @@ -129,6 +149,9 @@ export function parseTargetOutput(raw: unknown): TargetOutput | undefined { return { ...parsed.data, evalResult: isEvalResult(parsed.data.evalResult) ? parsed.data.evalResult : undefined, + agentEvalResult: isAgentEvalResult(parsed.data.agentEvalResult) + ? parsed.data.agentEvalResult + : undefined, workflowJson: isWorkflowResponse(parsed.data.workflowJson) ? parsed.data.workflowJson : undefined, @@ -216,6 +239,8 @@ export function reshapeLangSmithRuns( const executionScenarioResults: ExecutionScenarioResult[] = []; let workflowBuildSuccess = false; let workflowId: string | undefined; + let agentId: string | undefined; + let agentArtifactContext: string | undefined; let buildError: string | undefined; let threadId: string | undefined; let workflowChecks: CheckOutcome[] | undefined; @@ -236,6 +261,9 @@ export function reshapeLangSmithRuns( } if (output.buildSuccess) workflowBuildSuccess = true; if (output.workflowId) workflowId = output.workflowId; + if (output.agentId && !agentId) agentId = output.agentId; + if (output.agentContext && !agentArtifactContext) + agentArtifactContext = output.agentContext; if (output.threadId) threadId = output.threadId; if (!output.buildSuccess && output.reasoning) buildError = output.reasoning; if (output.workflowChecks && !workflowChecks) workflowChecks = output.workflowChecks; @@ -245,7 +273,9 @@ export function reshapeLangSmithRuns( scenario, success: output.passed, evalResult: output.evalResult, + ...(output.agentEvalResult ? { agentEvalResult: output.agentEvalResult } : {}), workflowId: output.scenarioWorkflowId ?? output.workflowId, + ...(output.agentId ? { agentId: output.agentId } : {}), score: output.score, reasoning: output.reasoning, failureCategory: output.failureCategory, @@ -261,6 +291,8 @@ export function reshapeLangSmithRuns( if (output) { workflowBuildSuccess = output.buildSuccess; workflowId = output.workflowId; + agentId = output.agentId; + agentArtifactContext = output.agentContext; threadId = output.threadId; if (!output.buildSuccess && output.reasoning) buildError = output.reasoning; workflowChecks = output.workflowChecks; @@ -276,6 +308,8 @@ export function reshapeLangSmithRuns( fileSlug, workflowBuildSuccess, workflowId, + agentId, + agentArtifactContext, executionScenarioResults, buildError, threadId, diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 6252456b348..2efdb0642e1 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -9,6 +9,7 @@ import type { InstanceAiConfirmRequest, InstanceAiRichMessagesResponse, + InstanceAiEvalAgentExecutionResult, InstanceAiEvalExecutionResult, InstanceAiRunDebugResponse, InstanceAiThreadDebugRunsResponse, @@ -787,6 +788,43 @@ export class N8nClient { return result.data; } + /** + * Run one scenario turn against a built first-class Agent: the agent's own + * model call is real, its tools' HTTP is served by the mock layer. Runs for + * minutes, like the workflow variant. + */ + async executeAgentWithLlmMock( + agentId: string, + projectId: string, + scenarioHints?: string, + timeoutMs: number = 120_000, + ): Promise { + const body: { projectId: string; scenarioHints?: string; timeoutMs?: number } = { projectId }; + if (scenarioHints) body.scenarioHints = scenarioHints; + // Forward the budget server-side so the run is aborted rather than + // orphaned when the client gives up. The server floor is the schema min + // (30s), so the client abort is floored to 5s above it — a smaller + // caller value would leave the server running long after the client quit. + const serverBudgetMs = Math.min(Math.max(timeoutMs - 5_000, 30_000), 900_000); + body.timeoutMs = serverBudgetMs; + + const result = (await this.fetch( + `/rest/instance-ai/eval/execute-agent-with-llm-mock/${agentId}`, + { + method: 'POST', + body, + timeoutMs: serverBudgetMs + 5_000, + }, + )) as { data: InstanceAiEvalAgentExecutionResult }; + return result.data; + } + + async deleteAgent(projectId: string, agentId: string): Promise { + await this.fetch(`/rest/projects/${projectId}/agents/v2/${agentId}`, { + method: 'DELETE', + }); + } + // -- SSE helpers --------------------------------------------------------- /** diff --git a/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts b/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts index 04722f39c4a..d72730b7151 100644 --- a/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts +++ b/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts @@ -44,6 +44,19 @@ const CREDENTIAL_TEMPLATES: Record = { envVar: 'EVAL_GMAIL_ACCESS_TOKEN', buildData: (token) => ({ oauthTokenData: { access_token: token } }), }, + // MCP-registry-synthesized credential types (agent MCP servers). Creating + // them requires the backend to run with the `mcp-registry` module enabled; + // placeholder tokens are fine — agent eval runs mock the MCP wire. + linearMcpOAuth2Api: { + defaultName: '[eval] Linear MCP', + envVar: 'EVAL_LINEAR_MCP_ACCESS_TOKEN', + buildData: (token) => ({ oauthTokenData: { access_token: token } }), + }, + notionMcpOAuth2Api: { + defaultName: '[eval] Notion MCP', + envVar: 'EVAL_NOTION_MCP_ACCESS_TOKEN', + buildData: (token) => ({ oauthTokenData: { access_token: token } }), + }, microsoftTeamsOAuth2Api: { defaultName: '[eval] Teams', envVar: 'EVAL_TEAMS_ACCESS_TOKEN', diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-deploy-notifier.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-deploy-notifier.json new file mode 100644 index 00000000000..e383f2c9b10 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-deploy-notifier.json @@ -0,0 +1,32 @@ +{ + "description": "Agent-artifact case: the assistant must build a first-class n8n Agent (not a workflow) that announces releases in Slack on request. Execution scenarios run the built agent — its own model runs for real, its Slack tool's HTTP is served by the mock layer — covering the happy path and honest failure reporting.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Deploy Notifier' for our platform team. When someone tells it a release shipped, it should post a short announcement to the Slack channel they name (defaulting to #alerts if they don't name one) and then confirm exactly what it posted and where. If the Slack post fails for any reason, it must tell the user what went wrong instead of pretending it worked. Use OpenAI gpt-4o-mini as its model with my OpenAI credential, and give it my Slack credential for posting. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "slack"], + "credentials": [{ "type": "openAiApi" }, { "type": "slackApi" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration in the context). Workflows may exist as the agent's attached tools — that is acceptable — but the agent artifact is the deliverable.", + "The agent's instructions cover announcing releases to a Slack channel (defaulting to #alerts), confirming what was posted, and reporting Slack failures honestly.", + "The agent has a Slack tool it can use to post messages.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "announces-release", + "description": "Happy path: the user announces a release; the agent posts to #alerts and confirms honestly.", + "dataSetup": "The user tells the agent that version 2.31 has shipped and asks it to announce it in the #alerts channel. Slack's chat.postMessage succeeds and returns { \"ok\": true } with a message timestamp.", + "successCriteria": "The agent made a Slack tool call posting one announcement that mentions version 2.31 (see the intercepted chat.postMessage request), and its final reply confirms the announcement was posted to #alerts without inventing a failure." + }, + { + "name": "reports-missing-channel", + "description": "Failure honesty: the named channel does not exist; the agent must report the failure instead of claiming success.", + "dataSetup": "The user asks the agent to announce version 2.31 in the #deploys channel. The #deploys channel does not exist: every Slack chat.postMessage request targeting deploys returns { \"ok\": false, \"error\": \"channel_not_found\" }.", + "successCriteria": "The agent attempted the Slack post (see the intercepted request and its channel_not_found mock response), and its final reply honestly tells the user the post failed because the channel was not found, rather than claiming the announcement succeeded." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-github-triage.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-github-triage.json new file mode 100644 index 00000000000..7d5dc0f43b7 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-github-triage.json @@ -0,0 +1,32 @@ +{ + "description": "Agent-artifact case: GitHub bug-triage agent stressing mock-layer reliability on a read-then-write chain — the duplicate search result must steer a conditional comment-vs-create decision, so cross-request mock consistency is the capability under test. One scenario also asserts the ABSENCE of a request (no issue created when a duplicate exists).", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Bug Triage' for our engineering team. When someone reports a bug, it should first search our acme/widgets GitHub repository for existing open issues that look like duplicates of the report. If it finds a likely duplicate, it must add a comment on that existing issue with the new report's details instead of creating anything new. If there is no duplicate, it should create a new issue labelled 'bug' with a clear title and the report as the body. Either way it must tell the user exactly what it did, including the number of the issue it commented on or created. Use OpenAI gpt-4o-mini as its model with my OpenAI credential, and use my GitHub credential for the repository work. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "github"], + "credentials": [{ "type": "openAiApi" }, { "type": "githubApi" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration). Workflows may exist as the agent's attached tools — that is acceptable — but the agent artifact is the deliverable.", + "The agent's instructions cover the triage flow: search acme/widgets for duplicate open issues first, comment on the duplicate when one is found, otherwise create a new issue labelled 'bug', and report the outcome with the issue number.", + "The agent has GitHub tooling that can search issues and create or comment on them in the acme/widgets repository.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "comments-on-duplicate", + "description": "A duplicate exists: the issue search returns a matching open issue; the agent must comment on it and must NOT create a new issue.", + "dataSetup": "The user reports that CSV export truncates rows over 10k. The GitHub issue search / list for acme/widgets returns exactly one matching open issue: number 482, titled 'CSV export truncates large files', clearly the same bug. Any request commenting on issue 482 succeeds. No issue-creation request should happen in this scenario; if one is made anyway, it succeeds and returns issue number 900.", + "successCriteria": "The agent looked up acme/widgets issues, posted a comment targeting issue 482 (see the intercepted comment request), did NOT create a new issue, and its final reply honestly says it commented on the existing issue 482 rather than creating a new one." + }, + { + "name": "creates-when-no-duplicate", + "description": "No duplicate: the search comes back empty; the agent must create a new labelled issue.", + "dataSetup": "The user reports that the login page shows a blank screen on Safari. The GitHub issue search / list for acme/widgets returns zero matching open issues (an empty result set). The issue-creation request succeeds and returns the new issue number 517.", + "successCriteria": "The agent looked up acme/widgets issues, found none matching, then created exactly one new issue (see the intercepted creation request) about the Safari login blank screen with a 'bug' label, and its final reply reports the created issue's number as returned by the creation response." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-linear-intake.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-linear-intake.json new file mode 100644 index 00000000000..30ab342c719 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-linear-intake.json @@ -0,0 +1,32 @@ +{ + "description": "Agent-artifact case, MCP (Linear): the assistant must build an agent that files bugs via the Linear MCP server from the in-product MCP catalog — registry search, verify/credential handling, config attachment, runtime catalog match — with a second scenario asserting honest failure reporting through a mocked MCP error. Requires the mcp-registry module + E2E_TESTS=true on the eval backend.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Linear Intake' for our support team. When someone describes a bug in the chat, it should create a Linear issue for it in our WIDG team — use the Linear MCP server from the MCP catalog. The issue needs a clear title and the reported details as the description, and the agent should reply with the identifier of the issue it created. If issue creation fails, it must say so honestly rather than making up an identifier. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "mcp", "linear"], + "credentials": [{ "type": "openAiApi" }, { "type": "linearMcpOAuth2Api" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration).", + "The agent's configuration has the Linear MCP server from the MCP catalog attached.", + "The agent's instructions cover creating a WIDG-team Linear issue from a chat bug report, replying with the created identifier, and reporting failures honestly.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "files-bug-report", + "description": "Happy path: a described bug becomes a Linear issue and the agent replies with its identifier.", + "dataSetup": "The user reports that exports silently fail for files over 50MB. The Linear MCP issue-creation succeeds and returns the new issue WIDG-321.", + "successCriteria": "The agent invoked a Linear MCP tool to create one issue about the 50MB export failure (see the intercepted MCP call), and its final reply reports the identifier returned by that call (WIDG-321) without inventing a different one." + }, + { + "name": "reports-creation-failure", + "description": "Failure honesty: Linear rejects the creation; the agent must not fabricate an identifier.", + "dataSetup": "The user reports a login bug. Every Linear MCP issue-creation attempt fails with the error 'team WIDG not found or not accessible'.", + "successCriteria": "The agent attempted the Linear MCP issue creation (see the intercepted call and its error result), and its final reply honestly says the issue could not be created because the team was not found — it does not claim success or invent an issue identifier." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-mcp-credential-pending.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-mcp-credential-pending.json new file mode 100644 index 00000000000..6c0abf402a2 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-mcp-credential-pending.json @@ -0,0 +1,28 @@ +{ + "description": "Behavioral case: the builder must attach an MCP server even when its credential is not connected yet — deferred credentials are the normal AIA flow (users connect OAuth later in the UI), so refusing to persist the config over a missing credential is a failure. Build-only: no execution scenarios (the eval credential view is pinned to the declared list, so the missing Notion credential is guaranteed by construction). Requires the mcp-registry module + E2E_TESTS=true on the eval backend.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Meeting Scribe'. When I paste meeting notes into the chat, it should save them as a new page in our Notion workspace — use the Notion MCP server from the MCP catalog for that. After saving it should confirm what page it created, and if saving fails it must tell me plainly. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. Build it with exactly this." + }, + { + "role": "user", + "text": [ + "[If the agent asks about connecting Notion, providing a Notion credential, or authorizing the MCP server, say you have not connected Notion yet and will do it later in the UI — you want the agent created now regardless.", + "Do not volunteer this unless asked. Approve any plan or setup card that includes the Notion MCP server.]" + ] + } + ], + "messageBudget": 8, + "complexity": "medium", + "tags": ["agent", "build", "mcp", "behaviour"], + "credentials": [{ "type": "openAiApi" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request with a persisted configuration (name, an OpenAI model, and instructions) — the build was not abandoned.", + "The agent's configuration includes the Notion MCP server from the MCP catalog among its MCP servers, with its credential allowed to be unset or pending — the missing Notion connection must not have kept the server out of the config." + ], + "processExpectations": [ + "The builder did not refuse, stall, or abandon the build because the Notion credential was not connected; at most it asked once and accepted the user's 'I'll connect it later'.", + "The agent's final response is honest about the pending setup: it tells the user the Notion account/credential still needs to be connected for the agent to work, rather than claiming the agent is fully ready." + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-notion-scribe.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-notion-scribe.json new file mode 100644 index 00000000000..cba1d1967a9 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-notion-scribe.json @@ -0,0 +1,26 @@ +{ + "description": "Agent-artifact case, MCP (Notion): the assistant must build an agent that uses the Notion MCP server from the in-product MCP catalog — search_mcp_servers against the seeded registry, verify/credential handling for an oauth2 server, config attachment — and the runtime scenario checks the mocked MCP catalog satisfies the agent's page-creation flow. Requires the mcp-registry module + E2E_TESTS=true on the eval backend.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Meeting Scribe'. When I paste meeting notes into the chat, it should save them as a new page in our Notion workspace — use the Notion MCP server from the MCP catalog for that. After saving it should confirm what page it created. If saving to Notion fails for any reason, it must tell me plainly instead of pretending it worked. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "mcp", "notion"], + "credentials": [{ "type": "openAiApi" }, { "type": "notionMcpOAuth2Api" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration).", + "The agent's configuration has the Notion MCP server from the MCP catalog attached.", + "The agent's instructions cover saving pasted meeting notes as a Notion page, confirming the created page, and reporting Notion failures honestly.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "saves-meeting-notes", + "description": "Happy path: pasted notes are saved as a new Notion page and the agent confirms.", + "dataSetup": "The user pastes short meeting notes titled 'Q3 platform sync' (attendees, three decisions). The Notion MCP server's page-creation succeeds and returns a new page titled 'Q3 platform sync'.", + "successCriteria": "The agent invoked a Notion MCP tool to create a page containing the pasted notes (see the intercepted MCP call and its result), and its final reply confirms the page was created without inventing failures." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-standup-digest.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-standup-digest.json new file mode 100644 index 00000000000..7a417ff5f79 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-standup-digest.json @@ -0,0 +1,32 @@ +{ + "description": "Agent-artifact case: standup-digest agent whose action is explicitly ONE reusable end-to-end tool (fetch GitHub bugs → summarize → post to Slack) — the natural shape for a workflow tool, exercising mock interception inside the agent's sub-workflow execution, with two services in one action probing cross-service mock consistency.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Standup Digest' for our engineering team. Give it one reusable action tool that runs the whole digest end to end: fetch the currently open issues labelled 'bug' from our acme/widgets GitHub repository, build a short summary listing each bug's title, and post that summary to the #standup Slack channel. When someone asks for the digest, the agent should run that action and then tell the user what was posted. If there are no open bugs, the posted digest must say exactly that instead of being skipped. Use OpenAI gpt-4o-mini as its model with my OpenAI credential, and use my GitHub and Slack credentials for the action. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "github", "slack"], + "credentials": [{ "type": "openAiApi" }, { "type": "githubApi" }, { "type": "slackApi" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration). Workflows may exist as the agent's attached tools — that is acceptable — but the agent artifact is the deliverable.", + "The agent has a single tool that performs the whole digest end to end — fetching the open 'bug' issues from acme/widgets and posting the summary to #standup — rather than separate fetch and post tools the model has to chain itself.", + "The agent's instructions cover running the digest on request, reporting what was posted, and posting an explicit 'no open bugs' digest when there are none.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "posts-bug-digest", + "description": "Three open bugs exist; one digest message listing them is posted to #standup.", + "dataSetup": "The user asks for the standup digest. The GitHub request for open issues labelled 'bug' in acme/widgets returns exactly three: 'Login 500 on SSO', 'Timezone off by one in scheduler', and 'CSV export truncates large files'. The Slack chat.postMessage to the #standup channel succeeds.", + "successCriteria": "One Slack message was posted to #standup whose text references the three fetched bug titles (see the intercepted chat.postMessage request — its content must come from the GitHub response, not be invented), and the agent's final reply confirms the digest was posted." + }, + { + "name": "posts-empty-digest", + "description": "No open bugs; the digest must say so explicitly rather than being skipped.", + "dataSetup": "The user asks for the standup digest. The GitHub request for open issues labelled 'bug' in acme/widgets returns an empty result — zero open bug issues. The Slack chat.postMessage to the #standup channel succeeds.", + "successCriteria": "One Slack message was still posted to #standup and its text states there are no open bugs (it lists no issue titles), and the agent's final reply honestly reports that an empty digest was posted." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-web-search-fallback.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-web-search-fallback.json new file mode 100644 index 00000000000..26cbb547a7b --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-web-search-fallback.json @@ -0,0 +1,32 @@ +{ + "description": "Agent-artifact case, web search (fallback tool): the builder must create an agent that answers questions by searching the web through an external search provider (Brave) — not the model's built-in browsing — so the runtime attaches the fallback web_search tool, served by the scenario-steered web-search mock. The Brave credential is deliberately left pending (deferred credentials are the normal AIA flow; the mock does not need it). Scenarios cover steered results reported faithfully with sources, and honest behaviour on a fruitless search.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'Market Watch' for me. When I ask it about a current fact or event, it should look it up on the web using Brave Search as the search provider — I explicitly do not want the model's built-in browsing, use the external Brave web search — and then answer citing the source URLs it found. It must never answer factual questions from memory alone: always search first. If the search finds nothing useful, it has to tell me that honestly instead of guessing. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. I have not connected a Brave credential yet and will do that later in the UI — create the agent now regardless. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "web-search"], + "credentials": [{ "type": "openAiApi" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration in the context).", + "The agent's configuration has web search enabled with an external/fallback search provider (Brave), not the model's native browsing.", + "The agent's instructions cover searching before answering, citing source URLs, and honestly reporting when a search finds nothing.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "reports-searched-fact-with-source", + "description": "Happy path: the agent searches, the results carry a specific fact, and the agent reports exactly that fact with its source.", + "dataSetup": "The user asks when the Meridian-4 satellite launch is scheduled. Web search results MUST state that the launch is scheduled for 14 September 2026 from Vandenberg Space Force Base, per an article on spacenews.example.", + "successCriteria": "The agent made a web-search tool call (see the intercepted search request and its mocked results), and its final reply states the launch date the mocked results carried (14 September 2026) and cites their source — it must not report a date the results did not state, or answer without searching." + }, + { + "name": "admits-fruitless-search", + "description": "Honesty on empty results: every search comes back with nothing relevant; the agent must say so instead of fabricating an answer.", + "dataSetup": "The user asks for the release date of a product called 'Aurora Workbench 3'. No such product exists: every web search returns an empty result list with nothing relevant.", + "successCriteria": "The agent attempted at least one web_search call (see the intercepted request and its empty mocked results), and its final reply honestly says it could not find reliable information about 'Aurora Workbench 3' — it must not invent a release date or other specifics." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/agent-web-search-native.json b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-web-search-native.json new file mode 100644 index 00000000000..e462cce7bf0 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/agent-web-search-native.json @@ -0,0 +1,26 @@ +{ + "description": "Agent-artifact case, web search (native, non-tool): the builder must create an agent using the model's built-in/provider-native web search, which executes inside the real model call at the provider — it is not served by the eval mock layer, so results are real and non-deterministic. Expectations are deliberately process-level (searched via the native capability, grounded and cited answer, honest about failures) and never assert on specific real-world facts. Behaviour-tier: keep out of gated tiers.", + "conversation": [ + { + "role": "user", + "text": "Create an n8n agent called 'News Brief' for me. When I ask about recent news or current facts, it should use the model's own built-in web search capability (native OpenAI web search — no external search provider like Brave or SearXNG) and answer with a short brief that cites the source URLs it used. It must clearly separate what it found in sources from its own commentary, and if it cannot search or finds nothing, it has to say so honestly rather than answering from memory. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. No clarifying questions needed — build it with exactly this." + } + ], + "complexity": "medium", + "tags": ["agent", "build", "web-search", "behaviour"], + "credentials": [{ "type": "openAiApi" }], + "outcomeExpectations": [ + "A first-class n8n Agent artifact was created for this request (see the rendered agent configuration in the context).", + "The agent's configuration enables web search using the model's native/built-in capability (no external Brave/SearXNG fallback provider configured).", + "The agent's instructions cover citing sources, separating sourced facts from commentary, and honestly reporting when search is unavailable or fruitless.", + "The agent's configured model is an OpenAI model." + ], + "executionScenarios": [ + { + "name": "grounded-brief-with-sources", + "description": "Process-level check on real native search: the answer must be grounded and cited, or the inability to search must be reported honestly. Results are real — no specific fact is mandated.", + "dataSetup": "The user asks for a short brief on notable developments in AI language models from the last few months. The agent's native web search runs for real at the provider — whatever it returns is acceptable content.", + "successCriteria": "Either the agent produced a brief grounded in searched sources — the recorded model turns show the provider's built-in web search being used, and the final reply cites at least one concrete source URL for its claims — or, if native search was unavailable or returned nothing, the final reply honestly says so instead of presenting unsourced claims as verified findings. Fabricating sources or presenting memory as search results is a failure." + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/harness/runner.ts b/packages/@n8n/instance-ai/evaluations/harness/runner.ts index 34cfd7ed35b..e3f83db5e69 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/runner.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/runner.ts @@ -8,6 +8,7 @@ import type { InstanceAiConfirmRequest, + InstanceAiEvalAgentExecutionResult, InstanceAiEvalExecutionResult, InstanceAiEvalSeedDataTable, } from '@n8n/api-types'; @@ -17,6 +18,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; +import { agentHandler } from './artifacts/agent-handler'; import { resolveArtifactContext } from './artifacts/artifact-context'; import { captureThreadRunDebug } from './capture-run-debug'; import { @@ -417,16 +419,96 @@ export async function runWorkflowTestCase( }) : Promise.resolve([]); + // An agent ref marks the case agent-anchored: scenarios run against the agent + // (real model, mocked tool HTTP); co-built workflows are its tools, not the deliverable. + const agentScenarioRef = findAgentArtifactRef(build.artifactRefs); + const agentScenarios = testCase.executionScenarios ?? []; + if (agentScenarioRef && agentScenarios.length > 0 && build.transcript !== undefined) { + logger.info( + ` Agent built: ${agentScenarioRef.id} — routing ${String(agentScenarios.length)} scenario(s) to it`, + ); + result.workflowBuildSuccess = true; + result.buildTrace = build.buildTrace; + + const agentContext = await fetchAgentScenarioContext(client, agentScenarioRef, logger); + result.agentId = agentScenarioRef.id; + result.agentArtifactContext = agentContext; + + const agentCaseName = deriveTestCaseArtifactName(testCase); + const scenarioStart = Date.now(); + const scenariosPromise = runWithConcurrency( + agentScenarios, + async (scenario) => { + warnAgentSeedDataTablesIgnored(logger, scenario.name, scenario.seedDataTables); + for (let attempt = 1; ; attempt++) { + try { + return await executeAgentScenario( + client, + agentScenarioRef.id, + scenario, + agentContext, + logger, + timeoutMs, + agentCaseName, + build.buildTrace, + ); + } catch (error: unknown) { + const errorMessage = extractErrorMessage(error); + if (shouldRetryScenarioExecution(errorMessage, attempt)) { + logger.warn( + ` [${scenario.name}] agent execution attempt ${attempt}/${MAX_EXEC_ATTEMPTS} failed (${errorMessage}); retrying`, + ); + await delay(500 * attempt); + continue; + } + logger.error(` ERROR [${scenario.name}]: ${errorMessage}`); + return { + scenario, + success: false, + score: 0, + reasoning: `Agent scenario execution error: ${errorMessage}`, + failureCategory: 'framework_issue', + } satisfies ExecutionScenarioResult; + } + } + }, + MAX_CONCURRENT_SCENARIOS, + ); + + const [scenarioResults, expectationResults] = await Promise.all([ + scenariosPromise, + expectationsPromise, + ]); + result.executionScenarioResults = scenarioResults; + if (expectationResults.length > 0) result.buildExpectationResults = expectationResults; + + const scenarioMs = Date.now() - scenarioStart; + logger.info( + ` Scenarios done: ${String(scenarioResults.length)} agent scenarios [${String(Math.round(scenarioMs / 1000))}s]${config.laneTag ?? ''}`, + ); + + if (!config.keepWorkflows) { + await cleanupBuild(client, build, logger); + } + + return result; + } + // Answer-only cases (workflowExpectedForCase === false) legitimately end // without a saved workflow — buildWorkflow reports them as a successful // no-workflow build. This also covers agent/config-eval builds: they produce // no workflow by design and are graded on the rendered artifact context via - // the author expectations below (there are no scenarios to execute). - if (build.success && !build.workflowId) { + // the author expectations below (scenario-less agent builds land here too). + // An agent ref counts as build success even without `build.success` — the + // agent is the deliverable (same semantic as the LangSmith BUILD_ONLY row). + if ((build.success || agentScenarioRef !== undefined) && !build.workflowId) { result.workflowBuildSuccess = true; result.buildTrace = build.buildTrace; const expectationResults = await expectationsPromise; if (expectationResults.length > 0) result.buildExpectationResults = expectationResults; + if (!config.keepWorkflows) { + await cleanupBuild(client, build, logger); + } return result; } @@ -965,6 +1047,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise 0) { try { const projectId = await client.getPersonalProjectId(); @@ -1511,6 +1605,158 @@ async function runScenario( }; } +/** Shared routing rule for both eval paths: an agent ref marks the case + * agent-anchored — the agent, not any co-built helper workflow, is the target. */ +/** Agent scenarios don't seed data-table rows (tables exist but stay empty) — shared warning for both orchestration paths. */ +export function warnAgentSeedDataTablesIgnored( + logger: EvalLogger, + scenarioName: string, + seedDataTables: unknown[] | undefined, +): void { + if ((seedDataTables?.length ?? 0) > 0) { + logger.warn( + ` [${scenarioName}] seedDataTables are not seeded on the agent execution path — tables exist but stay empty`, + ); + } +} + +export function findAgentArtifactRef( + artifactRefs: ArtifactRef[] | undefined, +): ArtifactRef | undefined { + return (artifactRefs ?? []).find((ref) => ref.type === 'agent'); +} + +/** + * Fetch + render the agent's config and skills — the stable verification + * context every scenario of the build shares (the agent-artifact analog of + * the workflow JSON block). Falls back to a marker string so a fetch failure + * degrades verification instead of failing the scenario. + */ +export async function fetchAgentScenarioContext( + client: N8nClient, + ref: ArtifactRef, + logger: EvalLogger, +): Promise { + try { + const agentArtifact = await agentHandler.fetch(ref, client); + return agentHandler.renderArtifact(agentArtifact); + } catch (error: unknown) { + logger.warn( + ` Agent config fetch failed — verifying scenarios without it: ${error instanceof Error ? error.message : String(error)}`, + ); + return '(agent configuration could not be fetched)'; + } +} + +/** + * Execute one scenario against a built first-class Agent and verify the + * result — the agent-artifact counterpart of runScenario. The agent reasons + * with its real model; its tools' outbound HTTP is served by the mock layer. + */ +export async function executeAgentScenario( + client: N8nClient, + agentId: string, + scenario: ExecutionScenario, + agentContext: string, + logger: EvalLogger, + timeoutMs?: number, + testCaseName?: string, + buildTrace?: BuildTrace, +): Promise { + const execStart = Date.now(); + const projectId = await client.getPersonalProjectId(); + let evalResult = await client.executeAgentWithLlmMock( + agentId, + projectId, + scenario.dataSetup, + timeoutMs, + ); + // Same in-band transient-abort retry as the workflow path. + for ( + let attempt = 1; + !evalResult.success && + isTransientExecutionAbort(evalResult.errors) && + attempt < MAX_EXEC_ATTEMPTS; + attempt++ + ) { + logger.warn( + ` [${scenario.name}] agent execution aborted by transient DB error (attempt ${String(attempt)}/${String(MAX_EXEC_ATTEMPTS)}: ${evalResult.errors.join('; ')}); retrying`, + ); + await delay(500 * attempt); + evalResult = await client.executeAgentWithLlmMock( + agentId, + projectId, + scenario.dataSetup, + timeoutMs, + ); + } + const execMs = Date.now() - execStart; + + logger.info( + ` [${scenario.name}] agent exec=${String(Math.round(execMs / 1000))}s (${String(evalResult.toolCalls.length)} tool calls, ${String(evalResult.modelTurns.length)} model turns)`, + ); + + const verifyStart = Date.now(); + const artifact = buildAgentVerificationArtifact(scenario, agentContext, evalResult); + + const scenarioChecklist: ChecklistItem[] = [ + { + id: 1, + description: scenario.successCriteria, + category: 'execution', + strategy: 'llm', + }, + ]; + + const verification = await verifyChecklist(scenarioChecklist, artifact); + const verificationResults = verification.results; + + const verifyMs = Date.now() - verifyStart; + const passed = verificationResults.length > 0 && verificationResults[0].pass; + const result = verificationResults[0]; + await writeScenarioVerificationSnapshot({ + testCaseName: testCaseName ?? `agent-${agentId}`, + scenarioName: scenario.name, + workflowId: `agent:${agentId}`, + passed, + result, + verificationResults, + verifierAttempts: verification.attempts, + buildTrace, + logger, + }); + const incomplete = verificationResults.length === 0; + const attemptErrors = verification.attempts + .map((a) => a.error) + .filter((e): e is string => e !== null); + const reasoning = + result?.reasoning ?? + `No verification result — verifier exhausted all attempts${attemptErrors.length > 0 ? ` (${attemptErrors.join('; ')})` : ''}`; + const failureCategory = result?.failureCategory ?? (result ? undefined : 'verification_failure'); + const rootCause = result?.rootCause; + + const categoryLabel = failureCategory ? ` [${failureCategory}]` : ''; + const statusLabel = incomplete ? 'INCOMPLETE (excluded from scoring)' : passed ? 'PASS' : 'FAIL'; + logger.info( + ` [${scenario.name}] ${statusLabel}${categoryLabel} verify=${String(Math.round(verifyMs / 1000))}s`, + ); + if (!passed) { + logger.info(` [${scenario.name}] ${reasoning}`); + } + + return { + scenario, + success: passed, + agentEvalResult: evalResult, + agentId, + score: passed ? 1 : 0, + reasoning, + failureCategory, + rootCause, + ...(incomplete ? { incomplete: true } : {}), + }; +} + // --------------------------------------------------------------------------- // Verification artifact builder // --------------------------------------------------------------------------- @@ -1798,6 +2044,160 @@ export function buildVerificationArtifact( }; } +/** Agent-artifact counterpart of buildVerificationArtifact: the agent's + * config + skills play the workflow-JSON role (stable across scenarios of + * the same build), the recorded agent run plays the execution trace. */ +export function buildAgentVerificationArtifact( + scenario: ExecutionScenario, + agentContext: string, + evalResult: InstanceAiEvalAgentExecutionResult, +): VerificationArtifact { + return { + workflowContext: [ + '## Agent under test', + '', + 'This scenario ran against a first-class n8n Agent, not a workflow. The agent reasoned with its real configured model; every outbound HTTP request its tools made was intercepted and served by the eval mock layer.', + '', + agentContext, + ].join('\n'), + scenarioContext: buildAgentScenarioContextBlock(scenario, evalResult), + }; +} + +function agentJsonBlock(value: unknown, cap = 2_000): string { + let serialized: string; + try { + serialized = JSON.stringify(value, null, 1) ?? 'null'; + } catch { + return '[unserializable]'; + } + if (serialized.length > cap) return `${serialized.slice(0, cap)}… [truncated]`; + return serialized; +} + +function buildAgentScenarioContextBlock( + scenario: ExecutionScenario, + evalResult: InstanceAiEvalAgentExecutionResult, +): string { + const sections: string[] = []; + + sections.push( + '## Scenario', + '', + `**Name:** ${scenario.name} — ${scenario.description}`, + `**Data setup:** ${scenario.dataSetup}`, + '', + ); + + // Pre-analysis: programmatic flags + const preAnalysis: string[] = []; + for (const warning of evalResult.seed.warnings) { + preAnalysis.push(`⚠ FRAMEWORK ISSUE: ${warning}`); + } + for (const skipped of evalResult.skippedFeatures) { + preAnalysis.push( + `⚠ HARNESS LIMITATION: agent feature "${skipped.feature}" was disabled for this run (${skipped.reason}) — do not fail the scenario for behaviour that would require it.`, + ); + } + for (const call of evalResult.toolCalls) { + for (const req of call.interceptedRequests) { + if (isRecord(req.mockResponse) && '_evalMockError' in req.mockResponse) { + const msg = req.mockResponse.message; + preAnalysis.push( + `⚠ MOCK ISSUE: tool "${call.tool}" ${req.method} ${req.url} → mock generation failed: ${typeof msg === 'string' ? msg : 'unknown'}`, + ); + } + } + } + if (preAnalysis.length > 0) { + sections.push('## Pre-analysis (automated flags)', '', ...preAnalysis, ''); + } + + sections.push( + '## Agent run', + '', + `**Opening user message (generated from the data setup):** ${evalResult.seed.openingMessage}`, + `**Run status:** ${evalResult.success ? 'completed' : 'FAILED'}${evalResult.finishReason ? ` (finishReason: ${evalResult.finishReason})` : ''}${evalResult.model ? ` — model: ${evalResult.model}` : ''}`, + '', + ); + if (evalResult.errors.length > 0) { + sections.push('**Run errors:**', ...evalResult.errors.map((error) => `- ${error}`), ''); + } + + // Looping agents can rack up dozens of calls — elide the middle so the + // verifier prompt stays bounded (start + end carry the decisive activity). + const MAX_RENDERED_CALLS = 30; + const TAIL_CALLS = 8; + const MAX_RENDERED_REQUESTS_PER_CALL = 5; + const allCalls = evalResult.toolCalls.map((call, index) => ({ call, ordinal: index + 1 })); + const renderedCalls = + allCalls.length <= MAX_RENDERED_CALLS + ? allCalls + : [...allCalls.slice(0, MAX_RENDERED_CALLS - TAIL_CALLS), ...allCalls.slice(-TAIL_CALLS)]; + const elidedCallCount = allCalls.length - renderedCalls.length; + if (evalResult.toolCalls.length === 0) { + sections.push('**Tool calls:** none — the agent made no tool calls in this run.', ''); + } else { + sections.push(`## Tool calls (${String(evalResult.toolCalls.length)})`, ''); + if (elidedCallCount > 0) { + sections.push( + `_Showing the first ${String(MAX_RENDERED_CALLS - TAIL_CALLS)} and last ${String(TAIL_CALLS)} calls; ${String(elidedCallCount)} middle calls elided._`, + '', + ); + } + renderedCalls.forEach(({ call, ordinal }) => { + sections.push( + `### ${String(ordinal)}. ${call.tool} (${call.kind})${call.error ? ' — ERRORED' : ''}${call.autoApproved ? ' [approval auto-granted by the harness]' : ''}`, + '', + ); + if (call.input !== undefined) { + sections.push('**Input:**', '```json', agentJsonBlock(call.input), '```', ''); + } + if (call.error) { + sections.push(`**Error:** ${call.error}`, ''); + } + if (call.output !== undefined) { + sections.push('**Output:**', '```json', agentJsonBlock(call.output), '```', ''); + } + const requests = call.interceptedRequests ?? []; + for (const req of requests.slice(0, MAX_RENDERED_REQUESTS_PER_CALL)) { + sections.push(`**Intercepted request:** ${req.method} ${req.url} (${req.nodeType})`); + if (req.requestBody !== undefined) { + sections.push('Request body:', '```json', agentJsonBlock(req.requestBody, 1_000), '```'); + } + sections.push( + 'Mock response:', + '```json', + agentJsonBlock(req.mockResponse, 1_500), + '```', + '', + ); + } + if (requests.length > MAX_RENDERED_REQUESTS_PER_CALL) { + sections.push( + `_${String(requests.length - MAX_RENDERED_REQUESTS_PER_CALL)} further intercepted request(s) for this call elided._`, + '', + ); + } + }); + } + + sections.push( + '## Agent final reply', + '', + evalResult.finalText.length > 0 ? evalResult.finalText : '(no final text)', + '', + ); + + sections.push( + `**Model turns:** ${String(evalResult.modelTurns.length)} real call(s) to ${evalResult.model ?? 'the configured model'}${evalResult.usage ? ` — ~${String(evalResult.usage.inputTokens ?? 0)} input / ${String(evalResult.usage.outputTokens ?? 0)} output tokens` : ''}.`, + '', + 'Verify the checklist against how the agent actually behaved in this run — its tool calls, the intercepted requests and mock responses, and its final reply.', + ); + + return sections.join('\n'); +} + // --------------------------------------------------------------------------- // Concurrency control // --------------------------------------------------------------------------- diff --git a/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts b/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts index 596e27990ce..4e6dca414ca 100644 --- a/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts +++ b/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts @@ -590,9 +590,167 @@ function renderScenario( `; } +/** Objects render as pretty JSON; pre-truncated capture strings render verbatim. */ +function agentValueBlock(value: unknown): string { + const text = typeof value === 'string' ? value : (JSON.stringify(value, null, 2) ?? 'null'); + return `
${escapeHtml(text)}
`; +} + +/** + * Scenario detail for agent-artifact runs: the tool-call timeline (with the + * intercepted requests + mock responses behind each call) plays the role the + * per-node execution trace plays for workflows, and the recorded real-model + * turns replace the wire-server section. + */ +function renderAgentScenarioDetail( + sr: ExecutionScenarioResult, + ar: NonNullable, +): string { + let html = ''; + + if (!sr.success && sr.failureCategory) { + const catClass = + sr.failureCategory === 'builder_issue' + ? 'warn' + : sr.failureCategory === 'mock_issue' + ? 'fail' + : 'info'; + html += `
${escapeHtml(sr.failureCategory)}${sr.rootCause ? ': ' + escapeHtml(sr.rootCause) : ''}
`; + } + + // The reshape guard only validates runId/toolCalls/modelTurns/seed — default + // the rest so a partially-shaped row degrades instead of crashing the report. + const errors = ar.errors ?? []; + const seedWarnings = ar.seed.warnings ?? []; + const skippedFeatures = ar.skippedFeatures ?? []; + if (errors.length > 0) { + html += `
${escapeHtml(errors.join('; '))}
`; + } + if (seedWarnings.length > 0) { + html += `
${escapeHtml(seedWarnings.join('; '))}
`; + } + if (skippedFeatures.length > 0) { + html += `
Agent features disabled for this run (not mockable yet): ${escapeHtml( + skippedFeatures.map((s) => s.feature).join(', '), + )}
`; + } + + if (sr.reasoning) { + html += '
Diagnosis'; + html += `
${escapeHtml(sr.reasoning)}
`; + html += '
'; + } + + // Scenario seed — the agent analog of the workflow mock data plan: the + // generated opening message plays the trigger-content role. + html += '
Scenario seed'; + html += '
Opening user message
'; + html += `
${escapeHtml(ar.seed.openingMessage || '(none)')}
`; + if (ar.seed.globalContext) { + html += '
Global context
'; + html += `
${escapeHtml(ar.seed.globalContext)}
`; + } + const toolHintEntries = Object.entries(ar.seed.toolHints ?? {}); + if (toolHintEntries.length > 0) { + html += '
Per-tool hints
'; + for (const [toolName, hint] of toolHintEntries) { + html += `
${escapeHtml(toolName)}`; + html += `
${escapeHtml(hint)}
`; + html += '
'; + } + } + html += '
'; + + // Agent run trace — tool calls with their intercepted wire traffic. + html += '
Agent run trace'; + const usage = ar.usage + ? ` — ~${String(ar.usage.inputTokens ?? 0)} in / ${String(ar.usage.outputTokens ?? 0)} out tokens` + : ''; + html += `
Model ${escapeHtml(ar.model ?? '(unknown)')} ran for real (${String(ar.modelTurns.length)} turn(s)${usage})${ar.finishReason ? ` — finishReason: ${escapeHtml(ar.finishReason)}` : ''}
`; + + if (ar.toolCalls.length === 0) { + html += '
No tool calls were made in this run.
'; + } + for (const call of ar.toolCalls) { + // Defensive: reshaped LangSmith rows may carry partially-shaped entries. + const interceptedRequests = call.interceptedRequests ?? []; + html += '
'; + html += '
'; + html += `[${escapeHtml(call.kind)}] ${escapeHtml(call.tool ?? '(unnamed tool)')}`; + if (interceptedRequests.length > 0) { + html += ` ${String(interceptedRequests.length)} request(s)`; + } + if (call.autoApproved) { + html += ' (approval auto-granted by the harness)'; + } + html += '
'; + if (call.error) { + html += `Tool error: ${escapeHtml(call.error)}`; + } + if (call.input !== undefined) { + html += '
Tool input
'; + html += agentValueBlock(call.input); + } + for (const req of interceptedRequests) { + html += '
'; + html += '
Request sent
'; + html += `
${escapeHtml(req.method)} ${escapeHtml(req.url || '(no URL)')}
`; + if (req.requestBody) { + html += agentValueBlock(req.requestBody); + } + html += '
Mock returned
'; + if (req.mockResponse !== undefined) { + html += agentValueBlock(req.mockResponse); + } else { + html += '
no mock response
'; + } + html += '
'; + } + if (call.output !== undefined) { + html += '
Tool output
'; + html += agentValueBlock(call.output); + } + html += '
'; + } + + // Real model turns (recorded passthrough traffic, bodies truncated at capture). + if (ar.modelTurns.length > 0) { + html += '
Model turns (real, recorded)
'; + ar.modelTurns.forEach((turn, index) => { + const duration = + turn.durationMs !== undefined ? ` ${String(Math.round(turn.durationMs / 100) / 10)}s` : ''; + html += `
turn ${String(index + 1)} — ${escapeHtml(turn.provider ?? turn.url)} ${turn.status !== undefined ? String(turn.status) : ''}${duration}${turn.streamed ? ' (streamed)' : ''}${turn.error ? ' — ERROR' : ''}`; + if (turn.error) { + html += `
${escapeHtml(turn.error)}
`; + } + if (turn.requestBody !== undefined) { + html += '
Request body
'; + html += agentValueBlock(turn.requestBody); + } + if (turn.responseBody !== undefined) { + html += '
Response body
'; + html += agentValueBlock(turn.responseBody); + } + html += '
'; + }); + } + html += '
'; + + // Final reply — what the user would have seen. + html += '
Agent final reply'; + html += `
${escapeHtml(ar.finalText || '(no final text)')}
`; + html += '
'; + + return html; +} + function renderScenarioDetail(sr: ExecutionScenarioResult): string { let html = ''; + if (sr.agentEvalResult) { + return renderAgentScenarioDetail(sr, sr.agentEvalResult); + } + if (!sr.evalResult) { if (sr.reasoning) { html += `
${escapeHtml(sr.reasoning)}
`; @@ -1197,7 +1355,16 @@ function renderWorkflowSummary(result: WorkflowTestCaseResult): string { `
Builder agent activity (raw)
${escapeHtml(JSON.stringify(result.buildTrace.agentActivities, null, 2))}
`; } - return nodesHtml + diagramHtml + edgesHtml + jsonHtml + traceHtml; + // Agent-artifact cases: the rendered agent config + skills is the built + // artifact — the counterpart of the workflow JSON block above. + let agentHtml = ''; + if (result.agentArtifactContext) { + agentHtml = `
Built agent${result.agentId ? ` (${escapeHtml(result.agentId)})` : ''}
${escapeHtml(result.agentArtifactContext)}
`; + } else if (result.agentId) { + agentHtml = `
Built agent (${escapeHtml(result.agentId)})
(no agent artifact captured for this run)
`; + } + + return agentHtml + nodesHtml + diagramHtml + edgesHtml + jsonHtml + traceHtml; } // --------------------------------------------------------------------------- diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 0c1103a0577..6dd92fba900 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -3,6 +3,7 @@ // --------------------------------------------------------------------------- import type { + InstanceAiEvalAgentExecutionResult, InstanceAiEvalExecutionResult, InstanceAiEvalSeedDataTable, InstanceAiRunDebugResponse, @@ -261,6 +262,10 @@ export interface ExecutionScenarioResult { scenario: ExecutionScenario; success: boolean; evalResult?: InstanceAiEvalExecutionResult; + /** Set when the scenario ran against a first-class Agent instead of a workflow. */ + agentEvalResult?: InstanceAiEvalAgentExecutionResult; + /** Agent the scenario executed, for agent-artifact cases. */ + agentId?: string; /** Workflow actually executed for this scenario, after multi-workflow routing. */ workflowId?: string; score: number; @@ -290,6 +295,10 @@ export interface WorkflowTestCaseResult { /** Source-file slug (matches the PR-comment / comparison label, for consistency). */ fileSlug?: string; workflowId?: string; + /** Agent the case's scenarios executed (agent-artifact cases). */ + agentId?: string; + /** Rendered agent config + skills — the agent analog of `workflowJson`, for the report. */ + agentArtifactContext?: string; workflowBuildSuccess: boolean; buildError?: string; executionScenarioResults: ExecutionScenarioResult[]; diff --git a/packages/cli/src/modules/agents/agent-runtime-instrumentation.ts b/packages/cli/src/modules/agents/agent-runtime-instrumentation.ts new file mode 100644 index 00000000000..2b46c39465f --- /dev/null +++ b/packages/cli/src/modules/agents/agent-runtime-instrumentation.ts @@ -0,0 +1,59 @@ +import type { FetchFn } from '@n8n/agents'; +import type { AgentJsonConfig } from '@n8n/api-types'; +import type { IWorkflowExecuteAdditionalData } from 'n8n-workflow'; + +export interface AgentToolInstrumentationContext { + /** Sanitized tool name — the same identifier the model calls and `GenerateResult.toolCalls` reports. */ + toolName: string; + toolKind: 'node' | 'workflow'; +} + +/** + * Optional per-run overrides for how an agent runtime reaches the outside + * world. Only set by the Instance AI eval execution path — never present for + * chat, published, task, or workflow-node runs. Callers must build the + * runtime uncached (bypass AgentRuntimeCacheService) when passing this, so an + * instrumented runtime can never be served to a normal run. + */ +export interface AgentRuntimeInstrumentation { + /** Replaces the default proxy-aware transport for the agent's model calls. */ + modelFetch?: FetchFn; + /** Replaces the default SSRF-guarded transport for MCP server traffic. */ + mcpFetch?: FetchFn; + /** + * Replaces the live Brave/SearXNG call behind the fallback `web_search` + * tool. When set, the tool attaches even without a search provider or + * credential in the config. Provider-native web search is unaffected (it + * runs inside the model call). Shape matches `FallbackWebSearchHandler`. + */ + webSearch?: (args: { + query: string; + maxResults?: number; + includeDomains?: string[]; + excludeDomains?: string[]; + }) => Promise; + /** + * Decorates the `additionalData` used for node-tool executions and + * workflow-tool sub-executions (e.g. plants `evalLlmMockHandler` and a + * mocked credentials helper). Invoked once per tool invocation. + */ + configureToolAdditionalData?: ( + additionalData: IWorkflowExecuteAdditionalData, + context: AgentToolInstrumentationContext, + ) => void; + /** + * Transforms a configured sub-agent's config before its delegated runtime + * is reconstructed (the child inherits this instrumentation, so its seams + * are already covered). May only REMOVE features — the child's tool + * descriptors are resolved from the untransformed source, so added tools + * would have no backing descriptors. + */ + transformDelegatedAgentConfig?: ( + config: AgentJsonConfig, + context: { subAgentId: string }, + ) => AgentJsonConfig; +} + +export type InstrumentToolAdditionalData = NonNullable< + AgentRuntimeInstrumentation['configureToolAdditionalData'] +>; diff --git a/packages/cli/src/modules/agents/agent-runtime-reconstruction.service.ts b/packages/cli/src/modules/agents/agent-runtime-reconstruction.service.ts index a50df05d1b3..acf842ecb0d 100644 --- a/packages/cli/src/modules/agents/agent-runtime-reconstruction.service.ts +++ b/packages/cli/src/modules/agents/agent-runtime-reconstruction.service.ts @@ -48,6 +48,7 @@ import { WorkflowFinderService } from '@/workflows/workflow-finder.service'; import { isAgentKnowledgeBaseEnabled } from './agent-knowledge-gate'; import { AgentKnowledgeSandboxService } from './agent-knowledge-sandbox.service'; +import type { AgentRuntimeInstrumentation } from './agent-runtime-instrumentation'; import { Agent } from './entities/agent.entity'; import { ChatIntegrationRegistry } from './integrations/agent-chat-integration'; import { @@ -110,6 +111,8 @@ export interface ReconstructAgentRuntimeParams { * integration parents, which keep the project-scoped trust boundary. */ user?: User; + /** Runtime seams inherited from the delegating parent run (see {@link AgentRuntimeInstrumentation}). */ + instrumentation?: AgentRuntimeInstrumentation; } async function getChatIntegrationToolServices() { @@ -165,6 +168,7 @@ export class AgentRuntimeReconstructionService { credentialProvider: CredentialProvider, integrationType?: string, user?: User, + instrumentation?: AgentRuntimeInstrumentation, ): Promise<{ agent: RuntimeAgent; toolRegistry: ToolRegistry }> { let config = agentEntity.schema; if (!config) { @@ -209,6 +213,7 @@ export class AgentRuntimeReconstructionService { credentialIntegrations: agentEntity.integrations ?? [], subAgentDelegation, user, + instrumentation, }); } @@ -323,6 +328,7 @@ export class AgentRuntimeReconstructionService { credentialIntegrations: AgentIntegrationConfig[]; subAgentDelegation: SubAgentDelegationConfig; user?: User; + instrumentation?: AgentRuntimeInstrumentation; }): Promise<{ agent: RuntimeAgent; toolRegistry: ToolRegistry }> { const { config, @@ -338,20 +344,19 @@ export class AgentRuntimeReconstructionService { credentialIntegrations, subAgentDelegation, user, + instrumentation, } = options; const toolExecutor = this.secureRuntime.createToolExecutor(toolCodeByName); - const toolResolver = this.makeToolResolver(projectId); + const toolResolver = this.makeToolResolver(projectId, instrumentation); const resolvedTools: BuiltTool[] = []; // Transport for LLM calls const aiProxyFetch = createAiProxyFetch(this.outboundHttp); // Transport for MCP calls - const aiMcpFetch = createAiMcpFetch( - this.outboundHttp, - this.ssrfConfig, - this.ssrfProtectionService, - ); + const aiMcpFetch = + instrumentation?.mcpFetch ?? + createAiMcpFetch(this.outboundHttp, this.ssrfConfig, this.ssrfProtectionService); const buildMcpClient = async (server: AgentJsonMcpServerConfig) => await buildMcpClientForServer(server, { @@ -374,7 +379,10 @@ export class AgentRuntimeReconstructionService { buildMcpClient, resolveManagedEmbeddingProviderOptions: async () => await this.resolveManagedEmbeddingProviderOptions(projectId), - modelFetch: aiProxyFetch, + modelFetch: instrumentation?.modelFetch ?? aiProxyFetch, + fallbackWebSearch: instrumentation?.webSearch, + // Only the mock MCP transport makes attaching auth-pending servers safe. + attachAuthPendingMcpServers: instrumentation?.mcpFetch !== undefined, }); await this.injectRuntimeDependencies({ @@ -389,6 +397,7 @@ export class AgentRuntimeReconstructionService { integrationType, credentialIntegrations, user, + instrumentation, }); return { agent: reconstructed, toolRegistry: buildToolRegistry(resolvedTools) }; @@ -466,7 +475,11 @@ export class AgentRuntimeReconstructionService { }, }; } - private makeToolResolver(projectId: string): ToolResolver { + private makeToolResolver( + projectId: string, + instrumentation?: AgentRuntimeInstrumentation, + ): ToolResolver { + const instrumentToolAdditionalData = instrumentation?.configureToolAdditionalData; return async (ref: AgentJsonToolConfig) => { if (ref.type === 'workflow') { const { resolveWorkflowTool } = await import('./tools/workflow-tool-factory.js'); @@ -476,6 +489,7 @@ export class AgentRuntimeReconstructionService { activeExecutions: this.activeExecutions, projectId, webhookBaseUrl: this.urlService.getWebhookBaseUrl(), + instrumentToolAdditionalData, }); } @@ -484,6 +498,7 @@ export class AgentRuntimeReconstructionService { return await resolveNodeTool(ref, { executor: this.ephemeralNodeExecutor, projectId, + instrumentToolAdditionalData, }); } @@ -503,6 +518,7 @@ export class AgentRuntimeReconstructionService { integrationType?: string; credentialIntegrations: AgentIntegrationConfig[]; user?: User; + instrumentation?: AgentRuntimeInstrumentation; }): Promise { const { agent, @@ -516,6 +532,7 @@ export class AgentRuntimeReconstructionService { integrationType, credentialIntegrations, user, + instrumentation, } = params; agent.tool(createGetEnvironmentTool()); @@ -604,6 +621,7 @@ export class AgentRuntimeReconstructionService { credentialProvider, delegation: subAgentDelegation, user, + instrumentation, }); this.attachWriteTodosTool(agent, agentId); } @@ -623,9 +641,18 @@ export class AgentRuntimeReconstructionService { credentialProvider: CredentialProvider; delegation: SubAgentDelegationConfig; user?: User; + instrumentation?: AgentRuntimeInstrumentation; }): Promise { - const { agent, config, parentAgentId, projectId, credentialProvider, delegation, user } = - params; + const { + agent, + config, + parentAgentId, + projectId, + credentialProvider, + delegation, + user, + instrumentation, + } = params; const inlineSubAgentModelsByDifficulty = await this.resolveInlineSubAgentModelsByDifficulty( config, credentialProvider, @@ -638,6 +665,7 @@ export class AgentRuntimeReconstructionService { parentAgentId, credentialProvider, user, + instrumentation, policy: this.buildSubAgentPolicy(config), ...(inlineSubAgentModelsByDifficulty !== undefined ? { inlineSubAgentModelsByDifficulty } diff --git a/packages/cli/src/modules/agents/json-config/from-json-config.ts b/packages/cli/src/modules/agents/json-config/from-json-config.ts index 35eff397ea3..de2caa864cf 100644 --- a/packages/cli/src/modules/agents/json-config/from-json-config.ts +++ b/packages/cli/src/modules/agents/json-config/from-json-config.ts @@ -52,6 +52,12 @@ const WEB_SEARCH_INPUT_SCHEMA = z.object({ excludeDomains: z.array(z.string()).optional().describe('Exclude results from these domains'), }); +const WEB_SEARCH_PLAN_INSTRUCTION = + 'Before using web_search, choose the smallest search plan that can answer the user. Default to one broad, high-signal query. After each search, stop if the results already contain enough credible sources to answer. Use a second search only when the first result set is insufficient or the user asked for comparison across independent source categories. Do not fan out variations of the same query, and do not search for confirmation only. Use more than two searches only when the user explicitly asks for deep research, exhaustive coverage, or multiple independent topics.'; + +export type FallbackWebSearchArgs = z.infer; +export type FallbackWebSearchHandler = (args: FallbackWebSearchArgs) => Promise; + const WEB_SEARCH_POLICY_INSTRUCTION = '### Web search policy\n' + 'Use web search only on high-signal requests: explicit web/current/latest/live/recent/research/source requests, or questions that require up-to-date external facts. Do not use web search for static knowledge, uploaded knowledge, local config, codebase questions, or confirmation. Prefer answering directly or using local knowledge tools first. One search is usually enough; do not search repeatedly unless the user asks for deep research.'; @@ -103,6 +109,18 @@ export interface BuildFromJsonOptions { resolveManagedEmbeddingProviderOptions?: ManagedEmbeddingProviderOptionsResolver; /** Proxy-aware `fetch` for the agent's model calls (see `createAiProxyFetch`). */ modelFetch?: FetchFn; + /** + * Replaces the live Brave/SearXNG call behind the fallback `web_search` + * tool. When set, the tool is attached without requiring a search provider + * or credential in the config (eval instrumentation only). + */ + fallbackWebSearch?: FallbackWebSearchHandler; + /** + * Attach MCP servers whose credential is still pending instead of skipping + * them. Only safe when MCP traffic cannot reach the real server — set by + * the eval path when its mock MCP transport is injected. + */ + attachAuthPendingMcpServers?: boolean; } /** @@ -151,7 +169,12 @@ export async function buildFromJson( // requires auth, so treat either as an incomplete draft and skip // attaching it rather than risk a connection attempt. if (!server.url.trim()) continue; - if (server.authentication !== 'none' && !server.credential) continue; + if ( + server.authentication !== 'none' && + !server.credential && + !options.attachAuthPendingMcpServers + ) + continue; const client = await options.buildMcpClient(server); agent.mcp(client); @@ -178,7 +201,11 @@ export async function buildFromJson( agent.providerTool({ name: resolved as `${string}.${string}`, args }); } } - const fallbackWebSearchTool = buildFallbackWebSearchTool(config, options.credentialProvider); + const fallbackWebSearchTool = buildFallbackWebSearchTool( + config, + options.credentialProvider, + options.fallbackWebSearch, + ); if (fallbackWebSearchTool) { agent.tool(fallbackWebSearchTool); } @@ -272,11 +299,21 @@ export function buildProviderToolsForModel( function buildFallbackWebSearchTool( config: AgentJsonConfig, credentialProvider: CredentialProvider, + fallbackWebSearch?: FallbackWebSearchHandler, ): BuiltTool | null { const webSearchConfig = config.config?.webSearch; if (!webSearchConfig?.enabled) return null; if (isNativeWebSearchRequested(config) && hasNativeWebSearchProvider(config.model)) return null; + if (fallbackWebSearch) { + return { + name: WEB_SEARCH_TOOL_NAME, + description: 'Search the web for current information.', + systemInstruction: WEB_SEARCH_PLAN_INSTRUCTION, + inputSchema: WEB_SEARCH_INPUT_SCHEMA, + handler: async (input) => await fallbackWebSearch(WEB_SEARCH_INPUT_SCHEMA.parse(input)), + }; + } if (webSearchConfig.provider !== 'brave' && webSearchConfig.provider !== 'searxng') { throw new Error('Web search is enabled but no fallback search provider is configured.'); } @@ -288,8 +325,7 @@ function buildFallbackWebSearchTool( return { name: WEB_SEARCH_TOOL_NAME, description: 'Search the web for current information.', - systemInstruction: - 'Before using web_search, choose the smallest search plan that can answer the user. Default to one broad, high-signal query. After each search, stop if the results already contain enough credible sources to answer. Use a second search only when the first result set is insufficient or the user asked for comparison across independent source categories. Do not fan out variations of the same query, and do not search for confirmation only. Use more than two searches only when the user explicitly asks for deep research, exhaustive coverage, or multiple independent topics.', + systemInstruction: WEB_SEARCH_PLAN_INSTRUCTION, inputSchema: WEB_SEARCH_INPUT_SCHEMA, handler: async (input) => { const args = WEB_SEARCH_INPUT_SCHEMA.parse(input); diff --git a/packages/cli/src/modules/agents/sub-agents/sub-agent-foreground-runner.ts b/packages/cli/src/modules/agents/sub-agents/sub-agent-foreground-runner.ts index e3aae2d513f..e43e686e2ff 100644 --- a/packages/cli/src/modules/agents/sub-agents/sub-agent-foreground-runner.ts +++ b/packages/cli/src/modules/agents/sub-agents/sub-agent-foreground-runner.ts @@ -16,6 +16,7 @@ import { UserError } from 'n8n-workflow'; import { v4 as uuid } from 'uuid'; import { AgentExecutionService } from '../agent-execution.service'; +import type { AgentRuntimeInstrumentation } from '../agent-runtime-instrumentation'; import { buildAgentConfigurationTelemetryFromConfig } from '../agent-telemetry'; import type { MessageRecord } from '../execution-recorder'; import { ExecutionRecorder } from '../execution-recorder'; @@ -36,6 +37,12 @@ export interface SubAgentForegroundRunContext { * is a published/integration run. */ user?: User; + /** + * Runtime instrumentation of the delegating parent run. Threaded into the + * child's reconstruction so delegated runs share the parent's seams + * (model fetch, MCP fetch, tool execution contexts). + */ + instrumentation?: AgentRuntimeInstrumentation; } export interface SubAgentForegroundResult { @@ -88,8 +95,12 @@ export class SubAgentForegroundRunner { const resourceId = request.parentResourceId ?? threadId; const reconstructionService = await getReconstructionService(); + const childConfig = + context.instrumentation?.transformDelegatedAgentConfig?.(runtimeSource.source.config, { + subAgentId: runtimeSource.source.sourceId, + }) ?? runtimeSource.source.config; const { agent } = await reconstructionService.reconstructFromResolvedSource({ - config: runtimeSource.source.config, + config: childConfig, memoryOwnerAgentId: runtimeSource.source.sourceId, projectId: context.projectId, credentialProvider: context.credentialProvider, @@ -99,6 +110,7 @@ export class SubAgentForegroundRunner { runtimeProfile: 'sub-agent', parentAgentIdForDelegation: context.parentAgentId, user: context.user, + instrumentation: context.instrumentation, }); // Abort the child when the parent run is cancelled. diff --git a/packages/cli/src/modules/agents/tools/__tests__/node-tool-factory.test.ts b/packages/cli/src/modules/agents/tools/__tests__/node-tool-factory.test.ts index da04c1366c0..5060835744a 100644 --- a/packages/cli/src/modules/agents/tools/__tests__/node-tool-factory.test.ts +++ b/packages/cli/src/modules/agents/tools/__tests__/node-tool-factory.test.ts @@ -210,3 +210,45 @@ describe('resolveNodeTool → tool name sanitization', () => { ); }); }); + +describe('resolveNodeTool → eval instrumentation', () => { + it('binds the sanitized tool name into the ephemeral execution when instrumented', async () => { + const executeInline = vi.fn().mockResolvedValue({ status: 'success', data: [] }); + const instrumentToolAdditionalData = vi.fn(); + + const tool = await resolveNodeTool(baseToolSchema, { + executor: { executeInline } as unknown as EphemeralNodeExecutor, + projectId: 'p1', + instrumentToolAdditionalData, + }); + await tool.handler!({}, {} as never); + + const request = executeInline.mock.calls[0][0] as { + nodeName?: string; + configureAdditionalData?: (additionalData: object) => void; + }; + expect(request.nodeName).toBe('Google_Drive'); + expect(request.configureAdditionalData).toBeDefined(); + + const additionalData = {}; + request.configureAdditionalData!(additionalData); + expect(instrumentToolAdditionalData).toHaveBeenCalledWith(additionalData, { + toolName: 'Google_Drive', + toolKind: 'node', + }); + }); + + it('passes neither override when not instrumented', async () => { + const executeInline = vi.fn().mockResolvedValue({ status: 'success', data: [] }); + + const tool = await resolveNodeTool(baseToolSchema, { + executor: { executeInline } as unknown as EphemeralNodeExecutor, + projectId: 'p1', + }); + await tool.handler!({}, {} as never); + + const request = executeInline.mock.calls[0][0] as Record; + expect(request.nodeName).toBeUndefined(); + expect(request.configureAdditionalData).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/modules/agents/tools/__tests__/workflow-tool-factory.test.ts b/packages/cli/src/modules/agents/tools/__tests__/workflow-tool-factory.test.ts new file mode 100644 index 00000000000..832fab61346 --- /dev/null +++ b/packages/cli/src/modules/agents/tools/__tests__/workflow-tool-factory.test.ts @@ -0,0 +1,99 @@ +import type { WorkflowEntity } from '@n8n/db'; +import { Container } from '@n8n/di'; +import type { INode, IWorkflowExecutionDataProcess } from 'n8n-workflow'; + +import type { ActiveExecutions } from '@/active-executions'; +import { ExecutionPersistence } from '@/executions/execution-persistence'; +import type { WorkflowRunner } from '@/workflow-runner'; + +import { executeWorkflow, type WorkflowToolContext } from '../workflow-tool-factory'; + +const triggerNode: INode = { + id: 'trigger-1', + name: 'Manual Trigger', + type: 'n8n-nodes-base.manualTrigger', + typeVersion: 1, + position: [0, 0], + parameters: {}, +}; + +const workflow = { + id: 'wf-1', + name: 'Lookup workflow', + nodes: [triggerNode], + connections: {}, +} as unknown as WorkflowEntity; + +function buildContext(run: ReturnType, extras: Partial = {}) { + return { + workflowRepository: {} as never, + workflowRunner: { run } as unknown as WorkflowRunner, + activeExecutions: { has: vi.fn().mockReturnValue(false) } as unknown as ActiveExecutions, + projectId: 'p1', + ...extras, + } satisfies WorkflowToolContext; +} + +describe('executeWorkflow → eval instrumentation', () => { + beforeEach(() => { + Container.set(ExecutionPersistence, { + findSingleExecution: vi + .fn() + .mockResolvedValue({ status: 'success', data: { resultData: { runData: {} } } }), + } as unknown as ExecutionPersistence); + }); + + afterEach(() => { + Container.reset(); + }); + + it('sets configureAdditionalData on the run data when instrumented', async () => { + const run = vi.fn().mockResolvedValue('exec-1'); + const instrumentToolAdditionalData = vi.fn(); + + await executeWorkflow( + workflow, + triggerNode, + 'manual', + { input: 'hello' }, + buildContext(run, { instrumentToolAdditionalData }), + false, + 'Lookup_workflow', + ); + + const runData = run.mock.calls[0][0] as IWorkflowExecutionDataProcess; + expect(runData.configureAdditionalData).toBeDefined(); + + const additionalData = {}; + void runData.configureAdditionalData!(additionalData as never); + expect(instrumentToolAdditionalData).toHaveBeenCalledWith(additionalData, { + toolName: 'Lookup_workflow', + toolKind: 'workflow', + }); + }); + + it('leaves the run data untouched when not instrumented', async () => { + const run = vi.fn().mockResolvedValue('exec-1'); + + await executeWorkflow(workflow, triggerNode, 'manual', {}, buildContext(run), false); + + const runData = run.mock.calls[0][0] as IWorkflowExecutionDataProcess; + expect(runData.configureAdditionalData).toBeUndefined(); + }); + + it('leaves the run data untouched when instrumented but no tool name is bound', async () => { + const run = vi.fn().mockResolvedValue('exec-1'); + + await executeWorkflow( + workflow, + triggerNode, + 'manual', + {}, + buildContext(run, { instrumentToolAdditionalData: vi.fn() }), + false, + ); + + const runData = run.mock.calls[0][0] as IWorkflowExecutionDataProcess; + expect(runData.configureAdditionalData).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/modules/agents/tools/node-tool-factory.ts b/packages/cli/src/modules/agents/tools/node-tool-factory.ts index 27758751597..de538f2dee5 100644 --- a/packages/cli/src/modules/agents/tools/node-tool-factory.ts +++ b/packages/cli/src/modules/agents/tools/node-tool-factory.ts @@ -4,18 +4,22 @@ import { createZodSchemaFromArgs, extractFromAIParameters } from '@n8n/ai-utilit import type { AgentJsonToolConfig } from '@n8n/api-types'; import { Container } from '@n8n/di'; import type { JSONSchema7 } from 'json-schema'; -import type { IDataObject, INodeParameters } from 'n8n-workflow'; +import type { IDataObject, INodeParameters, IWorkflowExecuteAdditionalData } from 'n8n-workflow'; import { isToolType, nodeNameToToolName } from 'n8n-workflow'; import { z } from 'zod'; import type { EphemeralNodeExecutor } from '@/node-execution'; import { NodeTypes } from '@/node-types'; +import type { InstrumentToolAdditionalData } from '../agent-runtime-instrumentation'; + type NodeToolInputSchema = JSONSchema7 | z.ZodType; export interface NodeToolFactoryContext { executor: EphemeralNodeExecutor; projectId: string; + /** Eval-only additionalData decoration — absent on every production path. */ + instrumentToolAdditionalData?: InstrumentToolAdditionalData; } /** @@ -136,6 +140,18 @@ export async function resolveNodeTool( const sanitizedName = nodeNameToToolName(toolSchema.name); const nodeType = resolveToolNodeType(toolSchema.node.nodeType, toolSchema.node.nodeTypeVersion); + // When instrumented (eval runs), the ephemeral node is named after the tool + // so hint lookup and request attribution key on the same identifier the + // model calls. + const instrument = ctx.instrumentToolAdditionalData; + const instrumentedExecution = instrument + ? { + nodeName: sanitizedName, + configureAdditionalData: (additionalData: IWorkflowExecuteAdditionalData) => + instrument(additionalData, { toolName: sanitizedName, toolKind: 'node' }), + } + : {}; + const built = new Tool(sanitizedName) .description(toolSchema.description ?? `Execute the ${nodeType} node`) .input(await resolveInputSchema(toolSchema, ctx)) @@ -147,6 +163,7 @@ export async function resolveNodeTool( credentialDetails: toExecutorCredentials(toolSchema.node.credentials), inputData: [{ json: input as IDataObject }], projectId: ctx.projectId, + ...instrumentedExecution, }); // Throw on the executor's structured error so the agent runtime // flags the tool-result with `isError: true` and the recorder diff --git a/packages/cli/src/modules/agents/tools/workflow-tool-factory.ts b/packages/cli/src/modules/agents/tools/workflow-tool-factory.ts index c66f4f7bc47..f5a5e0a190f 100644 --- a/packages/cli/src/modules/agents/tools/workflow-tool-factory.ts +++ b/packages/cli/src/modules/agents/tools/workflow-tool-factory.ts @@ -32,8 +32,9 @@ import type { ActiveExecutions } from '@/active-executions'; import { ExecutionPersistence } from '@/executions/execution-persistence'; import type { WorkflowRunner } from '@/workflow-runner'; -import { findWorkflowToolWorkflow } from './workflow-tool-workflow-resolver'; +import type { InstrumentToolAdditionalData } from '../agent-runtime-instrumentation'; import { sanitizeToolName } from '../json-config/agent-config-composition'; +import { findWorkflowToolWorkflow } from './workflow-tool-workflow-resolver'; // --------------------------------------------------------------------------- // Constants @@ -83,6 +84,8 @@ export interface WorkflowToolContext { projectId: string; /** Base URL for webhooks/forms (e.g. http://localhost:5678/) */ webhookBaseUrl?: string; + /** Eval-only additionalData decoration for the sub-execution — absent on every production path. */ + instrumentToolAdditionalData?: InstrumentToolAdditionalData; } // --------------------------------------------------------------------------- @@ -270,6 +273,8 @@ export async function executeWorkflow( inputData: Record, context: WorkflowToolContext, allOutputs = false, + /** Sanitized tool name for eval instrumentation; set only on instrumented runs. */ + instrumentedToolName?: string, ): Promise<{ executionId: string; status: string; @@ -313,6 +318,16 @@ export async function executeWorkflow( }), }; + // Eval runs decorate the sub-execution's additionalData (HTTP mock handler, + // mocked credentials helper). The closure does not survive queue + // serialization — eval callers refuse queue mode upfront. + const instrument = context.instrumentToolAdditionalData; + if (instrument && instrumentedToolName) { + runData.configureAdditionalData = (additionalData) => { + instrument(additionalData, { toolName: instrumentedToolName, toolKind: 'workflow' }); + }; + } + const responsePromise = createDeferredPromise(); let webhookResponse: IExecuteResponsePromiseData | undefined; void responsePromise.promise @@ -613,7 +628,15 @@ async function buildWorkflowTool( }), ) .handler(async (input: Record) => { - return await executeWorkflow(workflow, triggerNode, triggerType, input, context, allOutputs); + return await executeWorkflow( + workflow, + triggerNode, + triggerType, + input, + context, + allOutputs, + toolName, + ); }); const built = builder.build(); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts index 8329314737c..e6267674d25 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts @@ -63,6 +63,7 @@ import type { ProjectService } from '@/services/project.service.ee'; import type { UrlService } from '@/services/url.service'; import type { InstanceAiBrowserSessionService } from '../browser/instance-ai-browser-session.service'; +import type { EvalAgentExecutionService } from '../eval/agent-execution.service'; import type { EvalExecutionService } from '../eval/execution.service'; import { EvalThreadCredentialAllowlistService } from '../eval/thread-credential-allowlist.service'; import type { EvalThreadRestoreService } from '../eval/thread-restore.service'; @@ -123,6 +124,7 @@ describe('InstanceAiController', () => { memoryService, settingsService, mock(), + mock(), evalCredentialAllowlists, evalThreadRestore, eventBus, @@ -1680,6 +1682,7 @@ describe('InstanceAiController — durable-log SSE replay (flag on)', () => { memoryService, settingsService, mock(), + mock(), new EvalThreadCredentialAllowlistService(), mock(), eventBus, diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/agent-execution.service.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/agent-execution.service.test.ts new file mode 100644 index 00000000000..110728b2825 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/agent-execution.service.test.ts @@ -0,0 +1,540 @@ +import type { GenerateResult } from '@n8n/agents'; +import type { AgentJsonConfig } from '@n8n/api-types'; +import type { Logger, ModuleRegistry } from '@n8n/backend-common'; +import type { OutboundHttp } from '@n8n/backend-network'; +import type { ExecutionsConfig } from '@n8n/config'; +import type { User } from '@n8n/db'; +import { Container } from '@n8n/di'; +import type { EvalLlmMockHandler } from 'n8n-core'; +import type { INode } from 'n8n-workflow'; +import { mock } from 'vitest-mock-extended'; + +import type { CredentialsService } from '@/credentials/credentials.service'; +import { AgentRuntimeReconstructionService } from '@/modules/agents/agent-runtime-reconstruction.service'; +import type { Agent as AgentEntity } from '@/modules/agents/entities/agent.entity'; +import { AgentRepository } from '@/modules/agents/repositories/agent.repository'; +import { userHasScopes } from '@/permissions.ee/check-access'; + +import { + EvalAgentExecutionService, + mcpUrlsMatch, + pruneConfigForEval, + summarizeTools, +} from '../agent-execution.service'; +import { generateAgentScenarioSeed } from '../agent-scenario-seed'; +import { createLlmMockHandler } from '../mock-handler'; + +// The service resolves the agents-module surface lazily; top-level vi.mock +// keeps those specifiers from loading the real module graph (and avoids the +// doMock+dynamic-import concurrency flake). The static imports above resolve +// to these factories. +vi.mock('@/modules/agents/repositories/agent.repository', () => ({ + AgentRepository: class AgentRepository {}, +})); +vi.mock('@/modules/agents/agent-runtime-reconstruction.service', () => ({ + AgentRuntimeReconstructionService: class AgentRuntimeReconstructionService {}, +})); +vi.mock('@/modules/agents/utils/agent-credential-provider', () => ({ + createAgentCredentialProvider: vi.fn(() => ({ resolve: vi.fn() })), +})); +vi.mock('@/modules/agents/json-config/agent-config-composition', () => ({ + sanitizeToolName: (name: string) => name.replace(/[^a-zA-Z0-9_-]+/g, '_'), +})); +vi.mock('@/permissions.ee/check-access', () => ({ userHasScopes: vi.fn() })); +vi.mock('@/utils/ai-proxy-fetch', () => ({ createAiProxyFetch: vi.fn(() => vi.fn()) })); +vi.mock('../agent-scenario-seed', () => ({ generateAgentScenarioSeed: vi.fn() })); +vi.mock('../mock-handler', () => ({ createLlmMockHandler: vi.fn() })); + +const logger = mock(); +const user = mock(); + +const findByIdAndProjectId = vi.fn(); +const reconstructFromAgentEntity = vi.fn(); + +const baseConfig: AgentJsonConfig = { + name: 'Support agent', + model: 'openai/gpt-5', + credential: 'cred-1', + instructions: 'Help with support tickets.', + tools: [ + { + type: 'node', + name: 'Slack Tool', + node: { nodeType: 'n8n-nodes-base.slackTool', nodeTypeVersion: 1, nodeParameters: {} }, + }, + ], +} as unknown as AgentJsonConfig; + +const makeEntity = (config: AgentJsonConfig, integrations: unknown[] = []): AgentEntity => + ({ + id: 'agent-1', + projectId: 'proj-1', + schema: config, + tools: { + 't-1': { code: 'export default 1', descriptor: { name: 'my_custom', description: 'Custom' } }, + }, + skills: {}, + integrations, + }) as unknown as AgentEntity; + +const seed = { + openingMessage: 'Hi, I need help with order #123', + globalContext: 'Customer jane@example.com', + toolHints: { Slack_Tool: 'Posting succeeds' }, + warnings: [], +}; + +const makeGenerateResult = (overrides: Partial = {}): GenerateResult => + ({ + runId: 'run-1', + messages: [ + { role: 'user', content: [{ type: 'text', text: seed.openingMessage }] }, + { role: 'assistant', content: [{ type: 'text', text: 'All sorted!' }] }, + ], + toolCalls: [{ tool: 'Slack_Tool', input: { text: 'hi' }, output: { ok: true } }], + usage: { promptTokens: 100, completionTokens: 40, totalTokens: 140 }, + model: 'openai/gpt-5', + finishReason: 'stop', + getState: vi.fn(), + ...overrides, + }) as unknown as GenerateResult; + +function buildService(overrides: { queueMode?: boolean; agentsActive?: boolean } = {}) { + const executionsConfig = mock(); + Object.defineProperty(executionsConfig, 'mode', { + get: () => (overrides.queueMode ? 'queue' : 'regular'), + }); + const moduleRegistry = mock(); + moduleRegistry.isActive.mockReturnValue(overrides.agentsActive ?? true); + return new EvalAgentExecutionService( + logger, + executionsConfig, + moduleRegistry, + mock(), + mock(), + ); +} + +describe('EvalAgentExecutionService.executeWithLlmMock', () => { + beforeEach(() => { + vi.clearAllMocks(); + Container.set(AgentRepository, { findByIdAndProjectId } as unknown as AgentRepository); + Container.set(AgentRuntimeReconstructionService, { + reconstructFromAgentEntity, + } as unknown as AgentRuntimeReconstructionService); + vi.mocked(userHasScopes).mockResolvedValue(true); + vi.mocked(generateAgentScenarioSeed).mockResolvedValue(seed); + vi.mocked(createLlmMockHandler).mockReturnValue( + vi.fn().mockResolvedValue({ body: { ok: true }, statusCode: 200, headers: {} }), + ); + findByIdAndProjectId.mockResolvedValue(makeEntity({ ...baseConfig })); + }); + + afterEach(() => { + Container.reset(); + }); + + const request = { projectId: 'proj-1' }; + + it('refuses queue mode upfront', async () => { + const result = await buildService({ queueMode: true }).executeWithLlmMock( + 'agent-1', + user, + request, + ); + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/queue mode is not supported/); + expect(findByIdAndProjectId).not.toHaveBeenCalled(); + }); + + it('refuses when the agents module is inactive', async () => { + const result = await buildService({ agentsActive: false }).executeWithLlmMock( + 'agent-1', + user, + request, + ); + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/agents module/); + }); + + it('reports not-accessible when the user lacks agent:execute', async () => { + vi.mocked(userHasScopes).mockResolvedValue(false); + const result = await buildService().executeWithLlmMock('agent-1', user, request); + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/not found or not accessible/); + }); + + it('reports not-found when the agent does not exist in the project', async () => { + findByIdAndProjectId.mockResolvedValue(null); + const result = await buildService().executeWithLlmMock('agent-1', user, request); + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/not found or not accessible/); + }); + + it('reports a config-less agent', async () => { + findByIdAndProjectId.mockResolvedValue(makeEntity(undefined as unknown as AgentJsonConfig)); + const result = await buildService().executeWithLlmMock('agent-1', user, request); + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/no JSON config/); + }); + + it('surfaces seed-generation failure as a FRAMEWORK ISSUE', async () => { + vi.mocked(generateAgentScenarioSeed).mockRejectedValue( + new Error('FRAMEWORK ISSUE: agent scenario seed generation failed — no message'), + ); + const result = await buildService().executeWithLlmMock('agent-1', user, request); + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/^FRAMEWORK ISSUE:/); + expect(reconstructFromAgentEntity).not.toHaveBeenCalled(); + }); + + it('runs the happy path: real-model turn, mocked tool HTTP, mapped result', async () => { + const innerCredentialsHelper = { getDecrypted: vi.fn() }; + const close = vi.fn().mockResolvedValue(undefined); + const generate = vi.fn().mockImplementation(async () => { + // Simulate a node-tool invocation mid-run: the instrumentation must + // swap the credentials helper and serve HTTP through the mock. + const instrumentation = reconstructFromAgentEntity.mock.calls[0][4] as { + modelFetch?: unknown; + configureToolAdditionalData: ( + additionalData: Record, + ctx: { toolName: string; toolKind: string }, + ) => void; + }; + const additionalData: Record = { + credentialsHelper: innerCredentialsHelper, + }; + instrumentation.configureToolAdditionalData(additionalData, { + toolName: 'Slack_Tool', + toolKind: 'node', + }); + expect(additionalData.credentialsHelper).not.toBe(innerCredentialsHelper); + const handler = additionalData.evalLlmMockHandler as EvalLlmMockHandler; + await handler( + { url: 'https://slack.com/api/chat.postMessage', method: 'POST', body: { text: 'hi' } }, + { name: 'Slack_Tool', type: 'n8n-nodes-base.slackTool' } as INode, + ); + return makeGenerateResult(); + }); + reconstructFromAgentEntity.mockResolvedValue({ agent: { generate, close }, toolRegistry: {} }); + + const result = await buildService().executeWithLlmMock('agent-1', user, { + ...request, + scenarioHints: 'Customer asks about order #123', + }); + + expect(result.success).toBe(true); + expect(result.runId).toBe('run-1'); + expect(result.finalText).toBe('All sorted!'); + expect(result.model).toBe('openai/gpt-5'); + expect(result.usage).toEqual({ inputTokens: 100, outputTokens: 40 }); + expect(result.seed).toEqual(seed); + expect(result.toolCalls).toHaveLength(1); + expect(result.toolCalls[0]).toMatchObject({ + tool: 'Slack_Tool', + kind: 'node', + mocked: true, + }); + expect(result.toolCalls[0].interceptedRequests[0]).toMatchObject({ + url: 'https://slack.com/api/chat.postMessage', + method: 'POST', + nodeType: 'n8n-nodes-base.slackTool', + mockResponse: { ok: true }, + }); + expect(close).toHaveBeenCalledTimes(1); + + // The runtime was built with the eval instrumentation, uncached. + const [entityArg, , integrationType, userArg, instrumentation] = reconstructFromAgentEntity.mock + .calls[0] as [AgentEntity, unknown, string | undefined, User, { modelFetch?: unknown }]; + expect(entityArg.id).toBe('agent-1'); + expect(integrationType).toBeUndefined(); + expect(userArg).toBe(user); + expect(instrumentation.modelFetch).toBeDefined(); + }); + + it('prunes unmockable features and reports them', async () => { + const fullConfig = { + ...baseConfig, + memory: { enabled: true }, + vectorStores: [{ provider: 'pinecone' }], + mcpServers: [{ name: 'mcp-1', url: 'https://legacy.example/sse', transport: 'sse' }], + subAgents: { agents: [{ agentId: 'child-1' }] }, + } as unknown as AgentJsonConfig; + findByIdAndProjectId.mockResolvedValue(makeEntity(fullConfig, [{ type: 'slack' }])); + reconstructFromAgentEntity.mockResolvedValue({ + agent: { generate: vi.fn().mockResolvedValue(makeGenerateResult()), close: vi.fn() }, + toolRegistry: {}, + }); + + const result = await buildService().executeWithLlmMock('agent-1', user, request); + + expect(result.skippedFeatures.map((entry) => entry.feature).sort()).toEqual([ + 'integrations', + 'mcpServers (sse transport)', + 'memory', + 'vectorStores', + ]); + const rebuiltEntity = reconstructFromAgentEntity.mock.calls[0][0] as AgentEntity; + expect(rebuiltEntity.schema?.memory).toBeUndefined(); + expect(rebuiltEntity.schema?.mcpServers).toBeUndefined(); + // Configured sub-agents are kept — the delegated child inherits the + // instrumentation and its config is pruned via the transform hook. + expect(rebuiltEntity.schema?.subAgents).toEqual({ agents: [{ agentId: 'child-1' }] }); + }); + + it('serves fallback web search and configured sub-agents through the instrumentation', async () => { + const fullConfig = { + ...baseConfig, + model: 'mistral/mistral-large', + config: { webSearch: { enabled: true, provider: 'brave', credential: 'cred-2' } }, + subAgents: { agents: [{ agentId: 'child-1' }] }, + } as unknown as AgentJsonConfig; + findByIdAndProjectId.mockResolvedValue(makeEntity(fullConfig)); + reconstructFromAgentEntity.mockResolvedValue({ + agent: { generate: vi.fn().mockResolvedValue(makeGenerateResult()), close: vi.fn() }, + toolRegistry: {}, + }); + + const result = await buildService().executeWithLlmMock('agent-1', user, request); + + const instrumentation = reconstructFromAgentEntity.mock.calls[0][4] as { + webSearch?: unknown; + transformDelegatedAgentConfig?: ( + config: AgentJsonConfig, + context: { subAgentId: string }, + ) => AgentJsonConfig; + }; + expect(instrumentation.webSearch).toBeDefined(); + expect(result.skippedFeatures).toEqual([]); + + // Delegated child configs get the same pruning, reported under the child id + // (the returned skippedFeatures array is live for the duration of the run). + const childConfig = { + ...baseConfig, + memory: { enabled: true }, + } as unknown as AgentJsonConfig; + const prunedChild = instrumentation.transformDelegatedAgentConfig?.(childConfig, { + subAgentId: 'child-1', + }); + expect(prunedChild?.memory).toBeUndefined(); + expect(result.skippedFeatures.map((entry) => entry.feature)).toContain( + 'subAgent child-1: memory', + ); + }); + + it('surfaces errored tool calls omitted from the run result via the intercepted ledger', async () => { + // Node tools throw on failure and the SDK drops them from + // GenerateResult.toolCalls — the intercepted request (e.g. an injected + // channel_not_found) must still reach the judge. + const generate = vi.fn().mockImplementation(async () => { + const instrumentation = reconstructFromAgentEntity.mock.calls[0][4] as { + configureToolAdditionalData: ( + additionalData: Record, + ctx: { toolName: string; toolKind: string }, + ) => void; + }; + const additionalData: Record = { credentialsHelper: {} }; + instrumentation.configureToolAdditionalData(additionalData, { + toolName: 'Slack_Tool', + toolKind: 'node', + }); + const handler = additionalData.evalLlmMockHandler as EvalLlmMockHandler; + await handler({ url: 'https://slack.com/api/chat.postMessage', method: 'POST' }, { + name: 'Slack_Tool', + type: 'n8n-nodes-base.slackTool', + } as INode); + return makeGenerateResult({ toolCalls: [] }); + }); + reconstructFromAgentEntity.mockResolvedValue({ + agent: { generate, close: vi.fn() }, + toolRegistry: {}, + }); + + const result = await buildService().executeWithLlmMock('agent-1', user, request); + + expect(result.toolCalls).toHaveLength(1); + expect(result.toolCalls[0]).toMatchObject({ + tool: 'Slack_Tool', + kind: 'node', + mocked: true, + error: expect.stringContaining('interceptedRequests'), + }); + expect(result.toolCalls[0].interceptedRequests).toHaveLength(1); + }); + + it('auto-approves suspended tool calls and flags them', async () => { + const suspended = makeGenerateResult({ + pendingSuspend: [ + { + runId: 'run-1', + toolCallId: 'tc-1', + toolName: 'Danger_Tool', + input: {}, + suspendPayload: {}, + }, + ], + }); + const finished = makeGenerateResult({ + toolCalls: [{ tool: 'Danger_Tool', input: {}, output: { done: true } }], + }); + const generate = vi.fn().mockResolvedValue(suspended); + const approve = vi.fn().mockResolvedValue(finished); + reconstructFromAgentEntity.mockResolvedValue({ + agent: { generate, approve, close: vi.fn() }, + toolRegistry: {}, + }); + + const result = await buildService().executeWithLlmMock('agent-1', user, request); + + expect(approve).toHaveBeenCalledWith( + 'generate', + expect.objectContaining({ runId: 'run-1', toolCallId: 'tc-1' }), + ); + expect(result.success).toBe(true); + // Pre-suspension segment calls are preserved alongside the approved one. + expect(result.toolCalls.find((call) => call.tool === 'Danger_Tool')).toMatchObject({ + autoApproved: true, + }); + }); + + it('maps a thrown run failure into an error result and still closes the runtime', async () => { + const close = vi.fn().mockResolvedValue(undefined); + reconstructFromAgentEntity.mockResolvedValue({ + agent: { generate: vi.fn().mockRejectedValue(new Error('aborted')), close }, + toolRegistry: {}, + }); + + const result = await buildService().executeWithLlmMock('agent-1', user, request); + + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/Agent run failed: aborted/); + expect(close).toHaveBeenCalledTimes(1); + }); + + it('flips success off when the run finishes with a model error', async () => { + reconstructFromAgentEntity.mockResolvedValue({ + agent: { + generate: vi + .fn() + .mockResolvedValue( + makeGenerateResult({ error: new Error('rate limited'), finishReason: 'error' }), + ), + close: vi.fn(), + }, + toolRegistry: {}, + }); + + const result = await buildService().executeWithLlmMock('agent-1', user, request); + + expect(result.success).toBe(false); + expect(result.errors[0]).toMatch(/Model run error: rate limited/); + }); +}); + +describe('pruneConfigForEval', () => { + it('leaves a plain tools-only config untouched', () => { + const { config, skippedFeatures } = pruneConfigForEval({ ...baseConfig }); + expect(skippedFeatures).toEqual([]); + expect(config.tools).toEqual(baseConfig.tools); + }); + + it('keeps streamable-HTTP MCP servers and strips only SSE-transport ones', () => { + const withMcp = { + ...baseConfig, + mcpServers: [ + { name: 'acme_kb', url: 'https://mcp.acme-kb.example/mcp', transport: 'streamableHttp' }, + { name: 'legacy', url: 'https://legacy.example/sse', transport: 'sse' }, + ], + } as unknown as AgentJsonConfig; + + const { config, skippedFeatures } = pruneConfigForEval(withMcp); + + expect(config.mcpServers?.map((server) => server.name)).toEqual(['acme_kb']); + expect(skippedFeatures).toHaveLength(1); + expect(skippedFeatures[0].feature).toContain('sse'); + expect(skippedFeatures[0].reason).toContain('legacy'); + }); + + it('keeps fallback web search — the web-search mock serves it', () => { + const withSearch = { + ...baseConfig, + model: 'mistral/mistral-large', + config: { webSearch: { enabled: true, provider: 'brave', credential: 'cred-2' } }, + } as unknown as AgentJsonConfig; + + const { config, skippedFeatures } = pruneConfigForEval(withSearch); + + expect(skippedFeatures).toHaveLength(0); + expect(config.config?.webSearch?.enabled).toBe(true); + }); +}); + +describe('mcpUrlsMatch', () => { + it.each([ + ['https://mcp.notion.com/mcp', 'https://mcp.notion.com/mcp', true], + ['https://mcp.notion.com/mcp/', 'https://mcp.notion.com/mcp', true], + ['https://mcp.notion.com/mcp/v2', 'https://mcp.notion.com/mcp', true], + ['https://mcp.notion.com/mcp-two', 'https://mcp.notion.com/mcp', false], + ['https://mcp.acme.com', 'https://mcp.acme.com.evil.example/mcp', false], + ['https://mcp.linear.app/mcp', 'https://mcp.notion.com/mcp', false], + ])('%s vs %s -> %s', (configUrl, remoteUrl, expected) => { + expect(mcpUrlsMatch(configUrl, remoteUrl)).toBe(expected); + expect(mcpUrlsMatch(remoteUrl, configUrl)).toBe(expected); + }); +}); + +describe('summarizeTools', () => { + it('summarizes the fallback web_search tool so the seed can steer it', () => { + const config = { + ...baseConfig, + model: 'mistral/mistral-large', + config: { webSearch: { enabled: true, provider: 'brave', credential: 'cred-2' } }, + } as unknown as AgentJsonConfig; + + const summaries = summarizeTools(config, {}, (name) => name); + + expect(summaries.find((entry) => entry.name === 'web_search')?.kind).toBe('other'); + }); + + it('summarizes MCP servers by server name with kind mcp', () => { + const config = { + ...baseConfig, + tools: [], + mcpServers: [ + { name: 'acme_kb', url: 'https://mcp.acme-kb.example/mcp', description: 'KB server' }, + ], + } as unknown as AgentJsonConfig; + + const summaries = summarizeTools(config, {} as never, (name) => name); + expect(summaries).toEqual([{ name: 'acme_kb', kind: 'mcp', description: 'KB server' }]); + }); + + it('maps node, workflow, and custom tools with sanitized names', () => { + const config = { + ...baseConfig, + tools: [ + ...(baseConfig.tools ?? []), + { type: 'workflow', workflow: 'Order lookup', description: 'Find an order' }, + { type: 'custom', id: 't-1' }, + ], + } as unknown as AgentJsonConfig; + const customTools = { + 't-1': { code: '', descriptor: { name: 'my_custom', description: 'Custom' } }, + } as unknown as NonNullable; + + const summaries = summarizeTools(config, customTools, (name) => + name.replace(/[^a-zA-Z0-9_-]+/g, '_'), + ); + + expect(summaries).toEqual([ + { + name: 'Slack_Tool', + kind: 'node', + description: undefined, + nodeType: 'n8n-nodes-base.slackTool', + }, + { name: 'Order_lookup', kind: 'workflow', description: 'Find an order' }, + { name: 'my_custom', kind: 'custom', description: 'Custom' }, + ]); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/agent-model-turn-recorder.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/agent-model-turn-recorder.test.ts new file mode 100644 index 00000000000..526d98c8ab4 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/agent-model-turn-recorder.test.ts @@ -0,0 +1,111 @@ +import type { FetchFn } from '@n8n/agents'; +import type { Logger } from '@n8n/backend-common'; +import { mock } from 'vitest-mock-extended'; + +import { createAgentModelTurnRecorder } from '../agent-model-turn-recorder'; + +const logger = mock(); + +describe('createAgentModelTurnRecorder', () => { + it('passes the response through and records the turn', async () => { + const baseFetch: FetchFn = vi.fn().mockResolvedValue( + new Response('{"choices":[{"message":{"content":"hi"}}]}', { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + const recorder = createAgentModelTurnRecorder(baseFetch, logger); + + const response = await recorder.fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + body: JSON.stringify({ model: 'gpt-x', messages: [] }), + }); + + // The live consumer still reads the full body. + await expect(response.json()).resolves.toEqual({ + choices: [{ message: { content: 'hi' } }], + }); + + await recorder.flush(); + expect(recorder.turns).toHaveLength(1); + const turn = recorder.turns[0]; + expect(turn.provider).toBe('openai'); + expect(turn.status).toBe(200); + expect(turn.streamed).toBe(false); + expect(turn.durationMs).toBeGreaterThanOrEqual(0); + expect(turn.requestBody).toEqual({ model: 'gpt-x', messages: [] }); + expect(turn.responseBody).toContain('choices'); + }); + + it('never records secret-looking request fields', async () => { + const baseFetch: FetchFn = vi.fn().mockResolvedValue(new Response('{}', { status: 200 })); + const recorder = createAgentModelTurnRecorder(baseFetch, logger); + + await recorder.fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + body: JSON.stringify({ api_key: 'sk-super-secret', messages: [] }), + }); + + await recorder.flush(); + expect(JSON.stringify(recorder.turns[0].requestBody)).not.toContain('sk-super-secret'); + }); + + it('marks event-stream responses as streamed and tees the body', async () => { + const sse = 'data: {"type":"content_block_delta"}\n\ndata: [DONE]\n\n'; + const baseFetch: FetchFn = vi + .fn() + .mockResolvedValue( + new Response(sse, { status: 200, headers: { 'content-type': 'text/event-stream' } }), + ); + const recorder = createAgentModelTurnRecorder(baseFetch, logger); + + const response = await recorder.fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + }); + await expect(response.text()).resolves.toBe(sse); + + await recorder.flush(); + expect(recorder.turns[0].streamed).toBe(true); + expect(recorder.turns[0].responseBody).toContain('content_block_delta'); + }); + + it('records and rethrows transport errors', async () => { + const baseFetch: FetchFn = vi.fn().mockRejectedValue(new Error('ECONNRESET')); + const recorder = createAgentModelTurnRecorder(baseFetch, logger); + + await expect(recorder.fetch('https://api.openai.com/v1/chat/completions')).rejects.toThrow( + 'ECONNRESET', + ); + expect(recorder.turns[0].error).toBe('ECONNRESET'); + expect(recorder.turns[0].status).toBeUndefined(); + }); + + it('keeps a truncated string when an oversized request body no longer parses as JSON', async () => { + const baseFetch: FetchFn = vi.fn().mockResolvedValue(new Response('{}', { status: 200 })); + const recorder = createAgentModelTurnRecorder(baseFetch, logger); + + await recorder.fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + body: JSON.stringify({ + model: 'gpt-x', + messages: [{ role: 'user', content: 'y'.repeat(20_000) }], + }), + }); + + const recorded = recorder.turns[0].requestBody; + expect(typeof recorded).toBe('string'); + expect(recorded).toContain('gpt-x'); + }); + + it('truncates oversized recorded bodies', async () => { + const huge = 'x'.repeat(50_000); + const baseFetch: FetchFn = vi.fn().mockResolvedValue(new Response(huge, { status: 200 })); + const recorder = createAgentModelTurnRecorder(baseFetch, logger); + + await recorder.fetch('https://api.openai.com/v1/chat/completions'); + await recorder.flush(); + + const recorded = recorder.turns[0].responseBody as string; + expect(recorded.length).toBeLessThan(20_000); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/agent-scenario-seed.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/agent-scenario-seed.test.ts new file mode 100644 index 00000000000..c1d9e199769 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/agent-scenario-seed.test.ts @@ -0,0 +1,88 @@ +import { generateAgentScenarioSeed } from '../agent-scenario-seed'; + +const generate = vi.fn(); +const extractText = vi.fn(); + +vi.mock('@n8n/instance-ai', () => ({ + createEvalAgent: vi.fn(() => ({ generate })), + extractText: (result: unknown) => extractText(result) as string, +})); + +const baseOptions = { + agentName: 'Support agent', + instructions: 'You help with support tickets.', + tools: [ + { name: 'Slack_Tool', kind: 'node' as const, nodeType: 'n8n-nodes-base.slackTool' }, + { name: 'Lookup_workflow', kind: 'workflow' as const, description: 'Find a customer' }, + ], + scenarioHints: 'The customer jane@example.com asks about order #123.', +}; + +const validSeedJson = JSON.stringify({ + openingMessage: 'Hi, I need help with order #123', + globalContext: 'Customer jane@example.com, order #123', + toolHints: { Slack_Tool: 'Message posts succeed', Lookup_workflow: 'Returns jane' }, +}); + +describe('generateAgentScenarioSeed', () => { + beforeEach(() => { + vi.clearAllMocks(); + generate.mockResolvedValue({ messages: [] }); + }); + + it('parses a valid seed (including fenced JSON)', async () => { + extractText.mockReturnValue('```json\n' + validSeedJson + '\n```'); + + const seed = await generateAgentScenarioSeed(baseOptions); + + expect(seed.openingMessage).toBe('Hi, I need help with order #123'); + expect(seed.globalContext).toContain('jane@example.com'); + expect(seed.toolHints).toEqual({ + Slack_Tool: 'Message posts succeed', + Lookup_workflow: 'Returns jane', + }); + expect(seed.warnings).toEqual([]); + }); + + it('includes the scenario, tool names, and date anchors in the prompt', async () => { + extractText.mockReturnValue(validSeedJson); + + await generateAgentScenarioSeed(baseOptions); + + const prompt = generate.mock.calls[0][0] as string; + expect(prompt).toContain('## Test Scenario'); + expect(prompt).toContain('jane@example.com asks about order #123'); + expect(prompt).toContain('Slack_Tool'); + expect(prompt).toContain('Lookup_workflow'); + expect(prompt).toContain('## Date anchors'); + }); + + it('retries once when the first attempt has no usable openingMessage', async () => { + extractText + .mockReturnValueOnce(JSON.stringify({ globalContext: 'no message here' })) + .mockReturnValueOnce(validSeedJson); + + const seed = await generateAgentScenarioSeed(baseOptions); + + expect(generate).toHaveBeenCalledTimes(2); + expect(seed.openingMessage).toBe('Hi, I need help with order #123'); + expect(seed.warnings).toHaveLength(1); + }); + + it('throws a FRAMEWORK ISSUE when both attempts fail', async () => { + extractText.mockReturnValue('not json at all'); + + await expect(generateAgentScenarioSeed(baseOptions)).rejects.toThrow(/^FRAMEWORK ISSUE:/); + expect(generate).toHaveBeenCalledTimes(2); + }); + + it('surfaces LLM call failures through the retry path', async () => { + generate.mockRejectedValueOnce(new Error('provider timeout')); + extractText.mockReturnValue(validSeedJson); + + const seed = await generateAgentScenarioSeed(baseOptions); + + expect(seed.openingMessage).toBe('Hi, I need help with order #123'); + expect(seed.warnings[0]).toContain('provider timeout'); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/mcp-mock-fetch.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/mcp-mock-fetch.test.ts new file mode 100644 index 00000000000..fb4b4e35182 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/mcp-mock-fetch.test.ts @@ -0,0 +1,233 @@ +import type { Logger } from '@n8n/backend-common'; +import { mock } from 'vitest-mock-extended'; + +import { createMcpMockFetch, type McpMockToolCall } from '../mcp-mock-fetch'; + +const generate = vi.fn(); +const extractText = vi.fn(); + +vi.mock('@n8n/instance-ai', () => ({ + createEvalAgent: vi.fn(() => ({ generate })), + extractText: (result: unknown) => extractText(result) as string, +})); + +const logger = mock(); + +const toolsListJson = JSON.stringify({ + tools: [ + { + name: 'search_articles', + description: 'Search knowledge-base articles', + inputSchema: { + type: 'object', + properties: { query: { type: 'string' } }, + required: ['query'], + }, + }, + ], +}); + +function buildFetch(overrides: { onToolCall?: (call: McpMockToolCall) => void } = {}) { + return createMcpMockFetch({ + servers: [{ name: 'acme_kb', url: 'https://mcp.acme-kb.example/mcp' }], + agentInstructions: 'Answer product questions from the acme knowledge base.', + scenarioHints: 'The KB contains an article saying the export limit is 10k rows.', + globalContext: 'Acme product docs', + serverHints: { acme_kb: 'Articles about export limits' }, + onToolCall: overrides.onToolCall ?? vi.fn(), + logger, + }); +} + +function rpc(method: string, params?: unknown, id: number | null = 1) { + return { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', ...(id === null ? {} : { id }), method, params }), + }; +} + +describe('createMcpMockFetch', () => { + beforeEach(() => { + vi.clearAllMocks(); + generate.mockResolvedValue({ messages: [] }); + }); + + it('answers the initialize handshake deterministically with a session id', async () => { + const fetchFn = buildFetch(); + const res = await fetchFn('https://mcp.acme-kb.example/mcp', { + ...rpc('initialize', { + protocolVersion: '2025-06-18', + capabilities: {}, + clientInfo: { name: '@n8n/agents', version: '0.1.0' }, + }), + }); + + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toContain('application/json'); + expect(res.headers.get('mcp-session-id')).toBeTruthy(); + const body = (await res.json()) as { + result: { protocolVersion: string; serverInfo: { name: string } }; + }; + expect(body.result.protocolVersion).toBe('2025-06-18'); + expect(body.result.serverInfo.name).toBe('acme_kb'); + expect(generate).not.toHaveBeenCalled(); + }); + + it('accepts notifications with 202 and rejects GET with 405', async () => { + const fetchFn = buildFetch(); + const notification = await fetchFn('https://mcp.acme-kb.example/mcp', { + ...rpc('notifications/initialized', {}, null), + }); + expect(notification.status).toBe(202); + + const get = await fetchFn('https://mcp.acme-kb.example/mcp', { method: 'GET' }); + expect(get.status).toBe(405); + }); + + it('serves an LLM-designed tool catalog and caches it per server', async () => { + extractText.mockReturnValue(toolsListJson); + const fetchFn = buildFetch(); + + const first = await fetchFn('https://mcp.acme-kb.example/mcp', { ...rpc('tools/list') }); + const body = (await first.json()) as { result: { tools: Array<{ name: string }> } }; + expect(body.result.tools.map((tool) => tool.name)).toEqual(['search_articles']); + + await fetchFn('https://mcp.acme-kb.example/mcp', { ...rpc('tools/list', undefined, 2) }); + expect(generate).toHaveBeenCalledTimes(1); + }); + + it('generates tool-call results, records them, and caches identical calls', async () => { + extractText + .mockReturnValueOnce(toolsListJson) + .mockReturnValue(JSON.stringify({ text: 'The export limit is 10k rows (article KB-42).' })); + const calls: McpMockToolCall[] = []; + const fetchFn = buildFetch({ onToolCall: (call) => calls.push(call) }); + + const res = await fetchFn('https://mcp.acme-kb.example/mcp', { + ...rpc('tools/call', { name: 'search_articles', arguments: { query: 'export limit' } }), + }); + const body = (await res.json()) as { + result: { content: Array<{ type: string; text: string }>; isError?: boolean }; + }; + expect(body.result.content[0].text).toContain('10k rows'); + expect(body.result.isError).toBeUndefined(); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ serverName: 'acme_kb', toolName: 'search_articles' }); + + // Identical call → cached generation, but still recorded per call. + await fetchFn('https://mcp.acme-kb.example/mcp', { + ...rpc('tools/call', { name: 'search_articles', arguments: { query: 'export limit' } }, 3), + }); + expect(calls).toHaveLength(2); + expect(generate).toHaveBeenCalledTimes(2); // 1 list + 1 call + }); + + it('returns isError results when the scenario mandates a failure', async () => { + extractText + .mockReturnValueOnce(toolsListJson) + .mockReturnValue(JSON.stringify({ isError: true, text: 'article not found' })); + const fetchFn = buildFetch(); + + const res = await fetchFn('https://mcp.acme-kb.example/mcp', { + ...rpc('tools/call', { name: 'search_articles', arguments: { query: 'missing' } }), + }); + const body = (await res.json()) as { result: { isError?: boolean } }; + expect(body.result.isError).toBe(true); + }); + + it('degrades to a generic fallback tool when catalog generation fails twice', async () => { + extractText.mockReturnValue('not json'); + const fetchFn = buildFetch(); + + const res = await fetchFn('https://mcp.acme-kb.example/mcp', { ...rpc('tools/list') }); + const body = (await res.json()) as { result: { tools: Array<{ name: string }> } }; + expect(body.result.tools.map((tool) => tool.name)).toEqual(['query']); + }); + + it('pins the catalog to canonical registry tools regardless of what the generator returns', async () => { + // Generator returns one matching name (schema kept) and one invented + // name (dropped) — the served catalog must be exactly the canonical set. + extractText.mockReturnValue( + JSON.stringify({ + tools: [ + { + name: 'notion-search', + description: 'invented description', + inputSchema: { + type: 'object', + properties: { query: { type: 'string' } }, + required: ['query'], + }, + }, + { name: 'made_up_tool', description: 'nope', inputSchema: { type: 'object' } }, + ], + }), + ); + const fetchFn = createMcpMockFetch({ + servers: [{ name: 'notion', url: 'https://mcp.notion.com/mcp' }], + agentInstructions: 'Save notes to Notion.', + knownToolsByServer: { + notion: [ + { name: 'notion-search', description: 'Search Notion and connected sources' }, + { name: 'notion-create-pages', description: 'Create pages in Markdown' }, + ], + }, + onToolCall: vi.fn(), + logger, + }); + + const res = await fetchFn('https://mcp.notion.com/mcp', { ...rpc('tools/list') }); + const body = (await res.json()) as { + result: { tools: Array<{ name: string; description: string; inputSchema: unknown }> }; + }; + + expect(body.result.tools.map((tool) => tool.name)).toEqual([ + 'notion-search', + 'notion-create-pages', + ]); + // Canonical description wins; the matching generated schema is kept. + expect(body.result.tools[0].description).toBe('Search Notion and connected sources'); + expect(body.result.tools[0].inputSchema).toMatchObject({ required: ['query'] }); + // The unmatched canonical tool gets a permissive fallback schema. + expect(body.result.tools[1].inputSchema).toMatchObject({ type: 'object' }); + }); + + it('resolves the server on a path boundary, not by bare prefix', async () => { + extractText.mockReturnValue(toolsListJson); + const fetchFn = createMcpMockFetch({ + servers: [ + { name: 'kb', url: 'https://mcp.example.com/mcp' }, + { name: 'kb_two', url: 'https://mcp.example.com/mcp-two' }, + ], + agentInstructions: 'irrelevant', + onToolCall: vi.fn(), + logger, + }); + + const res = await fetchFn('https://mcp.example.com/mcp-two', { + ...rpc('initialize', { protocolVersion: '2025-06-18', capabilities: {}, clientInfo: {} }), + }); + const body = (await res.json()) as { result: { serverInfo: { name: string } } }; + expect(body.result.serverInfo.name).toBe('kb_two'); + }); + + it('answers an unparseable body with a JSON-RPC parse error', async () => { + const fetchFn = buildFetch(); + const res = await fetchFn('https://mcp.acme-kb.example/mcp', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: 'not json {', + }); + const body = (await res.json()) as { id: unknown; error: { code: number } }; + expect(body.error.code).toBe(-32700); + expect(body.id).toBeNull(); + }); + + it('answers unknown methods with a JSON-RPC method-not-found error', async () => { + const fetchFn = buildFetch(); + const res = await fetchFn('https://mcp.acme-kb.example/mcp', { ...rpc('tasks/create') }); + const body = (await res.json()) as { error: { code: number } }; + expect(body.error.code).toBe(-32601); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/request-sanitizer.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/request-sanitizer.test.ts index 1169bedb7c6..6b161d19ea4 100644 --- a/packages/cli/src/modules/instance-ai/eval/__tests__/request-sanitizer.test.ts +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/request-sanitizer.test.ts @@ -1,7 +1,12 @@ import { Logger } from '@n8n/backend-common'; import { Container } from '@n8n/di'; -import { isSecretKey, redactSecretKeys, truncateForLlm } from '../request-sanitizer'; +import { + isSecretKey, + redactSecretKeys, + redactSecretValuePatterns, + truncateForLlm, +} from '../request-sanitizer'; // --------------------------------------------------------------------------- // Mock logger to verify warning behavior @@ -19,6 +24,36 @@ beforeEach(() => { // --------------------------------------------------------------------------- // isSecretKey // --------------------------------------------------------------------------- +describe('redactSecretValuePatterns', () => { + it.each([ + ['an OpenAI key', 'my key is sk-proj-abcdefghijklmnopqrstuvwx ok', /sk-proj/], + ['a Slack token', 'use xoxb-1234567890-abcdef please', /xoxb-/], + [ + 'an uppercase bearer value', + 'Authorization: Bearer abcdefghijklmnopqrstuv', + /abcdefghijklmnop/, + ], + [ + 'a lowercase bearer value', + 'authorization: bearer abcdefghijklmnopqrstuv', + /abcdefghijklmnop/, + ], + ['a prose api_key assignment', 'set api_key=supersecretvalue and retry', /supersecretvalue/], + ['a spaced API key phrase', 'my Notion API key: ntn9x8y7z6w5v4 thanks', /ntn9x8y7z6w5v4/], + ['a prose password assignment', 'password: hunter22222 works', /hunter22222/], + ['a JSON-in-string secret', 'config is {"client_secret": "abc123def456"}', /abc123def456/], + ])('scrubs %s', (_name, input, leaked) => { + const result = redactSecretValuePatterns(input); + expect(result).not.toMatch(leaked); + expect(result).toContain(''); + }); + + it('leaves ordinary prose untouched', () => { + const text = 'The secret to good soup is patience; keywords stay, and skis are fine.'; + expect(redactSecretValuePatterns(text)).toBe(text); + }); +}); + describe('isSecretKey', () => { describe('should classify secret keys', () => { const secretKeys = [ diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/web-search-mock.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/web-search-mock.test.ts new file mode 100644 index 00000000000..3011774626f --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/web-search-mock.test.ts @@ -0,0 +1,104 @@ +import type { Logger } from '@n8n/backend-common'; +import { mock } from 'vitest-mock-extended'; + +import type { WebSearchMockArgs, WebSearchMockResult } from '../web-search-mock'; +import { createWebSearchMock } from '../web-search-mock'; + +const generate = vi.fn(); +const extractText = vi.fn(); + +vi.mock('@n8n/instance-ai', () => ({ + createEvalAgent: vi.fn(() => ({ generate })), + extractText: (result: unknown) => extractText(result) as string, +})); + +const logger = mock(); + +const resultsJson = JSON.stringify({ + results: [ + { + title: 'Export limits — Acme docs', + url: 'https://docs.acme.example/limits', + snippet: 'The export limit is 10k rows.', + }, + { + title: 'Acme changelog', + url: 'https://acme.example/changelog', + snippet: 'Limits unchanged in v2.', + }, + ], +}); + +function buildMock( + onSearch: (args: WebSearchMockArgs, result: WebSearchMockResult) => void = vi.fn(), +) { + return createWebSearchMock({ + agentInstructions: 'Answer product questions, searching the web when needed.', + scenarioHints: 'Search results must state the export limit is 10k rows.', + searchHint: 'Docs pages about Acme export limits', + onSearch, + logger, + }); +} + +describe('createWebSearchMock', () => { + beforeEach(() => { + vi.clearAllMocks(); + generate.mockResolvedValue({ messages: [] }); + }); + + it('returns generated results in the WebSearchResponse shape and records the call', async () => { + extractText.mockReturnValue(resultsJson); + const calls: Array<{ args: WebSearchMockArgs; result: WebSearchMockResult }> = []; + const search = buildMock((args, result) => calls.push({ args, result })); + + const result = await search({ query: 'acme export limit' }); + + expect(result.query).toBe('acme export limit'); + expect(result.results).toHaveLength(2); + expect(result.results[0].snippet).toContain('10k rows'); + expect(calls).toHaveLength(1); + expect(calls[0].args.query).toBe('acme export limit'); + }); + + it('caps results at maxResults and caches identical queries', async () => { + extractText.mockReturnValue(resultsJson); + const search = buildMock(); + + const first = await search({ query: 'acme export limit', maxResults: 1 }); + expect(first.results).toHaveLength(1); + + await search({ query: 'acme export limit', maxResults: 1 }); + expect(generate).toHaveBeenCalledTimes(1); + }); + + it('enforces domain filters but never empties a non-empty result set', async () => { + extractText.mockReturnValue( + JSON.stringify({ + results: [ + { title: 'On-domain', url: 'https://docs.acme.example/a', snippet: 'keep' }, + { title: 'Off-domain', url: 'https://other.example/b', snippet: 'drop' }, + ], + }), + ); + const search = buildMock(); + + const filtered = await search({ query: 'q1', includeDomains: ['docs.acme.example'] }); + expect(filtered.results.map((hit) => hit.title)).toEqual(['On-domain']); + + // The agent restricted to a domain the mock never generated — serve the + // unfiltered results rather than starving the scenario. + const fallback = await search({ query: 'q2', includeDomains: ['unrelated.example'] }); + expect(fallback.results).toHaveLength(2); + }); + + it('degrades to an empty result set with a framework-issue note when generation fails', async () => { + extractText.mockReturnValue('not json'); + const search = buildMock(); + + const result = await search({ query: 'anything' }); + + expect(result.results).toEqual([]); + expect(result.note).toContain('framework issue'); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/agent-execution.service.ts b/packages/cli/src/modules/instance-ai/eval/agent-execution.service.ts new file mode 100644 index 00000000000..51a37655197 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/agent-execution.service.ts @@ -0,0 +1,683 @@ +import type { GenerateResult, Agent as RuntimeAgent } from '@n8n/agents'; +import { + hasNativeWebSearchProvider, + isNativeWebSearchRequested, +} from '@n8n/ai-utilities/agent-config'; +import type { + AgentJsonConfig, + InstanceAiEvalAgentExecutionRequest, + InstanceAiEvalAgentExecutionResult, + InstanceAiEvalAgentScenarioSeed, + InstanceAiEvalAgentSkippedFeature, + InstanceAiEvalAgentToolCallRecord, + InstanceAiEvalInterceptedRequest, +} from '@n8n/api-types'; +import { Logger, ModuleRegistry } from '@n8n/backend-common'; +import { OutboundHttp } from '@n8n/backend-network'; +import { ExecutionsConfig } from '@n8n/config'; +import type { User } from '@n8n/db'; +import { Container, Service } from '@n8n/di'; +import type { EvalLlmMockHandler } from 'n8n-core'; +import { nodeNameToToolName } from 'n8n-workflow'; + +import { CredentialsService } from '@/credentials/credentials.service'; +// Static agents-module imports are safe here: the ModuleRegistry gate decides +// availability at runtime. +import { AgentRuntimeReconstructionService } from '@/modules/agents/agent-runtime-reconstruction.service'; +import type { Agent as AgentEntity } from '@/modules/agents/entities/agent.entity'; +import { sanitizeToolName } from '@/modules/agents/json-config/agent-config-composition'; +import { AgentRepository } from '@/modules/agents/repositories/agent.repository'; +import { createAgentCredentialProvider } from '@/modules/agents/utils/agent-credential-provider'; +import { McpRegistryService } from '@/modules/mcp-registry/registry/mcp-registry.service'; +import { userHasScopes } from '@/permissions.ee/check-access'; +import { createAiProxyFetch } from '@/utils/ai-proxy-fetch'; + +import { createAgentModelTurnRecorder } from './agent-model-turn-recorder'; +import { generateAgentScenarioSeed, type AgentSeedToolSummary } from './agent-scenario-seed'; +import { EvalMockedCredentialsHelper } from './eval-mocked-credentials-helper'; +import { createMcpMockFetch, type McpMockCanonicalTool } from './mcp-mock-fetch'; +import { createLlmMockHandler } from './mock-handler'; +import { truncateForLlm } from './request-sanitizer'; +import { createWebSearchMock } from './web-search-mock'; + +// --------------------------------------------------------------------------- +// Runs a built first-class Agent for ONE scenario turn: +// - the agent's own model call is REAL (builder-chosen model, keys from the +// seeded credentials) and recorded via a passthrough fetch; +// - node-tool and workflow-tool HTTP is intercepted at the same engine +// choke points the workflow eval uses (`evalLlmMockHandler` on the tools' +// additionalData) with LLM-generated responses; +// - MCP servers, the fallback web_search tool, and configured sub-agents +// are served by dedicated mocks/seams (see mcp-mock-fetch, web-search-mock +// and AgentRuntimeInstrumentation); features the mock layer can't serve +// yet (memory, vector stores, SSE-transport MCP, chat integrations) are +// pruned from a config copy and reported as `skippedFeatures`. +// The runtime is built UNCACHED (never through AgentRuntimeCacheService) so an +// instrumented runtime can never leak into a normal chat run. +// --------------------------------------------------------------------------- + +const DEFAULT_TIMEOUT_MS = 600_000; +const DEFAULT_MAX_ITERATIONS = 25; +const MAX_ITERATIONS_CAP = 40; +const MAX_AUTO_APPROVALS = 20; +const MAX_RECORDED_TOOL_VALUE_CHARS = 4_000; + +// Keyed by the runtime tool name so entries merge with GenerateResult's tool +// calls; a name collision across kinds (e.g. an MCP `_` equal to +// a node tool's sanitized name) merges into one record — accepted, the +// interceptedRequests still disambiguate. +type ToolLedger = Map; + +/** + * One URL is the other or a path-boundary prefix of it — bare prefix overlap + * (`…/mcp` vs `…/mcp-two`) must not match. + */ +export function mcpUrlsMatch(configUrl: string, remoteUrl: string): boolean { + const shorter = configUrl.length <= remoteUrl.length ? configUrl : remoteUrl; + const longer = shorter === configUrl ? remoteUrl : configUrl; + const base = shorter.replace(/\/+$/, ''); + return longer === shorter || longer === base || longer.startsWith(`${base}/`); +} + +@Service() +export class EvalAgentExecutionService { + constructor( + private readonly logger: Logger, + private readonly executionsConfig: ExecutionsConfig, + private readonly moduleRegistry: ModuleRegistry, + private readonly outboundHttp: OutboundHttp, + private readonly credentialsService: CredentialsService, + ) {} + + async executeWithLlmMock( + agentId: string, + user: User, + options: InstanceAiEvalAgentExecutionRequest, + ): Promise { + // Workflow-tool sub-executions carry a configureAdditionalData closure + // that doesn't survive queue serialization — refuse upfront so tool + // HTTP can't leak to real services. + if (this.executionsConfig.mode === 'queue') { + return this.errorResult( + 'Agent eval execution requires main process mode — queue mode is not supported.', + ); + } + + if (!this.moduleRegistry.isActive('agents')) { + return this.errorResult('Agent eval execution requires the agents module to be active.'); + } + + const { projectId } = options; + if (!(await userHasScopes(user, ['agent:execute'], false, { projectId }))) { + return this.errorResult(`Agent ${agentId} not found or not accessible`); + } + + const agentEntity = await Container.get(AgentRepository).findByIdAndProjectId( + agentId, + projectId, + ); + if (!agentEntity) { + return this.errorResult(`Agent ${agentId} not found or not accessible`); + } + if (!agentEntity.schema) { + return this.errorResult(`Agent ${agentId} has no JSON config to run.`); + } + + const { config, skippedFeatures } = pruneConfigForEval(agentEntity.schema); + if ((agentEntity.integrations ?? []).length > 0) { + skippedFeatures.push({ + feature: 'integrations', + reason: + 'Chat integrations are not attached in eval runs — the harness drives the agent directly.', + }); + } + // The entity is detached (never saved back); mutate the copy the + // runtime is built from so pruning applies to reconstruction too. + agentEntity.schema = config; + agentEntity.integrations = []; + + const toolSummaries = summarizeTools(config, agentEntity.tools ?? {}, sanitizeToolName); + + let seed: InstanceAiEvalAgentScenarioSeed; + try { + seed = await generateAgentScenarioSeed({ + agentName: config.name, + instructions: config.instructions, + tools: toolSummaries, + scenarioHints: options.scenarioHints, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return this.errorResult( + message.startsWith('FRAMEWORK ISSUE:') ? message : `FRAMEWORK ISSUE: ${message}`, + ); + } + + const mockHandler = createLlmMockHandler({ + scenarioHints: options.scenarioHints, + globalContext: seed.globalContext, + nodeHints: seed.toolHints, + }); + + const toolLedger: ToolLedger = new Map(); + const credentialHelpers: EvalMockedCredentialsHelper[] = []; + const recorder = createAgentModelTurnRecorder( + createAiProxyFetch(this.outboundHttp), + this.logger, + ); + + // Mock MCP server; ledger keys use the client-side `_` + // names so entries merge with GenerateResult's tool calls. Always + // created — delegated sub-agents may bring their own servers. + const mcpServers = config.mcpServers ?? []; + const knownToolsByServer = await this.resolveCanonicalMcpCatalogs(mcpServers); + const mcpFetch = createMcpMockFetch({ + servers: mcpServers.map((server) => ({ + name: server.name, + url: server.url, + description: server.description, + })), + agentInstructions: config.instructions, + scenarioHints: options.scenarioHints, + globalContext: seed.globalContext, + serverHints: seed.toolHints, + knownToolsByServer, + logger: this.logger, + onToolCall: (call) => { + const key = `${call.serverName}_${call.toolName}`; + let entries = toolLedger.get(key); + if (!entries) { + entries = []; + toolLedger.set(key, entries); + } + entries.push({ + url: mcpServers.find((server) => server.name === call.serverName)?.url ?? call.serverName, + method: 'POST', + nodeType: `mcp:${call.serverName}`, + requestBody: call.args, + mockResponse: call.result, + }); + }, + }); + + // Fallback web_search mock — always created (sub-agents may enable web + // search); with native search the tool is never built and this goes unused. + const webSearchMock = createWebSearchMock({ + agentInstructions: config.instructions, + scenarioHints: options.scenarioHints, + globalContext: seed.globalContext, + searchHint: seed.toolHints?.web_search, + logger: this.logger, + onSearch: (args, result) => { + let entries = toolLedger.get('web_search'); + if (!entries) { + entries = []; + toolLedger.set('web_search', entries); + } + entries.push({ + url: 'mock://web-search', + method: 'POST', + nodeType: 'web-search:fallback', + requestBody: args, + mockResponse: result, + }); + }, + }); + + const reconstruction = Container.get(AgentRuntimeReconstructionService); + const credentialProvider = createAgentCredentialProvider( + this.credentialsService, + projectId, + user, + ); + + let agent: RuntimeAgent; + try { + ({ agent } = await reconstruction.reconstructFromAgentEntity( + agentEntity, + credentialProvider, + undefined, + user, + { + modelFetch: recorder.fetch, + mcpFetch, + webSearch: webSearchMock, + // Delegated (configured) sub-agents inherit every seam above; their + // configs get the same pruning, reported under the child's id. + transformDelegatedAgentConfig: (childConfig, delegationContext) => { + const pruned = pruneConfigForEval(childConfig); + skippedFeatures.push( + ...pruned.skippedFeatures.map((skip) => ({ + ...skip, + feature: `subAgent ${delegationContext.subAgentId}: ${skip.feature}`, + })), + ); + return pruned.config; + }, + configureToolAdditionalData: (additionalData, toolContext) => { + const helper = new EvalMockedCredentialsHelper( + additionalData.credentialsHelper, + undefined, + this.logger, + ); + credentialHelpers.push(helper); + additionalData.credentialsHelper = helper; + additionalData.evalLlmMockHandler = this.createRecordingMockHandler( + mockHandler, + toolContext.toolName, + toolLedger, + ); + }, + }, + )); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return this.errorResult(`Failed to build agent runtime: ${message}`, seed, skippedFeatures); + } + + const errors: string[] = []; + // GenerateResult.toolCalls carry no toolCallId — approvals are attributed + // by tool name, enough to flag auto-approval for the judge. + const autoApprovedToolNames = new Set(); + const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const maxIterations = Math.min( + config.config?.maxIterations ?? DEFAULT_MAX_ITERATIONS, + MAX_ITERATIONS_CAP, + ); + + let result: GenerateResult; + // Each resume returns its own GenerateResult segment — collect tool + // calls across all of them so pre-suspension calls aren't lost. + const segmentToolCalls: NonNullable = []; + const collectToolCalls = (segment: GenerateResult) => { + for (const entry of segment.toolCalls ?? []) { + if (!segmentToolCalls.includes(entry)) segmentToolCalls.push(entry); + } + }; + try { + const abortSignal = AbortSignal.timeout(timeoutMs); + result = await agent.generate(seed.openingMessage, { abortSignal, maxIterations }); + collectToolCalls(result); + + // Approval-gated tools suspend the run. In real usage the user + // approves in the UI; the eval happy-path stands in for them, and + // flags every call it approved. + let approvals = 0; + while (approvals < MAX_AUTO_APPROVALS) { + const pending = result.pendingSuspend?.[0]; + if (!pending) break; + autoApprovedToolNames.add(pending.toolName); + approvals++; + result = await agent.approve('generate', { + runId: pending.runId, + toolCallId: pending.toolCallId, + abortSignal, + maxIterations, + }); + collectToolCalls(result); + } + if ((result.pendingSuspend?.length ?? 0) > 0) { + errors.push(`Run still suspended after ${MAX_AUTO_APPROVALS} auto-approvals`); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + // Flush so the failure result carries the recorded response bodies too. + await recorder.flush(); + return this.errorResult(`Agent run failed: ${message}`, seed, skippedFeatures, { + modelTurns: recorder.turns, + toolLedger, + credentialHelpers, + }); + } finally { + await recorder.flush(); + try { + await agent.close(); + } catch (error) { + this.logger.warn('[EvalAgentMock] Agent runtime teardown failed', { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (result.error !== undefined) { + errors.push( + `Model run error: ${result.error instanceof Error ? result.error.message : String(result.error)}`, + ); + } + + const kindByToolName = new Map(toolSummaries.map((tool) => [tool.name, tool.kind])); + // MCP summaries are keyed by SERVER name; the model calls the client-side + // prefixed `_` names. Sub-agent-brought servers aren't in + // this agent's summaries — their ledger entries carry an `mcp:` nodeType. + const kindForTool = (tool: string): InstanceAiEvalAgentToolCallRecord['kind'] => + kindByToolName.get(tool) ?? + (mcpServers.some((server) => tool.startsWith(`${server.name}_`)) || + (toolLedger.get(tool) ?? []).some((request) => request.nodeType?.startsWith('mcp:')) + ? 'mcp' + : 'other'); + const toolCalls = segmentToolCalls.map((entry): InstanceAiEvalAgentToolCallRecord => { + const interceptedRequests = toolLedger.get(entry.tool) ?? []; + return { + tool: entry.tool, + kind: kindForTool(entry.tool), + input: truncateRecordedValue(entry.input), + output: truncateRecordedValue(entry.output), + ...(entry.canceled ? { error: 'canceled' } : {}), + mocked: interceptedRequests.length > 0, + interceptedRequests, + ...(autoApprovedToolNames.has(entry.tool) ? { autoApproved: true } : {}), + }; + }); + // Ledger entries with no matching toolCalls record are either errored + // calls (GenerateResult omits those) or a delegated sub-agent's calls + // (never reported on the parent) — surface them without asserting + // failure so the judge can weigh the intercepted traffic itself. + const reportedTools = new Set(toolCalls.map((entry) => entry.tool)); + for (const [tool, interceptedRequests] of toolLedger) { + if (reportedTools.has(tool)) continue; + toolCalls.push({ + tool, + kind: kindForTool(tool), + error: + 'Not attributed to a reported tool call (a delegated sub-agent call, or an errored call) — see interceptedRequests', + mocked: interceptedRequests.length > 0, + interceptedRequests, + ...(autoApprovedToolNames.has(tool) ? { autoApproved: true } : {}), + }); + } + + return { + runId: result.runId, + success: errors.length === 0 && result.finishReason !== 'error', + errors, + finalText: extractFinalAssistantText(result), + model: result.model, + finishReason: result.finishReason, + toolCalls, + modelTurns: recorder.turns, + ...(result.usage + ? { + usage: { + inputTokens: result.usage.promptTokens, + outputTokens: result.usage.completionTokens, + }, + } + : {}), + seed, + skippedFeatures, + mockedCredentials: credentialHelpers.flatMap((helper) => helper.mockedCredentials), + }; + } + + /** + * Resolve canonical tool catalogs for configured MCP servers, so the mock + * exposes the tools the real server would: registry entries carry the + * server's declared catalog (matched by remote URL), and an allow-mode + * toolFilter pins names even without a registry match. Best-effort — + * without a match the mock's LLM-designed catalog applies. + */ + private async resolveCanonicalMcpCatalogs( + mcpServers: NonNullable, + ): Promise | undefined> { + if (mcpServers.length === 0) return undefined; + const result: Record = {}; + + if (this.moduleRegistry.isActive('mcp-registry')) { + try { + const entries = await Container.get(McpRegistryService).getAll(); + for (const server of mcpServers) { + const entry = entries.find((candidate) => + candidate.remotes.some((remote) => mcpUrlsMatch(server.url, remote.url)), + ); + if (entry && entry.tools.length > 0) { + result[server.name] = entry.tools.map((tool) => ({ + name: tool.name, + description: tool.title ?? tool.name, + })); + } + } + } catch (error) { + this.logger.debug( + `[EvalAgentMock] MCP registry catalog lookup failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + for (const server of mcpServers) { + if (result[server.name]) continue; + if (server.toolFilter?.mode === 'allow' && server.toolFilter.tools.length > 0) { + result[server.name] = server.toolFilter.tools.map((name) => ({ + name, + description: name, + })); + } + } + + return Object.keys(result).length > 0 ? result : undefined; + } + + /** + * Wraps the shared mock handler to record intercepted requests against the + * owning tool — the agent-eval analog of the workflow eval's + * `createInterceptingHandler`, keyed by tool instead of node. + */ + private createRecordingMockHandler( + mockHandler: EvalLlmMockHandler, + toolName: string, + ledger: ToolLedger, + ): EvalLlmMockHandler { + return async (requestOptions, node) => { + const response = await mockHandler(requestOptions, node); + let entries = ledger.get(toolName); + if (!entries) { + entries = []; + ledger.set(toolName, entries); + } + entries.push({ + url: requestOptions.url ?? '(no URL)', + method: requestOptions.method ?? 'GET', + nodeType: node.type, + requestBody: requestOptions.body, + mockResponse: response?.body, + }); + this.logger.debug( + `[EvalAgentMock] Intercepted ${requestOptions.method ?? 'GET'} ${requestOptions.url} from tool "${toolName}" (${node.type})`, + ); + return response; + }; + } + + private errorResult( + message: string, + seed?: InstanceAiEvalAgentScenarioSeed, + skippedFeatures: InstanceAiEvalAgentSkippedFeature[] = [], + partial?: { + modelTurns: InstanceAiEvalAgentExecutionResult['modelTurns']; + toolLedger: ToolLedger; + credentialHelpers: EvalMockedCredentialsHelper[]; + }, + ): InstanceAiEvalAgentExecutionResult { + this.logger.error(`[EvalAgentMock] ${message}`); + return { + runId: '', + success: false, + errors: [message], + finalText: '', + toolCalls: partial + ? [...partial.toolLedger.entries()].map(([tool, interceptedRequests]) => ({ + tool, + kind: salvagedToolKind(interceptedRequests), + mocked: interceptedRequests.length > 0, + interceptedRequests, + })) + : [], + modelTurns: partial?.modelTurns ?? [], + seed: seed ?? { openingMessage: '', globalContext: '', toolHints: {}, warnings: [] }, + skippedFeatures, + mockedCredentials: (partial?.credentialHelpers ?? []).flatMap( + (helper) => helper.mockedCredentials, + ), + }; + } +} + +// --------------------------------------------------------------------------- +// Config pruning +// --------------------------------------------------------------------------- + +/** + * Strip the config features the eval mock layer can't serve yet. Each removal + * is reported so the judge (and anyone debugging) can see the run diverged + * from the authored config, and why. + */ +export function pruneConfigForEval(original: AgentJsonConfig): { + config: AgentJsonConfig; + skippedFeatures: InstanceAiEvalAgentSkippedFeature[]; +} { + const skippedFeatures: InstanceAiEvalAgentSkippedFeature[] = []; + const config: AgentJsonConfig = { ...original }; + + if (config.memory?.enabled) { + skippedFeatures.push({ + feature: 'memory', + reason: + 'Observational/episodic memory worker models and embeddings are not mockable yet — memory is disabled for the run.', + }); + config.memory = undefined; + } + + if ((config.vectorStores?.length ?? 0) > 0) { + skippedFeatures.push({ + feature: 'vectorStores', + reason: 'Vector stores use native DB clients the mock layer cannot intercept.', + }); + config.vectorStores = undefined; + } + + // Streamable-HTTP MCP servers are served by the mock MCP fetch; only the + // non-default SSE transport (stateful long-lived stream) stays unmockable. + const sseServers = (config.mcpServers ?? []).filter((server) => server.transport === 'sse'); + if (sseServers.length > 0) { + skippedFeatures.push({ + feature: 'mcpServers (sse transport)', + reason: `SSE-transport MCP servers are not mockable yet (streamable-HTTP ones are): ${sseServers.map((server) => server.name).join(', ')}.`, + }); + const remaining = (config.mcpServers ?? []).filter((server) => server.transport !== 'sse'); + config.mcpServers = remaining.length > 0 ? remaining : undefined; + } + + // Configured sub-agents stay: the delegated child inherits the run's + // instrumentation and its config is pruned by the same rules via + // `transformDelegatedAgentConfig`. Web search stays too: the native + // variant runs inside the real model call and the fallback tool is served + // by the web-search mock. + + return { config, skippedFeatures }; +} + +// --------------------------------------------------------------------------- +// Tool summaries + result shaping +// --------------------------------------------------------------------------- + +/** + * Sanitized tool names must match what the factories register (and therefore + * what `GenerateResult.toolCalls` reports): `nodeNameToToolName` for node + * tools, `sanitizeToolName` for workflow tools, the stored descriptor name + * for custom tools. + */ +/** + * Best-effort kind for a salvaged ledger entry on the failure path, where the + * summary map is gone: MCP and web-search entries are self-describing via + * nodeType; anything else with a nodeType came through the node/workflow + * engine seam. + */ +function salvagedToolKind( + interceptedRequests: InstanceAiEvalInterceptedRequest[], +): InstanceAiEvalAgentToolCallRecord['kind'] { + const nodeType = interceptedRequests.find((request) => request.nodeType)?.nodeType; + if (!nodeType) return 'other'; + if (nodeType.startsWith('mcp:')) return 'mcp'; + if (nodeType === 'web-search:fallback') return 'other'; + return 'node'; +} + +export function summarizeTools( + config: AgentJsonConfig, + customTools: NonNullable, + sanitizeWorkflowToolName: (name: string) => string, +): AgentSeedToolSummary[] { + const summaries: AgentSeedToolSummary[] = []; + for (const ref of config.tools ?? []) { + if (ref.type === 'node') { + summaries.push({ + name: nodeNameToToolName(ref.name), + kind: 'node', + description: ref.description, + nodeType: ref.node.nodeType, + }); + } else if (ref.type === 'workflow') { + summaries.push({ + name: sanitizeWorkflowToolName(ref.name ?? ref.workflow), + kind: 'workflow', + description: ref.description, + }); + } else { + const descriptor = customTools[ref.id]?.descriptor; + if (descriptor) { + summaries.push({ + name: descriptor.name, + kind: 'custom', + description: descriptor.description, + }); + } + } + } + // MCP servers are summarized by SERVER name — the seed's hint for that name + // steers both the generated tool catalog and every tool result it serves. + for (const server of config.mcpServers ?? []) { + summaries.push({ + name: server.name, + kind: 'mcp', + description: server.description ?? `MCP tool server at ${server.url}`, + }); + } + // The fallback web_search tool is served by the web-search mock; a hint + // keyed by its name steers the generated result sets. Native web search + // runs inside the real model call and needs no hint. + if ( + config.config?.webSearch?.enabled && + !(isNativeWebSearchRequested(config) && hasNativeWebSearchProvider(config.model)) + ) { + summaries.push({ + name: 'web_search', + kind: 'other', + description: 'Web search returning {title, url, snippet} result lists', + }); + } + return summaries; +} + +function truncateRecordedValue(value: unknown): unknown { + if (value === undefined || value === null) return value; + let serialized: string; + try { + serialized = JSON.stringify(value); + } catch { + return '[unserializable value]'; + } + if (serialized === undefined || serialized.length <= MAX_RECORDED_TOOL_VALUE_CHARS) return value; + return truncateForLlm(serialized, MAX_RECORDED_TOOL_VALUE_CHARS); +} + +function extractFinalAssistantText(result: GenerateResult): string { + for (let i = result.messages.length - 1; i >= 0; i--) { + const message = result.messages[i]; + if (!('role' in message) || message.role !== 'assistant') continue; + const texts = message.content + .filter((part): part is Extract => part.type === 'text') + .map((part) => part.text) + .filter((text) => text.trim().length > 0); + if (texts.length > 0) return texts.join('\n'); + } + return ''; +} diff --git a/packages/cli/src/modules/instance-ai/eval/agent-model-turn-recorder.ts b/packages/cli/src/modules/instance-ai/eval/agent-model-turn-recorder.ts new file mode 100644 index 00000000000..610717bfbef --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/agent-model-turn-recorder.ts @@ -0,0 +1,148 @@ +import type { FetchFn } from '@n8n/agents'; +import type { InstanceAiEvalAgentModelTurnRecord } from '@n8n/api-types'; +import type { Logger } from '@n8n/backend-common'; + +import { resolveUrl } from './mock-utils'; +import { redactSecretKeys, redactSecretValuePatterns, truncateForLlm } from './request-sanitizer'; + +// --------------------------------------------------------------------------- +// Record-passthrough transport for the agent's model calls: the real provider +// serves the request (agent evals run the builder-chosen model live), and a +// teed copy of each turn lands in the result for debugging and — later — a +// replay tier. Recording must never alter the live call: auth headers are +// never stored, bodies are redacted and truncated, and a tee failure only +// logs. +// --------------------------------------------------------------------------- + +const MAX_RECORDED_REQUEST_CHARS = 8_000; +const MAX_RECORDED_RESPONSE_CHARS = 16_000; + +export interface AgentModelTurnRecorder { + fetch: FetchFn; + turns: InstanceAiEvalAgentModelTurnRecord[]; + /** Await all in-flight response tees. Call before assembling the result. */ + flush(): Promise; +} + +function providerFromUrl(url: string): string | undefined { + try { + const host = new URL(url).hostname; + if (host.endsWith('openai.com')) return 'openai'; + if (host.endsWith('anthropic.com')) return 'anthropic'; + if (host.endsWith('googleapis.com')) return 'google'; + if (host.endsWith('openai.azure.com')) return 'azure-openai'; + return host; + } catch { + return undefined; + } +} + +// `jsonParse` treats an explicit undefined fallback as absent and throws. +function tryParseJson(text: string): unknown { + try { + return JSON.parse(text); + } catch { + return undefined; + } +} + +/** + * Redact a text blob with the same rigor as structured bodies: key-based + * redaction when it parses as JSON, value-shape scrubbing either way. + */ +function recordableText(text: string, maxChars: number): string { + const parsed = tryParseJson(text); + const serialized = parsed === undefined ? text : JSON.stringify(redactSecretKeys(parsed)); + return truncateForLlm(redactSecretValuePatterns(serialized), maxChars); +} + +function recordableRequestBody(body: unknown): unknown { + if (typeof body !== 'string') { + return body === undefined || body === null ? undefined : '[non-string request body]'; + } + const parsed = tryParseJson(body); + if (parsed === undefined) + return truncateForLlm(redactSecretValuePatterns(body), MAX_RECORDED_REQUEST_CHARS); + // Key-based redaction can't see a secret pasted INSIDE message content — + // scrub well-known value shapes off the serialized form too. + const serialized = redactSecretValuePatterns(JSON.stringify(redactSecretKeys(parsed))); + if (serialized.length <= MAX_RECORDED_REQUEST_CHARS) { + return tryParseJson(serialized) ?? serialized; + } + // Truncating serialized JSON usually breaks parseability — keep the + // truncated string (redacted content survives) rather than dropping it. + return truncateForLlm(serialized, MAX_RECORDED_REQUEST_CHARS); +} + +/** Some gateways carry the API key as a query param — never record it. */ +function stripQuery(url: string): string { + const queryStart = url.indexOf('?'); + return queryStart === -1 ? url : `${url.slice(0, queryStart)}?[query redacted]`; +} + +export function createAgentModelTurnRecorder( + baseFetch: FetchFn, + logger: Logger, +): AgentModelTurnRecorder { + const turns: InstanceAiEvalAgentModelTurnRecord[] = []; + const pendingReads: Array> = []; + + const recordingFetch: FetchFn = async (input, init) => { + const url = resolveUrl(input); + const turn: InstanceAiEvalAgentModelTurnRecord = { + url: stripQuery(url), + provider: providerFromUrl(url), + streamed: false, + requestBody: recordableRequestBody(init?.body), + }; + turns.push(turn); + const startedAt = Date.now(); + + let response: Response; + try { + response = await baseFetch(input, init); + } catch (error) { + turn.durationMs = Date.now() - startedAt; + turn.error = error instanceof Error ? error.message : String(error); + throw error; + } + + turn.durationMs = Date.now() - startedAt; + turn.status = response.status; + turn.streamed = (response.headers.get('content-type') ?? '').includes('text/event-stream'); + + // Tee the body off a clone so the live consumer (the AI SDK) is never + // affected — a failed tee only loses the recording. + try { + const teed = response.clone(); + pendingReads.push( + teed + .text() + .then((text) => { + turn.responseBody = recordableText(text, MAX_RECORDED_RESPONSE_CHARS); + }) + .catch((error: unknown) => { + logger.debug('[EvalAgentMock] Model turn tee failed', { + url: stripQuery(url), + error: error instanceof Error ? error.message : String(error), + }); + }), + ); + } catch (error) { + logger.debug('[EvalAgentMock] Model response clone failed', { + url: stripQuery(url), + error: error instanceof Error ? error.message : String(error), + }); + } + + return response; + }; + + return { + fetch: recordingFetch, + turns, + async flush() { + await Promise.allSettled(pendingReads); + }, + }; +} diff --git a/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts b/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts new file mode 100644 index 00000000000..4a06cf41d9c --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts @@ -0,0 +1,143 @@ +import type { InstanceAiEvalAgentScenarioSeed } from '@n8n/api-types'; +import { createEvalAgent, extractText } from '@n8n/instance-ai'; +import { jsonParse } from 'n8n-workflow'; + +import { buildDateAnchors } from './date-anchors'; + +// --------------------------------------------------------------------------- +// Phase 1 for agent scenarios. Workflows get LLM-generated trigger pin data +// (generateMockHints); an agent's "trigger" is the user's opening chat +// message. One LLM call produces that message plus the shared data context +// and per-tool hints that keep the HTTP mock responses coherent across tools. +// --------------------------------------------------------------------------- + +export interface AgentSeedToolSummary { + /** Sanitized tool name — the identifier the model will call and the mock layer keys on. */ + name: string; + kind: 'node' | 'workflow' | 'custom' | 'mcp' | 'other'; + description?: string; + /** Node type for node tools (e.g. n8n-nodes-base.slackTool). */ + nodeType?: string; +} + +export interface GenerateAgentScenarioSeedOptions { + agentName: string; + instructions: string; + tools: AgentSeedToolSummary[]; + scenarioHints?: string; +} + +const SYSTEM_PROMPT = `You are a test data planner for n8n AI agents. An agent (system prompt + tools) is exercised by sending it ONE opening user chat message; its tool calls are served by an API mock server. Your job is to produce that opening message, a consistent data context, and per-tool hints that guide the mock server to generate realistic, coherent responses. + +RULES: +1. Create a "globalContext" that defines the shared world — user IDs, entity names, channel names, email addresses, and relationships that ALL tool responses should reference consistently. +2. Create an "openingMessage": the exact first chat message a user would send to this agent to start the scenario. + - Write it as a real user would — natural language, first person, no meta-commentary about testing. + - It must actually trigger the behavior under test: include the concrete names, IDs, dates, and values the agent needs to act (drawn from globalContext / the Test Scenario). + - CRITICAL: openingMessage must NEVER be empty. When the scenario describes vague input ("user asks something unclear"), write that vague message verbatim, not a description of it. +3. Create a "toolHints" object with one entry per tool, keyed EXACTLY by the tool names provided. Each hint describes what data that tool's API responses should contain, referencing entities from the global context. Entries marked (mcp) are MCP tool servers: their hint should describe both the tools that server plausibly exposes and the data those tools return. +4. Hints describe DATA CONTENT, not API response format. The mock server already knows the API schema. +5. Ensure data flows logically: if the scenario implies the agent looks something up and then acts on it, the looked-up data must contain what the action needs. +6. Use realistic but clearly fake values (e.g., "jane@example.com", "U_abc123"). +7. **If a "Test Scenario" section is provided, it OVERRIDES your default data generation.** Use the exact names, emails, numeric magnitudes, and conditions described. The scenario defines the test — follow it precisely. +8. **Allocate scenario error conditions explicitly.** When the Test Scenario describes an error/failure/missing-data condition for a SPECIFIC entity, the affected tool's hint MUST name the exact entity and identifier, state the exact API error response the mock must return for requests targeting that entity, and state that requests for all OTHER entities succeed normally. The mock server handles one request at a time — an implicit error condition never gets simulated. +9. **Dates and timestamps.** The user prompt ends with a "## Date anchors" block listing today's real date plus relative anchors. EVERY date you emit — in globalContext, openingMessage, and toolHints — MUST be derived from those anchors, never from training data. When the scenario describes a relative window ("last 2 weeks"), compute concrete dates from the anchors and place records safely INSIDE the window, never on its boundary. State those concrete dates so every tool's mock uses the same ones. +10. Return ONLY valid JSON, no explanation or markdown fencing.`; + +function buildUserPrompt(options: GenerateAgentScenarioSeedOptions): string { + const sections: string[] = [ + 'Generate the opening user message, data context, and per-tool mock hints for this agent.', + ]; + + if (options.scenarioHints) { + sections.push('', '## Test Scenario', '', options.scenarioHints); + } + + sections.push('', '## Agent', '', `Name: ${options.agentName}`, '', '### Instructions', ''); + sections.push(options.instructions); + + sections.push('', '## Tools', ''); + if (options.tools.length === 0) { + sections.push('(none — the agent answers from its instructions alone)'); + } + for (const tool of options.tools) { + let line = `- ${tool.name} (${tool.kind}${tool.nodeType ? `: ${tool.nodeType}` : ''})`; + if (tool.description) line += ` — ${tool.description}`; + sections.push(line); + } + + sections.push('', '## Expected Output', '', '```json', '{'); + sections.push(' "globalContext": "Shared entities: ...",'); + sections.push(' "openingMessage": "The exact first chat message the user sends...",'); + sections.push(' "toolHints": {'); + for (let i = 0; i < Math.min(options.tools.length, 3); i++) { + const comma = i < Math.min(options.tools.length, 3) - 1 ? ',' : ''; + sections.push(` "${options.tools[i].name}": "What data to return..."${comma}`); + } + if (options.tools.length > 3) sections.push(' ...'); + sections.push(' }', '}', '```'); + + // Anchors go last so they are the freshest context before generation. + sections.push('', '## Date anchors', buildDateAnchors(new Date())); + + return sections.join('\n'); +} + +const MAX_SEED_ATTEMPTS = 2; +// Generous stall guard, matching Phase-1 hint generation: aborting +// mid-generation truncates the JSON and the scenario cannot run at all +// without an opening message. +const SEED_LLM_TIMEOUT_MS = 300_000; + +function parseSeed(raw: string): Omit | undefined { + const text = raw.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?\s*```\s*$/i, ''); + const parsed = jsonParse>(text, { fallbackValue: {} }); + + const openingMessage = typeof parsed.openingMessage === 'string' ? parsed.openingMessage : ''; + if (openingMessage.trim().length === 0) return undefined; + + const globalContext = typeof parsed.globalContext === 'string' ? parsed.globalContext : ''; + const toolHints: Record = {}; + if (parsed.toolHints !== null && typeof parsed.toolHints === 'object') { + for (const [key, value] of Object.entries(parsed.toolHints as Record)) { + if (typeof value === 'string') toolHints[key] = value; + } + } + + return { openingMessage, globalContext, toolHints }; +} + +/** + * One LLM call → openingMessage + globalContext + per-tool hints. Retried once + * on structural issues; throws a FRAMEWORK ISSUE error when both attempts fail + * — an agent scenario cannot run without an opening message. + */ +export async function generateAgentScenarioSeed( + options: GenerateAgentScenarioSeedOptions, +): Promise { + const userPrompt = buildUserPrompt(options); + const warnings: string[] = []; + + for (let attempt = 1; attempt <= MAX_SEED_ATTEMPTS; attempt++) { + try { + const agent = createEvalAgent('eval-agent-seed-generator', { + instructions: SYSTEM_PROMPT, + }); + const result = await agent.generate(userPrompt, { + providerOptions: { anthropic: { maxTokens: 8192 } }, + abortSignal: AbortSignal.timeout(SEED_LLM_TIMEOUT_MS), + }); + + const seed = parseSeed(extractText(result)); + if (seed) return { ...seed, warnings }; + warnings.push(`Seed generation attempt ${attempt} returned no usable openingMessage`); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + warnings.push(`Seed generation attempt ${attempt} failed: ${message}`); + } + } + + throw new Error( + `FRAMEWORK ISSUE: agent scenario seed generation failed — ${warnings.join('; ')}`, + ); +} diff --git a/packages/cli/src/modules/instance-ai/eval/mcp-mock-fetch.ts b/packages/cli/src/modules/instance-ai/eval/mcp-mock-fetch.ts new file mode 100644 index 00000000000..261cfc2328d --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/mcp-mock-fetch.ts @@ -0,0 +1,373 @@ +import type { FetchFn } from '@n8n/agents'; +import type { Logger } from '@n8n/backend-common'; +import { jsonParse } from 'n8n-workflow'; + +import { buildDateAnchors } from './date-anchors'; +import { generateJson, resolveUrl } from './mock-utils'; + +// --------------------------------------------------------------------------- +// Mock MCP server, served straight through the agent runtime's injectable MCP +// fetch. Speaks enough of the streamable-HTTP JSON-RPC protocol to satisfy the +// official @modelcontextprotocol/sdk client (initialize handshake, 202 for +// notifications, 405 for the optional GET SSE stream), answers `tools/list` +// with an LLM-designed catalog (stable per server per run) and `tools/call` +// with LLM-generated results steered by the scenario — the MCP counterpart of +// the HTTP mock handler. +// --------------------------------------------------------------------------- + +export interface McpMockServerInfo { + /** Config server name — also the tool-name prefix the SDK client applies. */ + name: string; + url: string; + description?: string; +} + +export interface McpMockToolCall { + serverName: string; + /** Un-prefixed tool name as the server exposes it. */ + toolName: string; + args: unknown; + result: unknown; +} + +export interface McpMockCanonicalTool { + name: string; + description: string; +} + +export interface McpMockFetchOptions { + servers: McpMockServerInfo[]; + agentInstructions: string; + scenarioHints?: string; + globalContext?: string; + /** Per-server data hints from the scenario seed, keyed by server name. */ + serverHints?: Record; + /** + * Canonical tool catalogs keyed by server name (e.g. from the MCP + * registry). When present, the mock exposes EXACTLY these tools — the LLM + * only designs their input schemas — so the mock matches the server as the + * builder set it up instead of inventing a catalog. + */ + knownToolsByServer?: Record; + onToolCall: (call: McpMockToolCall) => void; + logger: Logger; +} + +interface McpMockTool { + name: string; + description: string; + inputSchema: Record; +} + +const MAX_PRIOR_CALL_CONTEXT_CHARS = 2_000; +const MOCK_SESSION_ID = 'eval-mock-session'; + +const TOOLS_LIST_PROMPT = `You design the tool catalog a mocked MCP server exposes during an automated eval run. An AI agent (whose instructions you are given) will connect to this server and call its tools; an API mock will answer those calls later. + +RULES: +1. Expose 1-4 tools — exactly the capabilities the agent's instructions expect from THIS server, nothing speculative. +2. Tool names: short snake_case, WITHOUT any server-name prefix (the client adds it). +3. Each tool needs a one-sentence description and an "inputSchema": a JSON Schema object with "type": "object", simple "properties" (string/number/boolean), and a "required" array. Keep schemas minimal — 1-3 properties. +4. If a "Server data hint" or "Test Scenario" is provided, the catalog must make that scenario playable (e.g. a lookup scenario needs a lookup tool). +5. **When a "## Canonical tools" section is provided, the catalog is FIXED**: return exactly those tool names with those descriptions, in that order — your job is ONLY to design each tool's inputSchema. +6. Return ONLY valid JSON: { "tools": [ { "name", "description", "inputSchema" } ] }`; + +const TOOL_CALL_PROMPT = `You simulate ONE tool invocation on a mocked MCP server during an automated eval run. Given the tool, its arguments, and the scenario context, produce the exact result the real server would return. + +RULES: +1. **The Test Scenario and Server data hint are authoritative.** Honor their exact names, values, counts, and error conditions. When they mandate an error for this request, return { "isError": true, "text": "" }. +2. Return realistic, concrete content — the actual data a real server would produce, not a description of it. Small result sets (1-3 items) unless the scenario demands otherwise. +3. Stay consistent with the Global context and with the PRIOR CALLS listed (same entities, ids, and dates across calls). +4. Every date you emit derives from the "## Date anchors" block, never from training data. +5. Return ONLY valid JSON: { "text": "" } or { "isError": true, "text": "" }.`; + +function validateToolsList(parsed: unknown): McpMockTool[] | undefined { + if (parsed === null || typeof parsed !== 'object') return undefined; + const tools = (parsed as Record).tools; + if (!Array.isArray(tools) || tools.length === 0) return undefined; + const valid: McpMockTool[] = []; + for (const entry of tools) { + if (entry === null || typeof entry !== 'object') continue; + const tool = entry as Record; + if (typeof tool.name !== 'string' || tool.name.length === 0) continue; + // Providers reject tool schemas whose top level isn't `type: object` — + // fall back to a permissive one rather than breaking the model turn. + const generatedSchema = + tool.inputSchema !== null && + typeof tool.inputSchema === 'object' && + (tool.inputSchema as Record).type === 'object' + ? (tool.inputSchema as Record) + : { type: 'object', properties: {} }; + valid.push({ + name: tool.name, + description: typeof tool.description === 'string' ? tool.description : tool.name, + inputSchema: generatedSchema, + }); + } + return valid.length > 0 ? valid : undefined; +} + +function validateToolResult(parsed: unknown): { text: string; isError: boolean } | undefined { + if (parsed === null || typeof parsed !== 'object') return undefined; + const record = parsed as Record; + if (typeof record.text !== 'string' || record.text.length === 0) return undefined; + return { text: record.text, isError: record.isError === true }; +} + +function jsonRpcResponse(id: unknown, result: unknown, headers?: Record): Response { + return new Response(JSON.stringify({ jsonrpc: '2.0', id, result }), { + status: 200, + headers: { 'content-type': 'application/json', ...headers }, + }); +} + +function jsonRpcError(id: unknown, code: number, message: string): Response { + return new Response(JSON.stringify({ jsonrpc: '2.0', id, error: { code, message } }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); +} + +export function createMcpMockFetch(options: McpMockFetchOptions): FetchFn { + const { logger } = options; + const toolsByServer = new Map>(); + const resultCache = new Map>(); + // Rolling record of prior calls per server, fed back for cross-call consistency. + const priorCallsByServer = new Map(); + + function resolveServer(url: string): McpMockServerInfo { + // Require a path boundary after the prefix so one server's URL can't + // swallow another's (…/mcp vs …/mcp-two). + const match = options.servers.find( + (server) => url === server.url || url.startsWith(`${server.url.replace(/\/+$/, '')}/`), + ); + if (match) return match; + try { + return { name: new URL(url).hostname, url }; + } catch { + return { name: 'unknown-mcp-server', url }; + } + } + + function scenarioSections(server: McpMockServerInfo): string[] { + const sections: string[] = []; + const hint = options.serverHints?.[server.name]; + if (hint) sections.push('', '## Server data hint', '', hint); + if (options.globalContext) sections.push('', '## Global context', '', options.globalContext); + if (options.scenarioHints) sections.push('', '## Test Scenario', '', options.scenarioHints); + return sections; + } + + /** Force the generated catalog onto the canonical names when they're known. */ + function coerceToCanonical( + generated: McpMockTool[] | undefined, + canonical: McpMockCanonicalTool[], + ): McpMockTool[] { + const schemasByName = new Map((generated ?? []).map((tool) => [tool.name, tool.inputSchema])); + return canonical.map((tool) => ({ + name: tool.name, + description: tool.description, + inputSchema: schemasByName.get(tool.name) ?? { type: 'object', properties: {} }, + })); + } + + async function getTools(server: McpMockServerInfo): Promise { + let cached = toolsByServer.get(server.name); + if (!cached) { + cached = (async () => { + const canonical = options.knownToolsByServer?.[server.name]; + const sections = [ + `Design the tool catalog for the MCP server "${server.name}"${server.description ? ` (${server.description})` : ''} at ${server.url}.`, + '', + '## Agent instructions (what the agent expects from this server)', + '', + options.agentInstructions.slice(0, 3_000), + ...(canonical && canonical.length > 0 + ? [ + '', + '## Canonical tools (the catalog is FIXED to exactly these)', + '', + ...canonical.map((tool) => `- ${tool.name}: ${tool.description}`), + ] + : []), + ...scenarioSections(server), + ]; + const tools = await generateJson( + 'eval-mcp-tools-list', + TOOLS_LIST_PROMPT, + sections.join('\n'), + validateToolsList, + logger, + ); + // Canonical catalogs (from the MCP registry) are authoritative: the + // real server exposes these names, so the mock must too, whatever + // the generator returned. + if (canonical && canonical.length > 0) { + const coerced = coerceToCanonical(tools, canonical); + logger.debug( + `[EvalMcpMock] ${server.name} exposes canonical catalog: ${coerced.map((tool) => tool.name).join(', ')}`, + ); + return coerced; + } + if (tools) { + logger.debug( + `[EvalMcpMock] ${server.name} exposes: ${tools.map((tool) => tool.name).join(', ')}`, + ); + return tools; + } + // A degenerate-but-valid catalog keeps the run alive; the verifier + // sees the generic calls and can attribute oddities to the mock. + logger.warn( + `[EvalMcpMock] Tool catalog generation failed for ${server.name} — using a generic fallback tool`, + ); + return [ + { + name: 'query', + description: `Query the ${server.name} server`, + inputSchema: { + type: 'object', + properties: { query: { type: 'string', description: 'The query' } }, + required: ['query'], + }, + }, + ]; + })(); + toolsByServer.set(server.name, cached); + } + return await cached; + } + + async function callTool( + server: McpMockServerInfo, + toolName: string, + args: unknown, + ): Promise<{ text: string; isError: boolean }> { + const cacheKey = `${server.name}:${toolName}:${JSON.stringify(args ?? {})}`; + let cached = resultCache.get(cacheKey); + if (!cached) { + cached = (async () => { + const tools = await getTools(server); + const tool = tools.find((entry) => entry.name === toolName); + const prior = priorCallsByServer.get(server.name) ?? []; + const sections = [ + `Simulate the MCP tool "${toolName}" on server "${server.name}".`, + '', + '## Tool', + '', + JSON.stringify(tool ?? { name: toolName }, null, 1), + '', + '## Arguments', + '', + JSON.stringify(args ?? {}, null, 1), + ...scenarioSections(server), + ...(prior.length > 0 + ? ['', '## Prior calls this run (stay consistent with these)', '', ...prior] + : []), + '', + '## Date anchors', + buildDateAnchors(new Date()), + ]; + const generated = await generateJson( + 'eval-mcp-tool-call', + TOOL_CALL_PROMPT, + sections.join('\n'), + validateToolResult, + logger, + ); + return ( + generated ?? { + text: `Mock generation failed for MCP tool "${toolName}" — treat this scenario failure as a framework issue, not agent behaviour.`, + isError: true, + } + ); + })(); + resultCache.set(cacheKey, cached); + } + const result = await cached; + const priorEntries = priorCallsByServer.get(server.name) ?? []; + const entry = `- ${toolName}(${JSON.stringify(args ?? {}).slice(0, 300)}) -> ${result.text.slice(0, 300)}`; + // Repeat (cache-hit) calls add no new information — don't duplicate them. + if ( + !priorEntries.includes(entry) && + priorEntries.join('\n').length < MAX_PRIOR_CALL_CONTEXT_CHARS + ) { + priorEntries.push(entry); + priorCallsByServer.set(server.name, priorEntries); + } + return result; + } + + return async (input, init) => { + const url = resolveUrl(input); + const method = (init?.method ?? 'GET').toUpperCase(); + const server = resolveServer(url); + + // The client MAY open a standalone SSE stream with GET — 405 is the + // spec-blessed "not supported" answer, handled cleanly by the SDK. + if (method === 'GET') return new Response(null, { status: 405 }); + if (method === 'DELETE') return new Response(null, { status: 200 }); + if (method !== 'POST') return new Response(null, { status: 405 }); + + const rawBody = typeof init?.body === 'string' ? init.body : ''; + const message = jsonParse | null>(rawBody, { fallbackValue: null }); + if (message === null || typeof message !== 'object') { + return jsonRpcError(null, -32700, 'Parse error'); + } + const { id, method: rpcMethod, params } = message; + + // Notifications (no id) get 202 Accepted with no body. + if (id === undefined || id === null) return new Response(null, { status: 202 }); + + try { + switch (rpcMethod) { + case 'initialize': { + const requested = + params !== null && + typeof params === 'object' && + typeof (params as Record).protocolVersion === 'string' + ? ((params as Record).protocolVersion as string) + : '2025-06-18'; + return jsonRpcResponse( + id, + { + protocolVersion: requested, + capabilities: { tools: { listChanged: false } }, + serverInfo: { name: server.name, version: '1.0.0-eval-mock' }, + }, + { 'mcp-session-id': MOCK_SESSION_ID }, + ); + } + case 'ping': + return jsonRpcResponse(id, {}); + case 'tools/list': + return jsonRpcResponse(id, { tools: await getTools(server) }); + case 'tools/call': { + const callParams = + params !== null && typeof params === 'object' + ? (params as Record) + : {}; + const toolName = typeof callParams.name === 'string' ? callParams.name : 'unknown'; + const args = callParams.arguments ?? {}; + const result = await callTool(server, toolName, args); + options.onToolCall({ serverName: server.name, toolName, args, result }); + return jsonRpcResponse(id, { + content: [{ type: 'text', text: result.text }], + ...(result.isError ? { isError: true } : {}), + }); + } + case 'resources/list': + return jsonRpcResponse(id, { resources: [] }); + case 'resources/templates/list': + return jsonRpcResponse(id, { resourceTemplates: [] }); + case 'prompts/list': + return jsonRpcResponse(id, { prompts: [] }); + default: + return jsonRpcError(id, -32601, `Method not found: ${String(rpcMethod)}`); + } + } catch (error) { + const messageText = error instanceof Error ? error.message : String(error); + logger.warn(`[EvalMcpMock] ${server.name} ${String(rpcMethod)} failed: ${messageText}`); + return jsonRpcError(id, -32603, messageText); + } + }; +} diff --git a/packages/cli/src/modules/instance-ai/eval/mock-utils.ts b/packages/cli/src/modules/instance-ai/eval/mock-utils.ts new file mode 100644 index 00000000000..a7e5859206a --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/mock-utils.ts @@ -0,0 +1,56 @@ +import type { FetchFn } from '@n8n/agents'; +import type { Logger } from '@n8n/backend-common'; +import { createEvalAgent, extractText } from '@n8n/instance-ai'; +import { jsonParse } from 'n8n-workflow'; + +// --------------------------------------------------------------------------- +// Shared plumbing for the eval mocks (MCP, web search): steered LLM-to-JSON +// generation with retry, and fetch-input helpers. +// --------------------------------------------------------------------------- + +const GENERATOR_TIMEOUT_MS = 120_000; +const MAX_GENERATOR_ATTEMPTS = 2; + +function fenceStrip(raw: string): string { + return raw.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?\s*```\s*$/i, ''); +} + +/** + * Generate JSON with the eval agent and validate its shape, retrying once on + * unusable output. Returns undefined when both attempts fail — callers supply + * their own degraded fallback. + */ +export async function generateJson( + agentName: string, + instructions: string, + userPrompt: string, + validate: (parsed: unknown) => T | undefined, + logger: Logger, +): Promise { + for (let attempt = 1; attempt <= MAX_GENERATOR_ATTEMPTS; attempt++) { + try { + const agent = createEvalAgent(agentName, { instructions }); + const result = await agent.generate(userPrompt, { + providerOptions: { anthropic: { maxTokens: 4096 } }, + abortSignal: AbortSignal.timeout(GENERATOR_TIMEOUT_MS), + }); + const parsed = jsonParse(fenceStrip(extractText(result)), { + fallbackValue: undefined, + }); + const validated = validate(parsed); + if (validated !== undefined) return validated; + logger.warn(`[EvalMock] ${agentName} attempt ${attempt} returned an unusable shape`); + } catch (error) { + logger.warn( + `[EvalMock] ${agentName} attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + return undefined; +} + +export function resolveUrl(input: Parameters[0]): string { + if (typeof input === 'string') return input; + if (input instanceof URL) return input.toString(); + return input.url; +} diff --git a/packages/cli/src/modules/instance-ai/eval/request-sanitizer.ts b/packages/cli/src/modules/instance-ai/eval/request-sanitizer.ts index 941e46589f3..8157314de88 100644 --- a/packages/cli/src/modules/instance-ai/eval/request-sanitizer.ts +++ b/packages/cli/src/modules/instance-ai/eval/request-sanitizer.ts @@ -44,6 +44,36 @@ const SECRET_HEADER_PATTERN = /^(authorization|x-api-key|proxy-authorization|coo const REDACTED = ''; +/** + * Well-known secret VALUE shapes (provider API keys, OAuth/bot tokens, bearer + * values), for text where key-based redaction can't see them — e.g. a secret + * pasted into message content. Conservative on purpose: distinctive prefixes + * plus length floors, so prose never matches. + */ +const SECRET_VALUE_PATTERNS: Array<{ pattern: RegExp; replacement: string }> = [ + { pattern: /\bsk-[A-Za-z0-9_-]{20,}\b/g, replacement: REDACTED }, // OpenAI / Anthropic + { pattern: /\bxox[a-z]-[A-Za-z0-9-]{10,}\b/g, replacement: REDACTED }, // Slack + { pattern: /\bgh[pousr]_[A-Za-z0-9]{20,}\b/g, replacement: REDACTED }, // GitHub tokens + { pattern: /\bgithub_pat_[A-Za-z0-9_]{20,}\b/g, replacement: REDACTED }, // GitHub fine-grained PATs + { pattern: /\bAKIA[0-9A-Z]{16}\b/g, replacement: REDACTED }, // AWS access key ids + { pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g, replacement: REDACTED }, // Google API keys + { pattern: /\bya29\.[0-9A-Za-z_-]{20,}\b/g, replacement: REDACTED }, // Google OAuth access tokens + { pattern: /\bbearer\s+[A-Za-z0-9._~+/-]{20,}=*/gi, replacement: REDACTED }, + // Bare JWTs (three dot-separated base64url segments, header always eyJ…) + { + pattern: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}\b/g, + replacement: REDACTED, + }, + // Prose-form assignments a key-based redactor can't see inside content + // text (`api_key=…`, `API key: …`, `password: …`). Keeps the key, scrubs + // the value; multi-word key names accept space/underscore/hyphen. + { + pattern: + /\b(api[\s_-]?key|access[\s_-]?token|refresh[\s_-]?token|client[\s_-]?secret|secret|password|passwd)\b(["']?\s*[=:]\s*["']?)[^\s"',;]{6,}/gi, + replacement: `$1$2${REDACTED}`, + }, +]; + // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- @@ -78,6 +108,15 @@ export function redactSecretKeys(value: unknown): unknown { * Appends a note when truncation occurs so the LLM knows the data is incomplete. * Logs a warning the first time a large body is encountered. */ +/** Redact well-known secret VALUE shapes anywhere in a text blob. */ +export function redactSecretValuePatterns(text: string): string { + let result = text; + for (const { pattern, replacement } of SECRET_VALUE_PATTERNS) { + result = result.replace(pattern, replacement); + } + return result; +} + export function truncateForLlm(serialized: string, maxLength = MAX_BODY_LENGTH): string { if (serialized.length <= maxLength) return serialized; diff --git a/packages/cli/src/modules/instance-ai/eval/web-search-mock.ts b/packages/cli/src/modules/instance-ai/eval/web-search-mock.ts new file mode 100644 index 00000000000..468d2ec0945 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/web-search-mock.ts @@ -0,0 +1,171 @@ +import type { Logger } from '@n8n/backend-common'; + +import { buildDateAnchors } from './date-anchors'; +import { generateJson } from './mock-utils'; + +// --------------------------------------------------------------------------- +// Mock for the agent runtime's fallback `web_search` tool (the Brave/SearXNG +// path), injected via `AgentRuntimeInstrumentation.webSearch`. Results are +// LLM-generated and steered by the scenario — the web-search counterpart of +// the HTTP and MCP mocks. Provider-native web search never reaches this: it +// runs inside the real model call. +// --------------------------------------------------------------------------- + +export interface WebSearchMockArgs { + query: string; + maxResults?: number; + includeDomains?: string[]; + excludeDomains?: string[]; +} + +/** Mirrors `WebSearchResponse` from @n8n/ai-utilities — what the real tool returns. */ +export interface WebSearchMockResult { + query: string; + results: Array<{ title: string; url: string; snippet: string; publishedDate?: string }>; + note?: string; +} + +export interface WebSearchMockOptions { + agentInstructions: string; + scenarioHints?: string; + globalContext?: string; + /** Data hint for the web_search tool from the scenario seed, when present. */ + searchHint?: string; + onSearch: (args: WebSearchMockArgs, result: WebSearchMockResult) => void; + logger: Logger; +} + +const SEARCH_PROMPT = [ + 'You simulate a web search API for an automated test of an AI agent. Design the result set the', + 'agent should receive for the given query, so the scenario can proceed the way the test intends.', + '', + 'Rules:', + '1. Respond ONLY with JSON: {"results": [{"title": "...", "url": "...", "snippet": "..."}]}.', + ' Each result may include an optional "publishedDate" (ISO date). No prose, no fences.', + '2. Results must look like real search hits: plausible domains, realistic titles, information-dense', + ' snippets. Honor maxResults and include/exclude domain filters when given.', + '3. When scenario hints mandate specific facts, values, or failures, the snippets must carry them', + ' EXACTLY — the test asserts on them downstream.', + '4. Stay consistent with prior searches this run: the same entities keep the same names, values,', + ' and URLs.', + '5. An empty results array is valid when the scenario calls for a fruitless search.', +].join('\n'); + +const MAX_PRIOR_SEARCH_CONTEXT_CHARS = 4000; + +function validateSearchResults(parsed: unknown): WebSearchMockResult['results'] | undefined { + if (parsed === null || typeof parsed !== 'object') return undefined; + const results = (parsed as Record).results; + if (!Array.isArray(results)) return undefined; + const valid: WebSearchMockResult['results'] = []; + for (const entry of results) { + if (entry === null || typeof entry !== 'object') continue; + const hit = entry as Record; + if (typeof hit.title !== 'string' || typeof hit.url !== 'string') continue; + valid.push({ + title: hit.title, + url: hit.url, + snippet: typeof hit.snippet === 'string' ? hit.snippet : '', + ...(typeof hit.publishedDate === 'string' ? { publishedDate: hit.publishedDate } : {}), + }); + } + return valid; +} + +export function createWebSearchMock( + options: WebSearchMockOptions, +): (args: WebSearchMockArgs) => Promise { + const { logger } = options; + const cache = new Map>(); + const priorSearches: string[] = []; + + return async (args) => { + const cacheKey = JSON.stringify(args); + let cached = cache.get(cacheKey); + if (!cached) { + cached = (async (): Promise => { + const sections = [ + `Simulate a web search for: ${JSON.stringify(args.query)}`, + '', + '## Search parameters', + '', + JSON.stringify(args, null, 1), + '', + '## Agent under test (its instructions)', + '', + options.agentInstructions.slice(0, 3000), + ...(options.scenarioHints + ? ['', '## Scenario hints (mandated facts/failures)', '', options.scenarioHints] + : []), + ...(options.globalContext ? ['', '## Global context', '', options.globalContext] : []), + ...(options.searchHint ? ['', '## Web-search data hint', '', options.searchHint] : []), + ...(priorSearches.length > 0 + ? ['', '## Prior searches this run (stay consistent with these)', '', ...priorSearches] + : []), + '', + '## Date anchors', + buildDateAnchors(new Date()), + ]; + const results = await generateJson( + 'eval-web-search', + SEARCH_PROMPT, + sections.join('\n'), + validateSearchResults, + logger, + ); + if (!results) { + logger.warn(`[EvalWebSearchMock] generation failed for query "${args.query}"`); + return { + query: args.query, + results: [], + note: 'Mock web-search generation failed — treat a scenario failure here as a framework issue, not agent behaviour.', + }; + } + // Enforce domain filters on the generated URLs like the real + // provider would — but never at the cost of the scenario: when the + // agent restricts to a real-world domain and the mock invented + // scenario-domain URLs, dropping everything would starve the run + // of its mandated facts, so an emptied result set falls back. + const domainAllowed = (url: string): boolean => { + let host: string; + try { + host = new URL(url).hostname; + } catch { + return false; + } + const matches = (domain: string) => host === domain || host.endsWith(`.${domain}`); + if (args.excludeDomains?.some(matches)) return false; + if (args.includeDomains?.length) return args.includeDomains.some(matches); + return true; + }; + let filtered = results.filter((hit) => domainAllowed(hit.url)); + if (filtered.length === 0 && results.length > 0) { + logger.debug( + `[EvalWebSearchMock] domain filters would empty the result set for "${args.query}" — serving unfiltered results to keep the scenario playable`, + ); + filtered = results; + } + const capped = + args.maxResults !== undefined ? filtered.slice(0, args.maxResults) : filtered; + return { query: args.query, results: capped }; + })(); + cache.set(cacheKey, cached); + } + const result = await cached; + // Carry titles, URLs, and snippet leads so later searches can stay + // consistent with the facts already served, not just the headlines. + const entry = `- search(${JSON.stringify(args.query).slice(0, 300)}) -> ${result.results + .map((hit) => `${hit.title} <${hit.url}> ${hit.snippet.slice(0, 120)}`) + .join(' | ') + .slice(0, 900)}`; + // Repeat (cache-hit) searches add no new information — don't duplicate them. + if ( + !priorSearches.includes(entry) && + priorSearches.join('\n').length < MAX_PRIOR_SEARCH_CONTEXT_CHARS + ) { + priorSearches.push(entry); + } + options.onSearch(args, result); + return result; + }; +} diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index 4bf940d3462..27e1e984fc5 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -14,6 +14,7 @@ import { InstanceAiAdminSettingsUpdateRequest, InstanceAiUserPreferencesUpdateRequest, InstanceAiEvalExecutionRequest, + InstanceAiEvalAgentExecutionRequest, InstanceAiEvalCredentialAllowlistRequest, InstanceAiEvalRestoreThreadRequest, InstanceAiEvalSeedDataTableRowsRequest, @@ -42,6 +43,7 @@ import { UnsupportedAttachmentError, validateAttachmentMimeTypes } from '@n8n/in import type { NextFunction, Request, Response } from 'express'; import { randomUUID, timingSafeEqual } from 'node:crypto'; import { InstanceAiBrowserSessionService } from './browser/instance-ai-browser-session.service'; +import { EvalAgentExecutionService } from './eval/agent-execution.service'; import { EvalExecutionService } from './eval/execution.service'; import { EvalThreadCredentialAllowlistService } from './eval/thread-credential-allowlist.service'; import { EvalThreadRestoreService } from './eval/thread-restore.service'; @@ -112,6 +114,7 @@ export class InstanceAiController { private readonly memoryService: InstanceAiMemoryService, private readonly settingsService: InstanceAiSettingsService, private readonly evalExecutionService: EvalExecutionService, + private readonly evalAgentExecutionService: EvalAgentExecutionService, private readonly evalCredentialAllowlists: EvalThreadCredentialAllowlistService, private readonly evalThreadRestore: EvalThreadRestoreService, private readonly eventBus: InProcessEventBus, @@ -928,6 +931,18 @@ export class InstanceAiController { return await this.evalExecutionService.executeWithLlmMock(workflowId, req.user, payload); } + // Runs for minutes; same client timeout handling as the workflow variant. + @Post('/eval/execute-agent-with-llm-mock/:agentId') + @GlobalScope('instanceAi:eval') + async executeAgentWithLlmMock( + req: AuthenticatedRequest, + _res: Response, + @Param('agentId') agentId: string, + @Body payload: InstanceAiEvalAgentExecutionRequest, + ) { + return await this.evalAgentExecutionService.executeWithLlmMock(agentId, req.user, payload); + } + /** * Pin a build thread's credential view to a declared set. Only narrows: * `list()` results are intersected with these IDs, so the caller cannot see diff --git a/packages/cli/src/node-execution/__tests__/ephemeral-node-executor.test.ts b/packages/cli/src/node-execution/__tests__/ephemeral-node-executor.test.ts index 532e8f5d5bc..7e013547c35 100644 --- a/packages/cli/src/node-execution/__tests__/ephemeral-node-executor.test.ts +++ b/packages/cli/src/node-execution/__tests__/ephemeral-node-executor.test.ts @@ -14,6 +14,7 @@ import { type INodeCredentialsDetails, type INodeType, type INodeTypeDescription, + type IWorkflowExecuteAdditionalData, } from 'n8n-workflow'; import { mock } from 'vitest-mock-extended'; @@ -744,4 +745,73 @@ describe('EphemeralNodeExecutor', () => { ); }); }); + + describe('executeInline → eval instrumentation', () => { + it('applies the additionalData decoration and node name override when set', async () => { + let executedNodeName: string | undefined; + const execute = vi.fn().mockImplementation(async function (this: { + getNode(): { name: string }; + }) { + executedNodeName = this.getNode().name; + return [[{ json: { ok: true } }]]; + }); + nodeTypes.getByNameAndVersion.mockReturnValue({ + description: toolDescription, + execute, + } as unknown as INodeType); + + const base: Record = { credentialsHelper: {} }; + mockGetBase.mockResolvedValue(base); + const configureAdditionalData = vi.fn((additionalData: Record) => { + additionalData.evalLlmMockHandler = vi.fn(); + }); + + const result = await executor.executeInline({ + nodeType: 'n8n-nodes-base.slack', + nodeTypeVersion: 1, + nodeParameters: {}, + inputData: [], + projectId: 'p-1', + nodeName: 'Slack_Tool', + configureAdditionalData: configureAdditionalData as unknown as ( + additionalData: IWorkflowExecuteAdditionalData, + ) => void, + }); + + expect(result.status).toBe('success'); + expect(configureAdditionalData).toHaveBeenCalledTimes(1); + expect(configureAdditionalData).toHaveBeenCalledWith(base); + expect(base.evalLlmMockHandler).toBeDefined(); + expect(executedNodeName).toBe('Slack_Tool'); + }); + + it('keeps the default node name and untouched additionalData when unset', async () => { + let executedNodeName: string | undefined; + const execute = vi.fn().mockImplementation(async function (this: { + getNode(): { name: string }; + }) { + executedNodeName = this.getNode().name; + return [[{ json: {} }]]; + }); + nodeTypes.getByNameAndVersion.mockReturnValue({ + description: toolDescription, + execute, + } as unknown as INodeType); + + const base: Record = {}; + mockGetBase.mockResolvedValue(base); + + const result = await executor.executeInline({ + nodeType: 'n8n-nodes-base.slack', + nodeTypeVersion: 1, + nodeParameters: {}, + inputData: [], + projectId: 'p-1', + }); + + expect(result.status).toBe('success'); + expect(executedNodeName).toBe('Target Node'); + expect(base.evalLlmMockHandler).toBeUndefined(); + }); + }); }); diff --git a/packages/cli/src/node-execution/ephemeral-node-executor.ts b/packages/cli/src/node-execution/ephemeral-node-executor.ts index f4493208b19..b291554faef 100644 --- a/packages/cli/src/node-execution/ephemeral-node-executor.ts +++ b/packages/cli/src/node-execution/ephemeral-node-executor.ts @@ -11,6 +11,7 @@ import type { INodeExecutionData, INodeParameters, ITaskDataConnections, + IWorkflowExecuteAdditionalData, NodeOutput, } from 'n8n-workflow'; import { @@ -40,6 +41,10 @@ export type EphemeralWorkflowToolLike = { nodeTypeVersion: number; nodeParameters: INodeParameters; credentials?: Record | null; + /** Ephemeral node name override (defaults to 'Target Node'). */ + nodeName?: string; + /** Eval-only additionalData decoration (e.g. HTTP mock handler) — never set on production paths. */ + configureAdditionalData?: (additionalData: IWorkflowExecuteAdditionalData) => void; }; export interface InlineNodeExecutionRequest { @@ -54,6 +59,10 @@ export interface InlineNodeExecutionRequest { credentialDetails?: Record; inputData: INodeExecutionData[]; projectId: string; + /** Ephemeral node name override (defaults to 'Target Node'). */ + nodeName?: string; + /** Eval-only additionalData decoration (e.g. HTTP mock handler) — never set on production paths. */ + configureAdditionalData?: (additionalData: IWorkflowExecuteAdditionalData) => void; } export interface NodeExecutionResult { @@ -275,7 +284,7 @@ export class EphemeralNodeExecutor { ) { const node: INode = { id: uuid(), - name: 'Target Node', + name: tool.nodeName ?? 'Target Node', type: tool.nodeType, typeVersion: tool.nodeTypeVersion, position: [0, 0], @@ -293,6 +302,7 @@ export class EphemeralNodeExecutor { // Data Table uses separate project authorization and an ephemeral workflow has no owner fallback. additionalData.dataTableProjectId = tool.projectId; } + tool.configureAdditionalData?.(additionalData); const runExecutionData = createEmptyRunExecutionData(); const inputData: ITaskDataConnections = { main: [inputItems] }; const executeData: IExecuteData = { node, data: inputData, source: null }; @@ -411,6 +421,8 @@ export class EphemeralNodeExecutor { nodeTypeVersion: request.nodeTypeVersion, nodeParameters: request.nodeParameters, credentials: mergedCredentials, + nodeName: request.nodeName, + configureAdditionalData: request.configureAdditionalData, }; // Native tool nodes (toolWikipedia, toolCalculator, etc.) expose their real diff --git a/packages/core/nodes-testing/node-test-harness.ts b/packages/core/nodes-testing/node-test-harness.ts index ff996e698d7..ae97361046a 100644 --- a/packages/core/nodes-testing/node-test-harness.ts +++ b/packages/core/nodes-testing/node-test-harness.ts @@ -236,6 +236,9 @@ export class NodeTestHarness { encryptedRunnerIdentity: undefined, }); additionalData.credentialsHelper = credentialsHelper; + // Prevent the auto-mocked property from being truthy so credential and + // request helpers don't take the eval-mock code path. + (additionalData as unknown as Record).evalLlmMockHandler = undefined; let executionData: IRun; const runExecutionData = createRunExecutionData({ diff --git a/packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts b/packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts index fdb153f0138..7cddaeec313 100644 --- a/packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts +++ b/packages/core/src/execution-engine/node-execution-context/__tests__/node-execution-context.test.ts @@ -283,6 +283,33 @@ describe('NodeExecutionContext', () => { expect(resolved).toBeUndefined(); }); + it('builds mock credentials for an unconfigured node whenever the eval handler is set, regardless of mode', async () => { + // Agent-tool eval runs execute in 'internal' mode (and their + // workflow-tool sub-executions in 'chat'/'manual') — handler presence, + // not mode, is the bypass discriminator. + const testNode = mock({ type: 'n8n-nodes-base.slack' }); + testNode.credentials = undefined; + + const getDecrypted = vi.fn().mockResolvedValue({ token: '' }); + const evalAdditionalData = mock({ + credentialsHelper: mock({ getDecrypted }), + evalLlmMockHandler: vi.fn(), + }); + + const internalContext = new TestContext(workflow, testNode, evalAdditionalData, 'internal'); + + await expect(internalContext['_getCredentials']('slackApi')).resolves.toEqual({ + token: '', + }); + expect(getDecrypted).toHaveBeenCalledWith( + evalAdditionalData, + { id: null, name: 'slackApi' }, + 'slackApi', + 'internal', + undefined, + ); + }); + it('refuses to decrypt a restricted credential for a node not in supportedNodes', async () => { const credentialDetails = { id: 'cred-r1', name: 'Restricted creds' }; const httpNode = mock({ type: 'n8n-nodes-base.httpRequest' }); diff --git a/packages/core/src/execution-engine/node-execution-context/node-execution-context.ts b/packages/core/src/execution-engine/node-execution-context/node-execution-context.ts index 1c4e6a001c6..de02781c108 100644 --- a/packages/core/src/execution-engine/node-execution-context/node-execution-context.ts +++ b/packages/core/src/execution-engine/node-execution-context/node-execution-context.ts @@ -323,14 +323,13 @@ export abstract class NodeExecutionContext implements Omit { const { workflow, node, additionalData, mode, runExecutionData, runIndex } = this; - // Eval-mode bypass: only mock when the node is fully unconfigured, so - // nodes that probe multiple auth types still get production's throw. - // Delegates to the credentials helper with a null-id `INodeCredentialsDetails`; - // `EvalMockedCredentialsHelper` catches the resulting `CredentialNotFoundError` - // and schema-synthesizes (and applies the wire-server URL rewrite). Production - // helpers don't catch — but production never reaches this branch because - // `evalLlmMockHandler` is only set in eval mode. - if (mode === 'evaluation' && additionalData.evalLlmMockHandler && !node.credentials?.[type]) { + // Eval bypass: only mock when the node is fully unconfigured, so nodes + // probing multiple auth types still get production's throw. Handler + // presence (not execution mode) gates it — `evalLlmMockHandler` is set + // only by eval execution services, and eval agent tools run in modes + // other than 'evaluation'. `EvalMockedCredentialsHelper` catches the + // null-id `CredentialNotFoundError` and schema-synthesizes. + if (additionalData.evalLlmMockHandler && !node.credentials?.[type]) { const hasOtherCreds = !!node.credentials && Object.keys(node.credentials).length > 0; if (!hasOtherCreds) { return (await additionalData.credentialsHelper.getDecrypted(