From cd623fd6ca5644093d4ade45783960b972d89ce6 Mon Sep 17 00:00:00 2001 From: Predrag Ristic Date: Thu, 17 Sep 2026 13:07:18 +0000 Subject: [PATCH] fix(core): Simulate Agent write tools during workflow verification (#38553) --- .../src/__tests__/agent-tool-workflow.ts | 65 ++++ .../verify-built-workflow.tool.test.ts | 341 +++++++++++++++++- .../__tests__/analyze-result.test.ts | 38 +- .../tool-simulation-preflight.test.ts | 178 +++++++++ .../verification/analyze-result.ts | 58 +-- .../verification/finalize-result.ts | 36 +- .../verification/tool-simulation-preflight.ts | 105 ++++++ .../verify-built-workflow.tool.ts | 51 ++- ...ssify-node-destructiveness.service.test.ts | 106 ++++++ .../reconcile-simulation-plan.test.ts | 39 +- .../classify-node-destructiveness.service.ts | 70 ++-- .../workflows/plan-verification-simulation.ts | 5 +- .../workflows/reconcile-simulation-plan.ts | 15 +- .../src/tools/workflows/verification-graph.ts | 64 ++++ .../agent-tool-simulation.workflow.test.ts | 170 +++++++++ packages/@n8n/nodes-langchain/package.json | 1 + pnpm-lock.yaml | 3 + 17 files changed, 1223 insertions(+), 122 deletions(-) create mode 100644 packages/@n8n/instance-ai/src/__tests__/agent-tool-workflow.ts create mode 100644 packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/tool-simulation-preflight.test.ts create mode 100644 packages/@n8n/instance-ai/src/tools/orchestration/verification/tool-simulation-preflight.ts create mode 100644 packages/@n8n/instance-ai/src/tools/workflows/verification-graph.ts create mode 100644 packages/@n8n/nodes-langchain/nodes/agents/Agent/test/integration/agent-tool-simulation.workflow.test.ts diff --git a/packages/@n8n/instance-ai/src/__tests__/agent-tool-workflow.ts b/packages/@n8n/instance-ai/src/__tests__/agent-tool-workflow.ts new file mode 100644 index 00000000000..36f7a62b5a3 --- /dev/null +++ b/packages/@n8n/instance-ai/src/__tests__/agent-tool-workflow.ts @@ -0,0 +1,65 @@ +import type { WorkflowJSON } from '@n8n/workflow-sdk'; + +export function agentToolNode( + name: string, + type: string, + overrides: Partial = {}, +): WorkflowJSON['nodes'][number] { + return { id: name, name, type, typeVersion: 1, parameters: {}, position: [0, 0], ...overrides }; +} + +export function mcpToolWorkflow(): WorkflowJSON { + return { + name: 'MCP tools', + settings: { executionOrder: 'v1' }, + nodes: [ + agentToolNode('MCP Server', '@n8n/n8n-nodes-langchain.mcpTrigger'), + agentToolNode('Write', 'n8n-nodes-base.slackTool', { parameters: { operation: 'send' } }), + ], + connections: { + Write: { ai_tool: [[{ node: 'MCP Server', type: 'ai_tool', index: 0 }]] }, + }, + }; +} + +export function agentToolWorkflow(agentVersion = 3.1, nestedVersion?: number): WorkflowJSON { + const caller = nestedVersion === undefined ? 'Agent' : 'Nested'; + return { + name: 'Agent tools', + settings: { executionOrder: 'v1' }, + nodes: [ + agentToolNode('Trigger', 'n8n-nodes-base.manualTrigger'), + agentToolNode('Agent', '@n8n/n8n-nodes-langchain.agent', { typeVersion: agentVersion }), + agentToolNode('Write', 'n8n-nodes-base.slackTool', { parameters: { operation: 'send' } }), + agentToolNode('Model', '@n8n/n8n-nodes-langchain.lmChatOpenAi', { + credentials: { openAiApi: { id: 'model-credential', name: 'Model account' } }, + }), + ...(nestedVersion === undefined + ? [] + : [ + agentToolNode('Nested', '@n8n/n8n-nodes-langchain.agentTool', { + typeVersion: nestedVersion, + }), + ]), + ], + connections: { + Trigger: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] }, + Write: { ai_tool: [[{ node: caller, type: 'ai_tool', index: 0 }]] }, + Model: { + ai_languageModel: [ + [ + { node: 'Agent', type: 'ai_languageModel', index: 0 }, + ...(nestedVersion === undefined + ? [] + : [{ node: 'Nested', type: 'ai_languageModel', index: 0 }]), + ], + ], + }, + ...(nestedVersion === undefined + ? {} + : { + Nested: { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] }, + }), + }, + }; +} diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts index 572dd33977a..2596425290a 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts @@ -1,5 +1,10 @@ import type { Mock } from 'vitest'; +import { + agentToolNode, + agentToolWorkflow, + mcpToolWorkflow, +} from '../../../__tests__/agent-tool-workflow'; import { executeTool } from '../../../__tests__/tool-test-utils'; import { successfulVerification } from '../../../__tests__/verification-fixtures'; import type { WorkflowLoopStorage } from '../../../storage/workflow-loop-storage'; @@ -17,6 +22,7 @@ import type { WorkflowBuildOutcome, } from '../../../workflow-loop/workflow-loop-state'; import { WorkflowTaskCoordinator } from '../../../workflow-loop/workflow-task-service'; +import { CREDENTIALLESS_AI_ROOT_SIMULATION_REASON } from '../../workflows/plan-verification-simulation'; import { createVerifyBuiltWorkflowTool } from '../verify-built-workflow.tool'; type VerifyBuiltWorkflowOutput = { @@ -590,6 +596,7 @@ function makeContext( })) ?? [], connections: overrides.workflowConnections ?? {}, + settings: { executionOrder: 'v1' }, }; }), getWorkflowHead: vi.fn().mockResolvedValue({ @@ -1238,7 +1245,7 @@ describe('verify-built-workflow tool — node simulation plan', () => { { nodeName: 'Send Slack', reason: 'Sends a message to a Slack channel' }, ]); expect(result.simulationNote).toContain('Send Slack'); - expect(result.simulationNote).toContain('no real external writes'); + expect(result.simulationNote).toContain('with fixture output'); }); it('surfaces parameters of reached simulated nodes that resolved to empty or threw', async () => { @@ -1303,7 +1310,7 @@ describe('verify-built-workflow tool — node simulation plan', () => { issue: 'empty', }, ]); - expect(result.simulationNote).toContain('no real external writes'); + expect(result.simulationNote).toContain('with fixture output'); expect(result.simulationNote).toContain('Send Slack: `channel`'); expect(result.simulationNote).toContain('`text` (={{ $json.query.caller }}) resolved to empty'); }); @@ -1325,7 +1332,7 @@ describe('verify-built-workflow tool — node simulation plan', () => { { nodeName: 'Send Slack', executionId: 'exec-sim', reason: 'replay-failed' }, ]); expect(result.skippedParameterCheckCount).toBe(1); - expect(result.simulationNote).toContain('no real external writes'); + expect(result.simulationNote).toContain('with fixture output'); expect(result.simulationNote).toContain('Parameter check skipped'); expect(result.simulationNote).toContain('unchecked dynamic fields'); expect(ctx.logger.debug).toHaveBeenCalledWith( @@ -1369,7 +1376,7 @@ describe('verify-built-workflow tool — node simulation plan', () => { }), ); expect(ctx.logger.warn).toHaveBeenCalledWith( - expect.stringContaining('no simulation plan'), + expect.stringContaining('preflight blocked verification'), expect.objectContaining({ workItemId: 'wi-1' }), ); }); @@ -1499,10 +1506,14 @@ describe('verify-built-workflow tool — node simulation plan', () => { ]); expect(result.coverageNote).toContain('UNVERIFIED'); expect(result.coverageNote).toContain('Look Up Order'); - expect(result.coverageNote).toContain('Seed matching test data'); + expect(result.coverageNote).toContain('Agent tool calls'); + expect(result.coverageNote).toContain('If a live lookup or query returned no matching records'); + expect(result.coverageNote).toContain( + 'seed matching test data only when the user has authorized that write', + ); }); - it('flags an items-dropped collapse with $input.all() guidance when a collection ran dry', async () => { + it('reports incomplete coverage without assuming why a collection stopped', async () => { // INS-662: HTTP split a top-level array into N items, a Code node read $input.first() and emitted zero, downstream was skipped. const { ctx } = makeContext( makeBuildOutcome({ @@ -1529,10 +1540,11 @@ describe('verify-built-workflow tool — node simulation plan', () => { expect(result.success).toBe(true); expect(result.nodesNotReached).toEqual(['Post to Slack']); - expect(result.coverageNote).toContain('$input.all()'); - expect(result.coverageNote).toContain('$input.first()'); - // Must NOT misattribute the dead-end to an empty lookup. - expect(result.coverageNote).not.toContain('Seed matching test data'); + expect(result.coverageNote).toContain('input items'); + expect(result.coverageNote).toContain('If a Code node dropped a collection'); + expect(result.coverageNote).toContain('$input.first().json'); + expect(result.coverageNote).toContain('$input.all().map(i => i.json)'); + expect(result.coverageNote).toContain('Do not report unreached nodes as verified'); }); it('persists unreached nodes in the verification evidence', async () => { @@ -1665,7 +1677,7 @@ describe('verify-built-workflow tool — stale mocked-credential plan', () => { }); }); - it('proceeds with the stored plan when the live workflow cannot be loaded', async () => { + it('blocks when credential reconciliation cannot load the live workflow', async () => { const { ctx } = makeContext(staleOutcome(), { executionId: 'exec-1', status: 'success', @@ -1677,11 +1689,10 @@ describe('verify-built-workflow tool — stale mocked-credential plan', () => { const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); - expect(result.success).toBe(true); + expect(result.success).toBe(false); + expect(result.remediation?.reason).toBe('verification_graph_unavailable'); const run = vi.mocked(ctx.domainContext.executionService.run); - expect(run.mock.calls[0][2]).toMatchObject({ - verificationPinData: { Notion: [{ page: 'mock' }] }, - }); + expect(run).not.toHaveBeenCalled(); expect(ctx.logger.warn).toHaveBeenCalledWith( 'verify-built-workflow: could not reconcile mocked-credential plan', expect.objectContaining({ workItemId: 'wi-1' }), @@ -1830,7 +1841,7 @@ describe('verify-built-workflow tool — trigger selection', () => { ); }); - it('preserves verification state when the workflow graph cannot be loaded', async () => { + it('records a blocked result and preserves prior trigger passes when the graph cannot be loaded', async () => { const outcome = makeTrackedOutcome({ verificationProgress: passedA, verifyAttempts: 1 }); const { ctx, getOutcome } = makeContext(outcome, successfulA, { workflowConnections }); vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockRejectedValueOnce( @@ -1840,7 +1851,11 @@ describe('verify-built-workflow tool — trigger selection', () => { const result = await runTool(ctx, triggerAInput); expect(result.success).toBe(false); - expect(getOutcome()).toEqual(outcome); + expect(getOutcome().verificationProgress).toEqual(outcome.verificationProgress); + expect(getOutcome().verification).toMatchObject({ + success: false, + failureSignature: 'verification_graph_unavailable', + }); expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); }); @@ -1956,6 +1971,298 @@ describe('verify-built-workflow tool — trigger selection', () => { }); }); +describe('verify-built-workflow tool — attached tools', () => { + const verdict = (nodeName: string, simulated = false) => ({ + nodeName, + verdict: simulated ? ('simulate' as const) : ('execute' as const), + reason: simulated ? 'Writes data' : 'Reads data', + confidence: 'high' as const, + source: 'deterministic' as const, + }); + const plan = [verdict('Agent'), verdict('Write', true)]; + + it('blocks MCP tool simulation before execution without a named trigger', async () => { + const { ctx, getOutcome } = makeContext( + makeBuildOutcome({ nodeSimulationPlan: [verdict('Write', true)] }), + { status: 'success' }, + ); + vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue( + mcpToolWorkflow(), + ); + + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + + expect(result).toMatchObject({ + success: false, + remediation: { category: 'blocked', reason: 'unsupported_tool_simulation' }, + nodesNotReached: expect.arrayContaining(['MCP Server', 'Write']), + }); + expect(result.error).toContain('Write (called by MCP Server)'); + expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); + expect(ctx.workflowTaskService.startVerification).not.toHaveBeenCalled(); + expect(getOutcome().verification).toMatchObject({ + success: false, + failureSignature: 'unsupported_tool_simulation', + }); + }); + + it('records a blocked result when the saved pin summary cannot be loaded', async () => { + const { ctx, getOutcome } = makeContext(makeBuildOutcome({ nodeSimulationPlan: plan }), { + status: 'success', + }); + const workflowService = ctx.domainContext.workflowService!; + vi.mocked(workflowService.getAsWorkflowJSON).mockResolvedValue(agentToolWorkflow()); + workflowService.getPinnedDataSummary = vi + .fn>() + .mockRejectedValue(new Error('Pin summary unavailable')); + + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + + expect(result).toMatchObject({ + success: false, + status: 'unknown', + resolvedWorkItemId: 'wi-1', + remediation: { + category: 'blocked', + shouldEdit: false, + reason: 'verification_pin_summary_unavailable', + }, + }); + expect(result.error).toContain('Retry verification'); + expect(result.executionId).toBeUndefined(); + expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); + expect(ctx.workflowTaskService.startVerification).not.toHaveBeenCalled(); + expect(getOutcome().verification).toMatchObject({ + attempted: true, + success: false, + status: 'unknown', + failureSignature: 'verification_pin_summary_unavailable', + evidence: { errorMessage: result.error }, + }); + expect(ctx.workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith( + expect.objectContaining({ + verdict: 'failed_terminal', + failureSignature: 'verification_pin_summary_unavailable', + }), + ); + }); + + it.each([ + { scripted: false, incomplete: false }, + { scripted: true, incomplete: false }, + { scripted: false, incomplete: true }, + ])( + 'blocks after restoring model credentials, scripted=$scripted, incomplete=$incomplete', + async ({ scripted, incomplete }) => { + const workflow = agentToolWorkflow(2); + const reason = incomplete ? 'incomplete_tool_simulation_plan' : 'unsupported_tool_simulation'; + const buildOutcome = makeBuildOutcome({ + mockedNodeNames: ['Model'], + mockedCredentialsByNode: { Model: ['openAiApi'] }, + simulationFixtures: { Agent: [{ output: 'Fixture' }] }, + nodeSimulationPlan: [ + { ...verdict('Agent', true), reason: CREDENTIALLESS_AI_ROOT_SIMULATION_REASON }, + ...(incomplete ? [] : [verdict('Write', true)]), + ...(scripted ? [{ ...verdict('Gate', true), haltBranch: true }] : []), + ], + waitGateScripts: scripted + ? [ + { + nodeName: 'Gate', + cutEdge: { source: 'Revise', target: 'Format' }, + decisions: [{ label: 'approve', items: [{ approved: true }] }], + }, + ] + : undefined, + }); + const { ctx, getOutcome } = makeContext( + buildOutcome, + { status: 'success' }, + { + availableCredentials: [ + { id: 'model-credential', name: 'Model account', type: 'openAiApi' }, + ], + }, + ); + vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(workflow); + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + expect(result).toMatchObject({ + success: false, + remediation: { + category: 'blocked', + shouldEdit: false, + reason, + }, + }); + expect(result.error).toContain('Write'); + expect(result.executionId).toBeUndefined(); + expect(result.simulatedNodes).toBeUndefined(); + expect(result.nodesNotReached).toContain('Write'); + expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); + expect(getOutcome().nodeSimulationPlan).toContainEqual( + expect.objectContaining({ nodeName: 'Agent', verdict: 'execute' }), + ); + expect(getOutcome().verification).toMatchObject({ + success: false, + failureSignature: reason, + evidence: { nodesNotReached: expect.arrayContaining(['Write']) }, + }); + expect(ctx.workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith( + expect.objectContaining({ + verdict: 'failed_terminal', + failureSignature: reason, + }), + ); + }, + ); + + it.each([0, 1])( + 'keeps tools beneath a saved pinned Agent v2 unverified with %i pinned items', + async (itemCount) => { + const workflow = agentToolWorkflow(2); + const { ctx, getOutcome } = makeContext(makeBuildOutcome({ nodeSimulationPlan: plan }), { + executionId: 'exec-parent', + status: 'success', + executedNodeNames: ['Trigger', 'Agent'], + }); + const workflowService = ctx.domainContext.workflowService!; + vi.mocked(workflowService.getAsWorkflowJSON).mockResolvedValue(workflow); + workflowService.getPinnedDataSummary = vi + .fn>() + .mockResolvedValue([{ nodeName: 'Agent', itemCount }]); + + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + + expect(result.success).toBe(true); + expect(result.executionId).toBe('exec-parent'); + expect(result.nodesNotReached).toEqual(['Write']); + expect(result.simulatedNodes).toBeUndefined(); + expect(ctx.domainContext.executionService.run).toHaveBeenCalledWith( + 'wf-1', + undefined, + expect.objectContaining({ verificationPinData: { Write: [{}] } }), + ); + expect(getOutcome().verification).toMatchObject({ + success: true, + evidence: { nodesNotReached: ['Write'] }, + }); + }, + ); + + it('rejects a non-trigger with a simulated tool without recording a verification failure', async () => { + const workflow = agentToolWorkflow(2); + const { ctx, getOutcome, updateBuildOutcome } = makeContext( + makeBuildOutcome({ nodeSimulationPlan: plan }), + { status: 'success' }, + ); + vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(workflow); + + const result = await runTool(ctx, { + workItemId: 'wi-1', + workflowId: 'wf-1', + triggerNodeName: 'Agent', + }); + + expect(result.success).toBe(false); + expect(result.error).toContain('Could not find trigger "Agent" in this workflow.'); + expect(result.remediation).toBeUndefined(); + expect(result.executionId).toBeUndefined(); + expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); + expect(ctx.workflowTaskService.startVerification).not.toHaveBeenCalled(); + expect(ctx.workflowTaskService.reportVerificationVerdict).not.toHaveBeenCalled(); + expect(updateBuildOutcome).not.toHaveBeenCalled(); + expect(getOutcome().verification).toBeUndefined(); + }); + + it('keeps tools beneath a simulated parent unverified', async () => { + const workflow = agentToolWorkflow(2, 3); + const { ctx } = makeContext( + makeBuildOutcome({ + nodeSimulationPlan: [verdict('Agent', true), verdict('Nested'), verdict('Write', true)], + }), + { executionId: 'exec-parent', status: 'success', executedNodeNames: ['Trigger', 'Agent'] }, + ); + vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(workflow); + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + expect(result.success).toBe(true); + expect(result.nodesNotReached).toEqual(['Nested', 'Write']); + expect(result.simulatedNodes?.some((node) => node.nodeName === 'Write')).not.toBe(true); + expect(result.coverageNote).toContain('Agent tool calls, and simulated parents'); + expect(ctx.domainContext.executionService.run).toHaveBeenCalledTimes(1); + }); + + it('includes tools in scoped coverage and aggregates successful trigger passes', async () => { + const workflow = agentToolWorkflow(); + workflow.nodes.push( + agentToolNode('Other Trigger', 'n8n-nodes-base.manualTrigger'), + agentToolNode('Other Agent', '@n8n/n8n-nodes-langchain.agent', { typeVersion: 3.1 }), + agentToolNode('Other Write', 'n8n-nodes-base.slackTool'), + ); + workflow.connections['Other Trigger'] = { + main: [[{ node: 'Other Agent', type: 'main', index: 0 }]], + }; + workflow.connections['Other Write'] = { + ai_tool: [[{ node: 'Other Agent', type: 'ai_tool', index: 0 }]], + }; + const { ctx, getOutcome } = makeContext( + makeBuildOutcome({ + nodeSimulationPlan: [...plan, verdict('Other Agent'), verdict('Other Write', true)], + triggerNodes: [ + { nodeName: 'Trigger', nodeType: 'n8n-nodes-base.manualTrigger' }, + { nodeName: 'Other Trigger', nodeType: 'n8n-nodes-base.manualTrigger' }, + ], + verificationProgress: {}, + }), + { status: 'success' }, + { workflowNodes: workflow.nodes, workflowConnections: workflow.connections }, + ); + ctx.domainContext.executionService.run + .mockResolvedValueOnce({ + status: 'success', + executionId: 'exec-a', + executedNodeNames: ['Trigger', 'Agent'], + }) + .mockResolvedValueOnce({ + status: 'success', + executionId: 'exec-b', + executedNodeNames: ['Other Trigger', 'Other Agent', 'Other Write'], + }) + .mockResolvedValueOnce({ + status: 'success', + executionId: 'exec-c', + executedNodeNames: ['Trigger', 'Agent', 'Write'], + }); + const first = await runTool(ctx, { + workItemId: 'wi-1', + workflowId: 'wf-1', + triggerNodeName: 'Trigger', + }); + expect(first.nodesNotReached).toEqual(['Write']); + expect(first.simulatedNodes).toBeUndefined(); + expect(first.simulationNote).toBeUndefined(); + expect(ctx.domainContext.executionService.run.mock.calls[0][2].verificationPinData).toEqual({ + Write: [{}], + 'Other Write': [{}], + }); + const second = await runTool(ctx, { + workItemId: 'wi-1', + workflowId: 'wf-1', + triggerNodeName: 'Other Trigger', + }); + expect(second.simulatedNodes).toEqual([{ nodeName: 'Other Write', reason: 'Writes data' }]); + expect(second.simulationNote).toContain('Other Write'); + expect(second.simulationNote).not.toContain('no real external writes'); + expect(getOutcome().verification?.claim?.nodesNotReached).toEqual(['Write']); + await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1', triggerNodeName: 'Trigger' }); + expect(getOutcome().verification?.claim?.nodesNotReached).toEqual([]); + expect( + getOutcome() + .verification?.claim?.simulatedNodes.map((node) => node.nodeName) + .sort(), + ).toEqual(['Other Write', 'Write']); + }); +}); + describe('verify-built-workflow tool — publish state', () => { it('warns that the fix is not live when the published version is an older one', async () => { const { ctx, getOutcome } = makeContext( diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/analyze-result.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/analyze-result.test.ts index 7d2d1bb83ee..e7e530b7614 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/analyze-result.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/analyze-result.test.ts @@ -66,7 +66,10 @@ describe('analyzeVerificationResult — halted wait gates', () => { // covers time-based Wait and Form gates), and non-gate dead ends keep the // seed-and-re-run guidance instead of being attributed to the gate. expect(analysis.coverageNote).not.toContain('human decision'); - expect(analysis.coverageNote).toContain('NOT behind the gate'); + expect(analysis.coverageNote).toContain('Other unreached nodes remain unverified'); + expect(analysis.coverageNote).toContain('not behind a wait gate'); + expect(analysis.coverageNote).toContain('seed matching test data'); + expect(analysis.coverageNote).toContain('only when the user has authorized that write'); }); it('keeps the generic partial-coverage guidance when no gate halted the run', () => { @@ -410,6 +413,8 @@ describe('analyzeVerificationResult — trigger-scoped coverage', () => { expect(analysis.nodesNotReached).toEqual(['Post Summary']); expect(analysis.coverageNote).toContain('Every Weekday 9am'); expect(analysis.coverageNote).toContain('triggerNodeName'); + expect(analysis.coverageNote).toContain('union of those passes'); + expect(analysis.coverageNote).toContain('For unreached main-flow nodes on this branch'); // The generic "a lookup returned nothing" cause would send the agent // editing a workflow whose other branch is simply not on this path. expect(analysis.coverageNote).not.toContain('lookup or query returned nothing'); @@ -460,7 +465,7 @@ describe('analyzeVerificationResult — trigger-scoped coverage', () => { expect(analysis.coverageNote).toContain('1st of Month'); }); - it('keeps the generic zero-output guidance when no trigger was named', () => { + it('gives neutral coverage guidance when no trigger was named', () => { const analysis = analyzeVerificationResult({ result: weekdayPass, buildOutcome: twoBranchOutcome, @@ -469,6 +474,33 @@ describe('analyzeVerificationResult — trigger-scoped coverage', () => { runId: 'run-1', }); - expect(analysis.coverageNote).toContain('Seed matching test data'); + expect(analysis.coverageNote).toContain('Agent tool calls'); + }); +}); + +describe('analyzeVerificationResult — uncalled tools', () => { + it('keeps an uncalled tool unverified without diagnosing an empty Agent result', () => { + const analysis = analyzeVerificationResult({ + result: { + executionId: 'exec-agent', + status: 'success', + executedNodeNames: ['Agent'], + lastNodeExecuted: 'Agent', + data: { Agent: [{ output: 'No tool call needed' }] }, + }, + buildOutcome, + simulatedNodes: [{ nodeName: 'Publish', reason: 'Sends a message' }], + stateBefore: undefined, + runId: 'run-1', + }); + + expect(analysis.reachedSimulatedNodes).toEqual([]); + expect(analysis.nodesNotReached).toContain('Publish'); + expect(analysis.coverageNote).toContain( + 'A tool can remain uncalled even when its Agent succeeds', + ); + expect(analysis.coverageNote).not.toContain('because it produced no output items'); + expect(analysis.coverageNote).toContain('if a simulated or pinned lookup returned no items'); + expect(analysis.coverageNote).toContain('inspect its fixture'); }); }); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/tool-simulation-preflight.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/tool-simulation-preflight.test.ts new file mode 100644 index 00000000000..abf9ef352c7 --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/tool-simulation-preflight.test.ts @@ -0,0 +1,178 @@ +import { + agentToolNode, + agentToolWorkflow, + mcpToolWorkflow, +} from '../../../../__tests__/agent-tool-workflow'; +import type { NodeSimulationVerdict } from '../../../../workflow-loop/workflow-loop-state'; +import { checkToolSimulationSupport } from '../tool-simulation-preflight'; + +function verdict(nodeName: string, simulate = false): NodeSimulationVerdict { + return { + nodeName, + verdict: simulate ? 'simulate' : 'execute', + reason: 'Test classification', + confidence: 'high', + source: 'deterministic', + }; +} + +function check( + workflow = agentToolWorkflow(), + options: { + pins?: Record; + plan?: NodeSimulationVerdict[]; + triggerNodeName?: string; + } = {}, +) { + return checkToolSimulationSupport({ + workflow, + plan: options.plan ?? [verdict('Agent'), verdict('Nested'), verdict('Write', true)], + prepared: { + verificationPinData: options.pins ?? { Write: [{}] }, + simulatedNodes: [], + haltedGateNames: [], + }, + triggerNodeName: options.triggerNodeName, + }); +} + +describe('tool simulation preflight', () => { + it.each(['v0', undefined] as const)( + 'blocks tool simulation with execution order %s', + (executionOrder) => { + const workflow = agentToolWorkflow(); + workflow.settings = { executionOrder }; + expect(check(workflow)?.guidance).toContain( + 'Write (called by Agent; legacy execution order)', + ); + expect(check(workflow, { pins: { Agent: [{}] } })).toBeUndefined(); + }, + ); + + it.each([3, 3.1, 4])('allows main Agent %s to simulate tools', (version) => { + expect(check(agentToolWorkflow(version))).toBeUndefined(); + expect(check(agentToolWorkflow(version, 3))).toBeUndefined(); + }); + + it.each([3, 3.1, 4])('allows Agent Tool %s to simulate children', (version) => { + expect(check(agentToolWorkflow(3.1, version))).toBeUndefined(); + }); + + it.each([2, 2.3])('blocks a simulated tool called by Agent %s', (version) => { + expect(check(agentToolWorkflow(version))).toMatchObject({ + reason: 'unsupported_tool_simulation', + guidance: expect.stringContaining('Write (called by Agent)'), + }); + }); + + it.each([2, 2.2])('blocks children of Agent Tool %s', (version) => { + expect(check(agentToolWorkflow(3.1, version))?.guidance).toContain('Write (called by Nested)'); + }); + + it.each([undefined, 'MCP Server'])( + 'checks MCP tools without main connections with trigger %s', + (triggerNodeName) => { + const workflow = mcpToolWorkflow(); + expect(check(workflow, { triggerNodeName })).toMatchObject({ + reason: 'unsupported_tool_simulation', + nodesNotReached: expect.arrayContaining(['MCP Server', 'Write']), + }); + expect(check(workflow, { triggerNodeName, plan: [] })?.reason).toBe( + 'incomplete_tool_simulation_plan', + ); + }, + ); + + it('checks a shared MCP tool only when its trigger is in scope', () => { + const workflow = agentToolWorkflow(); + workflow.nodes.push(agentToolNode('MCP Server', '@n8n/n8n-nodes-langchain.mcpTrigger')); + workflow.connections.Write = { + ai_tool: [ + [ + { node: 'Agent', type: 'ai_tool', index: 0 }, + { node: 'MCP Server', type: 'ai_tool', index: 0 }, + ], + ], + }; + expect(check(workflow)?.guidance).toContain('Write (called by MCP Server)'); + expect(check(workflow, { triggerNodeName: 'Trigger' })).toBeUndefined(); + expect(check(workflow, { triggerNodeName: 'MCP Server' })?.guidance).toContain( + 'Write (called by MCP Server)', + ); + }); + + it('blocks tools attached to an unknown caller', () => { + const workflow = agentToolWorkflow(); + workflow.nodes[1].type = 'community.customAgent'; + expect(check(workflow)?.reason).toBe('unsupported_tool_simulation'); + }); + + it('allows reads on an older caller when no simulation is needed', () => { + expect(check(agentToolWorkflow(2), { plan: [verdict('Write')], pins: {} })).toBeUndefined(); + }); + + it('checks effective saved pins as well as planned simulation', () => { + const workflow = agentToolWorkflow(2); + workflow.pinData = { Write: [{ result: 'saved' }] }; + expect(check(workflow, { plan: [verdict('Write')], pins: {} })?.reason).toBe( + 'unsupported_tool_simulation', + ); + }); + + it.each([{ items: [] }, { items: [{ output: 'Fixture' }] }])( + 'stops at an engine-scheduled parent pin with %j', + ({ items }) => { + expect(check(agentToolWorkflow(2), { pins: { Agent: items } })).toBeUndefined(); + const workflow = agentToolWorkflow(2); + workflow.pinData = { Agent: items }; + expect(check(workflow)).toBeUndefined(); + }, + ); + + it('allows a pin on an engine-scheduled nested parent', () => { + expect(check(agentToolWorkflow(3.1, 2), { pins: { Nested: [{}] } })).toBeUndefined(); + }); + + it('does not trust a pin on an inline nested parent', () => { + const workflow = agentToolWorkflow(2, 3); + workflow.pinData = { Nested: [{ output: 'Saved fixture' }] }; + const blocked = check(workflow, { pins: { Nested: [{}], Write: [{}] } }); + expect(blocked?.guidance).toContain('Nested (called by Agent)'); + expect(blocked?.guidance).toContain('Write (called by Nested)'); + }); + + it('requires missing classifications even when a parent is pinned', () => { + expect( + check(agentToolWorkflow(), { plan: [verdict('Agent')], pins: { Agent: [{}] } })?.reason, + ).toBe('incomplete_tool_simulation_plan'); + }); + + it('checks each caller of a shared tool and respects trigger scope', () => { + const workflow = agentToolWorkflow(); + workflow.nodes.push( + agentToolNode('Other Trigger', 'n8n-nodes-base.manualTrigger'), + agentToolNode('Older', '@n8n/n8n-nodes-langchain.agent', { typeVersion: 2 }), + ); + workflow.connections['Other Trigger'] = { main: [[{ node: 'Older', type: 'main', index: 0 }]] }; + workflow.connections.Write = { + ai_tool: [ + [ + { node: 'Agent', type: 'ai_tool', index: 0 }, + { node: 'Older', type: 'ai_tool', index: 0 }, + ], + ], + }; + expect(check(workflow)?.guidance).toContain('Write (called by Older)'); + expect(check(workflow, { triggerNodeName: 'Trigger' })).toBeUndefined(); + expect(check(workflow, { pins: { Agent: [{}], Write: [{}] } })?.reason).toBe( + 'unsupported_tool_simulation', + ); + expect(check(workflow, { pins: { Older: [{}], Write: [{}] } })).toBeUndefined(); + }); + + it('handles cycles without losing an unsupported caller', () => { + const workflow = agentToolWorkflow(3.1, 3); + workflow.connections.Agent = { ai_tool: [[{ node: 'Nested', type: 'ai_tool', index: 0 }]] }; + expect(check(workflow)?.reason).toBe('unsupported_tool_simulation'); + }); +}); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts index 85d39423358..3edec2a2d1c 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts @@ -1,7 +1,5 @@ import { isRecord } from '@n8n/utils/is-record'; import { isPlaceholderValue } from '@n8n/utils/placeholder'; -import { toEngineConnections, type WorkflowJSON } from '@n8n/workflow-sdk'; -import { getChildNodes, NodeConnectionTypes } from 'n8n-workflow'; import type { ExecutionRunResult, VerificationNodePreview } from './types'; import { @@ -167,17 +165,6 @@ export function countProducedOutputRows( return count; } -function maxEmittedItemCount(resultData: Record | undefined): number { - if (!resultData) return 0; - - let max = 0; - for (const nodeOutput of Object.values(resultData)) { - const count = countOutputItems(nodeOutput) ?? 0; - if (count > max) max = count; - } - return max; -} - function messageMatchesAny(normalized: string, keywords: readonly string[]): boolean { return keywords.some((keyword) => normalized.includes(keyword)); } @@ -354,7 +341,7 @@ export function buildSimulationNote( ): string | undefined { if (reachedSimulatedNodes.length > 0) { return ( - `Simulated ${reachedSimulatedNodes.length} node(s) during verification — no real external writes happened: ` + + `Simulated ${reachedSimulatedNodes.length} node(s) with fixture output during verification: ` + reachedSimulatedNodes.map((n) => `${n.nodeName} (${n.reason})`).join('; ') + '. Relay this to the user when presenting the result.' ); @@ -388,6 +375,18 @@ function buildCoverageNote( 'treat coverage as the union of those passes. Do not edit, disable, reorder, or copy the ' + 'workflow to reach them.' : ''; + const guidance = success + ? ' Check branch selection, input items, Agent tool calls, and simulated parents. ' + + 'A tool can remain uncalled even when its Agent succeeds. ' + + 'For unreached main-flow nodes on this branch that are not behind a wait gate: ' + + 'if a simulated or pinned lookup returned no items, inspect its fixture and use ' + + 'representative test input. If a live lookup or query returned no matching records, ' + + 'seed matching test data only when the user has authorized that write, then re-run verification. ' + + 'If a Code node dropped a collection, inspect `$input.first().json`; use ' + + '`$input.all().map(i => i.json)` when it needs all items. ' + + 'HTTP Request splits a top-level array into separate items. ' + + 'Do not report unreached nodes as verified.' + : ''; if (success && reachedHaltedGates.length > 0) { return ( `Verification pauses at wait gate(s) ${reachedHaltedGates.join(', ')} — in a live run the ` + @@ -395,10 +394,9 @@ function buildCoverageNote( `gate is not simulated. ${nodesNotReached.length} planned node(s) were not reached: ` + `${nodesNotReached.join(', ')}. Nodes behind the gate are expected to be unreached — do ` + 'not edit the workflow or re-run verification to force coverage there; recommend a live ' + - 'end-to-end test instead. Any unreached node NOT behind the gate did not receive input ' + - 'items (usually an empty lookup or query) — seed matching test data and re-run before ' + - 'treating it as verified.' + - triggerScopeNote + 'end-to-end test instead. Other unreached nodes remain unverified.' + + triggerScopeNote + + guidance ); } if (success && triggerNodeName) { @@ -406,20 +404,12 @@ function buildCoverageNote( `Partial coverage by design: ${String(nodesNotReached.length)} planned node(s) were not ` + `reached: ${nodesNotReached.join(', ')}.` + triggerScopeNote + - " Any unreached node that IS on this trigger's branch did not receive input items " + - '(usually an empty lookup or query) — seed matching test data and re-run before treating ' + - 'it as verified.' + guidance ); } const ending = result.lastNodeExecuted - ? `. Execution ended at "${result.lastNodeExecuted}"${success ? ' because it produced no output items (empty item lists stop downstream nodes)' : ''}.` + ? `. Execution ended at "${result.lastNodeExecuted}".` : '.'; - const collapsedFromCollection = success && maxEmittedItemCount(result.data) >= 2; - const guidance = success - ? collapsedFromCollection - ? ' An upstream node emitted multiple items but the chain collapsed to zero before reaching them. The usual cause is a Code node reading `$input.first().json` (a single split item) instead of `$input.all().map(i => i.json)` — an HTTP Request node splits a top-level array (including a bare array of IDs) into one item per element. Fix the node that dropped the items to read every item, then re-run verification. Do NOT report the workflow as fully verified.' - : ' This usually means a lookup or query returned nothing. Seed matching test data and re-run verification, or tell the user the unreached part needs a manual test. Do NOT report the workflow as fully verified.' - : ''; return ( `Partial coverage: ${nodesNotReached.length} node(s) were never reached and remain UNVERIFIED: ` + nodesNotReached.join(', ') + @@ -462,16 +452,6 @@ export interface VerificationAnalysis { nodeErrors: ExecutionNodeError[]; } -export function getTriggerMainFlowScope( - connections: WorkflowJSON['connections'], - triggerNodeName: string, -): Set { - return new Set([ - triggerNodeName, - ...getChildNodes(toEngineConnections(connections), triggerNodeName, NodeConnectionTypes.Main), - ]); -} - export function analyzeVerificationResult(args: { result: ExecutionRunResult; buildOutcome: WorkflowBuildOutcome; @@ -490,7 +470,7 @@ export function analyzeVerificationResult(args: { chatModelRecovery?: ChatModelRecoveryOptions; /** Trigger this pass started from, when the caller named one. */ triggerNodeName?: string; - /** Main-flow nodes that belong to the selected trigger. */ + /** Main-flow nodes and attached tools that belong to the selected trigger. */ verificationScope?: ReadonlySet; }): VerificationAnalysis { const { diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts index e98cc00c8b9..c0ce9db5526 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts @@ -28,20 +28,34 @@ export async function handleMissingSimulationPlan(args: { workflowTaskService: WorkflowTaskService; workflowId: string; }): Promise { - const { input, context, workflowTaskService, workflowId } = args; const guidance = 'Verification was not run because the build outcome has no simulation plan. ' + 'Rebuild or resubmit the workflow so destructive nodes can be classified before verification.'; + return await handleBlockedVerification({ ...args, reason: 'missing_simulation_plan', guidance }); +} + +export async function handleBlockedVerification(args: { + input: ResolvedVerifyInput; + context: OrchestrationContext; + workflowTaskService: WorkflowTaskService; + workflowId: string; + reason: string; + guidance: string; + nodesNotReached?: string[]; +}): Promise { + const { input, context, workflowTaskService, workflowId, reason, guidance, nodesNotReached } = + args; const remediation = createRemediation({ category: 'blocked', shouldEdit: false, - reason: 'missing_simulation_plan', + reason, guidance, }); - context.logger.warn( - 'verify-built-workflow: build outcome has no simulation plan — refusing to run without simulation safeguards', - { workItemId: input.workItemId, workflowId }, - ); + context.logger.warn('verify-built-workflow: preflight blocked verification', { + workItemId: input.workItemId, + workflowId, + reason, + }); try { await workflowTaskService.updateBuildOutcome(input.workItemId, { remediation, @@ -49,8 +63,8 @@ export async function handleMissingSimulationPlan(args: { attempted: true, success: false, status: 'unknown', - failureSignature: 'missing_simulation_plan', - evidence: { errorMessage: guidance }, + failureSignature: reason, + evidence: { errorMessage: guidance, nodesNotReached }, verifiedAt: new Date().toISOString(), }, }); @@ -63,7 +77,7 @@ export async function handleMissingSimulationPlan(args: { runId: context.runId, workflowId, verdict: 'failed_terminal', - failureSignature: 'missing_simulation_plan', + failureSignature: reason, diagnosis: guidance, remediation, summary: guidance, @@ -80,6 +94,10 @@ export async function handleMissingSimulationPlan(args: { resolvedWorkItemId: input.workItemId, status: 'unknown', error: guidance, + nodesNotReached, + coverageNote: nodesNotReached?.length + ? 'Verification did not run. All listed nodes remain unverified.' + : undefined, remediation, guidance, }; diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/tool-simulation-preflight.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/tool-simulation-preflight.ts new file mode 100644 index 00000000000..177e996ae99 --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/tool-simulation-preflight.ts @@ -0,0 +1,105 @@ +import type { WorkflowJSON } from '@n8n/workflow-sdk'; + +import type { PreparedVerificationRun } from './prepare-run'; +import { itemsForNode } from '../../../utils/node-keyed-items'; +import type { NodeSimulationVerdict } from '../../../workflow-loop/workflow-loop-state'; +import { + AGENT_NODE_TYPE, + AGENT_TOOL_NODE_TYPE, + createVerificationGraph, + getTriggerMainFlowScope, +} from '../../workflows/verification-graph'; + +export interface ToolSimulationBlocker { + reason: string; + guidance: string; + nodesNotReached?: string[]; +} + +const GRAPH_UNAVAILABLE: ToolSimulationBlocker = { + reason: 'verification_graph_unavailable', + guidance: + 'Verification was not run because the current workflow graph could not be inspected. Read the workflow and retry verification.', +}; + +/** Check every caller path before verification can run a tool. */ +export function checkToolSimulationSupport(args: { + workflow: WorkflowJSON | undefined; + workflowPinnedNodeNames?: string[]; + plan: NodeSimulationVerdict[]; + prepared: PreparedVerificationRun; + triggerNodeName?: string; +}): ToolSimulationBlocker | undefined { + const { workflow, plan, prepared, triggerNodeName } = args; + if (!workflow || !Array.isArray(workflow.nodes)) { + return GRAPH_UNAVAILABLE; + } + let graph: ReturnType; + let mainScope: Set; + let scope: Set; + try { + graph = createVerificationGraph(workflow); + mainScope = triggerNodeName + ? getTriggerMainFlowScope(workflow.connections, triggerNodeName) + : graph.rootNodeNames; + scope = graph.withTools(mainScope); + } catch { + return GRAPH_UNAVAILABLE; + } + const verdicts = new Map(plan.map((verdict) => [verdict.nodeName, verdict])); + const missing = new Set( + [...scope].flatMap((name) => graph.toolsFor(name)).filter((name) => !verdicts.has(name)), + ); + if (missing.size > 0) { + return { + reason: 'incomplete_tool_simulation_plan', + guidance: `Verification was not run because attached tools have no classification: ${[...missing].join(', ')}. Resubmit the workflow to classify these tools before verification.`, + nodesNotReached: [...scope], + }; + } + const unsupported = new Set(); + const workflowPinnedNodeNames = new Set(args.workflowPinnedNodeNames); + const visited = new Set(); + const pending: Array<{ name: string; scheduled: boolean; caller?: string }> = [...mainScope].map( + (name) => ({ name, scheduled: true }), + ); + while (pending.length > 0) { + const { name, scheduled, caller } = pending.pop()!; + const node = graph.nodesByName.get(name); + if (!node) continue; + const verdict = verdicts.get(name); + const pinned = + workflowPinnedNodeNames.has(name) || + itemsForNode(prepared.verificationPinData, name) !== undefined || + itemsForNode(workflow.pinData, name) !== undefined; + if (caller !== undefined) { + // An unset order is also legacy. The engine can resume the Agent before its tool result. + const legacyOrder = workflow.settings?.executionOrder !== 'v1'; + if ((!scheduled || legacyOrder) && (verdict?.verdict === 'simulate' || pinned)) { + unsupported.add( + `${name} (called by ${caller}${legacyOrder ? '; legacy execution order' : ''})`, + ); + } + } + // Inline calls do not read pin data. Only an engine-scheduled pin skips children. + if (scheduled && pinned) continue; + const state = JSON.stringify([name, scheduled, caller === undefined]); + if (visited.has(state)) continue; + visited.add(state); + const supportsTools = + scheduled && + node.typeVersion >= 3 && + (caller === undefined ? node.type === AGENT_NODE_TYPE : node.type === AGENT_TOOL_NODE_TYPE); + for (const tool of graph.toolsFor(name)) { + pending.push({ name: tool, scheduled: supportsTools, caller: name }); + } + } + if (unsupported.size > 0) { + return { + reason: 'unsupported_tool_simulation', + guidance: `Verification was not run because these tools need simulation on an unsupported execution path: ${[...unsupported].join('; ')}. Report these tools as unverified. Keep the Agent versions unchanged.`, + nodesNotReached: [...scope], + }; + } + return undefined; +} diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts index d9828ccf954..156c4b5b88b 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts @@ -11,15 +11,12 @@ import { isTriggerNodeType } from 'n8n-workflow'; import { z } from 'zod'; import type { InstanceAiWorkflowService, OrchestrationContext } from '../../types'; -import { - analyzeVerificationResult, - buildNodePreviews, - getTriggerMainFlowScope, -} from './verification/analyze-result'; +import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result'; import { deriveVerificationClaim } from './verification/claim'; import type { VerificationPublishState } from './verification/claim'; import { handleMissingSimulationPlan, + handleBlockedVerification, persistVerificationOutcome, } from './verification/finalize-result'; import { prepareVerificationRun } from './verification/prepare-run'; @@ -32,6 +29,8 @@ import { skippedParameterCheckSchema, } from './verification/resolved-parameter-warnings'; import { runScriptedGateVerification } from './verification/scripted-gate-run'; +import { checkToolSimulationSupport } from './verification/tool-simulation-preflight'; +import { createVerificationGraph, getTriggerMainFlowScope } from '../workflows/verification-graph'; import { describeClaimLiveState } from '../../workflow-loop/render-claim'; import type { VerificationClaim } from '../../workflow-loop/workflow-loop-state'; import { @@ -286,8 +285,10 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { .getAsWorkflowJSON(workflowId) .catch(() => undefined); if ( + workflow && + Array.isArray(workflow.nodes) && resolvedInput.triggerNodeName !== undefined && - !workflow?.nodes.some( + !workflow.nodes.some( (node) => node.name === resolvedInput.triggerNodeName && isTriggerNodeType(node.type), ) ) { @@ -297,6 +298,40 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { error: `Could not find trigger "${resolvedInput.triggerNodeName}" in this workflow. Read the workflow. Select an existing trigger.`, }; } + // WorkflowJSON omits saved pins. The summary supplies names without their payloads. + let workflowPinnedNodeNames: string[] | undefined; + try { + const workflowPins = workflow + ? await target.domainContext.workflowService.getPinnedDataSummary?.(workflowId) + : undefined; + workflowPinnedNodeNames = workflowPins?.map(({ nodeName }) => nodeName); + } catch { + return await handleBlockedVerification({ + input: resolvedInput, + context, + workflowTaskService, + workflowId, + reason: 'verification_pin_summary_unavailable', + guidance: + 'Verification was not run because saved pinned data could not be inspected. Retry verification.', + }); + } + const blocker = checkToolSimulationSupport({ + workflow, + workflowPinnedNodeNames, + plan: buildOutcome.nodeSimulationPlan, + prepared, + triggerNodeName: resolvedInput.triggerNodeName, + }); + if (blocker) { + return await handleBlockedVerification({ + input: resolvedInput, + context, + workflowTaskService, + workflowId, + ...blocker, + }); + } const chatModelRecovery = workflow ? await collectChatModelRecoveryContext( target.domainContext, @@ -312,7 +347,9 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { : undefined; const verificationScope = buildOutcome.verificationProgress && selectedTriggerNodeName && workflow - ? getTriggerMainFlowScope(workflow.connections, selectedTriggerNodeName) + ? createVerificationGraph(workflow).withTools( + getTriggerMainFlowScope(workflow.connections, selectedTriggerNodeName), + ) : undefined; const previousProgress = await workflowTaskService.startVerification( resolvedInput.workItemId, diff --git a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/classify-node-destructiveness.service.test.ts b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/classify-node-destructiveness.service.test.ts index 3905a3cea84..0f0b6ad9fa2 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/classify-node-destructiveness.service.test.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/classify-node-destructiveness.service.test.ts @@ -6,6 +6,11 @@ vi.mock('../../../utils/eval-agents', async () => { return { ...actual, createEvalAgent: vi.fn(), extractText: vi.fn() }; }); +import { + agentToolNode, + agentToolWorkflow, + mcpToolWorkflow, +} from '../../../__tests__/agent-tool-workflow'; import { createEvalAgent, extractText } from '../../../utils/eval-agents'; import { classifyNodesForSimulation } from '../classify-node-destructiveness.service'; @@ -381,3 +386,104 @@ describe('classifyNodesForSimulation', () => { }); }); }); + +describe('attached tool classification', () => { + beforeEach(() => vi.clearAllMocks()); + + it('classifies direct reads and nested writes and keeps Agents executable', async () => { + const workflow = agentToolWorkflow(3.1, 3); + workflow.nodes.push( + agentToolNode('Read', 'n8n-nodes-base.slackTool', { parameters: { operation: 'get' } }), + ); + workflow.connections.Read = { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] }; + const plan = await classifyNodesForSimulation({ workflow }); + expect(Object.fromEntries(plan.map(({ nodeName, verdict }) => [nodeName, verdict]))).toEqual({ + Agent: 'execute', + Nested: 'execute', + Read: 'execute', + Write: 'simulate', + }); + expect(mockCreateEvalAgent).not.toHaveBeenCalled(); + }); + + it.each([ + ['n8n-nodes-base.httpRequestTool', { method: 'GET' }, 'execute'], + ['n8n-nodes-base.emailSendTool', {}, 'simulate'], + ['n8n-nodes-base.executeWorkflowTool', {}, 'simulate'], + ['@n8n/n8n-nodes-langchain.toolWorkflow', {}, 'simulate'], + ['@n8n/n8n-nodes-langchain.mcpClientTool', {}, 'simulate'], + ['@n8n/n8n-nodes-langchain.mcpRegistryClientTool', {}, 'simulate'], + ] as const)('uses the existing rules for %s', async (type, parameters, expected) => { + const workflow = agentToolWorkflow(); + workflow.nodes[2] = agentToolNode('Write', type, { parameters }); + const plan = await classifyNodesForSimulation({ workflow }); + expect(verdictOf(plan, 'Write')).toMatchObject({ verdict: expected, source: 'deterministic' }); + expect(mockCreateEvalAgent).not.toHaveBeenCalled(); + }); + + it('classifies MCP tools without main connections', async () => { + const workflow = mcpToolWorkflow(); + workflow.nodes.push( + agentToolNode('Read', 'n8n-nodes-base.slackTool', { parameters: { operation: 'get' } }), + ); + workflow.connections.Read = { ai_tool: [[{ node: 'MCP Server', type: 'ai_tool', index: 0 }]] }; + + const plan = await classifyNodesForSimulation({ workflow }); + + expect(plan.map(({ nodeName, verdict }) => ({ nodeName, verdict }))).toEqual([ + { nodeName: 'Write', verdict: 'simulate' }, + { nodeName: 'Read', verdict: 'execute' }, + ]); + expect(mockCreateEvalAgent).not.toHaveBeenCalled(); + }); + + it('uses conservative fallback for an unknown tool operation', async () => { + setupAgentFailure(); + const workflow = agentToolWorkflow(); + workflow.nodes[2].parameters = { operation: 'customAction' }; + expect(verdictOf(await classifyNodesForSimulation({ workflow }), 'Write')).toMatchObject({ + verdict: 'simulate', + source: 'fallback', + }); + }); + + it('classifies a shared tool once when the graph has a cycle', async () => { + const workflow = agentToolWorkflow(3.1, 3); + workflow.connections.Write = { + ai_tool: [ + [ + { node: 'Agent', type: 'ai_tool', index: 0 }, + { node: 'Nested', type: 'ai_tool', index: 0 }, + ], + ], + }; + workflow.connections.Agent = { ai_tool: [[{ node: 'Nested', type: 'ai_tool', index: 0 }]] }; + const plan = await classifyNodesForSimulation({ workflow }); + expect(plan.map((node) => node.nodeName).sort()).toEqual(['Agent', 'Nested', 'Write']); + }); + + it('excludes disabled and disconnected tools and other AI sub-nodes', async () => { + const workflow = agentToolWorkflow(); + workflow.nodes[2].disabled = true; + workflow.nodes.push( + agentToolNode('Disconnected', 'n8n-nodes-base.slackTool'), + agentToolNode('Memory', '@n8n/n8n-nodes-langchain.memoryBufferWindow'), + agentToolNode('Parser', '@n8n/n8n-nodes-langchain.outputParserStructured'), + ); + workflow.connections.Memory = { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] }; + workflow.connections.Parser = { + ai_outputParser: [[{ node: 'Agent', type: 'ai_outputParser', index: 0 }]], + }; + expect((await classifyNodesForSimulation({ workflow })).map((node) => node.nodeName)).toEqual([ + 'Agent', + ]); + }); + + it('excludes tools attached only to a disabled container', async () => { + const workflow = agentToolWorkflow(3.1, 3); + workflow.nodes.find((node) => node.name === 'Nested')!.disabled = true; + expect((await classifyNodesForSimulation({ workflow })).map((node) => node.nodeName)).toEqual([ + 'Agent', + ]); + }); +}); diff --git a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/reconcile-simulation-plan.test.ts b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/reconcile-simulation-plan.test.ts index 7b956a1978e..af4247640b3 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/reconcile-simulation-plan.test.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/reconcile-simulation-plan.test.ts @@ -1,10 +1,14 @@ import type { WorkflowJSON } from '@n8n/workflow-sdk'; +import { agentToolWorkflow } from '../../../__tests__/agent-tool-workflow'; import type { NodeSimulationVerdict, WorkflowBuildOutcome, } from '../../../workflow-loop/workflow-loop-state'; -import { CREDENTIALLESS_AI_ROOT_SIMULATION_REASON } from '../plan-verification-simulation'; +import { + CREDENTIALLESS_AI_ROOT_SIMULATION_REASON, + DECLARED_OUTPUT_SIMULATION_REASON, +} from '../plan-verification-simulation'; import { reconcileSimulationPlan } from '../reconcile-simulation-plan'; import { type CredentialEntry, type CredentialMap } from '../resolve-credentials'; @@ -70,6 +74,39 @@ function makeWorkflow(nodes: TestNode[]): WorkflowJSON { // --------------------------------------------------------------------------- describe('reconcileSimulationPlan', () => { + it.each([ + { operation: 'send', declared: false, expected: 'simulate' }, + { operation: 'get', declared: false, expected: 'execute' }, + { operation: 'get', declared: true, expected: 'simulate' }, + ])( + 'reconciles attached $operation tools with declared output: $declared', + async ({ operation, declared, expected }) => { + const workflow = agentToolWorkflow(3.1, 3); + workflow.nodes[2].parameters = { operation }; + workflow.nodes[2].credentials = { slackApi: { id: 'cred-1', name: 'Tool account' } }; + const reason = declared ? DECLARED_OUTPUT_SIMULATION_REASON : MOCKED_CREDENTIAL_REASON; + const patch = await reconcileSimulationPlan({ + workflow, + buildOutcome: makeBuildOutcome({ + mockedNodeNames: ['Write'], + mockedCredentialsByNode: { Write: ['slackApi'] }, + nodeSimulationPlan: [{ ...mockedVerdict('Write'), reason }], + simulationFixtures: { Write: [{ id: 'fixture' }] }, + }), + availableCredentials: makeCredentialMap([ + { id: 'cred-1', name: 'Tool account', type: 'slackApi' }, + ]), + }); + expect(patch?.nodeSimulationPlan).toEqual([ + expect.objectContaining({ nodeName: 'Write', verdict: expected }), + ]); + expect(patch?.nodeSimulationPlan?.[0].reason).not.toBe(MOCKED_CREDENTIAL_REASON); + expect(patch?.simulationFixtures?.Write).toEqual( + expected === 'simulate' ? [{ id: 'fixture' }] : undefined, + ); + }, + ); + it('returns undefined when the outcome has no mocked credentials', async () => { const patch = await reconcileSimulationPlan({ buildOutcome: makeBuildOutcome(), diff --git a/packages/@n8n/instance-ai/src/tools/workflows/classify-node-destructiveness.service.ts b/packages/@n8n/instance-ai/src/tools/workflows/classify-node-destructiveness.service.ts index d1eb82acd49..8ccb19343dc 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/classify-node-destructiveness.service.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/classify-node-destructiveness.service.ts @@ -1,7 +1,7 @@ /** * Node Destructiveness Classification * - * Decides, per main-flow node, whether a verification execution may run the + * Decides, per main-flow node and attached tool, whether verification may run the * node for real (`execute`) or must mock its output via per-execution pin * data (`simulate`) because the node would create, update, or delete data in * an external system. @@ -18,10 +18,10 @@ import { isRecord } from '@n8n/utils/is-record'; import { isAiRootNodeType, type WorkflowJSON } from '@n8n/workflow-sdk'; -import type { IConnections } from 'n8n-workflow'; import { z } from 'zod'; import { isTriggerNodeType } from './workflow-json-utils'; +import { AGENT_TOOL_NODE_TYPE, createVerificationGraph } from './verification-graph'; import type { ModelConfig } from '../../types'; import { HAIKU_MODEL } from '../../utils/eval-agents'; import { generateValidatedJson } from '../../utils/generate-validated-json'; @@ -33,6 +33,8 @@ export interface ClassifyNodesForSimulationInput { workflow: WorkflowJSON; /** Node names whose credentials were mocked — always simulated. */ mockedNodeNames?: string[]; + /** Classify only these names. Keep the full graph for tool discovery. */ + nodeNames?: ReadonlySet; /** Host-resolved model used when no eval model API key is configured in the environment. */ fallbackModelConfig?: ModelConfig; } @@ -74,12 +76,14 @@ const SAFE_NODE_TYPES = new Set([ * AI roots (agent/chain/extractor/classifier nodes) call an LLM but are * read-only towards user systems, so they classify as safe — the set lives in * `@n8n/workflow-sdk` (`isAiRootNodeType`), shared with fixture shaping. Write - * operations living as agent *tools* are a known gap. Roots whose LLM + * tools receive separate classifications. Roots whose LLM * sub-node has no usable credentials are overridden to `simulate` later, in * `plan-verification-simulation.ts`. */ function isSafeNodeType(nodeType: string): boolean { - return SAFE_NODE_TYPES.has(nodeType) || isAiRootNodeType(nodeType); + return ( + SAFE_NODE_TYPES.has(nodeType) || isAiRootNodeType(nodeType) || nodeType === AGENT_TOOL_NODE_TYPE + ); } /** Node types that are destructive by their very type (no operation param needed). */ @@ -89,6 +93,15 @@ const DESTRUCTIVE_NODE_TYPES = new Map([ ['n8n-nodes-base.ssh', 'Runs commands on a remote machine'], // Simulating the call skips the sub-workflow entirely, side effects included. ['n8n-nodes-base.executeWorkflow', 'Executes another workflow which may have side effects'], + [ + '@n8n/n8n-nodes-langchain.toolWorkflow', + 'Executes another workflow which may have side effects', + ], + ['@n8n/n8n-nodes-langchain.mcpClientTool', 'Calls remote MCP tools which may have side effects'], + [ + '@n8n/n8n-nodes-langchain.mcpRegistryClientTool', + 'Calls remote MCP tools which may have side effects', + ], ]); const CODE_NODE_TYPES = new Set([ @@ -184,27 +197,6 @@ function getStringParam(node: WorkflowNode, key: string): string | undefined { return typeof value === 'string' && value.length > 0 ? value : undefined; } -/** - * Node names reachable over `main` connections (sources and destinations). - * Sub-nodes (ai_tool, ai_languageModel, …) and disconnected nodes never carry - * pin data, so the plan excludes them. - */ -function collectMainFlowNodeNames(workflow: WorkflowJSON): Set { - const connections = (workflow.connections ?? {}) as IConnections; - const names = new Set(); - for (const [sourceName, outputs] of Object.entries(connections)) { - const main = outputs.main; - if (!main) continue; - names.add(sourceName); - for (const port of main) { - for (const target of port ?? []) { - names.add(target.node); - } - } - } - return names; -} - function deterministic( nodeName: string, verdict: 'simulate' | 'execute', @@ -217,20 +209,25 @@ function deterministicVerdict( node: WorkflowNode & { name: string }, mockedNodeNames: Set, ): NodeSimulationVerdict | 'ambiguous' { + // Generated node tools use the same operations as their base node. + const nodeType = + node.type.startsWith('n8n-nodes-base.') && node.type.endsWith('Tool') + ? node.type.slice(0, -4) + : node.type; if (mockedNodeNames.has(node.name)) { return deterministic(node.name, 'simulate', 'Credentials are not configured for this node'); } - if (isSafeNodeType(node.type)) { + if (isSafeNodeType(nodeType)) { return deterministic(node.name, 'execute', 'Transforms data without touching external systems'); } - const destructiveByType = DESTRUCTIVE_NODE_TYPES.get(node.type); + const destructiveByType = DESTRUCTIVE_NODE_TYPES.get(nodeType); if (destructiveByType) { return deterministic(node.name, 'simulate', destructiveByType); } - if (node.type === HTTP_REQUEST_NODE_TYPE) { + if (nodeType === HTTP_REQUEST_NODE_TYPE) { const method = (getStringParam(node, 'method') ?? 'GET').toUpperCase(); if (['GET', 'HEAD', 'OPTIONS'].includes(method)) { return deterministic( @@ -248,7 +245,7 @@ function deterministicVerdict( // parks the run in `waiting` and everything downstream is never verified. // Pinning them lets execution flow past (a pinned node never calls // putExecutionToWait). - if (node.type === FORM_NODE_TYPE) { + if (nodeType === FORM_NODE_TYPE) { return deterministic( node.name, 'simulate', @@ -256,7 +253,7 @@ function deterministicVerdict( ); } - if (node.type === WAIT_NODE_TYPE) { + if (nodeType === WAIT_NODE_TYPE) { // Wait node defaults: resume='timeInterval', amount=1, unit='hours'. const params = isRecord(node.parameters) ? node.parameters : {}; const resume = typeof params.resume === 'string' ? params.resume : 'timeInterval'; @@ -281,7 +278,7 @@ function deterministicVerdict( ); } - if (CODE_NODE_TYPES.has(node.type)) { + if (CODE_NODE_TYPES.has(nodeType)) { const source = ( getStringParam(node, 'jsCode') ?? getStringParam(node, 'pythonCode') ?? @@ -395,14 +392,14 @@ function nodeHasName(node: WorkflowNode): node is WorkflowNode & { name: string /** * Classify every main-flow, non-trigger node of the workflow. Returns one - * verdict per node, ordered as the nodes appear in the workflow. Triggers and - * user-action nodes are out of scope (Mechanism A handles them); sub-nodes - * and disconnected nodes are excluded because pin data cannot apply to them. + * verdict per node, ordered as the nodes appear in the workflow. Include + * attached tools. Exclude triggers, other sub-nodes, and disconnected nodes. */ export async function classifyNodesForSimulation( input: ClassifyNodesForSimulationInput, ): Promise { - const mainFlowNames = collectMainFlowNodeNames(input.workflow); + const graph = createVerificationGraph(input.workflow); + const candidateNames = graph.withTools(graph.rootNodeNames); const mockedNodeNames = new Set(input.mockedNodeNames ?? []); const candidates = (input.workflow.nodes ?? []).filter( @@ -411,7 +408,8 @@ export async function classifyNodesForSimulation( node.disabled !== true && node.type !== STICKY_NOTE_TYPE && !isTriggerNodeType(node.type) && - mainFlowNames.has(node.name), + candidateNames.has(node.name) && + (input.nodeNames === undefined || input.nodeNames.has(node.name)), ); if (candidates.length === 0) return []; diff --git a/packages/@n8n/instance-ai/src/tools/workflows/plan-verification-simulation.ts b/packages/@n8n/instance-ai/src/tools/workflows/plan-verification-simulation.ts index 77f302251f3..cbe2ee8e1c9 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/plan-verification-simulation.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/plan-verification-simulation.ts @@ -65,7 +65,8 @@ export interface VerificationSimulationPlan { simulationFixtures?: SimulationFixtures; } -const DECLARED_OUTPUT_SIMULATION_REASON = 'Source declares verification output for this node'; +export const DECLARED_OUTPUT_SIMULATION_REASON = + 'Source declares verification output for this node'; const TRIGGER_SIMULATION_REASON = 'Trigger event is simulated during verification'; @@ -177,7 +178,7 @@ export function findCredentiallessAiRoots( * credentials get their `execute` verdict overridden to `simulate`: the real * run would die at the root with a credential error ("Node does not have any * credentials set"), leaving every downstream simulated node unreached. The - * classifier marks AI roots safe by type and never visits `ai_*` sub-nodes, + * classifier marks AI roots and Agent Tool containers safe by type, * so — like the trigger pass above — this is the single injection point. * Roots already `simulate` (declared fixtures, mocked credentials) keep * their verdict and reason. diff --git a/packages/@n8n/instance-ai/src/tools/workflows/reconcile-simulation-plan.ts b/packages/@n8n/instance-ai/src/tools/workflows/reconcile-simulation-plan.ts index 7115c25d33c..facd18f530b 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/reconcile-simulation-plan.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/reconcile-simulation-plan.ts @@ -17,6 +17,7 @@ import { classifyNodesForSimulation } from './classify-node-destructiveness.serv import { isAiGatewayManagedCredential } from './credential-utils'; import { CREDENTIALLESS_AI_ROOT_SIMULATION_REASON, + DECLARED_OUTPUT_SIMULATION_REASON, findCredentiallessAiRoots, withWaitGateHaltVerdicts, } from './plan-verification-simulation'; @@ -126,14 +127,9 @@ export async function reconcileSimulationPlan(args: { // pass never re-judges the rest of the workflow. let freshVerdictByName = new Map(); if (satisfiedNames.size > 0) { - const scopedWorkflow: WorkflowJSON = { - ...workflow, - nodes: (workflow.nodes ?? []).filter( - (node) => typeof node.name === 'string' && satisfiedNames.has(node.name), - ), - }; const freshVerdicts = await classifyNodesForSimulation({ - workflow: scopedWorkflow, + workflow, + nodeNames: satisfiedNames, mockedNodeNames: remainingMockedNames, fallbackModelConfig, }); @@ -145,6 +141,7 @@ export async function reconcileSimulationPlan(args: { // simulation. Restored AI roots are deterministically safe by type, so // flipping straight to execute matches what a rebuild would classify. const reconciledPlan = buildOutcome.nodeSimulationPlan?.map((verdict) => { + if (verdict.reason === DECLARED_OUTPUT_SIMULATION_REASON) return verdict; if (satisfiedNames.has(verdict.nodeName)) { return freshVerdictByName.get(verdict.nodeName) ?? verdict; } @@ -164,7 +161,9 @@ export async function reconcileSimulationPlan(args: { const retainedFixtureEntries = Object.entries(buildOutcome.simulationFixtures ?? {}).filter( ([nodeName]) => - freshVerdictByName.get(nodeName)?.verdict !== 'execute' && !restoredAiRoots.has(nodeName), + nodeSimulationPlan?.some( + (verdict) => verdict.nodeName === nodeName && verdict.verdict === 'simulate', + ), ); const simulationFixtures = retainedFixtureEntries.length > 0 ? Object.fromEntries(retainedFixtureEntries) : undefined; diff --git a/packages/@n8n/instance-ai/src/tools/workflows/verification-graph.ts b/packages/@n8n/instance-ai/src/tools/workflows/verification-graph.ts new file mode 100644 index 00000000000..f40bc8ab8e4 --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/workflows/verification-graph.ts @@ -0,0 +1,64 @@ +import { toEngineConnections, type WorkflowJSON } from '@n8n/workflow-sdk'; +import { + getChildNodes, + getParentNodes, + isTriggerNodeType, + mapConnectionsByDestination, + NodeConnectionTypes, +} from 'n8n-workflow'; + +export const AGENT_NODE_TYPE = '@n8n/n8n-nodes-langchain.agent'; +export const AGENT_TOOL_NODE_TYPE = '@n8n/n8n-nodes-langchain.agentTool'; + +export function getTriggerMainFlowScope( + connections: WorkflowJSON['connections'], + triggerNodeName: string, +): Set { + return new Set([ + triggerNodeName, + ...getChildNodes(toEngineConnections(connections), triggerNodeName, NodeConnectionTypes.Main), + ]); +} + +/** Keep tool edges separate. They point from the tool to its caller. */ +export function createVerificationGraph(workflow: WorkflowJSON) { + const connections = toEngineConnections(workflow.connections); + const byDestination = mapConnectionsByDestination(connections); + const nodesByName = new Map( + workflow.nodes + .filter( + (node): node is WorkflowJSON['nodes'][number] & { name: string } => + typeof node.name === 'string' && !node.disabled, + ) + .map((node) => [node.name, node]), + ); + // Triggers such as MCP can have tools without any main connections. + const rootNodeNames = new Set( + [...nodesByName.values()] + .filter((node) => isTriggerNodeType(node.type)) + .map((node) => node.name), + ); + for (const name of Object.keys(connections)) { + const children = getChildNodes(connections, name, NodeConnectionTypes.Main, 1); + if (children.length === 0) continue; + for (const nodeName of [name, ...children]) { + if (nodesByName.has(nodeName)) rootNodeNames.add(nodeName); + } + } + const toolsFor = (name: string) => + getParentNodes(byDestination, name, NodeConnectionTypes.AiTool, 1).filter((toolName) => + nodesByName.has(toolName), + ); + const withTools = (mainScope: ReadonlySet) => { + const names = new Set(); + const pending = [...mainScope]; + while (pending.length > 0) { + const name = pending.pop()!; + if (names.has(name) || !nodesByName.has(name)) continue; + names.add(name); + pending.push(...toolsFor(name)); + } + return names; + }; + return { nodesByName, rootNodeNames, toolsFor, withTools }; +} diff --git a/packages/@n8n/nodes-langchain/nodes/agents/Agent/test/integration/agent-tool-simulation.workflow.test.ts b/packages/@n8n/nodes-langchain/nodes/agents/Agent/test/integration/agent-tool-simulation.workflow.test.ts new file mode 100644 index 00000000000..102f82b2269 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/agents/Agent/test/integration/agent-tool-simulation.workflow.test.ts @@ -0,0 +1,170 @@ +import { NodeTypes } from '@nodes-testing/node-types'; +import { NodeTestHarness } from '@nodes-testing/node-test-harness'; +import { WorkflowExecute } from 'n8n-core/dist/execution-engine/workflow-execute'; +import { + NodeConnectionTypes, + type INodeExecutionData, + type INodeType, + type WorkflowTestData, +} from 'n8n-workflow'; +import nock from 'nock'; +import path from 'node:path'; + +function mockCompletion( + content: string, + tool?: string, + requestBody?: (body: unknown) => boolean, +): NonNullable['mocks'][number] { + return { + method: 'post', + path: '/v1/chat/completions', + statusCode: 200, + requestBody, + responseBody: { + id: 'chatcmpl-test', + object: 'chat.completion', + created: 1700000000, + model: 'gpt-4o-mini', + choices: [ + { + index: 0, + message: tool + ? { + role: 'assistant', + content: null, + tool_calls: [ + { + id: `call_${tool}`, + type: 'function', + function: { + name: tool, + arguments: JSON.stringify({ input: 'Write a record' }), + }, + }, + ], + } + : { role: 'assistant', content }, + finish_reason: tool ? 'tool_calls' : 'stop', + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }, + }; +} + +describe('Agent tool simulation', () => { + const harness = new NodeTestHarness({ + additionalPackagePaths: [path.dirname(require.resolve('n8n-nodes-base'))], + }); + const fixture = { id: 'simulated-write-1', status: 'accepted' }; + const credentials = { openAiApi: { apiKey: 'test-api-key', url: 'https://api.openai.com/v1' } }; + + for (const agentVersion of [3, 3.1]) { + for (const nested of [false, true]) { + describe(`Agent ${agentVersion}, nested=${nested}`, () => { + const workflow = harness.readWorkflowJSON( + nested ? 'workflows/sub-agent-with-inner-tool.json' : 'workflows/agent-v3-with-tool.json', + ); + workflow.settings = { ...workflow.settings, executionOrder: 'v1' }; + const agentName = nested ? 'Parent Agent' : 'AI Agent'; + const callerName = nested ? 'MathSubAgent' : agentName; + workflow.nodes.find((node) => node.name === agentName)!.typeVersion = agentVersion; + const write = workflow.nodes.find((node) => node.name === 'Calculator')!; + write.name = 'Write'; + write.type = 'n8n-nodes-testing.write'; + write.typeVersion = 1; + write.parameters = {}; + workflow.connections.Write = workflow.connections.Calculator; + delete workflow.connections.Calculator; + delete workflow.pinData; + + const writeImplementation = vi.fn( + async (): Promise => [[{ json: { id: 'real-write' } }]], + ); + const writeNode: INodeType = { + description: { + displayName: 'Write', + name: 'write', + description: 'Writes a record', + group: ['output'], + version: 1, + defaults: { name: 'Write' }, + inputs: [NodeConnectionTypes.Main], + outputs: [NodeConnectionTypes.AiTool], + properties: [], + }, + execute: writeImplementation, + }; + const getNodeType = NodeTypes.prototype.getByNameAndVersion; + const runNode = WorkflowExecute.prototype.runNode; + let receivedFixture = false; + beforeEach(() => { + receivedFixture = false; + writeImplementation.mockClear(); + vi.spyOn(NodeTypes.prototype, 'getByNameAndVersion').mockImplementation(function ( + this: NodeTypes, + type, + version, + ) { + return type === write.type ? writeNode : getNodeType.call(this, type, version); + }); + let initialized = false; + vi.spyOn(WorkflowExecute.prototype, 'runNode').mockImplementation(async function ( + this: WorkflowExecute, + ...args + ) { + const [, executionData, runData, , , , , response] = args; + if (!initialized) { + initialized = true; + // The harness starts at the trigger. Add per-execution pins before the Agent runs. + runData.resultData.pinData = { Write: [{ json: fixture }] }; + } + if (executionData.node.name === callerName && response?.actionResponses.length) { + expect(response.actionResponses).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + action: expect.objectContaining({ nodeName: 'Write', type: 'ai_tool' }), + data: expect.objectContaining({ + data: { ai_tool: [[expect.objectContaining({ json: fixture })]] }, + }), + }), + ]), + ); + receivedFixture = true; + } + return await runNode.apply(this, args); + }); + }); + afterEach(() => { + vi.restoreAllMocks(); + nock.cleanAll(); + }); + + const mocks = [ + ...(nested ? [mockCompletion('', 'MathSubAgent')] : []), + mockCompletion('', 'Write'), + mockCompletion('Fixture received', undefined, (body) => + JSON.stringify(body).includes(fixture.id), + ), + ...(nested ? [mockCompletion('Fixture received')] : []), + ]; + harness.setupTest( + { + description: 'returns the write fixture to the Agent through ai_tool', + input: { workflowData: workflow }, + output: { nodeData: { [agentName]: [[{ json: { output: 'Fixture received' } }]] } }, + nock: { baseUrl: 'https://api.openai.com', mocks }, + }, + { + credentials, + customAssertions: () => { + expect(writeImplementation).not.toHaveBeenCalled(); + expect(receivedFixture).toBe(true); + expect(nock.isDone()).toBe(true); + }, + }, + ); + }); + } + } +}); diff --git a/packages/@n8n/nodes-langchain/package.json b/packages/@n8n/nodes-langchain/package.json index b2c47d6fb90..54dfeee6701 100644 --- a/packages/@n8n/nodes-langchain/package.json +++ b/packages/@n8n/nodes-langchain/package.json @@ -228,6 +228,7 @@ "@types/temp": "^0.9.1", "@vitest/coverage-v8": "catalog:", "@typescript/native": "catalog:typescript-tooling", + "nock": "catalog:", "typescript": "catalog:typescript-tooling", "vitest": "catalog:", "vitest-mock-extended": "catalog:" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a943e9fdd3d..cacc09e522b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3623,6 +3623,9 @@ importers: '@vitest/coverage-v8': specifier: 'catalog:' version: 4.1.9(@vitest/browser@4.1.10)(vitest@4.1.9) + nock: + specifier: 'catalog:' + version: 14.0.14 typescript: specifier: catalog:typescript-tooling version: '@typescript/typescript6@6.0.2'