mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
fix(core): Simulate Agent write tools during workflow verification (#38553)
This commit is contained in:
Predrag Ristic
GitHub
parent
9059a89078
commit
cd623fd6ca
|
|
@ -0,0 +1,65 @@
|
|||
import type { WorkflowJSON } from '@n8n/workflow-sdk';
|
||||
|
||||
export function agentToolNode(
|
||||
name: string,
|
||||
type: string,
|
||||
overrides: Partial<WorkflowJSON['nodes'][number]> = {},
|
||||
): WorkflowJSON['nodes'][number] {
|
||||
return { id: name, name, type, typeVersion: 1, parameters: {}, position: [0, 0], ...overrides };
|
||||
}
|
||||
|
||||
export function mcpToolWorkflow(): WorkflowJSON {
|
||||
return {
|
||||
name: 'MCP tools',
|
||||
settings: { executionOrder: 'v1' },
|
||||
nodes: [
|
||||
agentToolNode('MCP Server', '@n8n/n8n-nodes-langchain.mcpTrigger'),
|
||||
agentToolNode('Write', 'n8n-nodes-base.slackTool', { parameters: { operation: 'send' } }),
|
||||
],
|
||||
connections: {
|
||||
Write: { ai_tool: [[{ node: 'MCP Server', type: 'ai_tool', index: 0 }]] },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function agentToolWorkflow(agentVersion = 3.1, nestedVersion?: number): WorkflowJSON {
|
||||
const caller = nestedVersion === undefined ? 'Agent' : 'Nested';
|
||||
return {
|
||||
name: 'Agent tools',
|
||||
settings: { executionOrder: 'v1' },
|
||||
nodes: [
|
||||
agentToolNode('Trigger', 'n8n-nodes-base.manualTrigger'),
|
||||
agentToolNode('Agent', '@n8n/n8n-nodes-langchain.agent', { typeVersion: agentVersion }),
|
||||
agentToolNode('Write', 'n8n-nodes-base.slackTool', { parameters: { operation: 'send' } }),
|
||||
agentToolNode('Model', '@n8n/n8n-nodes-langchain.lmChatOpenAi', {
|
||||
credentials: { openAiApi: { id: 'model-credential', name: 'Model account' } },
|
||||
}),
|
||||
...(nestedVersion === undefined
|
||||
? []
|
||||
: [
|
||||
agentToolNode('Nested', '@n8n/n8n-nodes-langchain.agentTool', {
|
||||
typeVersion: nestedVersion,
|
||||
}),
|
||||
]),
|
||||
],
|
||||
connections: {
|
||||
Trigger: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
|
||||
Write: { ai_tool: [[{ node: caller, type: 'ai_tool', index: 0 }]] },
|
||||
Model: {
|
||||
ai_languageModel: [
|
||||
[
|
||||
{ node: 'Agent', type: 'ai_languageModel', index: 0 },
|
||||
...(nestedVersion === undefined
|
||||
? []
|
||||
: [{ node: 'Nested', type: 'ai_languageModel', index: 0 }]),
|
||||
],
|
||||
],
|
||||
},
|
||||
...(nestedVersion === undefined
|
||||
? {}
|
||||
: {
|
||||
Nested: { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -1,5 +1,10 @@
|
|||
import type { Mock } from 'vitest';
|
||||
|
||||
import {
|
||||
agentToolNode,
|
||||
agentToolWorkflow,
|
||||
mcpToolWorkflow,
|
||||
} from '../../../__tests__/agent-tool-workflow';
|
||||
import { executeTool } from '../../../__tests__/tool-test-utils';
|
||||
import { successfulVerification } from '../../../__tests__/verification-fixtures';
|
||||
import type { WorkflowLoopStorage } from '../../../storage/workflow-loop-storage';
|
||||
|
|
@ -17,6 +22,7 @@ import type {
|
|||
WorkflowBuildOutcome,
|
||||
} from '../../../workflow-loop/workflow-loop-state';
|
||||
import { WorkflowTaskCoordinator } from '../../../workflow-loop/workflow-task-service';
|
||||
import { CREDENTIALLESS_AI_ROOT_SIMULATION_REASON } from '../../workflows/plan-verification-simulation';
|
||||
import { createVerifyBuiltWorkflowTool } from '../verify-built-workflow.tool';
|
||||
|
||||
type VerifyBuiltWorkflowOutput = {
|
||||
|
|
@ -590,6 +596,7 @@ function makeContext(
|
|||
})) ??
|
||||
[],
|
||||
connections: overrides.workflowConnections ?? {},
|
||||
settings: { executionOrder: 'v1' },
|
||||
};
|
||||
}),
|
||||
getWorkflowHead: vi.fn().mockResolvedValue({
|
||||
|
|
@ -1238,7 +1245,7 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
{ nodeName: 'Send Slack', reason: 'Sends a message to a Slack channel' },
|
||||
]);
|
||||
expect(result.simulationNote).toContain('Send Slack');
|
||||
expect(result.simulationNote).toContain('no real external writes');
|
||||
expect(result.simulationNote).toContain('with fixture output');
|
||||
});
|
||||
|
||||
it('surfaces parameters of reached simulated nodes that resolved to empty or threw', async () => {
|
||||
|
|
@ -1303,7 +1310,7 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
issue: 'empty',
|
||||
},
|
||||
]);
|
||||
expect(result.simulationNote).toContain('no real external writes');
|
||||
expect(result.simulationNote).toContain('with fixture output');
|
||||
expect(result.simulationNote).toContain('Send Slack: `channel`');
|
||||
expect(result.simulationNote).toContain('`text` (={{ $json.query.caller }}) resolved to empty');
|
||||
});
|
||||
|
|
@ -1325,7 +1332,7 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
{ nodeName: 'Send Slack', executionId: 'exec-sim', reason: 'replay-failed' },
|
||||
]);
|
||||
expect(result.skippedParameterCheckCount).toBe(1);
|
||||
expect(result.simulationNote).toContain('no real external writes');
|
||||
expect(result.simulationNote).toContain('with fixture output');
|
||||
expect(result.simulationNote).toContain('Parameter check skipped');
|
||||
expect(result.simulationNote).toContain('unchecked dynamic fields');
|
||||
expect(ctx.logger.debug).toHaveBeenCalledWith(
|
||||
|
|
@ -1369,7 +1376,7 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
}),
|
||||
);
|
||||
expect(ctx.logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining('no simulation plan'),
|
||||
expect.stringContaining('preflight blocked verification'),
|
||||
expect.objectContaining({ workItemId: 'wi-1' }),
|
||||
);
|
||||
});
|
||||
|
|
@ -1499,10 +1506,14 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
]);
|
||||
expect(result.coverageNote).toContain('UNVERIFIED');
|
||||
expect(result.coverageNote).toContain('Look Up Order');
|
||||
expect(result.coverageNote).toContain('Seed matching test data');
|
||||
expect(result.coverageNote).toContain('Agent tool calls');
|
||||
expect(result.coverageNote).toContain('If a live lookup or query returned no matching records');
|
||||
expect(result.coverageNote).toContain(
|
||||
'seed matching test data only when the user has authorized that write',
|
||||
);
|
||||
});
|
||||
|
||||
it('flags an items-dropped collapse with $input.all() guidance when a collection ran dry', async () => {
|
||||
it('reports incomplete coverage without assuming why a collection stopped', async () => {
|
||||
// INS-662: HTTP split a top-level array into N items, a Code node read $input.first() and emitted zero, downstream was skipped.
|
||||
const { ctx } = makeContext(
|
||||
makeBuildOutcome({
|
||||
|
|
@ -1529,10 +1540,11 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.nodesNotReached).toEqual(['Post to Slack']);
|
||||
expect(result.coverageNote).toContain('$input.all()');
|
||||
expect(result.coverageNote).toContain('$input.first()');
|
||||
// Must NOT misattribute the dead-end to an empty lookup.
|
||||
expect(result.coverageNote).not.toContain('Seed matching test data');
|
||||
expect(result.coverageNote).toContain('input items');
|
||||
expect(result.coverageNote).toContain('If a Code node dropped a collection');
|
||||
expect(result.coverageNote).toContain('$input.first().json');
|
||||
expect(result.coverageNote).toContain('$input.all().map(i => i.json)');
|
||||
expect(result.coverageNote).toContain('Do not report unreached nodes as verified');
|
||||
});
|
||||
|
||||
it('persists unreached nodes in the verification evidence', async () => {
|
||||
|
|
@ -1665,7 +1677,7 @@ describe('verify-built-workflow tool — stale mocked-credential plan', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('proceeds with the stored plan when the live workflow cannot be loaded', async () => {
|
||||
it('blocks when credential reconciliation cannot load the live workflow', async () => {
|
||||
const { ctx } = makeContext(staleOutcome(), {
|
||||
executionId: 'exec-1',
|
||||
status: 'success',
|
||||
|
|
@ -1677,11 +1689,10 @@ describe('verify-built-workflow tool — stale mocked-credential plan', () => {
|
|||
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.remediation?.reason).toBe('verification_graph_unavailable');
|
||||
const run = vi.mocked(ctx.domainContext.executionService.run);
|
||||
expect(run.mock.calls[0][2]).toMatchObject({
|
||||
verificationPinData: { Notion: [{ page: 'mock' }] },
|
||||
});
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(ctx.logger.warn).toHaveBeenCalledWith(
|
||||
'verify-built-workflow: could not reconcile mocked-credential plan',
|
||||
expect.objectContaining({ workItemId: 'wi-1' }),
|
||||
|
|
@ -1830,7 +1841,7 @@ describe('verify-built-workflow tool — trigger selection', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('preserves verification state when the workflow graph cannot be loaded', async () => {
|
||||
it('records a blocked result and preserves prior trigger passes when the graph cannot be loaded', async () => {
|
||||
const outcome = makeTrackedOutcome({ verificationProgress: passedA, verifyAttempts: 1 });
|
||||
const { ctx, getOutcome } = makeContext(outcome, successfulA, { workflowConnections });
|
||||
vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockRejectedValueOnce(
|
||||
|
|
@ -1840,7 +1851,11 @@ describe('verify-built-workflow tool — trigger selection', () => {
|
|||
const result = await runTool(ctx, triggerAInput);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(getOutcome()).toEqual(outcome);
|
||||
expect(getOutcome().verificationProgress).toEqual(outcome.verificationProgress);
|
||||
expect(getOutcome().verification).toMatchObject({
|
||||
success: false,
|
||||
failureSignature: 'verification_graph_unavailable',
|
||||
});
|
||||
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -1956,6 +1971,298 @@ describe('verify-built-workflow tool — trigger selection', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('verify-built-workflow tool — attached tools', () => {
|
||||
const verdict = (nodeName: string, simulated = false) => ({
|
||||
nodeName,
|
||||
verdict: simulated ? ('simulate' as const) : ('execute' as const),
|
||||
reason: simulated ? 'Writes data' : 'Reads data',
|
||||
confidence: 'high' as const,
|
||||
source: 'deterministic' as const,
|
||||
});
|
||||
const plan = [verdict('Agent'), verdict('Write', true)];
|
||||
|
||||
it('blocks MCP tool simulation before execution without a named trigger', async () => {
|
||||
const { ctx, getOutcome } = makeContext(
|
||||
makeBuildOutcome({ nodeSimulationPlan: [verdict('Write', true)] }),
|
||||
{ status: 'success' },
|
||||
);
|
||||
vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(
|
||||
mcpToolWorkflow(),
|
||||
);
|
||||
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
remediation: { category: 'blocked', reason: 'unsupported_tool_simulation' },
|
||||
nodesNotReached: expect.arrayContaining(['MCP Server', 'Write']),
|
||||
});
|
||||
expect(result.error).toContain('Write (called by MCP Server)');
|
||||
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
|
||||
expect(ctx.workflowTaskService.startVerification).not.toHaveBeenCalled();
|
||||
expect(getOutcome().verification).toMatchObject({
|
||||
success: false,
|
||||
failureSignature: 'unsupported_tool_simulation',
|
||||
});
|
||||
});
|
||||
|
||||
it('records a blocked result when the saved pin summary cannot be loaded', async () => {
|
||||
const { ctx, getOutcome } = makeContext(makeBuildOutcome({ nodeSimulationPlan: plan }), {
|
||||
status: 'success',
|
||||
});
|
||||
const workflowService = ctx.domainContext.workflowService!;
|
||||
vi.mocked(workflowService.getAsWorkflowJSON).mockResolvedValue(agentToolWorkflow());
|
||||
workflowService.getPinnedDataSummary = vi
|
||||
.fn<NonNullable<InstanceAiWorkflowService['getPinnedDataSummary']>>()
|
||||
.mockRejectedValue(new Error('Pin summary unavailable'));
|
||||
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
status: 'unknown',
|
||||
resolvedWorkItemId: 'wi-1',
|
||||
remediation: {
|
||||
category: 'blocked',
|
||||
shouldEdit: false,
|
||||
reason: 'verification_pin_summary_unavailable',
|
||||
},
|
||||
});
|
||||
expect(result.error).toContain('Retry verification');
|
||||
expect(result.executionId).toBeUndefined();
|
||||
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
|
||||
expect(ctx.workflowTaskService.startVerification).not.toHaveBeenCalled();
|
||||
expect(getOutcome().verification).toMatchObject({
|
||||
attempted: true,
|
||||
success: false,
|
||||
status: 'unknown',
|
||||
failureSignature: 'verification_pin_summary_unavailable',
|
||||
evidence: { errorMessage: result.error },
|
||||
});
|
||||
expect(ctx.workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
verdict: 'failed_terminal',
|
||||
failureSignature: 'verification_pin_summary_unavailable',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ scripted: false, incomplete: false },
|
||||
{ scripted: true, incomplete: false },
|
||||
{ scripted: false, incomplete: true },
|
||||
])(
|
||||
'blocks after restoring model credentials, scripted=$scripted, incomplete=$incomplete',
|
||||
async ({ scripted, incomplete }) => {
|
||||
const workflow = agentToolWorkflow(2);
|
||||
const reason = incomplete ? 'incomplete_tool_simulation_plan' : 'unsupported_tool_simulation';
|
||||
const buildOutcome = makeBuildOutcome({
|
||||
mockedNodeNames: ['Model'],
|
||||
mockedCredentialsByNode: { Model: ['openAiApi'] },
|
||||
simulationFixtures: { Agent: [{ output: 'Fixture' }] },
|
||||
nodeSimulationPlan: [
|
||||
{ ...verdict('Agent', true), reason: CREDENTIALLESS_AI_ROOT_SIMULATION_REASON },
|
||||
...(incomplete ? [] : [verdict('Write', true)]),
|
||||
...(scripted ? [{ ...verdict('Gate', true), haltBranch: true }] : []),
|
||||
],
|
||||
waitGateScripts: scripted
|
||||
? [
|
||||
{
|
||||
nodeName: 'Gate',
|
||||
cutEdge: { source: 'Revise', target: 'Format' },
|
||||
decisions: [{ label: 'approve', items: [{ approved: true }] }],
|
||||
},
|
||||
]
|
||||
: undefined,
|
||||
});
|
||||
const { ctx, getOutcome } = makeContext(
|
||||
buildOutcome,
|
||||
{ status: 'success' },
|
||||
{
|
||||
availableCredentials: [
|
||||
{ id: 'model-credential', name: 'Model account', type: 'openAiApi' },
|
||||
],
|
||||
},
|
||||
);
|
||||
vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(workflow);
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
remediation: {
|
||||
category: 'blocked',
|
||||
shouldEdit: false,
|
||||
reason,
|
||||
},
|
||||
});
|
||||
expect(result.error).toContain('Write');
|
||||
expect(result.executionId).toBeUndefined();
|
||||
expect(result.simulatedNodes).toBeUndefined();
|
||||
expect(result.nodesNotReached).toContain('Write');
|
||||
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
|
||||
expect(getOutcome().nodeSimulationPlan).toContainEqual(
|
||||
expect.objectContaining({ nodeName: 'Agent', verdict: 'execute' }),
|
||||
);
|
||||
expect(getOutcome().verification).toMatchObject({
|
||||
success: false,
|
||||
failureSignature: reason,
|
||||
evidence: { nodesNotReached: expect.arrayContaining(['Write']) },
|
||||
});
|
||||
expect(ctx.workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
verdict: 'failed_terminal',
|
||||
failureSignature: reason,
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([0, 1])(
|
||||
'keeps tools beneath a saved pinned Agent v2 unverified with %i pinned items',
|
||||
async (itemCount) => {
|
||||
const workflow = agentToolWorkflow(2);
|
||||
const { ctx, getOutcome } = makeContext(makeBuildOutcome({ nodeSimulationPlan: plan }), {
|
||||
executionId: 'exec-parent',
|
||||
status: 'success',
|
||||
executedNodeNames: ['Trigger', 'Agent'],
|
||||
});
|
||||
const workflowService = ctx.domainContext.workflowService!;
|
||||
vi.mocked(workflowService.getAsWorkflowJSON).mockResolvedValue(workflow);
|
||||
workflowService.getPinnedDataSummary = vi
|
||||
.fn<NonNullable<InstanceAiWorkflowService['getPinnedDataSummary']>>()
|
||||
.mockResolvedValue([{ nodeName: 'Agent', itemCount }]);
|
||||
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.executionId).toBe('exec-parent');
|
||||
expect(result.nodesNotReached).toEqual(['Write']);
|
||||
expect(result.simulatedNodes).toBeUndefined();
|
||||
expect(ctx.domainContext.executionService.run).toHaveBeenCalledWith(
|
||||
'wf-1',
|
||||
undefined,
|
||||
expect.objectContaining({ verificationPinData: { Write: [{}] } }),
|
||||
);
|
||||
expect(getOutcome().verification).toMatchObject({
|
||||
success: true,
|
||||
evidence: { nodesNotReached: ['Write'] },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it('rejects a non-trigger with a simulated tool without recording a verification failure', async () => {
|
||||
const workflow = agentToolWorkflow(2);
|
||||
const { ctx, getOutcome, updateBuildOutcome } = makeContext(
|
||||
makeBuildOutcome({ nodeSimulationPlan: plan }),
|
||||
{ status: 'success' },
|
||||
);
|
||||
vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(workflow);
|
||||
|
||||
const result = await runTool(ctx, {
|
||||
workItemId: 'wi-1',
|
||||
workflowId: 'wf-1',
|
||||
triggerNodeName: 'Agent',
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('Could not find trigger "Agent" in this workflow.');
|
||||
expect(result.remediation).toBeUndefined();
|
||||
expect(result.executionId).toBeUndefined();
|
||||
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
|
||||
expect(ctx.workflowTaskService.startVerification).not.toHaveBeenCalled();
|
||||
expect(ctx.workflowTaskService.reportVerificationVerdict).not.toHaveBeenCalled();
|
||||
expect(updateBuildOutcome).not.toHaveBeenCalled();
|
||||
expect(getOutcome().verification).toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps tools beneath a simulated parent unverified', async () => {
|
||||
const workflow = agentToolWorkflow(2, 3);
|
||||
const { ctx } = makeContext(
|
||||
makeBuildOutcome({
|
||||
nodeSimulationPlan: [verdict('Agent', true), verdict('Nested'), verdict('Write', true)],
|
||||
}),
|
||||
{ executionId: 'exec-parent', status: 'success', executedNodeNames: ['Trigger', 'Agent'] },
|
||||
);
|
||||
vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockResolvedValue(workflow);
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.nodesNotReached).toEqual(['Nested', 'Write']);
|
||||
expect(result.simulatedNodes?.some((node) => node.nodeName === 'Write')).not.toBe(true);
|
||||
expect(result.coverageNote).toContain('Agent tool calls, and simulated parents');
|
||||
expect(ctx.domainContext.executionService.run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('includes tools in scoped coverage and aggregates successful trigger passes', async () => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes.push(
|
||||
agentToolNode('Other Trigger', 'n8n-nodes-base.manualTrigger'),
|
||||
agentToolNode('Other Agent', '@n8n/n8n-nodes-langchain.agent', { typeVersion: 3.1 }),
|
||||
agentToolNode('Other Write', 'n8n-nodes-base.slackTool'),
|
||||
);
|
||||
workflow.connections['Other Trigger'] = {
|
||||
main: [[{ node: 'Other Agent', type: 'main', index: 0 }]],
|
||||
};
|
||||
workflow.connections['Other Write'] = {
|
||||
ai_tool: [[{ node: 'Other Agent', type: 'ai_tool', index: 0 }]],
|
||||
};
|
||||
const { ctx, getOutcome } = makeContext(
|
||||
makeBuildOutcome({
|
||||
nodeSimulationPlan: [...plan, verdict('Other Agent'), verdict('Other Write', true)],
|
||||
triggerNodes: [
|
||||
{ nodeName: 'Trigger', nodeType: 'n8n-nodes-base.manualTrigger' },
|
||||
{ nodeName: 'Other Trigger', nodeType: 'n8n-nodes-base.manualTrigger' },
|
||||
],
|
||||
verificationProgress: {},
|
||||
}),
|
||||
{ status: 'success' },
|
||||
{ workflowNodes: workflow.nodes, workflowConnections: workflow.connections },
|
||||
);
|
||||
ctx.domainContext.executionService.run
|
||||
.mockResolvedValueOnce({
|
||||
status: 'success',
|
||||
executionId: 'exec-a',
|
||||
executedNodeNames: ['Trigger', 'Agent'],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
status: 'success',
|
||||
executionId: 'exec-b',
|
||||
executedNodeNames: ['Other Trigger', 'Other Agent', 'Other Write'],
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
status: 'success',
|
||||
executionId: 'exec-c',
|
||||
executedNodeNames: ['Trigger', 'Agent', 'Write'],
|
||||
});
|
||||
const first = await runTool(ctx, {
|
||||
workItemId: 'wi-1',
|
||||
workflowId: 'wf-1',
|
||||
triggerNodeName: 'Trigger',
|
||||
});
|
||||
expect(first.nodesNotReached).toEqual(['Write']);
|
||||
expect(first.simulatedNodes).toBeUndefined();
|
||||
expect(first.simulationNote).toBeUndefined();
|
||||
expect(ctx.domainContext.executionService.run.mock.calls[0][2].verificationPinData).toEqual({
|
||||
Write: [{}],
|
||||
'Other Write': [{}],
|
||||
});
|
||||
const second = await runTool(ctx, {
|
||||
workItemId: 'wi-1',
|
||||
workflowId: 'wf-1',
|
||||
triggerNodeName: 'Other Trigger',
|
||||
});
|
||||
expect(second.simulatedNodes).toEqual([{ nodeName: 'Other Write', reason: 'Writes data' }]);
|
||||
expect(second.simulationNote).toContain('Other Write');
|
||||
expect(second.simulationNote).not.toContain('no real external writes');
|
||||
expect(getOutcome().verification?.claim?.nodesNotReached).toEqual(['Write']);
|
||||
await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1', triggerNodeName: 'Trigger' });
|
||||
expect(getOutcome().verification?.claim?.nodesNotReached).toEqual([]);
|
||||
expect(
|
||||
getOutcome()
|
||||
.verification?.claim?.simulatedNodes.map((node) => node.nodeName)
|
||||
.sort(),
|
||||
).toEqual(['Other Write', 'Write']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('verify-built-workflow tool — publish state', () => {
|
||||
it('warns that the fix is not live when the published version is an older one', async () => {
|
||||
const { ctx, getOutcome } = makeContext(
|
||||
|
|
|
|||
|
|
@ -66,7 +66,10 @@ describe('analyzeVerificationResult — halted wait gates', () => {
|
|||
// covers time-based Wait and Form gates), and non-gate dead ends keep the
|
||||
// seed-and-re-run guidance instead of being attributed to the gate.
|
||||
expect(analysis.coverageNote).not.toContain('human decision');
|
||||
expect(analysis.coverageNote).toContain('NOT behind the gate');
|
||||
expect(analysis.coverageNote).toContain('Other unreached nodes remain unverified');
|
||||
expect(analysis.coverageNote).toContain('not behind a wait gate');
|
||||
expect(analysis.coverageNote).toContain('seed matching test data');
|
||||
expect(analysis.coverageNote).toContain('only when the user has authorized that write');
|
||||
});
|
||||
|
||||
it('keeps the generic partial-coverage guidance when no gate halted the run', () => {
|
||||
|
|
@ -410,6 +413,8 @@ describe('analyzeVerificationResult — trigger-scoped coverage', () => {
|
|||
expect(analysis.nodesNotReached).toEqual(['Post Summary']);
|
||||
expect(analysis.coverageNote).toContain('Every Weekday 9am');
|
||||
expect(analysis.coverageNote).toContain('triggerNodeName');
|
||||
expect(analysis.coverageNote).toContain('union of those passes');
|
||||
expect(analysis.coverageNote).toContain('For unreached main-flow nodes on this branch');
|
||||
// The generic "a lookup returned nothing" cause would send the agent
|
||||
// editing a workflow whose other branch is simply not on this path.
|
||||
expect(analysis.coverageNote).not.toContain('lookup or query returned nothing');
|
||||
|
|
@ -460,7 +465,7 @@ describe('analyzeVerificationResult — trigger-scoped coverage', () => {
|
|||
expect(analysis.coverageNote).toContain('1st of Month');
|
||||
});
|
||||
|
||||
it('keeps the generic zero-output guidance when no trigger was named', () => {
|
||||
it('gives neutral coverage guidance when no trigger was named', () => {
|
||||
const analysis = analyzeVerificationResult({
|
||||
result: weekdayPass,
|
||||
buildOutcome: twoBranchOutcome,
|
||||
|
|
@ -469,6 +474,33 @@ describe('analyzeVerificationResult — trigger-scoped coverage', () => {
|
|||
runId: 'run-1',
|
||||
});
|
||||
|
||||
expect(analysis.coverageNote).toContain('Seed matching test data');
|
||||
expect(analysis.coverageNote).toContain('Agent tool calls');
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyzeVerificationResult — uncalled tools', () => {
|
||||
it('keeps an uncalled tool unverified without diagnosing an empty Agent result', () => {
|
||||
const analysis = analyzeVerificationResult({
|
||||
result: {
|
||||
executionId: 'exec-agent',
|
||||
status: 'success',
|
||||
executedNodeNames: ['Agent'],
|
||||
lastNodeExecuted: 'Agent',
|
||||
data: { Agent: [{ output: 'No tool call needed' }] },
|
||||
},
|
||||
buildOutcome,
|
||||
simulatedNodes: [{ nodeName: 'Publish', reason: 'Sends a message' }],
|
||||
stateBefore: undefined,
|
||||
runId: 'run-1',
|
||||
});
|
||||
|
||||
expect(analysis.reachedSimulatedNodes).toEqual([]);
|
||||
expect(analysis.nodesNotReached).toContain('Publish');
|
||||
expect(analysis.coverageNote).toContain(
|
||||
'A tool can remain uncalled even when its Agent succeeds',
|
||||
);
|
||||
expect(analysis.coverageNote).not.toContain('because it produced no output items');
|
||||
expect(analysis.coverageNote).toContain('if a simulated or pinned lookup returned no items');
|
||||
expect(analysis.coverageNote).toContain('inspect its fixture');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
import {
|
||||
agentToolNode,
|
||||
agentToolWorkflow,
|
||||
mcpToolWorkflow,
|
||||
} from '../../../../__tests__/agent-tool-workflow';
|
||||
import type { NodeSimulationVerdict } from '../../../../workflow-loop/workflow-loop-state';
|
||||
import { checkToolSimulationSupport } from '../tool-simulation-preflight';
|
||||
|
||||
function verdict(nodeName: string, simulate = false): NodeSimulationVerdict {
|
||||
return {
|
||||
nodeName,
|
||||
verdict: simulate ? 'simulate' : 'execute',
|
||||
reason: 'Test classification',
|
||||
confidence: 'high',
|
||||
source: 'deterministic',
|
||||
};
|
||||
}
|
||||
|
||||
function check(
|
||||
workflow = agentToolWorkflow(),
|
||||
options: {
|
||||
pins?: Record<string, unknown[]>;
|
||||
plan?: NodeSimulationVerdict[];
|
||||
triggerNodeName?: string;
|
||||
} = {},
|
||||
) {
|
||||
return checkToolSimulationSupport({
|
||||
workflow,
|
||||
plan: options.plan ?? [verdict('Agent'), verdict('Nested'), verdict('Write', true)],
|
||||
prepared: {
|
||||
verificationPinData: options.pins ?? { Write: [{}] },
|
||||
simulatedNodes: [],
|
||||
haltedGateNames: [],
|
||||
},
|
||||
triggerNodeName: options.triggerNodeName,
|
||||
});
|
||||
}
|
||||
|
||||
describe('tool simulation preflight', () => {
|
||||
it.each(['v0', undefined] as const)(
|
||||
'blocks tool simulation with execution order %s',
|
||||
(executionOrder) => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.settings = { executionOrder };
|
||||
expect(check(workflow)?.guidance).toContain(
|
||||
'Write (called by Agent; legacy execution order)',
|
||||
);
|
||||
expect(check(workflow, { pins: { Agent: [{}] } })).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([3, 3.1, 4])('allows main Agent %s to simulate tools', (version) => {
|
||||
expect(check(agentToolWorkflow(version))).toBeUndefined();
|
||||
expect(check(agentToolWorkflow(version, 3))).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([3, 3.1, 4])('allows Agent Tool %s to simulate children', (version) => {
|
||||
expect(check(agentToolWorkflow(3.1, version))).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([2, 2.3])('blocks a simulated tool called by Agent %s', (version) => {
|
||||
expect(check(agentToolWorkflow(version))).toMatchObject({
|
||||
reason: 'unsupported_tool_simulation',
|
||||
guidance: expect.stringContaining('Write (called by Agent)'),
|
||||
});
|
||||
});
|
||||
|
||||
it.each([2, 2.2])('blocks children of Agent Tool %s', (version) => {
|
||||
expect(check(agentToolWorkflow(3.1, version))?.guidance).toContain('Write (called by Nested)');
|
||||
});
|
||||
|
||||
it.each([undefined, 'MCP Server'])(
|
||||
'checks MCP tools without main connections with trigger %s',
|
||||
(triggerNodeName) => {
|
||||
const workflow = mcpToolWorkflow();
|
||||
expect(check(workflow, { triggerNodeName })).toMatchObject({
|
||||
reason: 'unsupported_tool_simulation',
|
||||
nodesNotReached: expect.arrayContaining(['MCP Server', 'Write']),
|
||||
});
|
||||
expect(check(workflow, { triggerNodeName, plan: [] })?.reason).toBe(
|
||||
'incomplete_tool_simulation_plan',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('checks a shared MCP tool only when its trigger is in scope', () => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes.push(agentToolNode('MCP Server', '@n8n/n8n-nodes-langchain.mcpTrigger'));
|
||||
workflow.connections.Write = {
|
||||
ai_tool: [
|
||||
[
|
||||
{ node: 'Agent', type: 'ai_tool', index: 0 },
|
||||
{ node: 'MCP Server', type: 'ai_tool', index: 0 },
|
||||
],
|
||||
],
|
||||
};
|
||||
expect(check(workflow)?.guidance).toContain('Write (called by MCP Server)');
|
||||
expect(check(workflow, { triggerNodeName: 'Trigger' })).toBeUndefined();
|
||||
expect(check(workflow, { triggerNodeName: 'MCP Server' })?.guidance).toContain(
|
||||
'Write (called by MCP Server)',
|
||||
);
|
||||
});
|
||||
|
||||
it('blocks tools attached to an unknown caller', () => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes[1].type = 'community.customAgent';
|
||||
expect(check(workflow)?.reason).toBe('unsupported_tool_simulation');
|
||||
});
|
||||
|
||||
it('allows reads on an older caller when no simulation is needed', () => {
|
||||
expect(check(agentToolWorkflow(2), { plan: [verdict('Write')], pins: {} })).toBeUndefined();
|
||||
});
|
||||
|
||||
it('checks effective saved pins as well as planned simulation', () => {
|
||||
const workflow = agentToolWorkflow(2);
|
||||
workflow.pinData = { Write: [{ result: 'saved' }] };
|
||||
expect(check(workflow, { plan: [verdict('Write')], pins: {} })?.reason).toBe(
|
||||
'unsupported_tool_simulation',
|
||||
);
|
||||
});
|
||||
|
||||
it.each([{ items: [] }, { items: [{ output: 'Fixture' }] }])(
|
||||
'stops at an engine-scheduled parent pin with %j',
|
||||
({ items }) => {
|
||||
expect(check(agentToolWorkflow(2), { pins: { Agent: items } })).toBeUndefined();
|
||||
const workflow = agentToolWorkflow(2);
|
||||
workflow.pinData = { Agent: items };
|
||||
expect(check(workflow)).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it('allows a pin on an engine-scheduled nested parent', () => {
|
||||
expect(check(agentToolWorkflow(3.1, 2), { pins: { Nested: [{}] } })).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not trust a pin on an inline nested parent', () => {
|
||||
const workflow = agentToolWorkflow(2, 3);
|
||||
workflow.pinData = { Nested: [{ output: 'Saved fixture' }] };
|
||||
const blocked = check(workflow, { pins: { Nested: [{}], Write: [{}] } });
|
||||
expect(blocked?.guidance).toContain('Nested (called by Agent)');
|
||||
expect(blocked?.guidance).toContain('Write (called by Nested)');
|
||||
});
|
||||
|
||||
it('requires missing classifications even when a parent is pinned', () => {
|
||||
expect(
|
||||
check(agentToolWorkflow(), { plan: [verdict('Agent')], pins: { Agent: [{}] } })?.reason,
|
||||
).toBe('incomplete_tool_simulation_plan');
|
||||
});
|
||||
|
||||
it('checks each caller of a shared tool and respects trigger scope', () => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes.push(
|
||||
agentToolNode('Other Trigger', 'n8n-nodes-base.manualTrigger'),
|
||||
agentToolNode('Older', '@n8n/n8n-nodes-langchain.agent', { typeVersion: 2 }),
|
||||
);
|
||||
workflow.connections['Other Trigger'] = { main: [[{ node: 'Older', type: 'main', index: 0 }]] };
|
||||
workflow.connections.Write = {
|
||||
ai_tool: [
|
||||
[
|
||||
{ node: 'Agent', type: 'ai_tool', index: 0 },
|
||||
{ node: 'Older', type: 'ai_tool', index: 0 },
|
||||
],
|
||||
],
|
||||
};
|
||||
expect(check(workflow)?.guidance).toContain('Write (called by Older)');
|
||||
expect(check(workflow, { triggerNodeName: 'Trigger' })).toBeUndefined();
|
||||
expect(check(workflow, { pins: { Agent: [{}], Write: [{}] } })?.reason).toBe(
|
||||
'unsupported_tool_simulation',
|
||||
);
|
||||
expect(check(workflow, { pins: { Older: [{}], Write: [{}] } })).toBeUndefined();
|
||||
});
|
||||
|
||||
it('handles cycles without losing an unsupported caller', () => {
|
||||
const workflow = agentToolWorkflow(3.1, 3);
|
||||
workflow.connections.Agent = { ai_tool: [[{ node: 'Nested', type: 'ai_tool', index: 0 }]] };
|
||||
expect(check(workflow)?.reason).toBe('unsupported_tool_simulation');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,7 +1,5 @@
|
|||
import { isRecord } from '@n8n/utils/is-record';
|
||||
import { isPlaceholderValue } from '@n8n/utils/placeholder';
|
||||
import { toEngineConnections, type WorkflowJSON } from '@n8n/workflow-sdk';
|
||||
import { getChildNodes, NodeConnectionTypes } from 'n8n-workflow';
|
||||
|
||||
import type { ExecutionRunResult, VerificationNodePreview } from './types';
|
||||
import {
|
||||
|
|
@ -167,17 +165,6 @@ export function countProducedOutputRows(
|
|||
return count;
|
||||
}
|
||||
|
||||
function maxEmittedItemCount(resultData: Record<string, unknown> | undefined): number {
|
||||
if (!resultData) return 0;
|
||||
|
||||
let max = 0;
|
||||
for (const nodeOutput of Object.values(resultData)) {
|
||||
const count = countOutputItems(nodeOutput) ?? 0;
|
||||
if (count > max) max = count;
|
||||
}
|
||||
return max;
|
||||
}
|
||||
|
||||
function messageMatchesAny(normalized: string, keywords: readonly string[]): boolean {
|
||||
return keywords.some((keyword) => normalized.includes(keyword));
|
||||
}
|
||||
|
|
@ -354,7 +341,7 @@ export function buildSimulationNote(
|
|||
): string | undefined {
|
||||
if (reachedSimulatedNodes.length > 0) {
|
||||
return (
|
||||
`Simulated ${reachedSimulatedNodes.length} node(s) during verification — no real external writes happened: ` +
|
||||
`Simulated ${reachedSimulatedNodes.length} node(s) with fixture output during verification: ` +
|
||||
reachedSimulatedNodes.map((n) => `${n.nodeName} (${n.reason})`).join('; ') +
|
||||
'. Relay this to the user when presenting the result.'
|
||||
);
|
||||
|
|
@ -388,6 +375,18 @@ function buildCoverageNote(
|
|||
'treat coverage as the union of those passes. Do not edit, disable, reorder, or copy the ' +
|
||||
'workflow to reach them.'
|
||||
: '';
|
||||
const guidance = success
|
||||
? ' Check branch selection, input items, Agent tool calls, and simulated parents. ' +
|
||||
'A tool can remain uncalled even when its Agent succeeds. ' +
|
||||
'For unreached main-flow nodes on this branch that are not behind a wait gate: ' +
|
||||
'if a simulated or pinned lookup returned no items, inspect its fixture and use ' +
|
||||
'representative test input. If a live lookup or query returned no matching records, ' +
|
||||
'seed matching test data only when the user has authorized that write, then re-run verification. ' +
|
||||
'If a Code node dropped a collection, inspect `$input.first().json`; use ' +
|
||||
'`$input.all().map(i => i.json)` when it needs all items. ' +
|
||||
'HTTP Request splits a top-level array into separate items. ' +
|
||||
'Do not report unreached nodes as verified.'
|
||||
: '';
|
||||
if (success && reachedHaltedGates.length > 0) {
|
||||
return (
|
||||
`Verification pauses at wait gate(s) ${reachedHaltedGates.join(', ')} — in a live run the ` +
|
||||
|
|
@ -395,10 +394,9 @@ function buildCoverageNote(
|
|||
`gate is not simulated. ${nodesNotReached.length} planned node(s) were not reached: ` +
|
||||
`${nodesNotReached.join(', ')}. Nodes behind the gate are expected to be unreached — do ` +
|
||||
'not edit the workflow or re-run verification to force coverage there; recommend a live ' +
|
||||
'end-to-end test instead. Any unreached node NOT behind the gate did not receive input ' +
|
||||
'items (usually an empty lookup or query) — seed matching test data and re-run before ' +
|
||||
'treating it as verified.' +
|
||||
triggerScopeNote
|
||||
'end-to-end test instead. Other unreached nodes remain unverified.' +
|
||||
triggerScopeNote +
|
||||
guidance
|
||||
);
|
||||
}
|
||||
if (success && triggerNodeName) {
|
||||
|
|
@ -406,20 +404,12 @@ function buildCoverageNote(
|
|||
`Partial coverage by design: ${String(nodesNotReached.length)} planned node(s) were not ` +
|
||||
`reached: ${nodesNotReached.join(', ')}.` +
|
||||
triggerScopeNote +
|
||||
" Any unreached node that IS on this trigger's branch did not receive input items " +
|
||||
'(usually an empty lookup or query) — seed matching test data and re-run before treating ' +
|
||||
'it as verified.'
|
||||
guidance
|
||||
);
|
||||
}
|
||||
const ending = result.lastNodeExecuted
|
||||
? `. Execution ended at "${result.lastNodeExecuted}"${success ? ' because it produced no output items (empty item lists stop downstream nodes)' : ''}.`
|
||||
? `. Execution ended at "${result.lastNodeExecuted}".`
|
||||
: '.';
|
||||
const collapsedFromCollection = success && maxEmittedItemCount(result.data) >= 2;
|
||||
const guidance = success
|
||||
? collapsedFromCollection
|
||||
? ' An upstream node emitted multiple items but the chain collapsed to zero before reaching them. The usual cause is a Code node reading `$input.first().json` (a single split item) instead of `$input.all().map(i => i.json)` — an HTTP Request node splits a top-level array (including a bare array of IDs) into one item per element. Fix the node that dropped the items to read every item, then re-run verification. Do NOT report the workflow as fully verified.'
|
||||
: ' This usually means a lookup or query returned nothing. Seed matching test data and re-run verification, or tell the user the unreached part needs a manual test. Do NOT report the workflow as fully verified.'
|
||||
: '';
|
||||
return (
|
||||
`Partial coverage: ${nodesNotReached.length} node(s) were never reached and remain UNVERIFIED: ` +
|
||||
nodesNotReached.join(', ') +
|
||||
|
|
@ -462,16 +452,6 @@ export interface VerificationAnalysis {
|
|||
nodeErrors: ExecutionNodeError[];
|
||||
}
|
||||
|
||||
export function getTriggerMainFlowScope(
|
||||
connections: WorkflowJSON['connections'],
|
||||
triggerNodeName: string,
|
||||
): Set<string> {
|
||||
return new Set([
|
||||
triggerNodeName,
|
||||
...getChildNodes(toEngineConnections(connections), triggerNodeName, NodeConnectionTypes.Main),
|
||||
]);
|
||||
}
|
||||
|
||||
export function analyzeVerificationResult(args: {
|
||||
result: ExecutionRunResult;
|
||||
buildOutcome: WorkflowBuildOutcome;
|
||||
|
|
@ -490,7 +470,7 @@ export function analyzeVerificationResult(args: {
|
|||
chatModelRecovery?: ChatModelRecoveryOptions;
|
||||
/** Trigger this pass started from, when the caller named one. */
|
||||
triggerNodeName?: string;
|
||||
/** Main-flow nodes that belong to the selected trigger. */
|
||||
/** Main-flow nodes and attached tools that belong to the selected trigger. */
|
||||
verificationScope?: ReadonlySet<string>;
|
||||
}): VerificationAnalysis {
|
||||
const {
|
||||
|
|
|
|||
|
|
@ -28,20 +28,34 @@ export async function handleMissingSimulationPlan(args: {
|
|||
workflowTaskService: WorkflowTaskService;
|
||||
workflowId: string;
|
||||
}): Promise<VerifyBuiltWorkflowOutput> {
|
||||
const { input, context, workflowTaskService, workflowId } = args;
|
||||
const guidance =
|
||||
'Verification was not run because the build outcome has no simulation plan. ' +
|
||||
'Rebuild or resubmit the workflow so destructive nodes can be classified before verification.';
|
||||
return await handleBlockedVerification({ ...args, reason: 'missing_simulation_plan', guidance });
|
||||
}
|
||||
|
||||
export async function handleBlockedVerification(args: {
|
||||
input: ResolvedVerifyInput;
|
||||
context: OrchestrationContext;
|
||||
workflowTaskService: WorkflowTaskService;
|
||||
workflowId: string;
|
||||
reason: string;
|
||||
guidance: string;
|
||||
nodesNotReached?: string[];
|
||||
}): Promise<VerifyBuiltWorkflowOutput> {
|
||||
const { input, context, workflowTaskService, workflowId, reason, guidance, nodesNotReached } =
|
||||
args;
|
||||
const remediation = createRemediation({
|
||||
category: 'blocked',
|
||||
shouldEdit: false,
|
||||
reason: 'missing_simulation_plan',
|
||||
reason,
|
||||
guidance,
|
||||
});
|
||||
context.logger.warn(
|
||||
'verify-built-workflow: build outcome has no simulation plan — refusing to run without simulation safeguards',
|
||||
{ workItemId: input.workItemId, workflowId },
|
||||
);
|
||||
context.logger.warn('verify-built-workflow: preflight blocked verification', {
|
||||
workItemId: input.workItemId,
|
||||
workflowId,
|
||||
reason,
|
||||
});
|
||||
try {
|
||||
await workflowTaskService.updateBuildOutcome(input.workItemId, {
|
||||
remediation,
|
||||
|
|
@ -49,8 +63,8 @@ export async function handleMissingSimulationPlan(args: {
|
|||
attempted: true,
|
||||
success: false,
|
||||
status: 'unknown',
|
||||
failureSignature: 'missing_simulation_plan',
|
||||
evidence: { errorMessage: guidance },
|
||||
failureSignature: reason,
|
||||
evidence: { errorMessage: guidance, nodesNotReached },
|
||||
verifiedAt: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
|
@ -63,7 +77,7 @@ export async function handleMissingSimulationPlan(args: {
|
|||
runId: context.runId,
|
||||
workflowId,
|
||||
verdict: 'failed_terminal',
|
||||
failureSignature: 'missing_simulation_plan',
|
||||
failureSignature: reason,
|
||||
diagnosis: guidance,
|
||||
remediation,
|
||||
summary: guidance,
|
||||
|
|
@ -80,6 +94,10 @@ export async function handleMissingSimulationPlan(args: {
|
|||
resolvedWorkItemId: input.workItemId,
|
||||
status: 'unknown',
|
||||
error: guidance,
|
||||
nodesNotReached,
|
||||
coverageNote: nodesNotReached?.length
|
||||
? 'Verification did not run. All listed nodes remain unverified.'
|
||||
: undefined,
|
||||
remediation,
|
||||
guidance,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
import type { WorkflowJSON } from '@n8n/workflow-sdk';
|
||||
|
||||
import type { PreparedVerificationRun } from './prepare-run';
|
||||
import { itemsForNode } from '../../../utils/node-keyed-items';
|
||||
import type { NodeSimulationVerdict } from '../../../workflow-loop/workflow-loop-state';
|
||||
import {
|
||||
AGENT_NODE_TYPE,
|
||||
AGENT_TOOL_NODE_TYPE,
|
||||
createVerificationGraph,
|
||||
getTriggerMainFlowScope,
|
||||
} from '../../workflows/verification-graph';
|
||||
|
||||
export interface ToolSimulationBlocker {
|
||||
reason: string;
|
||||
guidance: string;
|
||||
nodesNotReached?: string[];
|
||||
}
|
||||
|
||||
const GRAPH_UNAVAILABLE: ToolSimulationBlocker = {
|
||||
reason: 'verification_graph_unavailable',
|
||||
guidance:
|
||||
'Verification was not run because the current workflow graph could not be inspected. Read the workflow and retry verification.',
|
||||
};
|
||||
|
||||
/** Check every caller path before verification can run a tool. */
|
||||
export function checkToolSimulationSupport(args: {
|
||||
workflow: WorkflowJSON | undefined;
|
||||
workflowPinnedNodeNames?: string[];
|
||||
plan: NodeSimulationVerdict[];
|
||||
prepared: PreparedVerificationRun;
|
||||
triggerNodeName?: string;
|
||||
}): ToolSimulationBlocker | undefined {
|
||||
const { workflow, plan, prepared, triggerNodeName } = args;
|
||||
if (!workflow || !Array.isArray(workflow.nodes)) {
|
||||
return GRAPH_UNAVAILABLE;
|
||||
}
|
||||
let graph: ReturnType<typeof createVerificationGraph>;
|
||||
let mainScope: Set<string>;
|
||||
let scope: Set<string>;
|
||||
try {
|
||||
graph = createVerificationGraph(workflow);
|
||||
mainScope = triggerNodeName
|
||||
? getTriggerMainFlowScope(workflow.connections, triggerNodeName)
|
||||
: graph.rootNodeNames;
|
||||
scope = graph.withTools(mainScope);
|
||||
} catch {
|
||||
return GRAPH_UNAVAILABLE;
|
||||
}
|
||||
const verdicts = new Map(plan.map((verdict) => [verdict.nodeName, verdict]));
|
||||
const missing = new Set(
|
||||
[...scope].flatMap((name) => graph.toolsFor(name)).filter((name) => !verdicts.has(name)),
|
||||
);
|
||||
if (missing.size > 0) {
|
||||
return {
|
||||
reason: 'incomplete_tool_simulation_plan',
|
||||
guidance: `Verification was not run because attached tools have no classification: ${[...missing].join(', ')}. Resubmit the workflow to classify these tools before verification.`,
|
||||
nodesNotReached: [...scope],
|
||||
};
|
||||
}
|
||||
const unsupported = new Set<string>();
|
||||
const workflowPinnedNodeNames = new Set(args.workflowPinnedNodeNames);
|
||||
const visited = new Set<string>();
|
||||
const pending: Array<{ name: string; scheduled: boolean; caller?: string }> = [...mainScope].map(
|
||||
(name) => ({ name, scheduled: true }),
|
||||
);
|
||||
while (pending.length > 0) {
|
||||
const { name, scheduled, caller } = pending.pop()!;
|
||||
const node = graph.nodesByName.get(name);
|
||||
if (!node) continue;
|
||||
const verdict = verdicts.get(name);
|
||||
const pinned =
|
||||
workflowPinnedNodeNames.has(name) ||
|
||||
itemsForNode(prepared.verificationPinData, name) !== undefined ||
|
||||
itemsForNode(workflow.pinData, name) !== undefined;
|
||||
if (caller !== undefined) {
|
||||
// An unset order is also legacy. The engine can resume the Agent before its tool result.
|
||||
const legacyOrder = workflow.settings?.executionOrder !== 'v1';
|
||||
if ((!scheduled || legacyOrder) && (verdict?.verdict === 'simulate' || pinned)) {
|
||||
unsupported.add(
|
||||
`${name} (called by ${caller}${legacyOrder ? '; legacy execution order' : ''})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Inline calls do not read pin data. Only an engine-scheduled pin skips children.
|
||||
if (scheduled && pinned) continue;
|
||||
const state = JSON.stringify([name, scheduled, caller === undefined]);
|
||||
if (visited.has(state)) continue;
|
||||
visited.add(state);
|
||||
const supportsTools =
|
||||
scheduled &&
|
||||
node.typeVersion >= 3 &&
|
||||
(caller === undefined ? node.type === AGENT_NODE_TYPE : node.type === AGENT_TOOL_NODE_TYPE);
|
||||
for (const tool of graph.toolsFor(name)) {
|
||||
pending.push({ name: tool, scheduled: supportsTools, caller: name });
|
||||
}
|
||||
}
|
||||
if (unsupported.size > 0) {
|
||||
return {
|
||||
reason: 'unsupported_tool_simulation',
|
||||
guidance: `Verification was not run because these tools need simulation on an unsupported execution path: ${[...unsupported].join('; ')}. Report these tools as unverified. Keep the Agent versions unchanged.`,
|
||||
nodesNotReached: [...scope],
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -11,15 +11,12 @@ import { isTriggerNodeType } from 'n8n-workflow';
|
|||
import { z } from 'zod';
|
||||
|
||||
import type { InstanceAiWorkflowService, OrchestrationContext } from '../../types';
|
||||
import {
|
||||
analyzeVerificationResult,
|
||||
buildNodePreviews,
|
||||
getTriggerMainFlowScope,
|
||||
} from './verification/analyze-result';
|
||||
import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result';
|
||||
import { deriveVerificationClaim } from './verification/claim';
|
||||
import type { VerificationPublishState } from './verification/claim';
|
||||
import {
|
||||
handleMissingSimulationPlan,
|
||||
handleBlockedVerification,
|
||||
persistVerificationOutcome,
|
||||
} from './verification/finalize-result';
|
||||
import { prepareVerificationRun } from './verification/prepare-run';
|
||||
|
|
@ -32,6 +29,8 @@ import {
|
|||
skippedParameterCheckSchema,
|
||||
} from './verification/resolved-parameter-warnings';
|
||||
import { runScriptedGateVerification } from './verification/scripted-gate-run';
|
||||
import { checkToolSimulationSupport } from './verification/tool-simulation-preflight';
|
||||
import { createVerificationGraph, getTriggerMainFlowScope } from '../workflows/verification-graph';
|
||||
import { describeClaimLiveState } from '../../workflow-loop/render-claim';
|
||||
import type { VerificationClaim } from '../../workflow-loop/workflow-loop-state';
|
||||
import {
|
||||
|
|
@ -286,8 +285,10 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
.getAsWorkflowJSON(workflowId)
|
||||
.catch(() => undefined);
|
||||
if (
|
||||
workflow &&
|
||||
Array.isArray(workflow.nodes) &&
|
||||
resolvedInput.triggerNodeName !== undefined &&
|
||||
!workflow?.nodes.some(
|
||||
!workflow.nodes.some(
|
||||
(node) => node.name === resolvedInput.triggerNodeName && isTriggerNodeType(node.type),
|
||||
)
|
||||
) {
|
||||
|
|
@ -297,6 +298,40 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
error: `Could not find trigger "${resolvedInput.triggerNodeName}" in this workflow. Read the workflow. Select an existing trigger.`,
|
||||
};
|
||||
}
|
||||
// WorkflowJSON omits saved pins. The summary supplies names without their payloads.
|
||||
let workflowPinnedNodeNames: string[] | undefined;
|
||||
try {
|
||||
const workflowPins = workflow
|
||||
? await target.domainContext.workflowService.getPinnedDataSummary?.(workflowId)
|
||||
: undefined;
|
||||
workflowPinnedNodeNames = workflowPins?.map(({ nodeName }) => nodeName);
|
||||
} catch {
|
||||
return await handleBlockedVerification({
|
||||
input: resolvedInput,
|
||||
context,
|
||||
workflowTaskService,
|
||||
workflowId,
|
||||
reason: 'verification_pin_summary_unavailable',
|
||||
guidance:
|
||||
'Verification was not run because saved pinned data could not be inspected. Retry verification.',
|
||||
});
|
||||
}
|
||||
const blocker = checkToolSimulationSupport({
|
||||
workflow,
|
||||
workflowPinnedNodeNames,
|
||||
plan: buildOutcome.nodeSimulationPlan,
|
||||
prepared,
|
||||
triggerNodeName: resolvedInput.triggerNodeName,
|
||||
});
|
||||
if (blocker) {
|
||||
return await handleBlockedVerification({
|
||||
input: resolvedInput,
|
||||
context,
|
||||
workflowTaskService,
|
||||
workflowId,
|
||||
...blocker,
|
||||
});
|
||||
}
|
||||
const chatModelRecovery = workflow
|
||||
? await collectChatModelRecoveryContext(
|
||||
target.domainContext,
|
||||
|
|
@ -312,7 +347,9 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
: undefined;
|
||||
const verificationScope =
|
||||
buildOutcome.verificationProgress && selectedTriggerNodeName && workflow
|
||||
? getTriggerMainFlowScope(workflow.connections, selectedTriggerNodeName)
|
||||
? createVerificationGraph(workflow).withTools(
|
||||
getTriggerMainFlowScope(workflow.connections, selectedTriggerNodeName),
|
||||
)
|
||||
: undefined;
|
||||
const previousProgress = await workflowTaskService.startVerification(
|
||||
resolvedInput.workItemId,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,11 @@ vi.mock('../../../utils/eval-agents', async () => {
|
|||
return { ...actual, createEvalAgent: vi.fn(), extractText: vi.fn() };
|
||||
});
|
||||
|
||||
import {
|
||||
agentToolNode,
|
||||
agentToolWorkflow,
|
||||
mcpToolWorkflow,
|
||||
} from '../../../__tests__/agent-tool-workflow';
|
||||
import { createEvalAgent, extractText } from '../../../utils/eval-agents';
|
||||
import { classifyNodesForSimulation } from '../classify-node-destructiveness.service';
|
||||
|
||||
|
|
@ -381,3 +386,104 @@ describe('classifyNodesForSimulation', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('attached tool classification', () => {
|
||||
beforeEach(() => vi.clearAllMocks());
|
||||
|
||||
it('classifies direct reads and nested writes and keeps Agents executable', async () => {
|
||||
const workflow = agentToolWorkflow(3.1, 3);
|
||||
workflow.nodes.push(
|
||||
agentToolNode('Read', 'n8n-nodes-base.slackTool', { parameters: { operation: 'get' } }),
|
||||
);
|
||||
workflow.connections.Read = { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] };
|
||||
const plan = await classifyNodesForSimulation({ workflow });
|
||||
expect(Object.fromEntries(plan.map(({ nodeName, verdict }) => [nodeName, verdict]))).toEqual({
|
||||
Agent: 'execute',
|
||||
Nested: 'execute',
|
||||
Read: 'execute',
|
||||
Write: 'simulate',
|
||||
});
|
||||
expect(mockCreateEvalAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['n8n-nodes-base.httpRequestTool', { method: 'GET' }, 'execute'],
|
||||
['n8n-nodes-base.emailSendTool', {}, 'simulate'],
|
||||
['n8n-nodes-base.executeWorkflowTool', {}, 'simulate'],
|
||||
['@n8n/n8n-nodes-langchain.toolWorkflow', {}, 'simulate'],
|
||||
['@n8n/n8n-nodes-langchain.mcpClientTool', {}, 'simulate'],
|
||||
['@n8n/n8n-nodes-langchain.mcpRegistryClientTool', {}, 'simulate'],
|
||||
] as const)('uses the existing rules for %s', async (type, parameters, expected) => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes[2] = agentToolNode('Write', type, { parameters });
|
||||
const plan = await classifyNodesForSimulation({ workflow });
|
||||
expect(verdictOf(plan, 'Write')).toMatchObject({ verdict: expected, source: 'deterministic' });
|
||||
expect(mockCreateEvalAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('classifies MCP tools without main connections', async () => {
|
||||
const workflow = mcpToolWorkflow();
|
||||
workflow.nodes.push(
|
||||
agentToolNode('Read', 'n8n-nodes-base.slackTool', { parameters: { operation: 'get' } }),
|
||||
);
|
||||
workflow.connections.Read = { ai_tool: [[{ node: 'MCP Server', type: 'ai_tool', index: 0 }]] };
|
||||
|
||||
const plan = await classifyNodesForSimulation({ workflow });
|
||||
|
||||
expect(plan.map(({ nodeName, verdict }) => ({ nodeName, verdict }))).toEqual([
|
||||
{ nodeName: 'Write', verdict: 'simulate' },
|
||||
{ nodeName: 'Read', verdict: 'execute' },
|
||||
]);
|
||||
expect(mockCreateEvalAgent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses conservative fallback for an unknown tool operation', async () => {
|
||||
setupAgentFailure();
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes[2].parameters = { operation: 'customAction' };
|
||||
expect(verdictOf(await classifyNodesForSimulation({ workflow }), 'Write')).toMatchObject({
|
||||
verdict: 'simulate',
|
||||
source: 'fallback',
|
||||
});
|
||||
});
|
||||
|
||||
it('classifies a shared tool once when the graph has a cycle', async () => {
|
||||
const workflow = agentToolWorkflow(3.1, 3);
|
||||
workflow.connections.Write = {
|
||||
ai_tool: [
|
||||
[
|
||||
{ node: 'Agent', type: 'ai_tool', index: 0 },
|
||||
{ node: 'Nested', type: 'ai_tool', index: 0 },
|
||||
],
|
||||
],
|
||||
};
|
||||
workflow.connections.Agent = { ai_tool: [[{ node: 'Nested', type: 'ai_tool', index: 0 }]] };
|
||||
const plan = await classifyNodesForSimulation({ workflow });
|
||||
expect(plan.map((node) => node.nodeName).sort()).toEqual(['Agent', 'Nested', 'Write']);
|
||||
});
|
||||
|
||||
it('excludes disabled and disconnected tools and other AI sub-nodes', async () => {
|
||||
const workflow = agentToolWorkflow();
|
||||
workflow.nodes[2].disabled = true;
|
||||
workflow.nodes.push(
|
||||
agentToolNode('Disconnected', 'n8n-nodes-base.slackTool'),
|
||||
agentToolNode('Memory', '@n8n/n8n-nodes-langchain.memoryBufferWindow'),
|
||||
agentToolNode('Parser', '@n8n/n8n-nodes-langchain.outputParserStructured'),
|
||||
);
|
||||
workflow.connections.Memory = { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] };
|
||||
workflow.connections.Parser = {
|
||||
ai_outputParser: [[{ node: 'Agent', type: 'ai_outputParser', index: 0 }]],
|
||||
};
|
||||
expect((await classifyNodesForSimulation({ workflow })).map((node) => node.nodeName)).toEqual([
|
||||
'Agent',
|
||||
]);
|
||||
});
|
||||
|
||||
it('excludes tools attached only to a disabled container', async () => {
|
||||
const workflow = agentToolWorkflow(3.1, 3);
|
||||
workflow.nodes.find((node) => node.name === 'Nested')!.disabled = true;
|
||||
expect((await classifyNodesForSimulation({ workflow })).map((node) => node.nodeName)).toEqual([
|
||||
'Agent',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
import type { WorkflowJSON } from '@n8n/workflow-sdk';
|
||||
|
||||
import { agentToolWorkflow } from '../../../__tests__/agent-tool-workflow';
|
||||
import type {
|
||||
NodeSimulationVerdict,
|
||||
WorkflowBuildOutcome,
|
||||
} from '../../../workflow-loop/workflow-loop-state';
|
||||
import { CREDENTIALLESS_AI_ROOT_SIMULATION_REASON } from '../plan-verification-simulation';
|
||||
import {
|
||||
CREDENTIALLESS_AI_ROOT_SIMULATION_REASON,
|
||||
DECLARED_OUTPUT_SIMULATION_REASON,
|
||||
} from '../plan-verification-simulation';
|
||||
import { reconcileSimulationPlan } from '../reconcile-simulation-plan';
|
||||
import { type CredentialEntry, type CredentialMap } from '../resolve-credentials';
|
||||
|
||||
|
|
@ -70,6 +74,39 @@ function makeWorkflow(nodes: TestNode[]): WorkflowJSON {
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('reconcileSimulationPlan', () => {
|
||||
it.each([
|
||||
{ operation: 'send', declared: false, expected: 'simulate' },
|
||||
{ operation: 'get', declared: false, expected: 'execute' },
|
||||
{ operation: 'get', declared: true, expected: 'simulate' },
|
||||
])(
|
||||
'reconciles attached $operation tools with declared output: $declared',
|
||||
async ({ operation, declared, expected }) => {
|
||||
const workflow = agentToolWorkflow(3.1, 3);
|
||||
workflow.nodes[2].parameters = { operation };
|
||||
workflow.nodes[2].credentials = { slackApi: { id: 'cred-1', name: 'Tool account' } };
|
||||
const reason = declared ? DECLARED_OUTPUT_SIMULATION_REASON : MOCKED_CREDENTIAL_REASON;
|
||||
const patch = await reconcileSimulationPlan({
|
||||
workflow,
|
||||
buildOutcome: makeBuildOutcome({
|
||||
mockedNodeNames: ['Write'],
|
||||
mockedCredentialsByNode: { Write: ['slackApi'] },
|
||||
nodeSimulationPlan: [{ ...mockedVerdict('Write'), reason }],
|
||||
simulationFixtures: { Write: [{ id: 'fixture' }] },
|
||||
}),
|
||||
availableCredentials: makeCredentialMap([
|
||||
{ id: 'cred-1', name: 'Tool account', type: 'slackApi' },
|
||||
]),
|
||||
});
|
||||
expect(patch?.nodeSimulationPlan).toEqual([
|
||||
expect.objectContaining({ nodeName: 'Write', verdict: expected }),
|
||||
]);
|
||||
expect(patch?.nodeSimulationPlan?.[0].reason).not.toBe(MOCKED_CREDENTIAL_REASON);
|
||||
expect(patch?.simulationFixtures?.Write).toEqual(
|
||||
expected === 'simulate' ? [{ id: 'fixture' }] : undefined,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('returns undefined when the outcome has no mocked credentials', async () => {
|
||||
const patch = await reconcileSimulationPlan({
|
||||
buildOutcome: makeBuildOutcome(),
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
/**
|
||||
* Node Destructiveness Classification
|
||||
*
|
||||
* Decides, per main-flow node, whether a verification execution may run the
|
||||
* Decides, per main-flow node and attached tool, whether verification may run the
|
||||
* node for real (`execute`) or must mock its output via per-execution pin
|
||||
* data (`simulate`) because the node would create, update, or delete data in
|
||||
* an external system.
|
||||
|
|
@ -18,10 +18,10 @@
|
|||
|
||||
import { isRecord } from '@n8n/utils/is-record';
|
||||
import { isAiRootNodeType, type WorkflowJSON } from '@n8n/workflow-sdk';
|
||||
import type { IConnections } from 'n8n-workflow';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { isTriggerNodeType } from './workflow-json-utils';
|
||||
import { AGENT_TOOL_NODE_TYPE, createVerificationGraph } from './verification-graph';
|
||||
import type { ModelConfig } from '../../types';
|
||||
import { HAIKU_MODEL } from '../../utils/eval-agents';
|
||||
import { generateValidatedJson } from '../../utils/generate-validated-json';
|
||||
|
|
@ -33,6 +33,8 @@ export interface ClassifyNodesForSimulationInput {
|
|||
workflow: WorkflowJSON;
|
||||
/** Node names whose credentials were mocked — always simulated. */
|
||||
mockedNodeNames?: string[];
|
||||
/** Classify only these names. Keep the full graph for tool discovery. */
|
||||
nodeNames?: ReadonlySet<string>;
|
||||
/** Host-resolved model used when no eval model API key is configured in the environment. */
|
||||
fallbackModelConfig?: ModelConfig;
|
||||
}
|
||||
|
|
@ -74,12 +76,14 @@ const SAFE_NODE_TYPES = new Set([
|
|||
* AI roots (agent/chain/extractor/classifier nodes) call an LLM but are
|
||||
* read-only towards user systems, so they classify as safe — the set lives in
|
||||
* `@n8n/workflow-sdk` (`isAiRootNodeType`), shared with fixture shaping. Write
|
||||
* operations living as agent *tools* are a known gap. Roots whose LLM
|
||||
* tools receive separate classifications. Roots whose LLM
|
||||
* sub-node has no usable credentials are overridden to `simulate` later, in
|
||||
* `plan-verification-simulation.ts`.
|
||||
*/
|
||||
function isSafeNodeType(nodeType: string): boolean {
|
||||
return SAFE_NODE_TYPES.has(nodeType) || isAiRootNodeType(nodeType);
|
||||
return (
|
||||
SAFE_NODE_TYPES.has(nodeType) || isAiRootNodeType(nodeType) || nodeType === AGENT_TOOL_NODE_TYPE
|
||||
);
|
||||
}
|
||||
|
||||
/** Node types that are destructive by their very type (no operation param needed). */
|
||||
|
|
@ -89,6 +93,15 @@ const DESTRUCTIVE_NODE_TYPES = new Map<string, string>([
|
|||
['n8n-nodes-base.ssh', 'Runs commands on a remote machine'],
|
||||
// Simulating the call skips the sub-workflow entirely, side effects included.
|
||||
['n8n-nodes-base.executeWorkflow', 'Executes another workflow which may have side effects'],
|
||||
[
|
||||
'@n8n/n8n-nodes-langchain.toolWorkflow',
|
||||
'Executes another workflow which may have side effects',
|
||||
],
|
||||
['@n8n/n8n-nodes-langchain.mcpClientTool', 'Calls remote MCP tools which may have side effects'],
|
||||
[
|
||||
'@n8n/n8n-nodes-langchain.mcpRegistryClientTool',
|
||||
'Calls remote MCP tools which may have side effects',
|
||||
],
|
||||
]);
|
||||
|
||||
const CODE_NODE_TYPES = new Set([
|
||||
|
|
@ -184,27 +197,6 @@ function getStringParam(node: WorkflowNode, key: string): string | undefined {
|
|||
return typeof value === 'string' && value.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Node names reachable over `main` connections (sources and destinations).
|
||||
* Sub-nodes (ai_tool, ai_languageModel, …) and disconnected nodes never carry
|
||||
* pin data, so the plan excludes them.
|
||||
*/
|
||||
function collectMainFlowNodeNames(workflow: WorkflowJSON): Set<string> {
|
||||
const connections = (workflow.connections ?? {}) as IConnections;
|
||||
const names = new Set<string>();
|
||||
for (const [sourceName, outputs] of Object.entries(connections)) {
|
||||
const main = outputs.main;
|
||||
if (!main) continue;
|
||||
names.add(sourceName);
|
||||
for (const port of main) {
|
||||
for (const target of port ?? []) {
|
||||
names.add(target.node);
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function deterministic(
|
||||
nodeName: string,
|
||||
verdict: 'simulate' | 'execute',
|
||||
|
|
@ -217,20 +209,25 @@ function deterministicVerdict(
|
|||
node: WorkflowNode & { name: string },
|
||||
mockedNodeNames: Set<string>,
|
||||
): NodeSimulationVerdict | 'ambiguous' {
|
||||
// Generated node tools use the same operations as their base node.
|
||||
const nodeType =
|
||||
node.type.startsWith('n8n-nodes-base.') && node.type.endsWith('Tool')
|
||||
? node.type.slice(0, -4)
|
||||
: node.type;
|
||||
if (mockedNodeNames.has(node.name)) {
|
||||
return deterministic(node.name, 'simulate', 'Credentials are not configured for this node');
|
||||
}
|
||||
|
||||
if (isSafeNodeType(node.type)) {
|
||||
if (isSafeNodeType(nodeType)) {
|
||||
return deterministic(node.name, 'execute', 'Transforms data without touching external systems');
|
||||
}
|
||||
|
||||
const destructiveByType = DESTRUCTIVE_NODE_TYPES.get(node.type);
|
||||
const destructiveByType = DESTRUCTIVE_NODE_TYPES.get(nodeType);
|
||||
if (destructiveByType) {
|
||||
return deterministic(node.name, 'simulate', destructiveByType);
|
||||
}
|
||||
|
||||
if (node.type === HTTP_REQUEST_NODE_TYPE) {
|
||||
if (nodeType === HTTP_REQUEST_NODE_TYPE) {
|
||||
const method = (getStringParam(node, 'method') ?? 'GET').toUpperCase();
|
||||
if (['GET', 'HEAD', 'OPTIONS'].includes(method)) {
|
||||
return deterministic(
|
||||
|
|
@ -248,7 +245,7 @@ function deterministicVerdict(
|
|||
// parks the run in `waiting` and everything downstream is never verified.
|
||||
// Pinning them lets execution flow past (a pinned node never calls
|
||||
// putExecutionToWait).
|
||||
if (node.type === FORM_NODE_TYPE) {
|
||||
if (nodeType === FORM_NODE_TYPE) {
|
||||
return deterministic(
|
||||
node.name,
|
||||
'simulate',
|
||||
|
|
@ -256,7 +253,7 @@ function deterministicVerdict(
|
|||
);
|
||||
}
|
||||
|
||||
if (node.type === WAIT_NODE_TYPE) {
|
||||
if (nodeType === WAIT_NODE_TYPE) {
|
||||
// Wait node defaults: resume='timeInterval', amount=1, unit='hours'.
|
||||
const params = isRecord(node.parameters) ? node.parameters : {};
|
||||
const resume = typeof params.resume === 'string' ? params.resume : 'timeInterval';
|
||||
|
|
@ -281,7 +278,7 @@ function deterministicVerdict(
|
|||
);
|
||||
}
|
||||
|
||||
if (CODE_NODE_TYPES.has(node.type)) {
|
||||
if (CODE_NODE_TYPES.has(nodeType)) {
|
||||
const source = (
|
||||
getStringParam(node, 'jsCode') ??
|
||||
getStringParam(node, 'pythonCode') ??
|
||||
|
|
@ -395,14 +392,14 @@ function nodeHasName(node: WorkflowNode): node is WorkflowNode & { name: string
|
|||
|
||||
/**
|
||||
* Classify every main-flow, non-trigger node of the workflow. Returns one
|
||||
* verdict per node, ordered as the nodes appear in the workflow. Triggers and
|
||||
* user-action nodes are out of scope (Mechanism A handles them); sub-nodes
|
||||
* and disconnected nodes are excluded because pin data cannot apply to them.
|
||||
* verdict per node, ordered as the nodes appear in the workflow. Include
|
||||
* attached tools. Exclude triggers, other sub-nodes, and disconnected nodes.
|
||||
*/
|
||||
export async function classifyNodesForSimulation(
|
||||
input: ClassifyNodesForSimulationInput,
|
||||
): Promise<NodeSimulationVerdict[]> {
|
||||
const mainFlowNames = collectMainFlowNodeNames(input.workflow);
|
||||
const graph = createVerificationGraph(input.workflow);
|
||||
const candidateNames = graph.withTools(graph.rootNodeNames);
|
||||
const mockedNodeNames = new Set(input.mockedNodeNames ?? []);
|
||||
|
||||
const candidates = (input.workflow.nodes ?? []).filter(
|
||||
|
|
@ -411,7 +408,8 @@ export async function classifyNodesForSimulation(
|
|||
node.disabled !== true &&
|
||||
node.type !== STICKY_NOTE_TYPE &&
|
||||
!isTriggerNodeType(node.type) &&
|
||||
mainFlowNames.has(node.name),
|
||||
candidateNames.has(node.name) &&
|
||||
(input.nodeNames === undefined || input.nodeNames.has(node.name)),
|
||||
);
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
|
|
|
|||
|
|
@ -65,7 +65,8 @@ export interface VerificationSimulationPlan {
|
|||
simulationFixtures?: SimulationFixtures;
|
||||
}
|
||||
|
||||
const DECLARED_OUTPUT_SIMULATION_REASON = 'Source declares verification output for this node';
|
||||
export const DECLARED_OUTPUT_SIMULATION_REASON =
|
||||
'Source declares verification output for this node';
|
||||
|
||||
const TRIGGER_SIMULATION_REASON = 'Trigger event is simulated during verification';
|
||||
|
||||
|
|
@ -177,7 +178,7 @@ export function findCredentiallessAiRoots(
|
|||
* credentials get their `execute` verdict overridden to `simulate`: the real
|
||||
* run would die at the root with a credential error ("Node does not have any
|
||||
* credentials set"), leaving every downstream simulated node unreached. The
|
||||
* classifier marks AI roots safe by type and never visits `ai_*` sub-nodes,
|
||||
* classifier marks AI roots and Agent Tool containers safe by type,
|
||||
* so — like the trigger pass above — this is the single injection point.
|
||||
* Roots already `simulate` (declared fixtures, mocked credentials) keep
|
||||
* their verdict and reason.
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import { classifyNodesForSimulation } from './classify-node-destructiveness.serv
|
|||
import { isAiGatewayManagedCredential } from './credential-utils';
|
||||
import {
|
||||
CREDENTIALLESS_AI_ROOT_SIMULATION_REASON,
|
||||
DECLARED_OUTPUT_SIMULATION_REASON,
|
||||
findCredentiallessAiRoots,
|
||||
withWaitGateHaltVerdicts,
|
||||
} from './plan-verification-simulation';
|
||||
|
|
@ -126,14 +127,9 @@ export async function reconcileSimulationPlan(args: {
|
|||
// pass never re-judges the rest of the workflow.
|
||||
let freshVerdictByName = new Map<string, NodeSimulationVerdict>();
|
||||
if (satisfiedNames.size > 0) {
|
||||
const scopedWorkflow: WorkflowJSON = {
|
||||
...workflow,
|
||||
nodes: (workflow.nodes ?? []).filter(
|
||||
(node) => typeof node.name === 'string' && satisfiedNames.has(node.name),
|
||||
),
|
||||
};
|
||||
const freshVerdicts = await classifyNodesForSimulation({
|
||||
workflow: scopedWorkflow,
|
||||
workflow,
|
||||
nodeNames: satisfiedNames,
|
||||
mockedNodeNames: remainingMockedNames,
|
||||
fallbackModelConfig,
|
||||
});
|
||||
|
|
@ -145,6 +141,7 @@ export async function reconcileSimulationPlan(args: {
|
|||
// simulation. Restored AI roots are deterministically safe by type, so
|
||||
// flipping straight to execute matches what a rebuild would classify.
|
||||
const reconciledPlan = buildOutcome.nodeSimulationPlan?.map((verdict) => {
|
||||
if (verdict.reason === DECLARED_OUTPUT_SIMULATION_REASON) return verdict;
|
||||
if (satisfiedNames.has(verdict.nodeName)) {
|
||||
return freshVerdictByName.get(verdict.nodeName) ?? verdict;
|
||||
}
|
||||
|
|
@ -164,7 +161,9 @@ export async function reconcileSimulationPlan(args: {
|
|||
|
||||
const retainedFixtureEntries = Object.entries(buildOutcome.simulationFixtures ?? {}).filter(
|
||||
([nodeName]) =>
|
||||
freshVerdictByName.get(nodeName)?.verdict !== 'execute' && !restoredAiRoots.has(nodeName),
|
||||
nodeSimulationPlan?.some(
|
||||
(verdict) => verdict.nodeName === nodeName && verdict.verdict === 'simulate',
|
||||
),
|
||||
);
|
||||
const simulationFixtures =
|
||||
retainedFixtureEntries.length > 0 ? Object.fromEntries(retainedFixtureEntries) : undefined;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
import { toEngineConnections, type WorkflowJSON } from '@n8n/workflow-sdk';
|
||||
import {
|
||||
getChildNodes,
|
||||
getParentNodes,
|
||||
isTriggerNodeType,
|
||||
mapConnectionsByDestination,
|
||||
NodeConnectionTypes,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
export const AGENT_NODE_TYPE = '@n8n/n8n-nodes-langchain.agent';
|
||||
export const AGENT_TOOL_NODE_TYPE = '@n8n/n8n-nodes-langchain.agentTool';
|
||||
|
||||
export function getTriggerMainFlowScope(
|
||||
connections: WorkflowJSON['connections'],
|
||||
triggerNodeName: string,
|
||||
): Set<string> {
|
||||
return new Set([
|
||||
triggerNodeName,
|
||||
...getChildNodes(toEngineConnections(connections), triggerNodeName, NodeConnectionTypes.Main),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Keep tool edges separate. They point from the tool to its caller. */
|
||||
export function createVerificationGraph(workflow: WorkflowJSON) {
|
||||
const connections = toEngineConnections(workflow.connections);
|
||||
const byDestination = mapConnectionsByDestination(connections);
|
||||
const nodesByName = new Map(
|
||||
workflow.nodes
|
||||
.filter(
|
||||
(node): node is WorkflowJSON['nodes'][number] & { name: string } =>
|
||||
typeof node.name === 'string' && !node.disabled,
|
||||
)
|
||||
.map((node) => [node.name, node]),
|
||||
);
|
||||
// Triggers such as MCP can have tools without any main connections.
|
||||
const rootNodeNames = new Set(
|
||||
[...nodesByName.values()]
|
||||
.filter((node) => isTriggerNodeType(node.type))
|
||||
.map((node) => node.name),
|
||||
);
|
||||
for (const name of Object.keys(connections)) {
|
||||
const children = getChildNodes(connections, name, NodeConnectionTypes.Main, 1);
|
||||
if (children.length === 0) continue;
|
||||
for (const nodeName of [name, ...children]) {
|
||||
if (nodesByName.has(nodeName)) rootNodeNames.add(nodeName);
|
||||
}
|
||||
}
|
||||
const toolsFor = (name: string) =>
|
||||
getParentNodes(byDestination, name, NodeConnectionTypes.AiTool, 1).filter((toolName) =>
|
||||
nodesByName.has(toolName),
|
||||
);
|
||||
const withTools = (mainScope: ReadonlySet<string>) => {
|
||||
const names = new Set<string>();
|
||||
const pending = [...mainScope];
|
||||
while (pending.length > 0) {
|
||||
const name = pending.pop()!;
|
||||
if (names.has(name) || !nodesByName.has(name)) continue;
|
||||
names.add(name);
|
||||
pending.push(...toolsFor(name));
|
||||
}
|
||||
return names;
|
||||
};
|
||||
return { nodesByName, rootNodeNames, toolsFor, withTools };
|
||||
}
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
import { NodeTypes } from '@nodes-testing/node-types';
|
||||
import { NodeTestHarness } from '@nodes-testing/node-test-harness';
|
||||
import { WorkflowExecute } from 'n8n-core/dist/execution-engine/workflow-execute';
|
||||
import {
|
||||
NodeConnectionTypes,
|
||||
type INodeExecutionData,
|
||||
type INodeType,
|
||||
type WorkflowTestData,
|
||||
} from 'n8n-workflow';
|
||||
import nock from 'nock';
|
||||
import path from 'node:path';
|
||||
|
||||
function mockCompletion(
|
||||
content: string,
|
||||
tool?: string,
|
||||
requestBody?: (body: unknown) => boolean,
|
||||
): NonNullable<WorkflowTestData['nock']>['mocks'][number] {
|
||||
return {
|
||||
method: 'post',
|
||||
path: '/v1/chat/completions',
|
||||
statusCode: 200,
|
||||
requestBody,
|
||||
responseBody: {
|
||||
id: 'chatcmpl-test',
|
||||
object: 'chat.completion',
|
||||
created: 1700000000,
|
||||
model: 'gpt-4o-mini',
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: tool
|
||||
? {
|
||||
role: 'assistant',
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_${tool}`,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool,
|
||||
arguments: JSON.stringify({ input: 'Write a record' }),
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
: { role: 'assistant', content },
|
||||
finish_reason: tool ? 'tool_calls' : 'stop',
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('Agent tool simulation', () => {
|
||||
const harness = new NodeTestHarness({
|
||||
additionalPackagePaths: [path.dirname(require.resolve('n8n-nodes-base'))],
|
||||
});
|
||||
const fixture = { id: 'simulated-write-1', status: 'accepted' };
|
||||
const credentials = { openAiApi: { apiKey: 'test-api-key', url: 'https://api.openai.com/v1' } };
|
||||
|
||||
for (const agentVersion of [3, 3.1]) {
|
||||
for (const nested of [false, true]) {
|
||||
describe(`Agent ${agentVersion}, nested=${nested}`, () => {
|
||||
const workflow = harness.readWorkflowJSON(
|
||||
nested ? 'workflows/sub-agent-with-inner-tool.json' : 'workflows/agent-v3-with-tool.json',
|
||||
);
|
||||
workflow.settings = { ...workflow.settings, executionOrder: 'v1' };
|
||||
const agentName = nested ? 'Parent Agent' : 'AI Agent';
|
||||
const callerName = nested ? 'MathSubAgent' : agentName;
|
||||
workflow.nodes.find((node) => node.name === agentName)!.typeVersion = agentVersion;
|
||||
const write = workflow.nodes.find((node) => node.name === 'Calculator')!;
|
||||
write.name = 'Write';
|
||||
write.type = 'n8n-nodes-testing.write';
|
||||
write.typeVersion = 1;
|
||||
write.parameters = {};
|
||||
workflow.connections.Write = workflow.connections.Calculator;
|
||||
delete workflow.connections.Calculator;
|
||||
delete workflow.pinData;
|
||||
|
||||
const writeImplementation = vi.fn(
|
||||
async (): Promise<INodeExecutionData[][]> => [[{ json: { id: 'real-write' } }]],
|
||||
);
|
||||
const writeNode: INodeType = {
|
||||
description: {
|
||||
displayName: 'Write',
|
||||
name: 'write',
|
||||
description: 'Writes a record',
|
||||
group: ['output'],
|
||||
version: 1,
|
||||
defaults: { name: 'Write' },
|
||||
inputs: [NodeConnectionTypes.Main],
|
||||
outputs: [NodeConnectionTypes.AiTool],
|
||||
properties: [],
|
||||
},
|
||||
execute: writeImplementation,
|
||||
};
|
||||
const getNodeType = NodeTypes.prototype.getByNameAndVersion;
|
||||
const runNode = WorkflowExecute.prototype.runNode;
|
||||
let receivedFixture = false;
|
||||
beforeEach(() => {
|
||||
receivedFixture = false;
|
||||
writeImplementation.mockClear();
|
||||
vi.spyOn(NodeTypes.prototype, 'getByNameAndVersion').mockImplementation(function (
|
||||
this: NodeTypes,
|
||||
type,
|
||||
version,
|
||||
) {
|
||||
return type === write.type ? writeNode : getNodeType.call(this, type, version);
|
||||
});
|
||||
let initialized = false;
|
||||
vi.spyOn(WorkflowExecute.prototype, 'runNode').mockImplementation(async function (
|
||||
this: WorkflowExecute,
|
||||
...args
|
||||
) {
|
||||
const [, executionData, runData, , , , , response] = args;
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
// The harness starts at the trigger. Add per-execution pins before the Agent runs.
|
||||
runData.resultData.pinData = { Write: [{ json: fixture }] };
|
||||
}
|
||||
if (executionData.node.name === callerName && response?.actionResponses.length) {
|
||||
expect(response.actionResponses).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
action: expect.objectContaining({ nodeName: 'Write', type: 'ai_tool' }),
|
||||
data: expect.objectContaining({
|
||||
data: { ai_tool: [[expect.objectContaining({ json: fixture })]] },
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
receivedFixture = true;
|
||||
}
|
||||
return await runNode.apply(this, args);
|
||||
});
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
nock.cleanAll();
|
||||
});
|
||||
|
||||
const mocks = [
|
||||
...(nested ? [mockCompletion('', 'MathSubAgent')] : []),
|
||||
mockCompletion('', 'Write'),
|
||||
mockCompletion('Fixture received', undefined, (body) =>
|
||||
JSON.stringify(body).includes(fixture.id),
|
||||
),
|
||||
...(nested ? [mockCompletion('Fixture received')] : []),
|
||||
];
|
||||
harness.setupTest(
|
||||
{
|
||||
description: 'returns the write fixture to the Agent through ai_tool',
|
||||
input: { workflowData: workflow },
|
||||
output: { nodeData: { [agentName]: [[{ json: { output: 'Fixture received' } }]] } },
|
||||
nock: { baseUrl: 'https://api.openai.com', mocks },
|
||||
},
|
||||
{
|
||||
credentials,
|
||||
customAssertions: () => {
|
||||
expect(writeImplementation).not.toHaveBeenCalled();
|
||||
expect(receivedFixture).toBe(true);
|
||||
expect(nock.isDone()).toBe(true);
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
@ -228,6 +228,7 @@
|
|||
"@types/temp": "^0.9.1",
|
||||
"@vitest/coverage-v8": "catalog:",
|
||||
"@typescript/native": "catalog:typescript-tooling",
|
||||
"nock": "catalog:",
|
||||
"typescript": "catalog:typescript-tooling",
|
||||
"vitest": "catalog:",
|
||||
"vitest-mock-extended": "catalog:"
|
||||
|
|
|
|||
|
|
@ -3623,6 +3623,9 @@ importers:
|
|||
'@vitest/coverage-v8':
|
||||
specifier: 'catalog:'
|
||||
version: 4.1.9(@vitest/browser@4.1.10)(vitest@4.1.9)
|
||||
nock:
|
||||
specifier: 'catalog:'
|
||||
version: 14.0.14
|
||||
typescript:
|
||||
specifier: catalog:typescript-tooling
|
||||
version: '@typescript/typescript6@6.0.2'
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user