github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-10-18 05:59:42 +02:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity Graph

fix(core): Require verification coverage for every workflow trigger (#37961)

Browse Source
This commit is contained in:
Predrag Ristic 2026-09-10 13:43:01 +00:00 committed by GitHub
parent 1bee3bcac7
commit 429116f985
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1003 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
packages/@n8n/instance-ai/docs/tools.md
View File

@ -196,8 +196,18 @@ waiting-with-output-as-success fallback.
| Chat Trigger | `{chatInput: "..."}` | `{ sessionId, action, chatInput }` |
| Schedule | omit | synthetic timestamp fields |
For reusable workflows with multiple enabled triggers, pass `triggerNodeName`
and run verification once for each trigger. Successful runs accumulate node
coverage per trigger. Each retry reserves an attempt and clears that trigger's
old pass before execution. A successful result restores the combined coverage.
If either write fails, the tool reports an error. The attempt limit still applies.
The returned and saved `claim` use the same cumulative evidence. Pending triggers
and nodes without real coverage prevent a `verified` claim. Publishing during a
retry requires explicit acknowledgement through `acknowledgeUnverified: true`.
**Writes on success/failure**: the tool persists a structured `verification`
record (`{ attempted, success, executionId, status, evidence, verifiedAt }`) onto
record (`{ attempted, success, executionId, status, claim, evidence, verifiedAt }`) onto
the build outcome so workflow-verification follow-ups and exceptional checkpoint
turns can reuse it without re-running verify.

29
packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md
View File

@ -85,7 +85,8 @@ obligation is `ready_to_verify` or `verifying`, call `verify-built-workflow`. Do
**not** call `workflows(action="setup")` in this turn and do **not** declare the
workflow finished if `outcome.setupRequirement.status === "required"` — setup is
routed automatically as a separate `<workflow-setup-required>` step after
verification.
verification. For a multi-trigger outcome, verify every trigger that does not
yet have a recorded successful verification. Make all of these calls in this turn.
## Setup follow-up
@ -304,13 +305,14 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries),
entry in `triggerNodes`. Naming no trigger verifies only the auto-detected
one. An unresolvable name is rejected outright, so a rejected call means the
name is wrong — re-read `triggerNodes`, never fall back to editing.
- Each pass covers its own trigger's branch, so its `nodesNotReached` will list
the other triggers' nodes. That is expected, not a defect: coverage is the
**union** across passes. Only treat a node as unverified once no pass reached
it.
- Report per-trigger coverage — name each trigger and whether its branch ran.
Claim the workflow is verified only when every trigger's branch has a
successful pass.
- Each pass reports `nodesNotReached` only for its selected trigger's main-flow
branch. Coverage is the **union** across successful passes. Run every trigger
before you report a workflow coverage gap. Different triggers can select
different outputs of a shared Switch or If node.
- A failed rerun removes that trigger's earlier coverage. Verify that trigger
again before you claim that the workflow is verified.
- Report each trigger and whether its branch ran. Use the combined `claim` to
describe the result (see "Claiming success").
- When the user asked for a live run, pass `triggerNodeName` to
`executions(action="run")` the same way — one run per trigger — and report
each branch's result.
@ -330,8 +332,10 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries),
`workflow-builder` skill and patch the same workflow with `build-workflow`
using the existing `workflowId` and `workItemId`; then inspect and verify
again.
- If `verificationReadiness.status === "already_verified"`, treat the
workflow as verified and do **not** call `verify-built-workflow` again.
- If `verificationReadiness.status === "already_verified"`, do not repeat
automatic verification. Read the saved claim before describing the workflow
as verified. For tracked multi-trigger builds, follow the verification
obligation until every trigger has a successful pass.
- If `verificationReadiness.status === "ready"`, call
`verify-built-workflow` with the `workflowId`, the `workItemId` when you
@ -540,6 +544,11 @@ result.
## Claiming success
For tracked multi-trigger builds, `claim` combines the saved successful passes.
A `verified` claim requires every trigger to pass and real coverage for every planned node.
Verify `claim.pendingTriggers` within the attempt limit. `nodesNotReached`
outside the claim describes only the current trigger's branch.
`verify-built-workflow` returns a `claim`, and its `level` decides what you may
say:

24
packages/@n8n/instance-ai/src/__tests__/verification-fixtures.ts Normal file
View File

@ -0,0 +1,24 @@
import type { WorkflowTriggerVerificationProgress } from '../workflow-loop/workflow-loop-state';
export function successfulVerification(
triggerNodeName: string,
nodesExecuted = [triggerNodeName],
): WorkflowTriggerVerificationProgress[number] {
return {
attempted: true,
success: true,
executionId: `exec-${triggerNodeName}`,
evidence: { triggerNodeName, nodesExecuted },
claim: {
level: 'verified',
plannedNodeCount: nodesExecuted.length,
reachedNodeCount: nodesExecuted.length,
nodesNotReached: [],
simulatedNodes: [],
pinnedNodes: [],
unprovenTargets: [],
publishReady: true,
liveTestRecommended: false,
},
};
}

58
packages/@n8n/instance-ai/src/storage/__tests__/workflow-loop-storage.test.ts
View File

@ -83,6 +83,64 @@ describe('WorkflowLoopStorage', () => {
storage = new WorkflowLoopStorage(memory);
});
it('updates the latest outcome without replacing other work items', async () => {
const current = {
...baseThread,
metadata: {
otherMetadata: true,
instanceAiWorkflowLoop: {
'wi-1': {
state: makeState(),
attempts: [],
lastBuildOutcome: makeOutcome({ verifyAttempts: 3 }),
},
'wi-2': { state: makeState({ workItemId: 'wi-2' }), attempts: [] },
},
},
};
mockedPatchThread.mockImplementationOnce(
async (_memory, { update }) =>
await Promise.resolve({
...current,
...update(current),
}),
);
await storage.updateBuildOutcome('thread-1', 'wi-1', (outcome) => {
expect(outcome.verifyAttempts).toBe(3);
return { ...outcome, verifyAttempts: 4 };
});
const saved: unknown = await mockedPatchThread.mock.results[0].value;
expect(saved).toMatchObject({
metadata: {
otherMetadata: true,
instanceAiWorkflowLoop: {
'wi-1': { lastBuildOutcome: { verifyAttempts: 4 } },
'wi-2': { state: { workItemId: 'wi-2' } },
},
},
});
});
it('rejects a verification update when the thread disappeared', async () => {
mockedPatchThread.mockResolvedValueOnce(null);
await expect(
storage.updateBuildOutcome('thread-1', 'wi-1', (outcome) => outcome),
).rejects.toThrow('could not be saved');
});
it('rejects a verification update when the build outcome disappeared', async () => {
mockedPatchThread.mockImplementationOnce(
async (_memory, { update }) =>
await Promise.resolve({
...baseThread,
...update(baseThread),
}),
);
await expect(
storage.updateBuildOutcome('thread-1', 'wi-1', (outcome) => outcome),
).rejects.toThrow('state is unavailable');
});
it('rejects an update when saving its patch fails', async () => {
const state = makeState();
mockedPatchThread.mockImplementation((_memory, { update }) => {

21
packages/@n8n/instance-ai/src/storage/workflow-loop-storage.ts
View File

@ -1,3 +1,4 @@
import { OperationalError } from 'n8n-workflow';
import { z } from 'zod';
import { getThread, patchThread, type PatchableThreadMemory } from './thread-patch';
@ -93,7 +94,7 @@ export class WorkflowLoopStorage {
update: (record: WorkflowLoopWorkItemRecord) => WorkflowLoopWorkItemRecord | null,
): Promise<boolean> {
let updated = false;
await patchThread(this.memory, {
const thread = await patchThread(this.memory, {
threadId,
update: ({ metadata = {} }) => {
const all = this.parse(metadata[METADATA_KEY]);
@ -108,7 +109,7 @@ export class WorkflowLoopStorage {
return { metadata: { ...metadata, [METADATA_KEY]: all } };
},
});
return updated;
return updated && thread !== null;
}
async getActiveWorkItem(threadId: string): Promise<WorkflowLoopWorkItemRecord | null> {
@ -121,6 +122,22 @@ export class WorkflowLoopStorage {
return null;
}
async updateBuildOutcome(
threadId: string,
workItemId: string,
update: (outcome: WorkflowBuildOutcome) => WorkflowBuildOutcome,
): Promise<void> {
const updated = await this.updateWorkItem(threadId, workItemId, (record) => {
if (!record.lastBuildOutcome) return null;
return { ...record, lastBuildOutcome: update(record.lastBuildOutcome) };
});
if (!updated) {
throw new OperationalError(
'Verification state is unavailable or could not be saved. Rebuild the workflow.',
);
}
}
async listWorkItems(threadId: string): Promise<WorkflowLoopWorkItemRecord[]> {
const all = await this.loadAll(threadId);
return Object.values(all);

2
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts
View File

@ -12,6 +12,8 @@ function createWorkflowTaskService(reportVerificationVerdict = vi.fn()) {
beginVerification: vi.fn(),
getWorkflowLoopState: vi.fn(),
updateBuildOutcome: vi.fn(),
startVerification: vi.fn(),
recordVerification: vi.fn(),
};
}

315
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts
View File

@ -1,6 +1,8 @@
import type { Mock } from 'vitest';
import { executeTool } from '../../../__tests__/tool-test-utils';
import { successfulVerification } from '../../../__tests__/verification-fixtures';
import type { WorkflowLoopStorage } from '../../../storage/workflow-loop-storage';
import type {
InstanceAiDataTableService,
InstanceAiWorkflowService,
@ -8,7 +10,9 @@ import type {
WorkflowTaskService,
} from '../../../types';
import { createRemediation, MAX_VERIFY_ATTEMPTS } from '../../../workflow-loop/remediation';
import { deriveWorkflowVerificationObligationFromOutcome } from '../../../workflow-loop/verification-obligation';
import type { WorkflowBuildOutcome } from '../../../workflow-loop/workflow-loop-state';
import { WorkflowTaskCoordinator } from '../../../workflow-loop/workflow-task-service';
import { createVerifyBuiltWorkflowTool } from '../verify-built-workflow.tool';
type VerifyBuiltWorkflowOutput = {
@ -55,6 +59,8 @@ function createContext(overrides: Partial<OrchestrationContext> = {}): Orchestra
beginVerification: vi.fn(),
getWorkflowLoopState: vi.fn(),
updateBuildOutcome: vi.fn(),
startVerification: vi.fn(),
recordVerification: vi.fn(),
};
return {
@ -493,10 +499,18 @@ function makeContext(
} = {},
) {
const updateBuildOutcome = vi.fn(
async (_workItemId: string, _update: Partial<WorkflowBuildOutcome>) => {
async (_workItemId: string, update: Partial<WorkflowBuildOutcome>) => {
if (outcome) outcome = { ...outcome, ...update };
await Promise.resolve();
},
);
const coordinator = new WorkflowTaskCoordinator('thread-1', {
updateBuildOutcome: async (_threadId, workItemId, update) => {
if (!outcome) throw new Error('Missing outcome');
await updateBuildOutcome(workItemId, update(outcome));
},
} as WorkflowLoopStorage);
const run = vi.fn(
async (
_workflowId: string,
@ -538,7 +552,13 @@ function makeContext(
getAsWorkflowJSON: vi.fn(async () => {
await Promise.resolve();
return {
nodes: overrides.workflowNodes ?? [],
nodes:
overrides.workflowNodes ??
outcome?.triggerNodes?.map(({ nodeName, nodeType }) => ({
name: nodeName,
type: nodeType,
})) ??
[],
connections: overrides.workflowConnections ?? {},
};
}),
@ -578,6 +598,8 @@ function makeContext(
beginVerification: vi.fn(),
getWorkflowLoopState: vi.fn(),
updateBuildOutcome,
startVerification: vi.fn(coordinator.startVerification.bind(coordinator)),
recordVerification: vi.fn(coordinator.recordVerification.bind(coordinator)),
} as unknown as WorkflowTaskService,
domainContext: {
executionService: { run },
@ -587,7 +609,7 @@ function makeContext(
},
logger,
};
return { ctx, updateBuildOutcome, queryRows, deleteRows };
return { ctx, updateBuildOutcome, queryRows, deleteRows, getOutcome: () => outcome! };
}
async function runTool(
@ -621,8 +643,8 @@ describe('verify-built-workflow tool', () => {
});
expect(result.success).toBe(true);
expect(updateBuildOutcome).toHaveBeenCalledTimes(1);
const call = updateBuildOutcome.mock.calls[0];
expect(updateBuildOutcome).toHaveBeenCalledTimes(2);
const call = updateBuildOutcome.mock.calls.at(-1)!;
expect(call).toBeDefined();
const update = call[1];
expect(update.verification).toMatchObject({
@ -644,7 +666,7 @@ describe('verify-built-workflow tool', () => {
await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
expect(updateBuildOutcome.mock.calls[0][1].verifyAttempts).toBe(3);
expect(updateBuildOutcome.mock.calls.at(-1)![1].verifyAttempts).toBe(3);
});
it('blocks verification once the verify attempt budget is exhausted', async () => {
@ -734,8 +756,8 @@ describe('verify-built-workflow tool', () => {
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
expect(result.success).toBe(false);
expect(updateBuildOutcome).toHaveBeenCalledTimes(1);
const call = updateBuildOutcome.mock.calls[0];
expect(updateBuildOutcome).toHaveBeenCalledTimes(2);
const call = updateBuildOutcome.mock.calls.at(-1)!;
expect(call).toBeDefined();
const update = call[1];
expect(update.verification).toMatchObject({
@ -775,7 +797,7 @@ describe('verify-built-workflow tool', () => {
expect(result.resolvedWorkItemId).toBe('wi-latest');
expect(ctx.workflowTaskService.getBuildOutcome).not.toHaveBeenCalled();
expect(ctx.workflowTaskService.getLatestBuildOutcomeForWorkflow).toHaveBeenCalledWith('wf-1');
expect(updateBuildOutcome.mock.calls[0][0]).toBe('wi-latest');
expect(updateBuildOutcome.mock.calls.at(-1)![0]).toBe('wi-latest');
});
it('falls back to the workflow build outcome when the supplied work item ID is stale', async () => {
@ -794,7 +816,7 @@ describe('verify-built-workflow tool', () => {
expect(result.success).toBe(true);
expect(result.resolvedWorkItemId).toBe('wi-latest');
expect(ctx.workflowTaskService.getLatestBuildOutcomeForWorkflow).toHaveBeenCalledWith('wf-1');
expect(updateBuildOutcome.mock.calls[0][0]).toBe('wi-latest');
expect(updateBuildOutcome.mock.calls.at(-1)![0]).toBe('wi-latest');
});
it('rejects verification when the requested workflow does not match the build outcome', async () => {
@ -827,17 +849,17 @@ describe('verify-built-workflow tool', () => {
expect(updateBuildOutcome).not.toHaveBeenCalled();
});
it('swallows storage errors when persisting verification', async () => {
it('does not execute when the attempt cannot be saved', async () => {
const { ctx, updateBuildOutcome } = makeContext(makeBuildOutcome(), {
executionId: 'exec-3',
status: 'success',
});
updateBuildOutcome.mockRejectedValueOnce(new Error('storage unavailable'));
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
expect(result.success).toBe(true);
expect(result.executionId).toBe('exec-3');
await expect(runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' })).rejects.toThrow(
'storage unavailable',
);
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
});
it('counts wrapped execution output items in previews and persisted evidence', async () => {
@ -867,7 +889,9 @@ describe('verify-built-workflow tool', () => {
expect.objectContaining({ nodeName: 'Large Export', itemCount: 5 }),
expect.objectContaining({ nodeName: 'Large Transform', itemCount: 7 }),
]);
expect(updateBuildOutcome.mock.calls[0][1].verification?.evidence?.producedOutputRows).toBe(14);
expect(
updateBuildOutcome.mock.calls.at(-1)![1].verification?.evidence?.producedOutputRows,
).toBe(14);
});
it('treats a waiting status with output as a successful run (e.g. Form Trigger response page)', async () => {
@ -889,8 +913,8 @@ describe('verify-built-workflow tool', () => {
expect(result.success).toBe(true);
expect(result.status).toBe('waiting');
expect(updateBuildOutcome).toHaveBeenCalledTimes(1);
const update = updateBuildOutcome.mock.calls[0][1];
expect(updateBuildOutcome).toHaveBeenCalledTimes(2);
const update = updateBuildOutcome.mock.calls.at(-1)![1];
expect(update.verification).toMatchObject({
attempted: true,
success: true,
@ -1165,7 +1189,7 @@ describe('verify-built-workflow tool — node simulation plan', () => {
reason: 'missing_simulation_plan',
});
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
expect(updateBuildOutcome.mock.calls[0][1].verification).toMatchObject({
expect(updateBuildOutcome.mock.calls.at(-1)![1].verification).toMatchObject({
attempted: true,
success: false,
status: 'unknown',
@ -1230,7 +1254,7 @@ describe('verify-built-workflow tool — node simulation plan', () => {
expect(result.nodeErrors?.[0]?.nodeName).toBe('geocode_city');
expect(result.nodeErrors?.[0]?.message).toContain('supplyData');
expect(result.error).toContain('geocode_city');
const verification = updateBuildOutcome.mock.calls[0][1].verification;
const verification = updateBuildOutcome.mock.calls.at(-1)![1].verification;
expect(verification?.success).toBe(false);
expect(verification?.status).toBe('success');
expect(verification?.failureSignature).toContain('geocode_city');
@ -1360,9 +1384,9 @@ describe('verify-built-workflow tool — node simulation plan', () => {
await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
expect(updateBuildOutcome.mock.calls[0][1].verification?.evidence?.nodesNotReached).toEqual([
'Send Slack',
]);
expect(
updateBuildOutcome.mock.calls.at(-1)![1].verification?.evidence?.nodesNotReached,
).toEqual(['Send Slack']);
});
it('never deletes rows whose IDs come from fabricated fixture output', async () => {
@ -1499,13 +1523,248 @@ describe('verify-built-workflow tool — stale mocked-credential plan', () => {
});
describe('verify-built-workflow tool — trigger selection', () => {
it('starts verification from the named trigger', async () => {
const { ctx } = makeContext(makeBuildOutcome(), {
executionId: 'exec-monthly',
status: 'success',
data: { 'Post Summary': [{ ok: true }] },
type TriggerInput = { workItemId: string; workflowId: string; triggerNodeName?: string };
const triggerNodes = [
{ nodeName: 'Trigger A', nodeType: 'n8n-nodes-base.webhook' },
{ nodeName: 'Trigger B', nodeType: 'n8n-nodes-base.scheduleTrigger' },
];
const executeVerdict = (nodeName: string) => ({
nodeName,
verdict: 'execute' as const,
reason: 'Reads data',
confidence: 'high' as const,
source: 'deterministic' as const,
});
const workflowConnections = {
'Trigger A': { main: [[{ node: 'Step A', type: 'main', index: 0 }]] },
'Trigger B': { main: [[{ node: 'Step B', type: 'main', index: 0 }]] },
};
const triggerAInput = { workItemId: 'wi-1', workflowId: 'wf-1', triggerNodeName: 'Trigger A' };
const successfulA: ExecutionRunResult = {
executionId: 'exec-a',
status: 'success',
executedNodeNames: ['Trigger A', 'Step A'],
};
const successfulB: ExecutionRunResult = {
executionId: 'exec-b',
status: 'success',
executedNodeNames: ['Trigger B', 'Step B'],
};
const makeTrackedOutcome = (overrides: Partial<WorkflowBuildOutcome> = {}) =>
makeBuildOutcome({
triggerNodes,
verificationProgress: {},
nodeSimulationPlan: [executeVerdict('Step A'), executeVerdict('Step B')],
...overrides,
});
function makeSequenceContext(
initialOutcome: WorkflowBuildOutcome,
connections: Record<string, unknown> = workflowConnections,
) {
return makeContext(initialOutcome, { status: 'success' }, { workflowConnections: connections });
}
it.each(['Trigger A', undefined])(
'requires a successful retry after a previously verified trigger fails (%s)',
async (failedTriggerName) => {
const { ctx, getOutcome } = makeSequenceContext(makeTrackedOutcome());
const run = ctx.domainContext.executionService.run;
const triggerBInput = { ...triggerAInput, triggerNodeName: 'Trigger B' };
run.mockResolvedValueOnce(successfulA).mockResolvedValueOnce(successfulB);
await runTool(ctx, triggerAInput);
await runTool(ctx, triggerBInput);
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'verified',
);
run.mockResolvedValueOnce({ ...successfulA, status: 'error', error: 'Step A failed' });
await runTool(ctx, { ...triggerAInput, triggerNodeName: failedTriggerName });
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'ready_to_verify',
);
run.mockResolvedValueOnce(successfulB);
await runTool(ctx, triggerBInput);
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'ready_to_verify',
);
run.mockResolvedValueOnce({ ...successfulA, executedNodeNames: ['Trigger A'] });
await runTool(ctx, triggerAInput);
const incompleteRetry = deriveWorkflowVerificationObligationFromOutcome(
'thread-1',
getOutcome(),
);
expect(incompleteRetry.status).toBe('not_verifiable');
expect(incompleteRetry.blockingReason).toContain('Step A');
expect(incompleteRetry.blockingReason).not.toContain('Step B');
run.mockResolvedValueOnce(successfulA);
await runTool(ctx, triggerAInput);
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'verified',
);
},
);
it('combines successful trigger runs through different Switch outputs', async () => {
const { ctx, getOutcome } = makeSequenceContext(
makeTrackedOutcome({
nodeSimulationPlan: ['Switch', 'Step A', 'Step B'].map(executeVerdict),
}),
{
'Trigger A': { main: [[{ node: 'Switch', type: 'main', index: 0 }]] },
'Trigger B': { main: [[{ node: 'Switch', type: 'main', index: 0 }]] },
Switch: {
main: [
[{ node: 'Step A', type: 'main', index: 0 }],
[{ node: 'Step B', type: 'main', index: 0 }],
],
},
},
);
ctx.domainContext.executionService.run
.mockResolvedValueOnce({
...successfulA,
executedNodeNames: ['Trigger A', 'Switch', 'Step A'],
})
.mockResolvedValueOnce({
...successfulB,
executedNodeNames: ['Trigger B', 'Switch', 'Step B'],
});
await runTool(ctx, triggerAInput);
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'ready_to_verify',
);
await runTool(ctx, { ...triggerAInput, triggerNodeName: 'Trigger B' });
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'verified',
);
});
it('records a fully covered pass without counting the other trigger branch as a gap', async () => {
const { ctx, updateBuildOutcome } = makeContext(makeTrackedOutcome(), successfulA, {
workflowConnections,
});
const result = await runTool(ctx, triggerAInput);
expect(result.nodesNotReached).toBeUndefined();
expect(updateBuildOutcome.mock.calls.at(-1)![1].verificationProgress).toMatchObject({
'Trigger A': [{ evidence: { nodesExecuted: ['Trigger A', 'Step A'] } }],
});
expect(updateBuildOutcome.mock.calls.at(-1)![1].verification?.evidence?.triggerNodeName).toBe(
'Trigger A',
);
});
it('preserves verification state when the workflow graph cannot be loaded', async () => {
const outcome = makeTrackedOutcome({ verificationProgress: passedA, verifyAttempts: 1 });
const { ctx, getOutcome } = makeContext(outcome, successfulA, { workflowConnections });
vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockRejectedValueOnce(
new Error('workflow unavailable'),
);
const result = await runTool(ctx, triggerAInput);
expect(result.success).toBe(false);
expect(getOutcome()).toEqual(outcome);
expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled();
});
const withoutTrigger = { workItemId: 'wi-1', workflowId: 'wf-1' };
const invalidPasses: Array<[string, ExecutionRunResult, TriggerInput]> = [
[
'a failed execution',
{ ...successfulA, status: 'error', error: 'Step A failed' },
triggerAInput,
],
['a missing execution ID', { ...successfulA, executionId: undefined }, triggerAInput],
[
'a trigger that was not reached',
{ ...successfulA, executedNodeNames: ['Step A'] },
triggerAInput,
],
['a pass without a named trigger', successfulA, withoutTrigger],
[
'a trigger outside the build outcome',
{ ...successfulA, executedNodeNames: ['Unknown Trigger'] },
{ ...withoutTrigger, triggerNodeName: 'Unknown Trigger' },
],
];
it.each(invalidPasses)(
'does not record %s as a successful trigger pass',
async (_name, runResult, input) => {
const { ctx, getOutcome } = makeContext(
makeTrackedOutcome({ nodeSimulationPlan: [executeVerdict('Step A')] }),
runResult,
{ workflowConnections },
);
await runTool(ctx, input);
expect(Object.keys(getOutcome().verificationProgress ?? {})).toEqual([]);
},
);
const passedA = {
'Trigger A': [successfulVerification('Trigger A', ['Trigger A', 'Step A'])],
};
it('keeps the attempt and removes old coverage when the result cannot be saved', async () => {
const { ctx, getOutcome } = makeSequenceContext(
makeTrackedOutcome({
verifyAttempts: MAX_VERIFY_ATTEMPTS - 1,
verificationProgress: passedA,
}),
);
ctx.domainContext.executionService.run.mockResolvedValueOnce(successfulA);
vi.mocked(ctx.workflowTaskService.recordVerification).mockRejectedValueOnce(
new Error('Save failed'),
);
await expect(runTool(ctx, triggerAInput)).rejects.toThrow('Save failed');
expect(getOutcome().verifyAttempts).toBe(MAX_VERIFY_ATTEMPTS);
expect(getOutcome().verificationProgress).toEqual({});
expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe(
'blocked',
);
await runTool(ctx, triggerAInput);
expect(ctx.domainContext.executionService.run).toHaveBeenCalledTimes(1);
});
it('combines repeated successful passes for the same trigger', async () => {
const { ctx, getOutcome } = makeSequenceContext(
makeTrackedOutcome({
verificationProgress: passedA,
}),
);
ctx.domainContext.executionService.run.mockResolvedValueOnce({
...successfulA,
executedNodeNames: ['Trigger A', 'Alternate Step'],
});
await runTool(ctx, triggerAInput);
expect(getOutcome().verificationProgress?.['Trigger A']).toMatchObject([
{ evidence: { nodesExecuted: ['Trigger A', 'Step A'] } },
{ evidence: { nodesExecuted: ['Trigger A', 'Alternate Step'] } },
]);
});
it('starts verification from the named trigger', async () => {
const { ctx } = makeContext(
makeBuildOutcome(),
{
executionId: 'exec-monthly',
status: 'success',
data: { 'Post Summary': [{ ok: true }] },
},
{ workflowNodes: [{ name: 'First of Month', type: 'n8n-nodes-base.scheduleTrigger' }] },
);
await runTool(ctx, {
workItemId: 'wi-1',
workflowId: 'wf-1',

16
packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts
View File

@ -1,5 +1,7 @@
import { isRecord } from '@n8n/utils/is-record';
import { isPlaceholderValue } from '@n8n/utils/placeholder';
import { toEngineConnections, type WorkflowJSON } from '@n8n/workflow-sdk';
import { getChildNodes, NodeConnectionTypes } from 'n8n-workflow';
import type { ExecutionRunResult, VerificationNodePreview } from './types';
import {
@ -427,6 +429,16 @@ export interface VerificationAnalysis {
nodeErrors: ExecutionNodeError[];
}
export function getTriggerMainFlowScope(
connections: WorkflowJSON['connections'],
triggerNodeName: string,
): Set<string> {
return new Set([
triggerNodeName,
...getChildNodes(toEngineConnections(connections), triggerNodeName, NodeConnectionTypes.Main),
]);
}
export function analyzeVerificationResult(args: {
result: ExecutionRunResult;
buildOutcome: WorkflowBuildOutcome;
@ -445,6 +457,8 @@ export function analyzeVerificationResult(args: {
chatModelRecovery?: ChatModelRecoveryOptions;
/** Trigger this pass started from, when the caller named one. */
triggerNodeName?: string;
/** Main-flow nodes that belong to the selected trigger. */
verificationScope?: ReadonlySet<string>;
}): VerificationAnalysis {
const {
result,
@ -456,6 +470,7 @@ export function analyzeVerificationResult(args: {
chatModelRelatedNodeNames,
chatModelRecovery,
triggerNodeName,
verificationScope,
} = args;
const nodeErrors = result.nodeErrors ?? [];
const reachedNames = new Set(
@ -474,6 +489,7 @@ export function analyzeVerificationResult(args: {
];
const nodesNotReached = (buildOutcome.nodeSimulationPlan ?? [])
.map((verdict) => verdict.nodeName)
.filter((name) => !verificationScope || verificationScope.has(name))
.filter((name) => !reachedNames.has(name));
const hasSimulationPlan = (buildOutcome.nodeSimulationPlan?.length ?? 0) > 0;
const hasOutput = result.data ? Object.keys(result.data).length > 0 : false;

11
packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts
View File

@ -13,7 +13,11 @@ import type {
} from '../../../workflow-loop/workflow-loop-state';
export interface DeriveVerificationClaimArgs {
analysis: VerificationAnalysis;
analysis: Pick<
VerificationAnalysis,
'success' | 'nodesNotReached' | 'reachedSimulatedNodes' | 'workflowPinnedNodeNames'
>;
pendingTriggers?: string[];
/** Planned nodes from the build outcome simulation plan. */
plannedNodeCount: number;
/**
@ -37,6 +41,7 @@ export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): Veri
hasUnreached: nodesNotReached.length > 0,
hasSimulated: simulatedNodes.length > 0,
hasUnprovenTarget: unprovenTargets.length > 0,
hasPendingTrigger: (args.pendingTriggers?.length ?? 0) > 0,
});
return {
@ -50,6 +55,7 @@ export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): Veri
simulatedNodes,
pinnedNodes: [...analysis.workflowPinnedNodeNames],
unprovenTargets,
...(args.pendingTriggers ? { pendingTriggers: args.pendingTriggers } : {}),
publishReady: level === 'verified',
liveTestRecommended: level === 'partial' || level === 'unproven',
};
@ -60,11 +66,12 @@ function resolveLevel(facts: {
hasUnreached: boolean;
hasSimulated: boolean;
hasUnprovenTarget: boolean;
hasPendingTrigger: boolean;
}): VerificationClaimLevel {
if (!facts.success) return 'failed';
// A named target that never ran for real outranks overall coverage: the user
// asked about that node, so a green run elsewhere does not answer them.
if (facts.hasUnprovenTarget) return 'unproven';
if (facts.hasUnreached || facts.hasSimulated) return 'partial';
if (facts.hasUnreached || facts.hasSimulated || facts.hasPendingTrigger) return 'partial';
return 'verified';
}

60
packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts
View File

@ -14,6 +14,7 @@ import { createRemediation } from '../../../workflow-loop/remediation';
import type {
RemediationMetadata,
VerificationClaim,
WorkflowTriggerVerificationProgress,
} from '../../../workflow-loop/workflow-loop-state';
/**
@ -91,11 +92,11 @@ export async function persistVerificationOutcome(args: {
workflowId: string;
result: ExecutionRunResult;
analysis: VerificationAnalysis;
scopedTriggerNodeName?: string;
previousProgress?: WorkflowTriggerVerificationProgress;
/** Deterministic verdict for this run, persisted so later turns cannot re-litigate it. */
claim: VerificationClaim;
/** Running count of verify runs for this build, used to enforce MAX_VERIFY_ATTEMPTS. */
verifyAttempts: number;
}): Promise<void> {
}): Promise<VerificationClaim | undefined> {
const {
input,
context,
@ -103,36 +104,34 @@ export async function persistVerificationOutcome(args: {
workflowId,
result,
analysis,
scopedTriggerNodeName,
previousProgress,
claim,
verifyAttempts,
} = args;
try {
const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data);
await workflowTaskService.updateBuildOutcome(input.workItemId, {
verifyAttempts,
verification: {
attempted: true,
success: analysis.success,
executionId: result.executionId || undefined,
status: result.status,
failureSignature: analysis.success ? undefined : analysis.errorMessage,
claim,
evidence: {
nodesExecuted:
executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined,
nodesNotReached:
analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined,
producedOutputRows: countProducedOutputRows(result.data),
errorNodeName: analysis.success ? undefined : analysis.nodeErrors[0]?.nodeName,
errorMessage: analysis.success ? undefined : analysis.errorMessage,
nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined,
},
verifiedAt: new Date().toISOString(),
const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data);
const storedClaim = await workflowTaskService.recordVerification(
input.workItemId,
{
attempted: true,
success: analysis.success,
executionId: result.executionId || undefined,
status: result.status,
failureSignature: analysis.success ? undefined : analysis.errorMessage,
claim,
evidence: {
...(scopedTriggerNodeName ? { triggerNodeName: scopedTriggerNodeName } : {}),
nodesExecuted:
executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined,
nodesNotReached: analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined,
producedOutputRows: countProducedOutputRows(result.data),
errorNodeName: analysis.success ? undefined : analysis.nodeErrors[0]?.nodeName,
errorMessage: analysis.success ? undefined : analysis.errorMessage,
nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined,
},
});
} catch {
// intentional: verification record persistence is advisory
}
verifiedAt: new Date().toISOString(),
},
previousProgress,
);
if (analysis.remediation && !analysis.remediation.shouldEdit) {
await reportTerminalRemediation({
@ -144,6 +143,7 @@ export async function persistVerificationOutcome(args: {
remediation: analysis.remediation,
});
}
return storedClaim;
}
async function reportTerminalRemediation(args: {

2
packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts
View File

@ -39,6 +39,7 @@ export interface ScriptedGateRunArgs {
runId: string;
chatModelRelatedNodeNames?: ReadonlySet<string>;
chatModelRecovery?: ChatModelRecoveryOptions;
verificationScope?: ReadonlySet<string>;
}
interface DecisionPass {
@ -82,6 +83,7 @@ export async function runScriptedGateVerification(
runId,
chatModelRelatedNodeNames,
chatModelRecovery,
verificationScope: args.verificationScope,
});
passes.push({ label: decision.label, result, analysis });
}

67
packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts
View File

@ -7,10 +7,15 @@
*/
import { Tool } from '@n8n/agents';
import { isTriggerNodeType } from 'n8n-workflow';
import { z } from 'zod';
import type { OrchestrationContext } from '../../types';
import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result';
import {
analyzeVerificationResult,
buildNodePreviews,
getTriggerMainFlowScope,
} from './verification/analyze-result';
import { deriveVerificationClaim } from './verification/claim';
import {
handleMissingSimulationPlan,
@ -194,18 +199,42 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
}
const { prepared } = preparedResult;
const chatModelRecovery = await target.domainContext.workflowService
const workflow = await target.domainContext.workflowService
.getAsWorkflowJSON(workflowId)
.then(
async (workflow) =>
await collectChatModelRecoveryContext(
target.domainContext,
workflow.nodes ?? [],
workflow.connections,
),
)
.catch(() => undefined);
if (
resolvedInput.triggerNodeName !== undefined &&
!workflow?.nodes.some(
(node) => node.name === resolvedInput.triggerNodeName && isTriggerNodeType(node.type),
)
) {
return {
success: false,
resolvedWorkItemId: resolvedInput.workItemId,
error: `Could not find trigger "${resolvedInput.triggerNodeName}" in this workflow. Read the workflow. Select an existing trigger.`,
};
}
const chatModelRecovery = workflow
? await collectChatModelRecoveryContext(
target.domainContext,
workflow.nodes ?? [],
workflow.connections,
).catch(() => undefined)
: undefined;
const chatModelRelatedNodeNames = chatModelRecovery?.relatedNodeNames;
const selectedTriggerNodeName = buildOutcome.triggerNodes?.some(
(trigger) => trigger.nodeName === resolvedInput.triggerNodeName,
)
? resolvedInput.triggerNodeName
: undefined;
const verificationScope =
buildOutcome.verificationProgress && selectedTriggerNodeName && workflow
? getTriggerMainFlowScope(workflow.connections, selectedTriggerNodeName)
: undefined;
const previousProgress = await workflowTaskService.startVerification(
resolvedInput.workItemId,
verificationScope ? selectedTriggerNodeName : undefined,
);
// A scripted gate replaces the halt with one loop-safe pass per decision;
// otherwise run the single standard pass (halted gates pin zero items).
@ -224,6 +253,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
runId: context.runId,
chatModelRelatedNodeNames,
chatModelRecovery,
verificationScope,
})
: await (async () => {
const runResult = await target.domainContext.executionService.run(
@ -249,6 +279,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
runId: context.runId,
chatModelRelatedNodeNames,
chatModelRecovery,
verificationScope,
}),
};
})();
@ -263,21 +294,27 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
].filter((name): name is string => name !== undefined),
),
];
const claim = deriveVerificationClaim({
analysis,
const runClaim = deriveVerificationClaim({
analysis: {
...analysis,
nodesNotReached: buildOutcome.nodeSimulationPlan
.map((node) => node.nodeName)
.filter((name) => !analysis.reachedNames.has(name)),
},
plannedNodeCount: buildOutcome.nodeSimulationPlan?.length ?? 0,
fixTargetNodeNames,
});
await persistVerificationOutcome({
const claim = await persistVerificationOutcome({
input: resolvedInput,
context,
workflowTaskService,
workflowId,
result,
analysis,
claim,
verifyAttempts: (buildOutcome.verifyAttempts ?? 0) + 1,
scopedTriggerNodeName: verificationScope ? selectedTriggerNodeName : undefined,
previousProgress,
claim: runClaim,
});
const maxDataChars = resolvedInput.maxDataChars ?? DEFAULT_NODE_PREVIEW_CHARS;

7
packages/@n8n/instance-ai/src/tools/workflows.tool.ts
View File

@ -1404,11 +1404,8 @@ async function resolveUnverifiedPublishDisclosure(
const claim = verification?.claim;
if (claim) return claim.publishReady ? undefined : formatClaimDisclosure(claim);
// A record with no claim means verification ran and could not produce one
// — it was refused for want of a simulation plan, or it failed before it
// reached a verdict. That is a failed verification, not an unverified
// workflow, and it must not pass as an absent record.
if (verification?.attempted) {
// A running or attempted verification must not pass as an absent record.
if (verification?.attempted || verification?.status === 'running') {
const cause = verification.failureSignature ?? verification.evidence?.errorMessage;
return (
'Verification ran but produced no verdict, so nothing about this workflow is proven.' +

26
packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts
View File

@ -2064,7 +2064,27 @@ describe('createBuildWorkflowTool', () => {
});
});
it('reports planned build outcomes without source artifact metadata', async () => {
it.each([false, true])('tracks only enabled triggers (disabled=%s)', async (disabled) => {
vi.mocked(compileWorkflowSource).mockResolvedValueOnce({
success: true,
workflow: {
...structuredClone(generatedWorkflow),
nodes: [
...structuredClone(generatedWorkflow.nodes),
{
id: 'schedule-1',
name: 'Schedule',
disabled,
type: 'n8n-nodes-base.scheduleTrigger',
typeVersion: 1,
position: [0, 100],
parameters: {},
},
],
},
warnings: [],
compiler: 'sandbox-tsx',
});
const reportBuildOutcome = vi.fn<
(outcome: WorkflowBuildOutcome) => Promise<{ type: 'verify'; workflowId: string }>
>(async () => await Promise.resolve({ type: 'verify', workflowId: 'wf-1' }));
@ -2105,7 +2125,11 @@ describe('createBuildWorkflowTool', () => {
owner: { type: 'planned', taskId: 'task-1' },
plannedTaskId: 'task-1',
sourceFilePath: filePath,
verificationProgress: disabled ? undefined : {},
});
expect(storedOutcome?.triggerNodes?.some((trigger) => trigger.nodeName === 'Schedule')).toBe(
!disabled,
);
expect(storedOutcome).not.toHaveProperty('sourceArtifact');
const reportedOutcome = reportBuildOutcome.mock.calls[0]?.[0] as

4
packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts
View File

@ -1150,7 +1150,7 @@ export function createBuildWorkflowTool(context: InstanceAiContext) {
const heldForNewCredentialTypes = mockResult.heldForNewCredentialTypes;
const referencedWorkflowIds = getReferencedWorkflowIds(json);
const triggerNodes = (json.nodes ?? [])
.filter((n) => isTriggerNodeType(n.type))
.filter((n) => !n.disabled && isTriggerNodeType(n.type))
.map((n) => ({ nodeName: n.name, nodeType: n.type }))
.filter(
(t): t is { nodeName: string; nodeType: string } =>
@ -1289,6 +1289,8 @@ export function createBuildWorkflowTool(context: InstanceAiContext) {
nodeSimulationPlan,
simulationFixtures,
waitGateScripts,
verificationProgress:
triggerNodes.length > 1 && executionIntent !== 'one-off' ? {} : undefined,
supportingWorkflowIds:
referencedWorkflowIds.length > 0 ? referencedWorkflowIds : undefined,
hasUnresolvedPlaceholders: hasPlaceholders || undefined,

12
packages/@n8n/instance-ai/src/types.ts
View File

@ -61,10 +61,13 @@ import type {
import type { BuilderRequiredArtifact } from './tools/orchestration/builder-required-artifact';
import type { IdRemapper, TraceIndex, TraceWriter } from './tracing/trace-replay';
import type {
VerificationClaim,
VerificationResult,
WorkflowBuildOutcome,
WorkflowLoopAction,
WorkflowLoopState,
WorkflowVerificationEvidence,
WorkflowTriggerVerificationProgress,
WorkflowVerificationObligation,
} from './workflow-loop/workflow-loop-state';
import type { BuilderTemplatesService } from './workspace/builder-templates-service';
@ -1914,6 +1917,15 @@ export interface WorkflowTaskService {
runId: string,
): Promise<boolean>;
updateBuildOutcome(workItemId: string, update: Partial<WorkflowBuildOutcome>): Promise<void>;
startVerification(
workItemId: string,
triggerNodeName?: string,
): Promise<WorkflowTriggerVerificationProgress | undefined>;
recordVerification(
workItemId: string,
verification: WorkflowVerificationEvidence & { claim: VerificationClaim },
previousProgress?: WorkflowTriggerVerificationProgress,
): Promise<VerificationClaim | undefined>;
}
// ── Orchestration context (plan tools) ──────────────────────────────────────

155
packages/@n8n/instance-ai/src/workflow-loop/__tests__/verification-obligation.test.ts
View File

@ -1,4 +1,9 @@
import { createRemediation, MAX_POST_SUBMIT_REMEDIATION_SUBMITS } from '../remediation';
import { successfulVerification } from '../../__tests__/verification-fixtures';
import {
createRemediation,
MAX_VERIFY_ATTEMPTS,
MAX_POST_SUBMIT_REMEDIATION_SUBMITS,
} from '../remediation';
import {
deriveWorkflowVerificationObligation,
deriveWorkflowVerificationObligationFromOutcome,
@ -8,6 +13,7 @@ import type {
AttemptRecord,
WorkflowBuildOutcome,
WorkflowLoopState,
WorkflowVerificationObligationStatus,
} from '../workflow-loop-state';
function makeState(overrides: Partial<WorkflowLoopState> = {}): WorkflowLoopState {
@ -52,6 +58,30 @@ function makeOutcome(overrides: Partial<WorkflowBuildOutcome> = {}): WorkflowBui
};
}
function makePlannedNode(nodeName: string) {
return {
nodeName,
verdict: 'execute' as const,
reason: 'Reads data',
confidence: 'high' as const,
source: 'deterministic' as const,
};
}
function makeMultiTriggerOutcome(
overrides: Partial<WorkflowBuildOutcome> = {},
): WorkflowBuildOutcome {
return makeOutcome({
triggerNodes: [
{ nodeName: 'Trigger A', nodeType: 'n8n-nodes-base.webhook' },
{ nodeName: 'Trigger B', nodeType: 'n8n-nodes-base.scheduleTrigger' },
],
nodeSimulationPlan: [makePlannedNode('Step A'), makePlannedNode('Step B')],
verificationProgress: {},
...overrides,
});
}
describe('deriveWorkflowVerificationObligation', () => {
it('keeps an exhausted setup verification budget blocked', () => {
const remediation = createRemediation({
@ -198,6 +228,129 @@ describe('deriveWorkflowVerificationObligation', () => {
expect(obligation.evidence?.executionId).toBe('exec-1');
});
const passedA = {
'Trigger A': [successfulVerification('Trigger A', ['Trigger A', 'Step A'])],
};
const passedAll = {
...passedA,
'Trigger B': [successfulVerification('Trigger B', ['Trigger B', 'Step B'])],
};
const successfulPass: WorkflowBuildOutcome['verification'] = {
attempted: true,
success: true,
executionId: 'exec-1',
status: 'success',
};
const verificationCases: Array<
[string, Partial<WorkflowBuildOutcome>, WorkflowVerificationObligationStatus]
> = [
[
'the first trigger passed',
{ verificationProgress: passedA, verification: successfulPass },
'ready_to_verify',
],
['all triggers and nodes passed', { verificationProgress: passedAll }, 'verified'],
[
'an unscoped partial pass was recorded',
{ verification: { ...successfulPass, evidence: { nodesNotReached: ['Step B'] } } },
'ready_to_verify',
],
[
'the first trigger has a coverage gap',
{
verificationProgress: passedA,
verification: {
...successfulPass,
evidence: { triggerNodeName: 'Trigger A', nodesNotReached: ['Shared Final Step'] },
},
},
'ready_to_verify',
],
[
'aggregate coverage has a gap',
{
verificationProgress: passedAll,
nodeSimulationPlan: ['Step A', 'Step B', 'Shared Final Step'].map(makePlannedNode),
},
'not_verifiable',
],
[
'the latest pass failed',
{ verification: { ...successfulPass, success: false, status: 'error' } },
'ready_to_verify',
],
[
'the latest failure contradicts stored coverage',
{
verificationProgress: passedAll,
verification: { ...successfulPass, success: false, status: 'error' },
},
'ready_to_verify',
],
[
'the attempt limit was reached before completion',
{ verificationProgress: passedA, verifyAttempts: MAX_VERIFY_ATTEMPTS },
'blocked',
],
[
'the final allowed attempt completed coverage',
{ verificationProgress: passedAll, verifyAttempts: MAX_VERIFY_ATTEMPTS },
'verified',
],
[
'a legacy outcome has no progress record',
{ verificationProgress: undefined, verification: successfulPass },
'verified',
],
];
it.each(verificationCases)(
'derives multi-trigger verification status when %s',
(_name, overrides, expectedStatus) => {
const obligation = deriveWorkflowVerificationObligation('thread-1', {
state: makeState(),
attempts: [makeAttempt()],
lastBuildOutcome: makeMultiTriggerOutcome(overrides),
});
expect(obligation.status).toBe(expectedStatus);
if (expectedStatus === 'blocked')
expect(obligation.blockingReason).toContain('attempt limit');
if (expectedStatus === 'not_verifiable') {
expect(obligation.policy).toBe('manual');
expect(obligation.blockingReason).toContain('Shared Final Step');
}
if (expectedStatus === 'verified') expect(obligation.blockingReason).toBeUndefined();
},
);
it('does not treat a reserved run as failed setup evidence', () => {
const obligation = deriveWorkflowVerificationObligationFromOutcome(
'thread-1',
makeMultiTriggerOutcome({
setupRequirement: {
status: 'required',
reason: 'mocked-credentials',
guidance: 'Connect the account after verification.',
},
verifyAttempts: 1,
verification: undefined,
}),
);
expect(obligation.status).toBe('ready_to_verify');
});
it('keeps incomplete multi-trigger progress blocked when the loop state is blocked', () => {
const obligation = deriveWorkflowVerificationObligation('thread-1', {
state: makeState({ status: 'blocked' }),
attempts: [makeAttempt()],
lastBuildOutcome: makeMultiTriggerOutcome({
verificationReadiness: { status: 'already_verified' },
}),
});
expect(obligation.status).toBe('blocked');
});
it('treats partial-coverage evidence as a manual warning completion', () => {
const obligation = deriveWorkflowVerificationObligation('thread-1', {
state: makeState(),

42
packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts
View File

@ -1,6 +1,7 @@
import { mock } from 'vitest-mock-extended';
import { executeTool } from '../../__tests__/tool-test-utils';
import { successfulVerification } from '../../__tests__/verification-fixtures';
import type { WorkflowLoopStorage } from '../../storage/workflow-loop-storage';
import { createReportVerificationVerdictTool } from '../../tools/orchestration/report-verification-verdict.tool';
import { createVerifyBuiltWorkflowTool } from '../../tools/orchestration/verify-built-workflow.tool';
@ -14,6 +15,21 @@ function createStorage() {
const records = new Map<string, Record<string, unknown>>();
const storage = {
updateBuildOutcome: vi.fn(
async (
_threadId: string,
workItemId: string,
update: (outcome: WorkflowBuildOutcome) => WorkflowBuildOutcome,
) => {
const record = records.get(workItemId);
if (!record?.lastBuildOutcome) throw new Error('Missing outcome');
records.set(workItemId, {
...record,
lastBuildOutcome: update(record.lastBuildOutcome as WorkflowBuildOutcome),
});
await Promise.resolve();
},
),
getWorkItem: vi.fn(async (_threadId: string, workItemId: string) => {
return await Promise.resolve(
(records.get(workItemId) ?? null) as Awaited<
@ -70,6 +86,32 @@ function createBuildOutcome(overrides: Partial<WorkflowBuildOutcome> = {}): Work
}
describe('WorkflowTaskCoordinator', () => {
it('counts reservations and stops at the attempt limit', async () => {
const { storage } = createStorage();
const coordinator = new WorkflowTaskCoordinator('thread-1', storage);
await coordinator.reportBuildOutcome(createBuildOutcome());
for (let attempt = 0; attempt < MAX_VERIFY_ATTEMPTS; attempt++) {
await coordinator.startVerification('wi_1');
}
await expect(coordinator.startVerification('wi_1')).rejects.toThrow('attempt limit');
expect((await coordinator.getBuildOutcome('wi_1'))?.verifyAttempts).toBe(MAX_VERIFY_ATTEMPTS);
});
it('merges completed trigger runs against the latest stored coverage', async () => {
const { storage } = createStorage();
const coordinator = new WorkflowTaskCoordinator('thread-1', storage);
await coordinator.reportBuildOutcome(createBuildOutcome({ verificationProgress: {} }));
await coordinator.startVerification('wi_1', 'A');
await coordinator.startVerification('wi_1', 'B');
for (const triggerNodeName of ['B', 'A']) {
await coordinator.recordVerification('wi_1', successfulVerification(triggerNodeName));
}
expect((await coordinator.getBuildOutcome('wi_1'))?.verificationProgress).toMatchObject({
A: [{ evidence: { nodesExecuted: ['A'] } }],
B: [{ evidence: { nodesExecuted: ['B'] } }],
});
});
function setupBlockedOutcome(overrides: Partial<WorkflowBuildOutcome> = {}) {
return createBuildOutcome({
runId: 'run-1',

6
packages/@n8n/instance-ai/src/workflow-loop/guidance.ts
View File

@ -33,7 +33,11 @@ function formatClaimLead(claim: VerificationClaim | undefined): string {
const rules = [
'Do NOT call the workflow verified, tested, working, or ready to publish.',
'Do NOT offer to publish it.',
claim.liveTestRecommended ? 'Offer a live end-to-end test instead.' : '',
(claim.pendingTriggers?.length ?? 0) > 0
? 'Follow the verification obligation for the remaining triggers. Respect its attempt limit.'
: claim.liveTestRecommended
? 'Offer a live end-to-end test instead.'
: '',
].filter((rule) => rule !== '');
return [formatClaimHeadline(claim), ...describeClaimCoverage(claim), ...rules].join(' ');

5
packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts
View File

@ -31,6 +31,11 @@ export function formatClaimHeadline(claim: VerificationClaim): string {
export function describeClaimCoverage(claim: VerificationClaim): string[] {
const facts: string[] = [];
if ((claim.pendingTriggers?.length ?? 0) > 0) {
facts.push(
`Verification evidence is incomplete for these triggers: ${formatClaimNodeList(claim.pendingTriggers ?? [])}.`,
);
}
if (claim.nodesNotReached.length > 0) {
facts.push(

72
packages/@n8n/instance-ai/src/workflow-loop/verification-obligation.ts
View File

@ -1,7 +1,9 @@
import { MAX_VERIFY_ATTEMPTS } from './remediation';
import {
setupRemediationBlocksVerification,
stateForPendingSetupVerification,
} from './setup-verification-policy';
import { getMultiTriggerCoverage } from './verification-progress';
import type {
AttemptRecord,
WorkflowBuildOwner,
@ -42,7 +44,20 @@ const UNSETTLED_OBLIGATION_STATUSES = new Set<WorkflowVerificationObligationStat
'verifying',
]);
function hasSuccessfulEvidence(outcome: WorkflowBuildOutcome): boolean {
type MultiTriggerCoverage = ReturnType<typeof getMultiTriggerCoverage>;
function hasSuccessfulEvidence(
outcome: WorkflowBuildOutcome,
coverage: MultiTriggerCoverage,
): boolean {
if (coverage) {
return (
coverage.allTriggersPassed &&
coverage.nodesNotReached.length === 0 &&
!hasFailedEvidence(outcome)
);
}
const nodesNotReached = outcome.verification?.evidence?.nodesNotReached;
return (
outcome.verification?.attempted === true &&
@ -52,7 +67,13 @@ function hasSuccessfulEvidence(outcome: WorkflowBuildOutcome): boolean {
);
}
function hasPartialSuccessfulCoverageEvidence(outcome: WorkflowBuildOutcome): boolean {
function hasPartialSuccessfulCoverageEvidence(
outcome: WorkflowBuildOutcome,
coverage: MultiTriggerCoverage,
): boolean {
if (coverage) {
return coverage.allTriggersPassed && coverage.nodesNotReached.length > 0;
}
return (
outcome.verification?.attempted === true &&
outcome.verification.success &&
@ -78,20 +99,17 @@ function hasSetupBlockingEvidence(
function deriveStatus(
state: WorkflowLoopState,
outcome: WorkflowBuildOutcome | undefined,
coverage: MultiTriggerCoverage,
): WorkflowVerificationObligationStatus {
if (!outcome) return 'pending_build';
if (!outcome.submitted) return 'blocked';
if (hasSuccessfulEvidence(outcome)) return 'verified';
if (hasSuccessfulEvidence(outcome, coverage)) return 'verified';
if (hasSetupBlockingEvidence(state, outcome)) return 'needs_setup';
if (hasPartialSuccessfulCoverageEvidence(outcome)) return 'not_verifiable';
// A verification that was attempted and failed has already run end-to-end and
// produced a concrete error (e.g. a missing credential or a runtime error).
// Re-issuing it just replays the same failure, so once setup-blocking and
// partial-success cases are ruled out above, settle it as a manual outcome.
// Without this the switch below falls through to `ready_to_verify` and the
// planned orchestrator re-issues verification forever.
if (hasFailedEvidence(outcome)) return 'not_verifiable';
if (hasPartialSuccessfulCoverageEvidence(outcome, coverage)) return 'not_verifiable';
// Keep tracked multi-trigger failures open so another trigger can contribute.
// The attempt limit below bounds these retries.
if (hasFailedEvidence(outcome) && !coverage) return 'not_verifiable';
// One-off builds: verification is optional, so the obligation settles
// immediately (after the evidence checks above, which still win). This must
@ -102,10 +120,14 @@ function deriveStatus(
if (outcome.executionIntent === 'one-off') {
return state.status === 'blocked' ? 'blocked' : 'not_verifiable';
}
if ((outcome.verifyAttempts ?? 0) >= MAX_VERIFY_ATTEMPTS) {
return 'blocked';
}
switch (outcome.verificationReadiness?.status) {
case 'already_verified':
return 'verified';
if (!coverage) return 'verified';
return state.status === 'blocked' ? 'blocked' : 'ready_to_verify';
case 'needs_setup':
return 'needs_setup';
case 'not_verifiable':
@ -131,6 +153,8 @@ function derivePolicy(
function deriveBlockingReason(
state: WorkflowLoopState,
outcome: WorkflowBuildOutcome | undefined,
coverage: MultiTriggerCoverage,
status: WorkflowVerificationObligationStatus,
): string | undefined {
if (!outcome) return undefined;
if (!outcome.submitted) {
@ -138,17 +162,24 @@ function deriveBlockingReason(
outcome.blockingReason ?? outcome.failureSignature ?? 'Builder did not submit a workflow.'
);
}
if (status === 'verified') return undefined;
if (outcome.verificationReadiness?.status === 'not_verifiable') {
return outcome.verificationReadiness.guidance;
}
if (outcome.verificationReadiness?.status === 'needs_setup') {
return outcome.verificationReadiness.guidance;
}
if (hasPartialSuccessfulCoverageEvidence(outcome)) {
if (hasPartialSuccessfulCoverageEvidence(outcome, coverage)) {
if (coverage) {
return (
'Automatic verification covered every trigger but did not reach all planned nodes. ' +
`Unreached nodes need manual testing: ${coverage.nodesNotReached.join(', ')}.`
);
}
const nodesNotReached = outcome.verification?.evidence?.nodesNotReached ?? [];
return `Automatic verification only covered part of the workflow. Unreached nodes need manual testing: ${nodesNotReached.join(', ')}.`;
}
if (hasFailedEvidence(outcome) && !hasSetupBlockingEvidence(state, outcome)) {
if (hasFailedEvidence(outcome) && !coverage && !hasSetupBlockingEvidence(state, outcome)) {
const failure =
outcome.verification?.failureSignature ??
outcome.verification?.evidence?.errorMessage ??
@ -158,6 +189,12 @@ function deriveBlockingReason(
nodesNotReached.length > 0 ? ` Nodes not reached: ${nodesNotReached.join(', ')}.` : '';
return `Automatic verification failed with: ${failure}. Re-running it will reproduce the same failure — explain this blocker to the user and have them resolve it (e.g. configure credentials or fix the data) before verifying manually.${unreached}`;
}
if (status === 'blocked' && (outcome.verifyAttempts ?? 0) >= MAX_VERIFY_ATTEMPTS) {
return (
'Automatic verification reached its attempt limit. ' +
'Report the remaining coverage. Ask the user to test the remaining nodes manually.'
);
}
const outcomeRemediation = outcome.remediation;
if (outcomeRemediation && setupRemediationBlocksVerification(outcomeRemediation, outcome)) {
return outcomeRemediation.guidance;
@ -172,7 +209,7 @@ function deriveBlockingReason(
// After the evidence checks: a one-off may have run an optional pre-flight
// verify, and a concrete failure message outranks the generic guidance. A
// verified one-off carries no blockingReason at all — nothing is blocked.
if (outcome.executionIntent === 'one-off' && !hasSuccessfulEvidence(outcome)) {
if (outcome.executionIntent === 'one-off') {
return ONE_OFF_VERIFICATION_GUIDANCE;
}
if (state.status === 'blocked') {
@ -200,7 +237,8 @@ export function deriveWorkflowVerificationObligation(
? { ...savedOutcome, verificationReadiness: { status: 'ready' } }
: savedOutcome;
const state = setupVerificationState ?? record.state;
const status = deriveStatus(state, outcome);
const coverage = getMultiTriggerCoverage(outcome);
const status = deriveStatus(state, outcome, coverage);
const updatedAt =
options.updatedAt ?? lastAttemptTimestamp(record.attempts) ?? new Date().toISOString();
const owner = resolveWorkflowBuildOwner(options, record.state, outcome);
@ -222,7 +260,7 @@ export function deriveWorkflowVerificationObligation(
setupRequirement: outcome?.setupRequirement,
evidence: outcome?.verification,
executionIntent: outcome?.executionIntent,
blockingReason: deriveBlockingReason(state, outcome),
blockingReason: deriveBlockingReason(state, outcome, coverage, status),
updatedAt,
};
}

71
packages/@n8n/instance-ai/src/workflow-loop/verification-progress.ts Normal file
View File

@ -0,0 +1,71 @@
import type {
VerificationClaim,
WorkflowBuildOutcome,
WorkflowVerificationEvidence,
} from './workflow-loop-state';
import { deriveVerificationClaim } from '../tools/orchestration/verification/claim';
export function getMultiTriggerCoverage(outcome: WorkflowBuildOutcome | undefined) {
const progress = outcome?.verificationProgress;
const triggers = outcome?.triggerNodes ?? [];
if (!outcome || outcome.executionIntent === 'one-off' || !progress || triggers.length < 2) {
return undefined;
}
const passedTriggers = triggers.filter((trigger) => Object.hasOwn(progress, trigger.nodeName));
const passes = passedTriggers.flatMap((trigger) => progress[trigger.nodeName]);
const coveredNodes = new Set(passes.flatMap((pass) => pass.evidence?.nodesExecuted ?? []));
return {
passes,
allTriggersPassed: passedTriggers.length === triggers.length,
pendingTriggers: triggers
.filter((trigger) => !Object.hasOwn(progress, trigger.nodeName))
.map((trigger) => trigger.nodeName),
nodesNotReached: (outcome.nodeSimulationPlan ?? [])
.map((node) => node.nodeName)
.filter((nodeName) => !coveredNodes.has(nodeName)),
};
}
export function deriveWorkflowVerificationClaim(
outcome: WorkflowBuildOutcome,
verification: WorkflowVerificationEvidence & { claim: VerificationClaim },
) {
const coverage = getMultiTriggerCoverage(outcome);
if (!coverage) return verification.claim;
const liveNodes = new Set(
coverage.passes.flatMap(({ evidence, claim }) => {
const simulated = new Set([
...claim.simulatedNodes.map((node) => node.nodeName),
...claim.pinnedNodes,
]);
return (evidence?.nodesExecuted ?? []).filter((name) => !simulated.has(name));
}),
);
// Only successful scoped passes contribute coverage. Keep the latest run's limitations.
const claims = [...coverage.passes.map((pass) => pass.claim), verification.claim];
return deriveVerificationClaim({
analysis: {
success: verification.success,
nodesNotReached: coverage.nodesNotReached,
reachedSimulatedNodes: [
...new Map(
claims
.flatMap((claim) => claim.simulatedNodes)
.filter((node) => !liveNodes.has(node.nodeName))
.map((node) => [node.nodeName, node]),
).values(),
],
workflowPinnedNodeNames: [...new Set(claims.flatMap((claim) => claim.pinnedNodes))].filter(
(name) => !liveNodes.has(name),
),
},
plannedNodeCount: outcome.nodeSimulationPlan?.length ?? 0,
fixTargetNodeNames: claims
.flatMap((claim) => claim.unprovenTargets)
.filter((name) => !liveNodes.has(name)),
pendingTriggers: coverage.pendingTriggers,
});
}

19
packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts
View File

@ -164,7 +164,7 @@ export const verificationClaimLevelSchema = z.enum(['verified', 'partial', 'unpr
export type VerificationClaimLevel = z.infer<typeof verificationClaimLevelSchema>;
/**
* The deterministic verdict for one verification run. Backend-only: it shapes
* The deterministic verdict from verification evidence. Backend-only: it shapes
* the tool result the model reads and gates the publish offer. It is
* deliberately not sent to the client — see INS-1308.
*/
@ -184,6 +184,8 @@ export const verificationClaimSchema = z.object({
* claim `verified` even when it ended without an error.
*/
unprovenTargets: z.array(z.string()),
/** Triggers with missing or incomplete scoped verification evidence. */
pendingTriggers: z.array(z.string()).optional(),
publishReady: z.boolean(),
liveTestRecommended: z.boolean(),
});
@ -205,6 +207,8 @@ export const workflowVerificationEvidenceSchema = z.object({
evidence: z
.object({
nodesExecuted: z.array(z.string()).optional(),
/** Trigger whose main-flow scope was used for this evidence. */
triggerNodeName: z.string().optional(),
/**
* Plan nodes the execution never reached (e.g. a lookup returned zero
* items and stopped the chain). Non-empty means partial coverage —
@ -222,6 +226,15 @@ export const workflowVerificationEvidenceSchema = z.object({
export type WorkflowVerificationEvidence = z.infer<typeof workflowVerificationEvidenceSchema>;
export const workflowVerificationProgressSchema = z.record(
z.string(),
z.array(workflowVerificationEvidenceSchema.required({ claim: true })),
);
export type WorkflowTriggerVerificationProgress = z.infer<
typeof workflowVerificationProgressSchema
>[string];
export const workflowVerificationReadinessSchema = z.discriminatedUnion('status', [
z.object({ status: z.literal('ready') }),
z.object({ status: z.literal('already_verified') }),
@ -334,7 +347,7 @@ export const workflowBuildOutcomeSchema = z.object({
submitted: z.boolean(),
triggerType: triggerTypeSchema,
/**
* Trigger nodes in the submitted workflow. Populated on successful submits;
* Enabled trigger nodes in the submitted workflow. Populated on successful submits;
* absent on failed or pre-submit outcomes. The orchestrator reads `nodeType`
* to pick a `verify-built-workflow` `inputData` shape for direct builds.
*/
@ -411,6 +424,8 @@ export const workflowBuildOutcomeSchema = z.object({
remediation: remediationMetadataSchema.optional(),
/** Count of verify-built-workflow runs for this build; capped by MAX_VERIFY_ATTEMPTS. */
verifyAttempts: z.number().int().min(0).optional(),
/** Successful verification runs by trigger. A failed rerun removes that trigger's entry. */
verificationProgress: workflowVerificationProgressSchema.optional(),
/**
* Structured verification record from the most recent `verify-built-workflow`
* tool call. This is tool evidence, not builder prose, so downstream checks may

70
packages/@n8n/instance-ai/src/workflow-loop/workflow-task-service.ts
View File

@ -1,14 +1,19 @@
import { UserError } from 'n8n-workflow';
import { isDeepStrictEqual } from 'node:util';
import type { WorkflowTaskService } from '../types';
import { terminalRemediationFromState } from './remediation';
import { MAX_VERIFY_ATTEMPTS, terminalRemediationFromState } from './remediation';
import { WorkflowLoopRuntime } from './runtime';
import {
isNeedsSetupRemediation,
stateForPendingSetupVerification,
} from './setup-verification-policy';
import { deriveWorkflowVerificationClaim } from './verification-progress';
import type {
VerificationResult,
VerificationClaim,
WorkflowVerificationEvidence,
WorkflowTriggerVerificationProgress,
WorkflowBuildOutcome,
WorkflowLoopAction,
WorkflowLoopState,
@ -112,4 +117,67 @@ export class WorkflowTaskCoordinator implements WorkflowTaskService {
};
});
}
/** Keep prior evidence outside storage until a successful retry restores it. */
async startVerification(
workItemId: string,
triggerNodeName?: string,
): Promise<WorkflowTriggerVerificationProgress | undefined> {
let previousProgress: WorkflowTriggerVerificationProgress | undefined;
await this.storage.updateBuildOutcome(this.threadId, workItemId, (outcome) => {
if ((outcome.verifyAttempts ?? 0) >= MAX_VERIFY_ATTEMPTS) {
throw new UserError(
'Verification reached its attempt limit. Test the remaining nodes manually.',
);
}
const progress = outcome.verificationProgress && { ...outcome.verificationProgress };
if (progress && triggerNodeName) {
previousProgress = Object.hasOwn(progress, triggerNodeName)
? progress[triggerNodeName]
: undefined;
delete progress[triggerNodeName];
}
return {
...outcome,
verifyAttempts: (outcome.verifyAttempts ?? 0) + 1,
// An unscoped retry cannot identify which pass it replaces.
verificationProgress: progress && !triggerNodeName ? {} : progress,
// Keep the obligation open and block publishing until the run completes.
verification: { attempted: false, success: false, status: 'running' },
};
});
return previousProgress;
}
async recordVerification(
workItemId: string,
verification: WorkflowVerificationEvidence & { claim: VerificationClaim },
previousProgress?: WorkflowTriggerVerificationProgress,
): Promise<VerificationClaim | undefined> {
let claim: VerificationClaim | undefined;
await this.storage.updateBuildOutcome(this.threadId, workItemId, (outcome) => {
const trigger = verification.evidence?.triggerNodeName;
const nodes = verification.evidence?.nodesExecuted ?? [];
const progress = outcome.verificationProgress && { ...outcome.verificationProgress };
if (progress && trigger) {
if (verification.success && verification.executionId && nodes.includes(trigger)) {
const priorProgress = Object.hasOwn(progress, trigger) ? progress[trigger] : [];
progress[trigger] = [
...new Map(
[...(previousProgress ?? []), ...priorProgress, verification].map((pass) => [
pass.executionId,
pass,
]),
).values(),
];
} else {
delete progress[trigger];
}
}
const next = { ...outcome, verificationProgress: progress };
claim = deriveWorkflowVerificationClaim(next, verification);
return { ...next, verification: { ...verification, claim } };
});
return claim;
}
}

14
packages/cli/src/modules/instance-ai/instance-ai.service.ts
View File

@ -2249,6 +2249,20 @@ export class InstanceAiService {
return resumed;
},
getBuildOutcome: async (workItemId) => await workflowTasks.getBuildOutcome(workItemId),
startVerification: async (workItemId, triggerNodeName) => {
const previousProgress = await workflowTasks.startVerification(workItemId, triggerNodeName);
await sync();
return previousProgress;
},
recordVerification: async (workItemId, verification, previousProgress) => {
const claim = await workflowTasks.recordVerification(
workItemId,
verification,
previousProgress,
);
await sync();
return claim;
},
getLatestBuildOutcomeForWorkflow: async (workflowId) =>
await workflowTasks.getLatestBuildOutcomeForWorkflow(workflowId),
getWorkflowLoopState: async (workItemId) =>