From 429116f985be8470de8eb77bf660734a4e2b5cb7 Mon Sep 17 00:00:00 2001 From: Predrag Ristic Date: Thu, 10 Sep 2026 13:43:01 +0000 Subject: [PATCH] fix(core): Require verification coverage for every workflow trigger (#37961) --- packages/@n8n/instance-ai/docs/tools.md | 12 +- .../skills/post-build-flow/SKILL.md | 29 +- .../src/__tests__/verification-fixtures.ts | 24 ++ .../__tests__/workflow-loop-storage.test.ts | 58 ++++ .../src/storage/workflow-loop-storage.ts | 21 +- .../report-verification-verdict.tool.test.ts | 2 + .../verify-built-workflow.tool.test.ts | 315 ++++++++++++++++-- .../verification/analyze-result.ts | 16 + .../tools/orchestration/verification/claim.ts | 11 +- .../verification/finalize-result.ts | 60 ++-- .../verification/scripted-gate-run.ts | 2 + .../verify-built-workflow.tool.ts | 67 +++- .../instance-ai/src/tools/workflows.tool.ts | 7 +- .../__tests__/build-workflow.tool.test.ts | 26 +- .../tools/workflows/build-workflow.tool.ts | 4 +- packages/@n8n/instance-ai/src/types.ts | 12 + .../__tests__/verification-obligation.test.ts | 155 ++++++++- .../__tests__/workflow-task-service.test.ts | 42 +++ .../instance-ai/src/workflow-loop/guidance.ts | 6 +- .../src/workflow-loop/render-claim.ts | 5 + .../workflow-loop/verification-obligation.ts | 72 +++- .../workflow-loop/verification-progress.ts | 71 ++++ .../src/workflow-loop/workflow-loop-state.ts | 19 +- .../workflow-loop/workflow-task-service.ts | 70 +++- .../instance-ai/instance-ai.service.ts | 14 + 25 files changed, 1003 insertions(+), 117 deletions(-) create mode 100644 packages/@n8n/instance-ai/src/__tests__/verification-fixtures.ts create mode 100644 packages/@n8n/instance-ai/src/workflow-loop/verification-progress.ts diff --git a/packages/@n8n/instance-ai/docs/tools.md b/packages/@n8n/instance-ai/docs/tools.md index 061d3a7749f..80c26e846f1 100644 --- a/packages/@n8n/instance-ai/docs/tools.md +++ b/packages/@n8n/instance-ai/docs/tools.md @@ -196,8 +196,18 @@ waiting-with-output-as-success fallback. | Chat Trigger | `{chatInput: "..."}` | `{ sessionId, action, chatInput }` | | Schedule | omit | synthetic timestamp fields | +For reusable workflows with multiple enabled triggers, pass `triggerNodeName` +and run verification once for each trigger. Successful runs accumulate node +coverage per trigger. Each retry reserves an attempt and clears that trigger's +old pass before execution. A successful result restores the combined coverage. +If either write fails, the tool reports an error. The attempt limit still applies. + +The returned and saved `claim` use the same cumulative evidence. Pending triggers +and nodes without real coverage prevent a `verified` claim. Publishing during a +retry requires explicit acknowledgement through `acknowledgeUnverified: true`. + **Writes on success/failure**: the tool persists a structured `verification` -record (`{ attempted, success, executionId, status, evidence, verifiedAt }`) onto +record (`{ attempted, success, executionId, status, claim, evidence, verifiedAt }`) onto the build outcome so workflow-verification follow-ups and exceptional checkpoint turns can reuse it without re-running verify. diff --git a/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md b/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md index 658098aeaed..c78565b3706 100644 --- a/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md +++ b/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md @@ -85,7 +85,8 @@ obligation is `ready_to_verify` or `verifying`, call `verify-built-workflow`. Do **not** call `workflows(action="setup")` in this turn and do **not** declare the workflow finished if `outcome.setupRequirement.status === "required"` — setup is routed automatically as a separate `` step after -verification. +verification. For a multi-trigger outcome, verify every trigger that does not +yet have a recorded successful verification. Make all of these calls in this turn. ## Setup follow-up @@ -304,13 +305,14 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries), entry in `triggerNodes`. Naming no trigger verifies only the auto-detected one. An unresolvable name is rejected outright, so a rejected call means the name is wrong — re-read `triggerNodes`, never fall back to editing. -- Each pass covers its own trigger's branch, so its `nodesNotReached` will list - the other triggers' nodes. That is expected, not a defect: coverage is the - **union** across passes. Only treat a node as unverified once no pass reached - it. -- Report per-trigger coverage — name each trigger and whether its branch ran. - Claim the workflow is verified only when every trigger's branch has a - successful pass. +- Each pass reports `nodesNotReached` only for its selected trigger's main-flow + branch. Coverage is the **union** across successful passes. Run every trigger + before you report a workflow coverage gap. Different triggers can select + different outputs of a shared Switch or If node. +- A failed rerun removes that trigger's earlier coverage. Verify that trigger + again before you claim that the workflow is verified. +- Report each trigger and whether its branch ran. Use the combined `claim` to + describe the result (see "Claiming success"). - When the user asked for a live run, pass `triggerNodeName` to `executions(action="run")` the same way — one run per trigger — and report each branch's result. @@ -330,8 +332,10 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries), `workflow-builder` skill and patch the same workflow with `build-workflow` using the existing `workflowId` and `workItemId`; then inspect and verify again. - - If `verificationReadiness.status === "already_verified"`, treat the - workflow as verified and do **not** call `verify-built-workflow` again. + - If `verificationReadiness.status === "already_verified"`, do not repeat + automatic verification. Read the saved claim before describing the workflow + as verified. For tracked multi-trigger builds, follow the verification + obligation until every trigger has a successful pass. - If `verificationReadiness.status === "ready"`, call `verify-built-workflow` with the `workflowId`, the `workItemId` when you @@ -540,6 +544,11 @@ result. ## Claiming success +For tracked multi-trigger builds, `claim` combines the saved successful passes. +A `verified` claim requires every trigger to pass and real coverage for every planned node. +Verify `claim.pendingTriggers` within the attempt limit. `nodesNotReached` +outside the claim describes only the current trigger's branch. + `verify-built-workflow` returns a `claim`, and its `level` decides what you may say: diff --git a/packages/@n8n/instance-ai/src/__tests__/verification-fixtures.ts b/packages/@n8n/instance-ai/src/__tests__/verification-fixtures.ts new file mode 100644 index 00000000000..57b713173f8 --- /dev/null +++ b/packages/@n8n/instance-ai/src/__tests__/verification-fixtures.ts @@ -0,0 +1,24 @@ +import type { WorkflowTriggerVerificationProgress } from '../workflow-loop/workflow-loop-state'; + +export function successfulVerification( + triggerNodeName: string, + nodesExecuted = [triggerNodeName], +): WorkflowTriggerVerificationProgress[number] { + return { + attempted: true, + success: true, + executionId: `exec-${triggerNodeName}`, + evidence: { triggerNodeName, nodesExecuted }, + claim: { + level: 'verified', + plannedNodeCount: nodesExecuted.length, + reachedNodeCount: nodesExecuted.length, + nodesNotReached: [], + simulatedNodes: [], + pinnedNodes: [], + unprovenTargets: [], + publishReady: true, + liveTestRecommended: false, + }, + }; +} diff --git a/packages/@n8n/instance-ai/src/storage/__tests__/workflow-loop-storage.test.ts b/packages/@n8n/instance-ai/src/storage/__tests__/workflow-loop-storage.test.ts index 87d064cceae..7c1c9612ce5 100644 --- a/packages/@n8n/instance-ai/src/storage/__tests__/workflow-loop-storage.test.ts +++ b/packages/@n8n/instance-ai/src/storage/__tests__/workflow-loop-storage.test.ts @@ -83,6 +83,64 @@ describe('WorkflowLoopStorage', () => { storage = new WorkflowLoopStorage(memory); }); + it('updates the latest outcome without replacing other work items', async () => { + const current = { + ...baseThread, + metadata: { + otherMetadata: true, + instanceAiWorkflowLoop: { + 'wi-1': { + state: makeState(), + attempts: [], + lastBuildOutcome: makeOutcome({ verifyAttempts: 3 }), + }, + 'wi-2': { state: makeState({ workItemId: 'wi-2' }), attempts: [] }, + }, + }, + }; + mockedPatchThread.mockImplementationOnce( + async (_memory, { update }) => + await Promise.resolve({ + ...current, + ...update(current), + }), + ); + await storage.updateBuildOutcome('thread-1', 'wi-1', (outcome) => { + expect(outcome.verifyAttempts).toBe(3); + return { ...outcome, verifyAttempts: 4 }; + }); + const saved: unknown = await mockedPatchThread.mock.results[0].value; + expect(saved).toMatchObject({ + metadata: { + otherMetadata: true, + instanceAiWorkflowLoop: { + 'wi-1': { lastBuildOutcome: { verifyAttempts: 4 } }, + 'wi-2': { state: { workItemId: 'wi-2' } }, + }, + }, + }); + }); + + it('rejects a verification update when the thread disappeared', async () => { + mockedPatchThread.mockResolvedValueOnce(null); + await expect( + storage.updateBuildOutcome('thread-1', 'wi-1', (outcome) => outcome), + ).rejects.toThrow('could not be saved'); + }); + + it('rejects a verification update when the build outcome disappeared', async () => { + mockedPatchThread.mockImplementationOnce( + async (_memory, { update }) => + await Promise.resolve({ + ...baseThread, + ...update(baseThread), + }), + ); + await expect( + storage.updateBuildOutcome('thread-1', 'wi-1', (outcome) => outcome), + ).rejects.toThrow('state is unavailable'); + }); + it('rejects an update when saving its patch fails', async () => { const state = makeState(); mockedPatchThread.mockImplementation((_memory, { update }) => { diff --git a/packages/@n8n/instance-ai/src/storage/workflow-loop-storage.ts b/packages/@n8n/instance-ai/src/storage/workflow-loop-storage.ts index a2be8db1165..7b6f20e55a9 100644 --- a/packages/@n8n/instance-ai/src/storage/workflow-loop-storage.ts +++ b/packages/@n8n/instance-ai/src/storage/workflow-loop-storage.ts @@ -1,3 +1,4 @@ +import { OperationalError } from 'n8n-workflow'; import { z } from 'zod'; import { getThread, patchThread, type PatchableThreadMemory } from './thread-patch'; @@ -93,7 +94,7 @@ export class WorkflowLoopStorage { update: (record: WorkflowLoopWorkItemRecord) => WorkflowLoopWorkItemRecord | null, ): Promise { let updated = false; - await patchThread(this.memory, { + const thread = await patchThread(this.memory, { threadId, update: ({ metadata = {} }) => { const all = this.parse(metadata[METADATA_KEY]); @@ -108,7 +109,7 @@ export class WorkflowLoopStorage { return { metadata: { ...metadata, [METADATA_KEY]: all } }; }, }); - return updated; + return updated && thread !== null; } async getActiveWorkItem(threadId: string): Promise { @@ -121,6 +122,22 @@ export class WorkflowLoopStorage { return null; } + async updateBuildOutcome( + threadId: string, + workItemId: string, + update: (outcome: WorkflowBuildOutcome) => WorkflowBuildOutcome, + ): Promise { + const updated = await this.updateWorkItem(threadId, workItemId, (record) => { + if (!record.lastBuildOutcome) return null; + return { ...record, lastBuildOutcome: update(record.lastBuildOutcome) }; + }); + if (!updated) { + throw new OperationalError( + 'Verification state is unavailable or could not be saved. Rebuild the workflow.', + ); + } + } + async listWorkItems(threadId: string): Promise { const all = await this.loadAll(threadId); return Object.values(all); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts index 7167f2cf12d..de2c1446dd3 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts @@ -12,6 +12,8 @@ function createWorkflowTaskService(reportVerificationVerdict = vi.fn()) { beginVerification: vi.fn(), getWorkflowLoopState: vi.fn(), updateBuildOutcome: vi.fn(), + startVerification: vi.fn(), + recordVerification: vi.fn(), }; } diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts index 6c5bb0cac1e..d7ae163eaf9 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts @@ -1,6 +1,8 @@ import type { Mock } from 'vitest'; import { executeTool } from '../../../__tests__/tool-test-utils'; +import { successfulVerification } from '../../../__tests__/verification-fixtures'; +import type { WorkflowLoopStorage } from '../../../storage/workflow-loop-storage'; import type { InstanceAiDataTableService, InstanceAiWorkflowService, @@ -8,7 +10,9 @@ import type { WorkflowTaskService, } from '../../../types'; import { createRemediation, MAX_VERIFY_ATTEMPTS } from '../../../workflow-loop/remediation'; +import { deriveWorkflowVerificationObligationFromOutcome } from '../../../workflow-loop/verification-obligation'; import type { WorkflowBuildOutcome } from '../../../workflow-loop/workflow-loop-state'; +import { WorkflowTaskCoordinator } from '../../../workflow-loop/workflow-task-service'; import { createVerifyBuiltWorkflowTool } from '../verify-built-workflow.tool'; type VerifyBuiltWorkflowOutput = { @@ -55,6 +59,8 @@ function createContext(overrides: Partial = {}): Orchestra beginVerification: vi.fn(), getWorkflowLoopState: vi.fn(), updateBuildOutcome: vi.fn(), + startVerification: vi.fn(), + recordVerification: vi.fn(), }; return { @@ -493,10 +499,18 @@ function makeContext( } = {}, ) { const updateBuildOutcome = vi.fn( - async (_workItemId: string, _update: Partial) => { + async (_workItemId: string, update: Partial) => { + if (outcome) outcome = { ...outcome, ...update }; await Promise.resolve(); }, ); + const coordinator = new WorkflowTaskCoordinator('thread-1', { + updateBuildOutcome: async (_threadId, workItemId, update) => { + if (!outcome) throw new Error('Missing outcome'); + await updateBuildOutcome(workItemId, update(outcome)); + }, + } as WorkflowLoopStorage); + const run = vi.fn( async ( _workflowId: string, @@ -538,7 +552,13 @@ function makeContext( getAsWorkflowJSON: vi.fn(async () => { await Promise.resolve(); return { - nodes: overrides.workflowNodes ?? [], + nodes: + overrides.workflowNodes ?? + outcome?.triggerNodes?.map(({ nodeName, nodeType }) => ({ + name: nodeName, + type: nodeType, + })) ?? + [], connections: overrides.workflowConnections ?? {}, }; }), @@ -578,6 +598,8 @@ function makeContext( beginVerification: vi.fn(), getWorkflowLoopState: vi.fn(), updateBuildOutcome, + startVerification: vi.fn(coordinator.startVerification.bind(coordinator)), + recordVerification: vi.fn(coordinator.recordVerification.bind(coordinator)), } as unknown as WorkflowTaskService, domainContext: { executionService: { run }, @@ -587,7 +609,7 @@ function makeContext( }, logger, }; - return { ctx, updateBuildOutcome, queryRows, deleteRows }; + return { ctx, updateBuildOutcome, queryRows, deleteRows, getOutcome: () => outcome! }; } async function runTool( @@ -621,8 +643,8 @@ describe('verify-built-workflow tool', () => { }); expect(result.success).toBe(true); - expect(updateBuildOutcome).toHaveBeenCalledTimes(1); - const call = updateBuildOutcome.mock.calls[0]; + expect(updateBuildOutcome).toHaveBeenCalledTimes(2); + const call = updateBuildOutcome.mock.calls.at(-1)!; expect(call).toBeDefined(); const update = call[1]; expect(update.verification).toMatchObject({ @@ -644,7 +666,7 @@ describe('verify-built-workflow tool', () => { await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); - expect(updateBuildOutcome.mock.calls[0][1].verifyAttempts).toBe(3); + expect(updateBuildOutcome.mock.calls.at(-1)![1].verifyAttempts).toBe(3); }); it('blocks verification once the verify attempt budget is exhausted', async () => { @@ -734,8 +756,8 @@ describe('verify-built-workflow tool', () => { const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); expect(result.success).toBe(false); - expect(updateBuildOutcome).toHaveBeenCalledTimes(1); - const call = updateBuildOutcome.mock.calls[0]; + expect(updateBuildOutcome).toHaveBeenCalledTimes(2); + const call = updateBuildOutcome.mock.calls.at(-1)!; expect(call).toBeDefined(); const update = call[1]; expect(update.verification).toMatchObject({ @@ -775,7 +797,7 @@ describe('verify-built-workflow tool', () => { expect(result.resolvedWorkItemId).toBe('wi-latest'); expect(ctx.workflowTaskService.getBuildOutcome).not.toHaveBeenCalled(); expect(ctx.workflowTaskService.getLatestBuildOutcomeForWorkflow).toHaveBeenCalledWith('wf-1'); - expect(updateBuildOutcome.mock.calls[0][0]).toBe('wi-latest'); + expect(updateBuildOutcome.mock.calls.at(-1)![0]).toBe('wi-latest'); }); it('falls back to the workflow build outcome when the supplied work item ID is stale', async () => { @@ -794,7 +816,7 @@ describe('verify-built-workflow tool', () => { expect(result.success).toBe(true); expect(result.resolvedWorkItemId).toBe('wi-latest'); expect(ctx.workflowTaskService.getLatestBuildOutcomeForWorkflow).toHaveBeenCalledWith('wf-1'); - expect(updateBuildOutcome.mock.calls[0][0]).toBe('wi-latest'); + expect(updateBuildOutcome.mock.calls.at(-1)![0]).toBe('wi-latest'); }); it('rejects verification when the requested workflow does not match the build outcome', async () => { @@ -827,17 +849,17 @@ describe('verify-built-workflow tool', () => { expect(updateBuildOutcome).not.toHaveBeenCalled(); }); - it('swallows storage errors when persisting verification', async () => { + it('does not execute when the attempt cannot be saved', async () => { const { ctx, updateBuildOutcome } = makeContext(makeBuildOutcome(), { executionId: 'exec-3', status: 'success', }); updateBuildOutcome.mockRejectedValueOnce(new Error('storage unavailable')); - const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); - - expect(result.success).toBe(true); - expect(result.executionId).toBe('exec-3'); + await expect(runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' })).rejects.toThrow( + 'storage unavailable', + ); + expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); }); it('counts wrapped execution output items in previews and persisted evidence', async () => { @@ -867,7 +889,9 @@ describe('verify-built-workflow tool', () => { expect.objectContaining({ nodeName: 'Large Export', itemCount: 5 }), expect.objectContaining({ nodeName: 'Large Transform', itemCount: 7 }), ]); - expect(updateBuildOutcome.mock.calls[0][1].verification?.evidence?.producedOutputRows).toBe(14); + expect( + updateBuildOutcome.mock.calls.at(-1)![1].verification?.evidence?.producedOutputRows, + ).toBe(14); }); it('treats a waiting status with output as a successful run (e.g. Form Trigger response page)', async () => { @@ -889,8 +913,8 @@ describe('verify-built-workflow tool', () => { expect(result.success).toBe(true); expect(result.status).toBe('waiting'); - expect(updateBuildOutcome).toHaveBeenCalledTimes(1); - const update = updateBuildOutcome.mock.calls[0][1]; + expect(updateBuildOutcome).toHaveBeenCalledTimes(2); + const update = updateBuildOutcome.mock.calls.at(-1)![1]; expect(update.verification).toMatchObject({ attempted: true, success: true, @@ -1165,7 +1189,7 @@ describe('verify-built-workflow tool — node simulation plan', () => { reason: 'missing_simulation_plan', }); expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); - expect(updateBuildOutcome.mock.calls[0][1].verification).toMatchObject({ + expect(updateBuildOutcome.mock.calls.at(-1)![1].verification).toMatchObject({ attempted: true, success: false, status: 'unknown', @@ -1230,7 +1254,7 @@ describe('verify-built-workflow tool — node simulation plan', () => { expect(result.nodeErrors?.[0]?.nodeName).toBe('geocode_city'); expect(result.nodeErrors?.[0]?.message).toContain('supplyData'); expect(result.error).toContain('geocode_city'); - const verification = updateBuildOutcome.mock.calls[0][1].verification; + const verification = updateBuildOutcome.mock.calls.at(-1)![1].verification; expect(verification?.success).toBe(false); expect(verification?.status).toBe('success'); expect(verification?.failureSignature).toContain('geocode_city'); @@ -1360,9 +1384,9 @@ describe('verify-built-workflow tool — node simulation plan', () => { await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); - expect(updateBuildOutcome.mock.calls[0][1].verification?.evidence?.nodesNotReached).toEqual([ - 'Send Slack', - ]); + expect( + updateBuildOutcome.mock.calls.at(-1)![1].verification?.evidence?.nodesNotReached, + ).toEqual(['Send Slack']); }); it('never deletes rows whose IDs come from fabricated fixture output', async () => { @@ -1499,13 +1523,248 @@ describe('verify-built-workflow tool — stale mocked-credential plan', () => { }); describe('verify-built-workflow tool — trigger selection', () => { - it('starts verification from the named trigger', async () => { - const { ctx } = makeContext(makeBuildOutcome(), { - executionId: 'exec-monthly', - status: 'success', - data: { 'Post Summary': [{ ok: true }] }, + type TriggerInput = { workItemId: string; workflowId: string; triggerNodeName?: string }; + + const triggerNodes = [ + { nodeName: 'Trigger A', nodeType: 'n8n-nodes-base.webhook' }, + { nodeName: 'Trigger B', nodeType: 'n8n-nodes-base.scheduleTrigger' }, + ]; + const executeVerdict = (nodeName: string) => ({ + nodeName, + verdict: 'execute' as const, + reason: 'Reads data', + confidence: 'high' as const, + source: 'deterministic' as const, + }); + const workflowConnections = { + 'Trigger A': { main: [[{ node: 'Step A', type: 'main', index: 0 }]] }, + 'Trigger B': { main: [[{ node: 'Step B', type: 'main', index: 0 }]] }, + }; + const triggerAInput = { workItemId: 'wi-1', workflowId: 'wf-1', triggerNodeName: 'Trigger A' }; + const successfulA: ExecutionRunResult = { + executionId: 'exec-a', + status: 'success', + executedNodeNames: ['Trigger A', 'Step A'], + }; + const successfulB: ExecutionRunResult = { + executionId: 'exec-b', + status: 'success', + executedNodeNames: ['Trigger B', 'Step B'], + }; + const makeTrackedOutcome = (overrides: Partial = {}) => + makeBuildOutcome({ + triggerNodes, + verificationProgress: {}, + nodeSimulationPlan: [executeVerdict('Step A'), executeVerdict('Step B')], + ...overrides, }); + function makeSequenceContext( + initialOutcome: WorkflowBuildOutcome, + connections: Record = workflowConnections, + ) { + return makeContext(initialOutcome, { status: 'success' }, { workflowConnections: connections }); + } + + it.each(['Trigger A', undefined])( + 'requires a successful retry after a previously verified trigger fails (%s)', + async (failedTriggerName) => { + const { ctx, getOutcome } = makeSequenceContext(makeTrackedOutcome()); + const run = ctx.domainContext.executionService.run; + const triggerBInput = { ...triggerAInput, triggerNodeName: 'Trigger B' }; + + run.mockResolvedValueOnce(successfulA).mockResolvedValueOnce(successfulB); + await runTool(ctx, triggerAInput); + await runTool(ctx, triggerBInput); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'verified', + ); + + run.mockResolvedValueOnce({ ...successfulA, status: 'error', error: 'Step A failed' }); + await runTool(ctx, { ...triggerAInput, triggerNodeName: failedTriggerName }); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'ready_to_verify', + ); + + run.mockResolvedValueOnce(successfulB); + await runTool(ctx, triggerBInput); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'ready_to_verify', + ); + + run.mockResolvedValueOnce({ ...successfulA, executedNodeNames: ['Trigger A'] }); + await runTool(ctx, triggerAInput); + const incompleteRetry = deriveWorkflowVerificationObligationFromOutcome( + 'thread-1', + getOutcome(), + ); + expect(incompleteRetry.status).toBe('not_verifiable'); + expect(incompleteRetry.blockingReason).toContain('Step A'); + expect(incompleteRetry.blockingReason).not.toContain('Step B'); + + run.mockResolvedValueOnce(successfulA); + await runTool(ctx, triggerAInput); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'verified', + ); + }, + ); + + it('combines successful trigger runs through different Switch outputs', async () => { + const { ctx, getOutcome } = makeSequenceContext( + makeTrackedOutcome({ + nodeSimulationPlan: ['Switch', 'Step A', 'Step B'].map(executeVerdict), + }), + { + 'Trigger A': { main: [[{ node: 'Switch', type: 'main', index: 0 }]] }, + 'Trigger B': { main: [[{ node: 'Switch', type: 'main', index: 0 }]] }, + Switch: { + main: [ + [{ node: 'Step A', type: 'main', index: 0 }], + [{ node: 'Step B', type: 'main', index: 0 }], + ], + }, + }, + ); + ctx.domainContext.executionService.run + .mockResolvedValueOnce({ + ...successfulA, + executedNodeNames: ['Trigger A', 'Switch', 'Step A'], + }) + .mockResolvedValueOnce({ + ...successfulB, + executedNodeNames: ['Trigger B', 'Switch', 'Step B'], + }); + + await runTool(ctx, triggerAInput); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'ready_to_verify', + ); + + await runTool(ctx, { ...triggerAInput, triggerNodeName: 'Trigger B' }); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'verified', + ); + }); + + it('records a fully covered pass without counting the other trigger branch as a gap', async () => { + const { ctx, updateBuildOutcome } = makeContext(makeTrackedOutcome(), successfulA, { + workflowConnections, + }); + + const result = await runTool(ctx, triggerAInput); + + expect(result.nodesNotReached).toBeUndefined(); + expect(updateBuildOutcome.mock.calls.at(-1)![1].verificationProgress).toMatchObject({ + 'Trigger A': [{ evidence: { nodesExecuted: ['Trigger A', 'Step A'] } }], + }); + expect(updateBuildOutcome.mock.calls.at(-1)![1].verification?.evidence?.triggerNodeName).toBe( + 'Trigger A', + ); + }); + + it('preserves verification state when the workflow graph cannot be loaded', async () => { + const outcome = makeTrackedOutcome({ verificationProgress: passedA, verifyAttempts: 1 }); + const { ctx, getOutcome } = makeContext(outcome, successfulA, { workflowConnections }); + vi.mocked(ctx.domainContext.workflowService!.getAsWorkflowJSON).mockRejectedValueOnce( + new Error('workflow unavailable'), + ); + + const result = await runTool(ctx, triggerAInput); + + expect(result.success).toBe(false); + expect(getOutcome()).toEqual(outcome); + expect(ctx.domainContext.executionService.run).not.toHaveBeenCalled(); + }); + + const withoutTrigger = { workItemId: 'wi-1', workflowId: 'wf-1' }; + const invalidPasses: Array<[string, ExecutionRunResult, TriggerInput]> = [ + [ + 'a failed execution', + { ...successfulA, status: 'error', error: 'Step A failed' }, + triggerAInput, + ], + ['a missing execution ID', { ...successfulA, executionId: undefined }, triggerAInput], + [ + 'a trigger that was not reached', + { ...successfulA, executedNodeNames: ['Step A'] }, + triggerAInput, + ], + ['a pass without a named trigger', successfulA, withoutTrigger], + [ + 'a trigger outside the build outcome', + { ...successfulA, executedNodeNames: ['Unknown Trigger'] }, + { ...withoutTrigger, triggerNodeName: 'Unknown Trigger' }, + ], + ]; + it.each(invalidPasses)( + 'does not record %s as a successful trigger pass', + async (_name, runResult, input) => { + const { ctx, getOutcome } = makeContext( + makeTrackedOutcome({ nodeSimulationPlan: [executeVerdict('Step A')] }), + runResult, + { workflowConnections }, + ); + + await runTool(ctx, input); + + expect(Object.keys(getOutcome().verificationProgress ?? {})).toEqual([]); + }, + ); + + const passedA = { + 'Trigger A': [successfulVerification('Trigger A', ['Trigger A', 'Step A'])], + }; + + it('keeps the attempt and removes old coverage when the result cannot be saved', async () => { + const { ctx, getOutcome } = makeSequenceContext( + makeTrackedOutcome({ + verifyAttempts: MAX_VERIFY_ATTEMPTS - 1, + verificationProgress: passedA, + }), + ); + ctx.domainContext.executionService.run.mockResolvedValueOnce(successfulA); + vi.mocked(ctx.workflowTaskService.recordVerification).mockRejectedValueOnce( + new Error('Save failed'), + ); + await expect(runTool(ctx, triggerAInput)).rejects.toThrow('Save failed'); + expect(getOutcome().verifyAttempts).toBe(MAX_VERIFY_ATTEMPTS); + expect(getOutcome().verificationProgress).toEqual({}); + expect(deriveWorkflowVerificationObligationFromOutcome('thread-1', getOutcome()).status).toBe( + 'blocked', + ); + await runTool(ctx, triggerAInput); + expect(ctx.domainContext.executionService.run).toHaveBeenCalledTimes(1); + }); + + it('combines repeated successful passes for the same trigger', async () => { + const { ctx, getOutcome } = makeSequenceContext( + makeTrackedOutcome({ + verificationProgress: passedA, + }), + ); + ctx.domainContext.executionService.run.mockResolvedValueOnce({ + ...successfulA, + executedNodeNames: ['Trigger A', 'Alternate Step'], + }); + await runTool(ctx, triggerAInput); + expect(getOutcome().verificationProgress?.['Trigger A']).toMatchObject([ + { evidence: { nodesExecuted: ['Trigger A', 'Step A'] } }, + { evidence: { nodesExecuted: ['Trigger A', 'Alternate Step'] } }, + ]); + }); + + it('starts verification from the named trigger', async () => { + const { ctx } = makeContext( + makeBuildOutcome(), + { + executionId: 'exec-monthly', + status: 'success', + data: { 'Post Summary': [{ ok: true }] }, + }, + { workflowNodes: [{ name: 'First of Month', type: 'n8n-nodes-base.scheduleTrigger' }] }, + ); + await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1', diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts index 9bda2bc7a9d..395ef43fcf9 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts @@ -1,5 +1,7 @@ import { isRecord } from '@n8n/utils/is-record'; import { isPlaceholderValue } from '@n8n/utils/placeholder'; +import { toEngineConnections, type WorkflowJSON } from '@n8n/workflow-sdk'; +import { getChildNodes, NodeConnectionTypes } from 'n8n-workflow'; import type { ExecutionRunResult, VerificationNodePreview } from './types'; import { @@ -427,6 +429,16 @@ export interface VerificationAnalysis { nodeErrors: ExecutionNodeError[]; } +export function getTriggerMainFlowScope( + connections: WorkflowJSON['connections'], + triggerNodeName: string, +): Set { + return new Set([ + triggerNodeName, + ...getChildNodes(toEngineConnections(connections), triggerNodeName, NodeConnectionTypes.Main), + ]); +} + export function analyzeVerificationResult(args: { result: ExecutionRunResult; buildOutcome: WorkflowBuildOutcome; @@ -445,6 +457,8 @@ export function analyzeVerificationResult(args: { chatModelRecovery?: ChatModelRecoveryOptions; /** Trigger this pass started from, when the caller named one. */ triggerNodeName?: string; + /** Main-flow nodes that belong to the selected trigger. */ + verificationScope?: ReadonlySet; }): VerificationAnalysis { const { result, @@ -456,6 +470,7 @@ export function analyzeVerificationResult(args: { chatModelRelatedNodeNames, chatModelRecovery, triggerNodeName, + verificationScope, } = args; const nodeErrors = result.nodeErrors ?? []; const reachedNames = new Set( @@ -474,6 +489,7 @@ export function analyzeVerificationResult(args: { ]; const nodesNotReached = (buildOutcome.nodeSimulationPlan ?? []) .map((verdict) => verdict.nodeName) + .filter((name) => !verificationScope || verificationScope.has(name)) .filter((name) => !reachedNames.has(name)); const hasSimulationPlan = (buildOutcome.nodeSimulationPlan?.length ?? 0) > 0; const hasOutput = result.data ? Object.keys(result.data).length > 0 : false; diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts index 039ee06d015..bea5bf8067b 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts @@ -13,7 +13,11 @@ import type { } from '../../../workflow-loop/workflow-loop-state'; export interface DeriveVerificationClaimArgs { - analysis: VerificationAnalysis; + analysis: Pick< + VerificationAnalysis, + 'success' | 'nodesNotReached' | 'reachedSimulatedNodes' | 'workflowPinnedNodeNames' + >; + pendingTriggers?: string[]; /** Planned nodes from the build outcome simulation plan. */ plannedNodeCount: number; /** @@ -37,6 +41,7 @@ export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): Veri hasUnreached: nodesNotReached.length > 0, hasSimulated: simulatedNodes.length > 0, hasUnprovenTarget: unprovenTargets.length > 0, + hasPendingTrigger: (args.pendingTriggers?.length ?? 0) > 0, }); return { @@ -50,6 +55,7 @@ export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): Veri simulatedNodes, pinnedNodes: [...analysis.workflowPinnedNodeNames], unprovenTargets, + ...(args.pendingTriggers ? { pendingTriggers: args.pendingTriggers } : {}), publishReady: level === 'verified', liveTestRecommended: level === 'partial' || level === 'unproven', }; @@ -60,11 +66,12 @@ function resolveLevel(facts: { hasUnreached: boolean; hasSimulated: boolean; hasUnprovenTarget: boolean; + hasPendingTrigger: boolean; }): VerificationClaimLevel { if (!facts.success) return 'failed'; // A named target that never ran for real outranks overall coverage: the user // asked about that node, so a green run elsewhere does not answer them. if (facts.hasUnprovenTarget) return 'unproven'; - if (facts.hasUnreached || facts.hasSimulated) return 'partial'; + if (facts.hasUnreached || facts.hasSimulated || facts.hasPendingTrigger) return 'partial'; return 'verified'; } diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts index 3b2577bde93..e98cc00c8b9 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts @@ -14,6 +14,7 @@ import { createRemediation } from '../../../workflow-loop/remediation'; import type { RemediationMetadata, VerificationClaim, + WorkflowTriggerVerificationProgress, } from '../../../workflow-loop/workflow-loop-state'; /** @@ -91,11 +92,11 @@ export async function persistVerificationOutcome(args: { workflowId: string; result: ExecutionRunResult; analysis: VerificationAnalysis; + scopedTriggerNodeName?: string; + previousProgress?: WorkflowTriggerVerificationProgress; /** Deterministic verdict for this run, persisted so later turns cannot re-litigate it. */ claim: VerificationClaim; - /** Running count of verify runs for this build, used to enforce MAX_VERIFY_ATTEMPTS. */ - verifyAttempts: number; -}): Promise { +}): Promise { const { input, context, @@ -103,36 +104,34 @@ export async function persistVerificationOutcome(args: { workflowId, result, analysis, + scopedTriggerNodeName, + previousProgress, claim, - verifyAttempts, } = args; - try { - const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data); - await workflowTaskService.updateBuildOutcome(input.workItemId, { - verifyAttempts, - verification: { - attempted: true, - success: analysis.success, - executionId: result.executionId || undefined, - status: result.status, - failureSignature: analysis.success ? undefined : analysis.errorMessage, - claim, - evidence: { - nodesExecuted: - executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined, - nodesNotReached: - analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined, - producedOutputRows: countProducedOutputRows(result.data), - errorNodeName: analysis.success ? undefined : analysis.nodeErrors[0]?.nodeName, - errorMessage: analysis.success ? undefined : analysis.errorMessage, - nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined, - }, - verifiedAt: new Date().toISOString(), + const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data); + const storedClaim = await workflowTaskService.recordVerification( + input.workItemId, + { + attempted: true, + success: analysis.success, + executionId: result.executionId || undefined, + status: result.status, + failureSignature: analysis.success ? undefined : analysis.errorMessage, + claim, + evidence: { + ...(scopedTriggerNodeName ? { triggerNodeName: scopedTriggerNodeName } : {}), + nodesExecuted: + executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined, + nodesNotReached: analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined, + producedOutputRows: countProducedOutputRows(result.data), + errorNodeName: analysis.success ? undefined : analysis.nodeErrors[0]?.nodeName, + errorMessage: analysis.success ? undefined : analysis.errorMessage, + nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined, }, - }); - } catch { - // intentional: verification record persistence is advisory - } + verifiedAt: new Date().toISOString(), + }, + previousProgress, + ); if (analysis.remediation && !analysis.remediation.shouldEdit) { await reportTerminalRemediation({ @@ -144,6 +143,7 @@ export async function persistVerificationOutcome(args: { remediation: analysis.remediation, }); } + return storedClaim; } async function reportTerminalRemediation(args: { diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts index 9aeaa2cf38b..fc56e7e4b3f 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts @@ -39,6 +39,7 @@ export interface ScriptedGateRunArgs { runId: string; chatModelRelatedNodeNames?: ReadonlySet; chatModelRecovery?: ChatModelRecoveryOptions; + verificationScope?: ReadonlySet; } interface DecisionPass { @@ -82,6 +83,7 @@ export async function runScriptedGateVerification( runId, chatModelRelatedNodeNames, chatModelRecovery, + verificationScope: args.verificationScope, }); passes.push({ label: decision.label, result, analysis }); } diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts index 03d86efdaa7..205b59c1816 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts @@ -7,10 +7,15 @@ */ import { Tool } from '@n8n/agents'; +import { isTriggerNodeType } from 'n8n-workflow'; import { z } from 'zod'; import type { OrchestrationContext } from '../../types'; -import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result'; +import { + analyzeVerificationResult, + buildNodePreviews, + getTriggerMainFlowScope, +} from './verification/analyze-result'; import { deriveVerificationClaim } from './verification/claim'; import { handleMissingSimulationPlan, @@ -194,18 +199,42 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { } const { prepared } = preparedResult; - const chatModelRecovery = await target.domainContext.workflowService + const workflow = await target.domainContext.workflowService .getAsWorkflowJSON(workflowId) - .then( - async (workflow) => - await collectChatModelRecoveryContext( - target.domainContext, - workflow.nodes ?? [], - workflow.connections, - ), - ) .catch(() => undefined); + if ( + resolvedInput.triggerNodeName !== undefined && + !workflow?.nodes.some( + (node) => node.name === resolvedInput.triggerNodeName && isTriggerNodeType(node.type), + ) + ) { + return { + success: false, + resolvedWorkItemId: resolvedInput.workItemId, + error: `Could not find trigger "${resolvedInput.triggerNodeName}" in this workflow. Read the workflow. Select an existing trigger.`, + }; + } + const chatModelRecovery = workflow + ? await collectChatModelRecoveryContext( + target.domainContext, + workflow.nodes ?? [], + workflow.connections, + ).catch(() => undefined) + : undefined; const chatModelRelatedNodeNames = chatModelRecovery?.relatedNodeNames; + const selectedTriggerNodeName = buildOutcome.triggerNodes?.some( + (trigger) => trigger.nodeName === resolvedInput.triggerNodeName, + ) + ? resolvedInput.triggerNodeName + : undefined; + const verificationScope = + buildOutcome.verificationProgress && selectedTriggerNodeName && workflow + ? getTriggerMainFlowScope(workflow.connections, selectedTriggerNodeName) + : undefined; + const previousProgress = await workflowTaskService.startVerification( + resolvedInput.workItemId, + verificationScope ? selectedTriggerNodeName : undefined, + ); // A scripted gate replaces the halt with one loop-safe pass per decision; // otherwise run the single standard pass (halted gates pin zero items). @@ -224,6 +253,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { runId: context.runId, chatModelRelatedNodeNames, chatModelRecovery, + verificationScope, }) : await (async () => { const runResult = await target.domainContext.executionService.run( @@ -249,6 +279,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { runId: context.runId, chatModelRelatedNodeNames, chatModelRecovery, + verificationScope, }), }; })(); @@ -263,21 +294,27 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { ].filter((name): name is string => name !== undefined), ), ]; - const claim = deriveVerificationClaim({ - analysis, + const runClaim = deriveVerificationClaim({ + analysis: { + ...analysis, + nodesNotReached: buildOutcome.nodeSimulationPlan + .map((node) => node.nodeName) + .filter((name) => !analysis.reachedNames.has(name)), + }, plannedNodeCount: buildOutcome.nodeSimulationPlan?.length ?? 0, fixTargetNodeNames, }); - await persistVerificationOutcome({ + const claim = await persistVerificationOutcome({ input: resolvedInput, context, workflowTaskService, workflowId, result, analysis, - claim, - verifyAttempts: (buildOutcome.verifyAttempts ?? 0) + 1, + scopedTriggerNodeName: verificationScope ? selectedTriggerNodeName : undefined, + previousProgress, + claim: runClaim, }); const maxDataChars = resolvedInput.maxDataChars ?? DEFAULT_NODE_PREVIEW_CHARS; diff --git a/packages/@n8n/instance-ai/src/tools/workflows.tool.ts b/packages/@n8n/instance-ai/src/tools/workflows.tool.ts index 59c06626d5e..eb07450bf5c 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows.tool.ts @@ -1404,11 +1404,8 @@ async function resolveUnverifiedPublishDisclosure( const claim = verification?.claim; if (claim) return claim.publishReady ? undefined : formatClaimDisclosure(claim); - // A record with no claim means verification ran and could not produce one - // — it was refused for want of a simulation plan, or it failed before it - // reached a verdict. That is a failed verification, not an unverified - // workflow, and it must not pass as an absent record. - if (verification?.attempted) { + // A running or attempted verification must not pass as an absent record. + if (verification?.attempted || verification?.status === 'running') { const cause = verification.failureSignature ?? verification.evidence?.errorMessage; return ( 'Verification ran but produced no verdict, so nothing about this workflow is proven.' + diff --git a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts index ced691c2583..4e0a26e469a 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts @@ -2064,7 +2064,27 @@ describe('createBuildWorkflowTool', () => { }); }); - it('reports planned build outcomes without source artifact metadata', async () => { + it.each([false, true])('tracks only enabled triggers (disabled=%s)', async (disabled) => { + vi.mocked(compileWorkflowSource).mockResolvedValueOnce({ + success: true, + workflow: { + ...structuredClone(generatedWorkflow), + nodes: [ + ...structuredClone(generatedWorkflow.nodes), + { + id: 'schedule-1', + name: 'Schedule', + disabled, + type: 'n8n-nodes-base.scheduleTrigger', + typeVersion: 1, + position: [0, 100], + parameters: {}, + }, + ], + }, + warnings: [], + compiler: 'sandbox-tsx', + }); const reportBuildOutcome = vi.fn< (outcome: WorkflowBuildOutcome) => Promise<{ type: 'verify'; workflowId: string }> >(async () => await Promise.resolve({ type: 'verify', workflowId: 'wf-1' })); @@ -2105,7 +2125,11 @@ describe('createBuildWorkflowTool', () => { owner: { type: 'planned', taskId: 'task-1' }, plannedTaskId: 'task-1', sourceFilePath: filePath, + verificationProgress: disabled ? undefined : {}, }); + expect(storedOutcome?.triggerNodes?.some((trigger) => trigger.nodeName === 'Schedule')).toBe( + !disabled, + ); expect(storedOutcome).not.toHaveProperty('sourceArtifact'); const reportedOutcome = reportBuildOutcome.mock.calls[0]?.[0] as diff --git a/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts b/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts index a2f95e961fe..ce921e22e1d 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts @@ -1150,7 +1150,7 @@ export function createBuildWorkflowTool(context: InstanceAiContext) { const heldForNewCredentialTypes = mockResult.heldForNewCredentialTypes; const referencedWorkflowIds = getReferencedWorkflowIds(json); const triggerNodes = (json.nodes ?? []) - .filter((n) => isTriggerNodeType(n.type)) + .filter((n) => !n.disabled && isTriggerNodeType(n.type)) .map((n) => ({ nodeName: n.name, nodeType: n.type })) .filter( (t): t is { nodeName: string; nodeType: string } => @@ -1289,6 +1289,8 @@ export function createBuildWorkflowTool(context: InstanceAiContext) { nodeSimulationPlan, simulationFixtures, waitGateScripts, + verificationProgress: + triggerNodes.length > 1 && executionIntent !== 'one-off' ? {} : undefined, supportingWorkflowIds: referencedWorkflowIds.length > 0 ? referencedWorkflowIds : undefined, hasUnresolvedPlaceholders: hasPlaceholders || undefined, diff --git a/packages/@n8n/instance-ai/src/types.ts b/packages/@n8n/instance-ai/src/types.ts index 0e2459e354b..a8df1c17a39 100644 --- a/packages/@n8n/instance-ai/src/types.ts +++ b/packages/@n8n/instance-ai/src/types.ts @@ -61,10 +61,13 @@ import type { import type { BuilderRequiredArtifact } from './tools/orchestration/builder-required-artifact'; import type { IdRemapper, TraceIndex, TraceWriter } from './tracing/trace-replay'; import type { + VerificationClaim, VerificationResult, WorkflowBuildOutcome, WorkflowLoopAction, WorkflowLoopState, + WorkflowVerificationEvidence, + WorkflowTriggerVerificationProgress, WorkflowVerificationObligation, } from './workflow-loop/workflow-loop-state'; import type { BuilderTemplatesService } from './workspace/builder-templates-service'; @@ -1914,6 +1917,15 @@ export interface WorkflowTaskService { runId: string, ): Promise; updateBuildOutcome(workItemId: string, update: Partial): Promise; + startVerification( + workItemId: string, + triggerNodeName?: string, + ): Promise; + recordVerification( + workItemId: string, + verification: WorkflowVerificationEvidence & { claim: VerificationClaim }, + previousProgress?: WorkflowTriggerVerificationProgress, + ): Promise; } // ── Orchestration context (plan tools) ────────────────────────────────────── diff --git a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/verification-obligation.test.ts b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/verification-obligation.test.ts index b182f21ee92..01e1afb5603 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/verification-obligation.test.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/verification-obligation.test.ts @@ -1,4 +1,9 @@ -import { createRemediation, MAX_POST_SUBMIT_REMEDIATION_SUBMITS } from '../remediation'; +import { successfulVerification } from '../../__tests__/verification-fixtures'; +import { + createRemediation, + MAX_VERIFY_ATTEMPTS, + MAX_POST_SUBMIT_REMEDIATION_SUBMITS, +} from '../remediation'; import { deriveWorkflowVerificationObligation, deriveWorkflowVerificationObligationFromOutcome, @@ -8,6 +13,7 @@ import type { AttemptRecord, WorkflowBuildOutcome, WorkflowLoopState, + WorkflowVerificationObligationStatus, } from '../workflow-loop-state'; function makeState(overrides: Partial = {}): WorkflowLoopState { @@ -52,6 +58,30 @@ function makeOutcome(overrides: Partial = {}): WorkflowBui }; } +function makePlannedNode(nodeName: string) { + return { + nodeName, + verdict: 'execute' as const, + reason: 'Reads data', + confidence: 'high' as const, + source: 'deterministic' as const, + }; +} + +function makeMultiTriggerOutcome( + overrides: Partial = {}, +): WorkflowBuildOutcome { + return makeOutcome({ + triggerNodes: [ + { nodeName: 'Trigger A', nodeType: 'n8n-nodes-base.webhook' }, + { nodeName: 'Trigger B', nodeType: 'n8n-nodes-base.scheduleTrigger' }, + ], + nodeSimulationPlan: [makePlannedNode('Step A'), makePlannedNode('Step B')], + verificationProgress: {}, + ...overrides, + }); +} + describe('deriveWorkflowVerificationObligation', () => { it('keeps an exhausted setup verification budget blocked', () => { const remediation = createRemediation({ @@ -198,6 +228,129 @@ describe('deriveWorkflowVerificationObligation', () => { expect(obligation.evidence?.executionId).toBe('exec-1'); }); + const passedA = { + 'Trigger A': [successfulVerification('Trigger A', ['Trigger A', 'Step A'])], + }; + const passedAll = { + ...passedA, + 'Trigger B': [successfulVerification('Trigger B', ['Trigger B', 'Step B'])], + }; + const successfulPass: WorkflowBuildOutcome['verification'] = { + attempted: true, + success: true, + executionId: 'exec-1', + status: 'success', + }; + const verificationCases: Array< + [string, Partial, WorkflowVerificationObligationStatus] + > = [ + [ + 'the first trigger passed', + { verificationProgress: passedA, verification: successfulPass }, + 'ready_to_verify', + ], + ['all triggers and nodes passed', { verificationProgress: passedAll }, 'verified'], + [ + 'an unscoped partial pass was recorded', + { verification: { ...successfulPass, evidence: { nodesNotReached: ['Step B'] } } }, + 'ready_to_verify', + ], + [ + 'the first trigger has a coverage gap', + { + verificationProgress: passedA, + verification: { + ...successfulPass, + evidence: { triggerNodeName: 'Trigger A', nodesNotReached: ['Shared Final Step'] }, + }, + }, + 'ready_to_verify', + ], + [ + 'aggregate coverage has a gap', + { + verificationProgress: passedAll, + nodeSimulationPlan: ['Step A', 'Step B', 'Shared Final Step'].map(makePlannedNode), + }, + 'not_verifiable', + ], + [ + 'the latest pass failed', + { verification: { ...successfulPass, success: false, status: 'error' } }, + 'ready_to_verify', + ], + [ + 'the latest failure contradicts stored coverage', + { + verificationProgress: passedAll, + verification: { ...successfulPass, success: false, status: 'error' }, + }, + 'ready_to_verify', + ], + [ + 'the attempt limit was reached before completion', + { verificationProgress: passedA, verifyAttempts: MAX_VERIFY_ATTEMPTS }, + 'blocked', + ], + [ + 'the final allowed attempt completed coverage', + { verificationProgress: passedAll, verifyAttempts: MAX_VERIFY_ATTEMPTS }, + 'verified', + ], + [ + 'a legacy outcome has no progress record', + { verificationProgress: undefined, verification: successfulPass }, + 'verified', + ], + ]; + it.each(verificationCases)( + 'derives multi-trigger verification status when %s', + (_name, overrides, expectedStatus) => { + const obligation = deriveWorkflowVerificationObligation('thread-1', { + state: makeState(), + attempts: [makeAttempt()], + lastBuildOutcome: makeMultiTriggerOutcome(overrides), + }); + + expect(obligation.status).toBe(expectedStatus); + if (expectedStatus === 'blocked') + expect(obligation.blockingReason).toContain('attempt limit'); + if (expectedStatus === 'not_verifiable') { + expect(obligation.policy).toBe('manual'); + expect(obligation.blockingReason).toContain('Shared Final Step'); + } + if (expectedStatus === 'verified') expect(obligation.blockingReason).toBeUndefined(); + }, + ); + + it('does not treat a reserved run as failed setup evidence', () => { + const obligation = deriveWorkflowVerificationObligationFromOutcome( + 'thread-1', + makeMultiTriggerOutcome({ + setupRequirement: { + status: 'required', + reason: 'mocked-credentials', + guidance: 'Connect the account after verification.', + }, + verifyAttempts: 1, + verification: undefined, + }), + ); + expect(obligation.status).toBe('ready_to_verify'); + }); + + it('keeps incomplete multi-trigger progress blocked when the loop state is blocked', () => { + const obligation = deriveWorkflowVerificationObligation('thread-1', { + state: makeState({ status: 'blocked' }), + attempts: [makeAttempt()], + lastBuildOutcome: makeMultiTriggerOutcome({ + verificationReadiness: { status: 'already_verified' }, + }), + }); + + expect(obligation.status).toBe('blocked'); + }); + it('treats partial-coverage evidence as a manual warning completion', () => { const obligation = deriveWorkflowVerificationObligation('thread-1', { state: makeState(), diff --git a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts index 13c3192f0e7..69fd2ceb907 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts @@ -1,6 +1,7 @@ import { mock } from 'vitest-mock-extended'; import { executeTool } from '../../__tests__/tool-test-utils'; +import { successfulVerification } from '../../__tests__/verification-fixtures'; import type { WorkflowLoopStorage } from '../../storage/workflow-loop-storage'; import { createReportVerificationVerdictTool } from '../../tools/orchestration/report-verification-verdict.tool'; import { createVerifyBuiltWorkflowTool } from '../../tools/orchestration/verify-built-workflow.tool'; @@ -14,6 +15,21 @@ function createStorage() { const records = new Map>(); const storage = { + updateBuildOutcome: vi.fn( + async ( + _threadId: string, + workItemId: string, + update: (outcome: WorkflowBuildOutcome) => WorkflowBuildOutcome, + ) => { + const record = records.get(workItemId); + if (!record?.lastBuildOutcome) throw new Error('Missing outcome'); + records.set(workItemId, { + ...record, + lastBuildOutcome: update(record.lastBuildOutcome as WorkflowBuildOutcome), + }); + await Promise.resolve(); + }, + ), getWorkItem: vi.fn(async (_threadId: string, workItemId: string) => { return await Promise.resolve( (records.get(workItemId) ?? null) as Awaited< @@ -70,6 +86,32 @@ function createBuildOutcome(overrides: Partial = {}): Work } describe('WorkflowTaskCoordinator', () => { + it('counts reservations and stops at the attempt limit', async () => { + const { storage } = createStorage(); + const coordinator = new WorkflowTaskCoordinator('thread-1', storage); + await coordinator.reportBuildOutcome(createBuildOutcome()); + for (let attempt = 0; attempt < MAX_VERIFY_ATTEMPTS; attempt++) { + await coordinator.startVerification('wi_1'); + } + await expect(coordinator.startVerification('wi_1')).rejects.toThrow('attempt limit'); + expect((await coordinator.getBuildOutcome('wi_1'))?.verifyAttempts).toBe(MAX_VERIFY_ATTEMPTS); + }); + + it('merges completed trigger runs against the latest stored coverage', async () => { + const { storage } = createStorage(); + const coordinator = new WorkflowTaskCoordinator('thread-1', storage); + await coordinator.reportBuildOutcome(createBuildOutcome({ verificationProgress: {} })); + await coordinator.startVerification('wi_1', 'A'); + await coordinator.startVerification('wi_1', 'B'); + for (const triggerNodeName of ['B', 'A']) { + await coordinator.recordVerification('wi_1', successfulVerification(triggerNodeName)); + } + expect((await coordinator.getBuildOutcome('wi_1'))?.verificationProgress).toMatchObject({ + A: [{ evidence: { nodesExecuted: ['A'] } }], + B: [{ evidence: { nodesExecuted: ['B'] } }], + }); + }); + function setupBlockedOutcome(overrides: Partial = {}) { return createBuildOutcome({ runId: 'run-1', diff --git a/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts b/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts index efa603fcd27..57f4002d64c 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts @@ -33,7 +33,11 @@ function formatClaimLead(claim: VerificationClaim | undefined): string { const rules = [ 'Do NOT call the workflow verified, tested, working, or ready to publish.', 'Do NOT offer to publish it.', - claim.liveTestRecommended ? 'Offer a live end-to-end test instead.' : '', + (claim.pendingTriggers?.length ?? 0) > 0 + ? 'Follow the verification obligation for the remaining triggers. Respect its attempt limit.' + : claim.liveTestRecommended + ? 'Offer a live end-to-end test instead.' + : '', ].filter((rule) => rule !== ''); return [formatClaimHeadline(claim), ...describeClaimCoverage(claim), ...rules].join(' '); diff --git a/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts b/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts index 629774a60ed..a5de1ed9bab 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts @@ -31,6 +31,11 @@ export function formatClaimHeadline(claim: VerificationClaim): string { export function describeClaimCoverage(claim: VerificationClaim): string[] { const facts: string[] = []; + if ((claim.pendingTriggers?.length ?? 0) > 0) { + facts.push( + `Verification evidence is incomplete for these triggers: ${formatClaimNodeList(claim.pendingTriggers ?? [])}.`, + ); + } if (claim.nodesNotReached.length > 0) { facts.push( diff --git a/packages/@n8n/instance-ai/src/workflow-loop/verification-obligation.ts b/packages/@n8n/instance-ai/src/workflow-loop/verification-obligation.ts index 04fddf66a3c..fa944e6767b 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/verification-obligation.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/verification-obligation.ts @@ -1,7 +1,9 @@ +import { MAX_VERIFY_ATTEMPTS } from './remediation'; import { setupRemediationBlocksVerification, stateForPendingSetupVerification, } from './setup-verification-policy'; +import { getMultiTriggerCoverage } from './verification-progress'; import type { AttemptRecord, WorkflowBuildOwner, @@ -42,7 +44,20 @@ const UNSETTLED_OBLIGATION_STATUSES = new Set; + +function hasSuccessfulEvidence( + outcome: WorkflowBuildOutcome, + coverage: MultiTriggerCoverage, +): boolean { + if (coverage) { + return ( + coverage.allTriggersPassed && + coverage.nodesNotReached.length === 0 && + !hasFailedEvidence(outcome) + ); + } + const nodesNotReached = outcome.verification?.evidence?.nodesNotReached; return ( outcome.verification?.attempted === true && @@ -52,7 +67,13 @@ function hasSuccessfulEvidence(outcome: WorkflowBuildOutcome): boolean { ); } -function hasPartialSuccessfulCoverageEvidence(outcome: WorkflowBuildOutcome): boolean { +function hasPartialSuccessfulCoverageEvidence( + outcome: WorkflowBuildOutcome, + coverage: MultiTriggerCoverage, +): boolean { + if (coverage) { + return coverage.allTriggersPassed && coverage.nodesNotReached.length > 0; + } return ( outcome.verification?.attempted === true && outcome.verification.success && @@ -78,20 +99,17 @@ function hasSetupBlockingEvidence( function deriveStatus( state: WorkflowLoopState, outcome: WorkflowBuildOutcome | undefined, + coverage: MultiTriggerCoverage, ): WorkflowVerificationObligationStatus { if (!outcome) return 'pending_build'; if (!outcome.submitted) return 'blocked'; - if (hasSuccessfulEvidence(outcome)) return 'verified'; + if (hasSuccessfulEvidence(outcome, coverage)) return 'verified'; if (hasSetupBlockingEvidence(state, outcome)) return 'needs_setup'; - if (hasPartialSuccessfulCoverageEvidence(outcome)) return 'not_verifiable'; - // A verification that was attempted and failed has already run end-to-end and - // produced a concrete error (e.g. a missing credential or a runtime error). - // Re-issuing it just replays the same failure, so once setup-blocking and - // partial-success cases are ruled out above, settle it as a manual outcome. - // Without this the switch below falls through to `ready_to_verify` and the - // planned orchestrator re-issues verification forever. - if (hasFailedEvidence(outcome)) return 'not_verifiable'; + if (hasPartialSuccessfulCoverageEvidence(outcome, coverage)) return 'not_verifiable'; + // Keep tracked multi-trigger failures open so another trigger can contribute. + // The attempt limit below bounds these retries. + if (hasFailedEvidence(outcome) && !coverage) return 'not_verifiable'; // One-off builds: verification is optional, so the obligation settles // immediately (after the evidence checks above, which still win). This must @@ -102,10 +120,14 @@ function deriveStatus( if (outcome.executionIntent === 'one-off') { return state.status === 'blocked' ? 'blocked' : 'not_verifiable'; } + if ((outcome.verifyAttempts ?? 0) >= MAX_VERIFY_ATTEMPTS) { + return 'blocked'; + } switch (outcome.verificationReadiness?.status) { case 'already_verified': - return 'verified'; + if (!coverage) return 'verified'; + return state.status === 'blocked' ? 'blocked' : 'ready_to_verify'; case 'needs_setup': return 'needs_setup'; case 'not_verifiable': @@ -131,6 +153,8 @@ function derivePolicy( function deriveBlockingReason( state: WorkflowLoopState, outcome: WorkflowBuildOutcome | undefined, + coverage: MultiTriggerCoverage, + status: WorkflowVerificationObligationStatus, ): string | undefined { if (!outcome) return undefined; if (!outcome.submitted) { @@ -138,17 +162,24 @@ function deriveBlockingReason( outcome.blockingReason ?? outcome.failureSignature ?? 'Builder did not submit a workflow.' ); } + if (status === 'verified') return undefined; if (outcome.verificationReadiness?.status === 'not_verifiable') { return outcome.verificationReadiness.guidance; } if (outcome.verificationReadiness?.status === 'needs_setup') { return outcome.verificationReadiness.guidance; } - if (hasPartialSuccessfulCoverageEvidence(outcome)) { + if (hasPartialSuccessfulCoverageEvidence(outcome, coverage)) { + if (coverage) { + return ( + 'Automatic verification covered every trigger but did not reach all planned nodes. ' + + `Unreached nodes need manual testing: ${coverage.nodesNotReached.join(', ')}.` + ); + } const nodesNotReached = outcome.verification?.evidence?.nodesNotReached ?? []; return `Automatic verification only covered part of the workflow. Unreached nodes need manual testing: ${nodesNotReached.join(', ')}.`; } - if (hasFailedEvidence(outcome) && !hasSetupBlockingEvidence(state, outcome)) { + if (hasFailedEvidence(outcome) && !coverage && !hasSetupBlockingEvidence(state, outcome)) { const failure = outcome.verification?.failureSignature ?? outcome.verification?.evidence?.errorMessage ?? @@ -158,6 +189,12 @@ function deriveBlockingReason( nodesNotReached.length > 0 ? ` Nodes not reached: ${nodesNotReached.join(', ')}.` : ''; return `Automatic verification failed with: ${failure}. Re-running it will reproduce the same failure — explain this blocker to the user and have them resolve it (e.g. configure credentials or fix the data) before verifying manually.${unreached}`; } + if (status === 'blocked' && (outcome.verifyAttempts ?? 0) >= MAX_VERIFY_ATTEMPTS) { + return ( + 'Automatic verification reached its attempt limit. ' + + 'Report the remaining coverage. Ask the user to test the remaining nodes manually.' + ); + } const outcomeRemediation = outcome.remediation; if (outcomeRemediation && setupRemediationBlocksVerification(outcomeRemediation, outcome)) { return outcomeRemediation.guidance; @@ -172,7 +209,7 @@ function deriveBlockingReason( // After the evidence checks: a one-off may have run an optional pre-flight // verify, and a concrete failure message outranks the generic guidance. A // verified one-off carries no blockingReason at all — nothing is blocked. - if (outcome.executionIntent === 'one-off' && !hasSuccessfulEvidence(outcome)) { + if (outcome.executionIntent === 'one-off') { return ONE_OFF_VERIFICATION_GUIDANCE; } if (state.status === 'blocked') { @@ -200,7 +237,8 @@ export function deriveWorkflowVerificationObligation( ? { ...savedOutcome, verificationReadiness: { status: 'ready' } } : savedOutcome; const state = setupVerificationState ?? record.state; - const status = deriveStatus(state, outcome); + const coverage = getMultiTriggerCoverage(outcome); + const status = deriveStatus(state, outcome, coverage); const updatedAt = options.updatedAt ?? lastAttemptTimestamp(record.attempts) ?? new Date().toISOString(); const owner = resolveWorkflowBuildOwner(options, record.state, outcome); @@ -222,7 +260,7 @@ export function deriveWorkflowVerificationObligation( setupRequirement: outcome?.setupRequirement, evidence: outcome?.verification, executionIntent: outcome?.executionIntent, - blockingReason: deriveBlockingReason(state, outcome), + blockingReason: deriveBlockingReason(state, outcome, coverage, status), updatedAt, }; } diff --git a/packages/@n8n/instance-ai/src/workflow-loop/verification-progress.ts b/packages/@n8n/instance-ai/src/workflow-loop/verification-progress.ts new file mode 100644 index 00000000000..7f44a334108 --- /dev/null +++ b/packages/@n8n/instance-ai/src/workflow-loop/verification-progress.ts @@ -0,0 +1,71 @@ +import type { + VerificationClaim, + WorkflowBuildOutcome, + WorkflowVerificationEvidence, +} from './workflow-loop-state'; +import { deriveVerificationClaim } from '../tools/orchestration/verification/claim'; + +export function getMultiTriggerCoverage(outcome: WorkflowBuildOutcome | undefined) { + const progress = outcome?.verificationProgress; + const triggers = outcome?.triggerNodes ?? []; + if (!outcome || outcome.executionIntent === 'one-off' || !progress || triggers.length < 2) { + return undefined; + } + + const passedTriggers = triggers.filter((trigger) => Object.hasOwn(progress, trigger.nodeName)); + const passes = passedTriggers.flatMap((trigger) => progress[trigger.nodeName]); + const coveredNodes = new Set(passes.flatMap((pass) => pass.evidence?.nodesExecuted ?? [])); + return { + passes, + allTriggersPassed: passedTriggers.length === triggers.length, + pendingTriggers: triggers + .filter((trigger) => !Object.hasOwn(progress, trigger.nodeName)) + .map((trigger) => trigger.nodeName), + nodesNotReached: (outcome.nodeSimulationPlan ?? []) + .map((node) => node.nodeName) + .filter((nodeName) => !coveredNodes.has(nodeName)), + }; +} + +export function deriveWorkflowVerificationClaim( + outcome: WorkflowBuildOutcome, + verification: WorkflowVerificationEvidence & { claim: VerificationClaim }, +) { + const coverage = getMultiTriggerCoverage(outcome); + if (!coverage) return verification.claim; + + const liveNodes = new Set( + coverage.passes.flatMap(({ evidence, claim }) => { + const simulated = new Set([ + ...claim.simulatedNodes.map((node) => node.nodeName), + ...claim.pinnedNodes, + ]); + return (evidence?.nodesExecuted ?? []).filter((name) => !simulated.has(name)); + }), + ); + // Only successful scoped passes contribute coverage. Keep the latest run's limitations. + const claims = [...coverage.passes.map((pass) => pass.claim), verification.claim]; + + return deriveVerificationClaim({ + analysis: { + success: verification.success, + nodesNotReached: coverage.nodesNotReached, + reachedSimulatedNodes: [ + ...new Map( + claims + .flatMap((claim) => claim.simulatedNodes) + .filter((node) => !liveNodes.has(node.nodeName)) + .map((node) => [node.nodeName, node]), + ).values(), + ], + workflowPinnedNodeNames: [...new Set(claims.flatMap((claim) => claim.pinnedNodes))].filter( + (name) => !liveNodes.has(name), + ), + }, + plannedNodeCount: outcome.nodeSimulationPlan?.length ?? 0, + fixTargetNodeNames: claims + .flatMap((claim) => claim.unprovenTargets) + .filter((name) => !liveNodes.has(name)), + pendingTriggers: coverage.pendingTriggers, + }); +} diff --git a/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts b/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts index 8001f52f4ce..3dc38fcc52e 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts @@ -164,7 +164,7 @@ export const verificationClaimLevelSchema = z.enum(['verified', 'partial', 'unpr export type VerificationClaimLevel = z.infer; /** - * The deterministic verdict for one verification run. Backend-only: it shapes + * The deterministic verdict from verification evidence. Backend-only: it shapes * the tool result the model reads and gates the publish offer. It is * deliberately not sent to the client — see INS-1308. */ @@ -184,6 +184,8 @@ export const verificationClaimSchema = z.object({ * claim `verified` even when it ended without an error. */ unprovenTargets: z.array(z.string()), + /** Triggers with missing or incomplete scoped verification evidence. */ + pendingTriggers: z.array(z.string()).optional(), publishReady: z.boolean(), liveTestRecommended: z.boolean(), }); @@ -205,6 +207,8 @@ export const workflowVerificationEvidenceSchema = z.object({ evidence: z .object({ nodesExecuted: z.array(z.string()).optional(), + /** Trigger whose main-flow scope was used for this evidence. */ + triggerNodeName: z.string().optional(), /** * Plan nodes the execution never reached (e.g. a lookup returned zero * items and stopped the chain). Non-empty means partial coverage — @@ -222,6 +226,15 @@ export const workflowVerificationEvidenceSchema = z.object({ export type WorkflowVerificationEvidence = z.infer; +export const workflowVerificationProgressSchema = z.record( + z.string(), + z.array(workflowVerificationEvidenceSchema.required({ claim: true })), +); + +export type WorkflowTriggerVerificationProgress = z.infer< + typeof workflowVerificationProgressSchema +>[string]; + export const workflowVerificationReadinessSchema = z.discriminatedUnion('status', [ z.object({ status: z.literal('ready') }), z.object({ status: z.literal('already_verified') }), @@ -334,7 +347,7 @@ export const workflowBuildOutcomeSchema = z.object({ submitted: z.boolean(), triggerType: triggerTypeSchema, /** - * Trigger nodes in the submitted workflow. Populated on successful submits; + * Enabled trigger nodes in the submitted workflow. Populated on successful submits; * absent on failed or pre-submit outcomes. The orchestrator reads `nodeType` * to pick a `verify-built-workflow` `inputData` shape for direct builds. */ @@ -411,6 +424,8 @@ export const workflowBuildOutcomeSchema = z.object({ remediation: remediationMetadataSchema.optional(), /** Count of verify-built-workflow runs for this build; capped by MAX_VERIFY_ATTEMPTS. */ verifyAttempts: z.number().int().min(0).optional(), + /** Successful verification runs by trigger. A failed rerun removes that trigger's entry. */ + verificationProgress: workflowVerificationProgressSchema.optional(), /** * Structured verification record from the most recent `verify-built-workflow` * tool call. This is tool evidence, not builder prose, so downstream checks may diff --git a/packages/@n8n/instance-ai/src/workflow-loop/workflow-task-service.ts b/packages/@n8n/instance-ai/src/workflow-loop/workflow-task-service.ts index 9cd818141f5..11cd7285044 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/workflow-task-service.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/workflow-task-service.ts @@ -1,14 +1,19 @@ +import { UserError } from 'n8n-workflow'; import { isDeepStrictEqual } from 'node:util'; import type { WorkflowTaskService } from '../types'; -import { terminalRemediationFromState } from './remediation'; +import { MAX_VERIFY_ATTEMPTS, terminalRemediationFromState } from './remediation'; import { WorkflowLoopRuntime } from './runtime'; import { isNeedsSetupRemediation, stateForPendingSetupVerification, } from './setup-verification-policy'; +import { deriveWorkflowVerificationClaim } from './verification-progress'; import type { VerificationResult, + VerificationClaim, + WorkflowVerificationEvidence, + WorkflowTriggerVerificationProgress, WorkflowBuildOutcome, WorkflowLoopAction, WorkflowLoopState, @@ -112,4 +117,67 @@ export class WorkflowTaskCoordinator implements WorkflowTaskService { }; }); } + + /** Keep prior evidence outside storage until a successful retry restores it. */ + async startVerification( + workItemId: string, + triggerNodeName?: string, + ): Promise { + let previousProgress: WorkflowTriggerVerificationProgress | undefined; + await this.storage.updateBuildOutcome(this.threadId, workItemId, (outcome) => { + if ((outcome.verifyAttempts ?? 0) >= MAX_VERIFY_ATTEMPTS) { + throw new UserError( + 'Verification reached its attempt limit. Test the remaining nodes manually.', + ); + } + const progress = outcome.verificationProgress && { ...outcome.verificationProgress }; + if (progress && triggerNodeName) { + previousProgress = Object.hasOwn(progress, triggerNodeName) + ? progress[triggerNodeName] + : undefined; + delete progress[triggerNodeName]; + } + return { + ...outcome, + verifyAttempts: (outcome.verifyAttempts ?? 0) + 1, + // An unscoped retry cannot identify which pass it replaces. + verificationProgress: progress && !triggerNodeName ? {} : progress, + // Keep the obligation open and block publishing until the run completes. + verification: { attempted: false, success: false, status: 'running' }, + }; + }); + return previousProgress; + } + + async recordVerification( + workItemId: string, + verification: WorkflowVerificationEvidence & { claim: VerificationClaim }, + previousProgress?: WorkflowTriggerVerificationProgress, + ): Promise { + let claim: VerificationClaim | undefined; + await this.storage.updateBuildOutcome(this.threadId, workItemId, (outcome) => { + const trigger = verification.evidence?.triggerNodeName; + const nodes = verification.evidence?.nodesExecuted ?? []; + const progress = outcome.verificationProgress && { ...outcome.verificationProgress }; + if (progress && trigger) { + if (verification.success && verification.executionId && nodes.includes(trigger)) { + const priorProgress = Object.hasOwn(progress, trigger) ? progress[trigger] : []; + progress[trigger] = [ + ...new Map( + [...(previousProgress ?? []), ...priorProgress, verification].map((pass) => [ + pass.executionId, + pass, + ]), + ).values(), + ]; + } else { + delete progress[trigger]; + } + } + const next = { ...outcome, verificationProgress: progress }; + claim = deriveWorkflowVerificationClaim(next, verification); + return { ...next, verification: { ...verification, claim } }; + }); + return claim; + } } diff --git a/packages/cli/src/modules/instance-ai/instance-ai.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.service.ts index eea30a4f4ff..c31a1496c68 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.service.ts @@ -2249,6 +2249,20 @@ export class InstanceAiService { return resumed; }, getBuildOutcome: async (workItemId) => await workflowTasks.getBuildOutcome(workItemId), + startVerification: async (workItemId, triggerNodeName) => { + const previousProgress = await workflowTasks.startVerification(workItemId, triggerNodeName); + await sync(); + return previousProgress; + }, + recordVerification: async (workItemId, verification, previousProgress) => { + const claim = await workflowTasks.recordVerification( + workItemId, + verification, + previousProgress, + ); + await sync(); + return claim; + }, getLatestBuildOutcomeForWorkflow: async (workflowId) => await workflowTasks.getLatestBuildOutcomeForWorkflow(workflowId), getWorkflowLoopState: async (workItemId) =>