diff --git a/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md b/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md index 079b55d3431..15c9c2c6e57 100644 --- a/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md +++ b/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md @@ -201,6 +201,14 @@ publish-readiness evidence. If the user explicitly asks to publish before a live execution succeeds, warn that the live path remains untested, then follow the requested publish flow. +`workflows(action="publish")` enforces this. While the latest verification left +nodes unreached or simulated, the call is refused and returns +`verificationDisclosure` — the coverage facts, generated from the run. Relay +those facts to the user and offer a live end-to-end test. Publish only if they +still ask, by calling publish again with `acknowledgeUnverified: true`. Never +set that flag to skip the disclosure. The user also sees the same facts in the +publish approval prompt, so a summary that contradicts them is visible to them. + Execution evidence can come from a run you started or a run the user started. If the user says they ran the workflow manually, call `executions(action="list", workflowId)`, identify the relevant run, and inspect @@ -476,9 +484,24 @@ result. ## Claiming success -Do not tell the user a workflow is "fixed", "verified", "tested", "working", or -has "no errors" unless you have a passing `verify-built-workflow`, -`executions(action="run")`, or inspected user-run execution that exercised the +`verify-built-workflow` returns a `claim`, and its `level` decides what you may +say: + +- `verified` — you may call the workflow verified, tested, or working. +- `partial`, `unproven`, or `failed` — you may NOT. Name what is unconfirmed + instead. + +**`success: true` does not mean verified.** It means the run ended without an +error, and a run with every write simulated also ends without an error. Read +`claim.level`, not `success`. + +Nothing else tells the user how strong the claim is. Your message is the only +place they learn it, so name the unreached and simulated nodes yourself, and say +what stays unconfigured and what that means when the workflow runs. + +Without a claim, do not tell the user a workflow is "fixed", "verified", +"tested", "working", or has "no errors" unless you have a passing +`executions(action="run")` or an inspected user-run execution that exercised the path being claimed. Do not call a workflow "ready to use" or "ready to publish" unless a passing execution met the publish-readiness requirement above. A successful `build-workflow`/save, a static `workflows(action="validate")`, or diff --git a/packages/@n8n/instance-ai/src/tools/__tests__/workflows.tool.test.ts b/packages/@n8n/instance-ai/src/tools/__tests__/workflows.tool.test.ts index 1f461d86bb5..71df1513076 100644 --- a/packages/@n8n/instance-ai/src/tools/__tests__/workflows.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/__tests__/workflows.tool.test.ts @@ -1753,6 +1753,236 @@ describe('workflows tool', () => { }); }); + function contextWithClaim(claim: unknown, permissions?: Partial) { + return createMockContext({ + ...(permissions ? { permissions } : {}), + workflowBuildContext: { + threadId: 't1', + runId: 'run-1', + taskId: 'task-1', + workItemId: 'wi-1', + workflowTaskService: { + getLatestBuildOutcomeForWorkflow: vi + .fn() + .mockResolvedValue({ runId: 'run-1', verification: { claim } }), + }, + } as never, + }); + } + + const partialClaim = { + level: 'partial', + plannedNodeCount: 12, + reachedNodeCount: 5, + nodesNotReached: ['Send Email', 'Log Row'], + simulatedNodes: [{ nodeName: 'Create Event', reason: 'Creates a record' }], + pinnedNodes: [], + unprovenTargets: [], + publishReady: false, + liveTestRecommended: true, + }; + + it('should refuse to publish an unverified workflow and disclose the coverage', async () => { + const context = contextWithClaim(partialClaim); + const suspend = vi.fn(); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, { + suspend, + resumeData: undefined, + } as never); + + expect(context.workflowService.publish).not.toHaveBeenCalled(); + // Refused before the dialog, so an `always_allow` instance is covered too. + expect(suspend).not.toHaveBeenCalled(); + expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' }); + const { verificationDisclosure } = result as { verificationDisclosure: string }; + expect(verificationDisclosure).toContain('NOT fully verified'); + expect(verificationDisclosure).toContain('Send Email, Log Row'); + expect(verificationDisclosure).toContain('Create Event'); + }); + + it('should refuse an unverified publish even when approval is not required', async () => { + const context = contextWithClaim(partialClaim, { publishWorkflow: 'always_allow' }); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {} as never); + + expect(context.workflowService.publish).not.toHaveBeenCalled(); + expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' }); + }); + + it('still asks the user to approve an acknowledged unverified publish under always_allow', async () => { + // The acknowledgement is only the model's word that the user asked, so + // it must not be the sole gate: without the dialog an `always_allow` + // instance would publish an unverified workflow with no disclosure + // anywhere the user could see. + const context = contextWithClaim(partialClaim, { publishWorkflow: 'always_allow' }); + const suspend = vi.fn(); + + const tool = createWorkflowsTool(context, 'full'); + await executeTool( + tool, + { action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true }, + { suspend, resumeData: undefined } as never, + ); + + expect(context.workflowService.publish).not.toHaveBeenCalled(); + expect(suspend).toHaveBeenCalledTimes(1); + const { message } = (suspend as Mock).mock.calls[0][0] as { message: string }; + expect(message).toContain('NOT fully verified'); + }); + + it('keeps always_allow routine for a verified publish', async () => { + const context = contextWithClaim( + { ...partialClaim, level: 'verified', publishReady: true }, + { publishWorkflow: 'always_allow' }, + ); + (context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' }); + const suspend = vi.fn(); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, { + suspend, + } as never); + + expect(suspend).not.toHaveBeenCalled(); + expect(result).toMatchObject({ success: true }); + }); + + it('should disclose the coverage in the approval prompt once acknowledged', async () => { + const context = contextWithClaim(partialClaim); + const suspend = vi.fn(); + + const tool = createWorkflowsTool(context, 'full'); + await executeTool( + tool, + { action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true }, + { suspend, resumeData: undefined } as never, + ); + + expect(suspend).toHaveBeenCalledTimes(1); + const { message } = (suspend as Mock).mock.calls[0][0] as { message: string }; + expect(message).toContain('Publish'); + expect(message).toContain('NOT fully verified'); + expect(message).toContain('Send Email, Log Row'); + }); + + it('should publish once acknowledged and approved', async () => { + const context = contextWithClaim(partialClaim); + (context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' }); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool( + tool, + { action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true }, + { resumeData: { approved: true } } as never, + ); + + expect(context.workflowService.publish).toHaveBeenCalledWith('wf1', { + versionId: undefined, + }); + expect(result).toMatchObject({ success: true, activeVersionId: 'v2' }); + }); + + it('should not gate a fully verified workflow', async () => { + const context = contextWithClaim({ ...partialClaim, level: 'verified', publishReady: true }); + (context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' }); + const suspend = vi.fn(); + + const tool = createWorkflowsTool(context, 'full'); + await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, { + suspend, + resumeData: undefined, + } as never); + + const { message } = (suspend as Mock).mock.calls[0][0] as { message: string }; + expect(message).not.toContain('NOT'); + }); + + it('refuses a publish when verification ran but produced no verdict', async () => { + // `handleMissingSimulationPlan` writes an attempted record with no + // claim. Reading that as an absent record let a failed verification + // publish as freely as a workflow nobody ever verified. + const context = createMockContext({ + workflowBuildContext: { + threadId: 't1', + runId: 'run-1', + taskId: 'task-1', + workItemId: 'wi-1', + workflowTaskService: { + getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue({ + runId: 'run-1', + verification: { + attempted: true, + success: false, + failureSignature: 'missing_simulation_plan', + }, + }), + }, + } as never, + }); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, { + suspend: vi.fn(), + resumeData: undefined, + } as never); + + expect(context.workflowService.publish).not.toHaveBeenCalled(); + expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' }); + const { verificationDisclosure } = result as { verificationDisclosure: string }; + expect(verificationDisclosure).toContain('produced no verdict'); + expect(verificationDisclosure).toContain('missing_simulation_plan'); + }); + + it('does not gate a workflow that was never verified', async () => { + // An absent record is unknown, not unverified. Gating it would stop + // publishing every workflow built before the claim existed. + const context = createMockContext({ + workflowBuildContext: { + threadId: 't1', + runId: 'run-1', + taskId: 'task-1', + workItemId: 'wi-1', + workflowTaskService: { + getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue({ runId: 'run-1' }), + }, + } as never, + }); + (context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' }); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, { + resumeData: { approved: true }, + } as never); + + expect(result).toMatchObject({ success: true }); + }); + + it('should fail open when the build outcome cannot be read', async () => { + const context = createMockContext({ + workflowBuildContext: { + threadId: 't1', + runId: 'run-1', + taskId: 'task-1', + workItemId: 'wi-1', + workflowTaskService: { + getLatestBuildOutcomeForWorkflow: vi.fn().mockRejectedValue(new Error('storage down')), + }, + } as never, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() } as never, + }); + (context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' }); + + const tool = createWorkflowsTool(context, 'full'); + const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, { + resumeData: { approved: true }, + } as never); + + expect(result).toMatchObject({ success: true }); + }); + it('should suspend for confirmation and then publish when approved', async () => { const context = createMockContext(); (context.workflowService.publish as Mock).mockResolvedValue({ diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts index 78214430dd1..e8504ca89ff 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts @@ -60,7 +60,7 @@ describe('report-verification-verdict tool', () => { expect((result as { guidance: string }).guidance).toContain('Error'); }); - it('returns done guidance when verdict is verified', async () => { + it('forwards a verified verdict and reports the workflow', async () => { const doneAction: WorkflowLoopAction = { type: 'done', workflowId: 'wf-123', @@ -83,10 +83,66 @@ describe('report-verification-verdict tool', () => { 'Inspected persisted workflow wf-123; the saved graph matches the requested outcome.', }), ); - expect((result as { guidance: string }).guidance).toContain('verified successfully'); expect((result as { guidance: string }).guidance).toContain('wf-123'); }); + it('forwards no claim when no run recorded one', async () => { + // `getBuildOutcome` returns nothing, so the tool must forward `claim: + // undefined` and let the controller decide. Asserting the guidance here + // would prove nothing: it is rendered from the mocked action, so it stays + // claimless even if the tool invented a verified claim. The wording for a + // missing claim is covered in `workflow-loop/__tests__/guidance.test.ts`. + const workflowTaskService = createWorkflowTaskService( + vi.fn().mockResolvedValue({ type: 'done', workflowId: 'wf-123', summary: 'All good' }), + ); + const context = createMockContext({ workflowTaskService }); + const tool = createReportVerificationVerdictTool(context); + + await executeTool(tool, baseInput, {} as never); + + expect(workflowTaskService.getBuildOutcome).toHaveBeenCalledWith('wi_test1234'); + const [reported] = workflowTaskService.reportVerificationVerdict.mock.calls[0] as [ + { verdict: string; claim?: unknown }, + ]; + expect(reported.claim).toBeUndefined(); + // It does not refuse the verdict: a live run the model inspected is still + // valid evidence, so the controller settles and the guidance carries the + // caveat. + expect(reported.verdict).toBe('verified'); + }); + + it('keeps the verified wording when a run recorded a verified claim', async () => { + const claim = { + level: 'verified' as const, + plannedNodeCount: 2, + reachedNodeCount: 2, + nodesNotReached: [], + simulatedNodes: [], + pinnedNodes: [], + unprovenTargets: [], + publishReady: true, + liveTestRecommended: false, + }; + const workflowTaskService = createWorkflowTaskService( + vi.fn().mockResolvedValue({ + type: 'done', + workflowId: 'wf-123', + summary: 'All good', + claim, + }), + ); + workflowTaskService.getBuildOutcome.mockResolvedValue({ verification: { claim } }); + const context = createMockContext({ workflowTaskService }); + const tool = createReportVerificationVerdictTool(context); + + const result = await executeTool(tool, baseInput, {} as never); + + expect(workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith( + expect.objectContaining({ claim }), + ); + expect((result as { guidance: string }).guidance).toContain('verified successfully'); + }); + it('returns verify guidance when action is verify', async () => { const verifyAction: WorkflowLoopAction = { type: 'verify', workflowId: 'wf-123' }; const reportVerificationVerdict = vi.fn().mockResolvedValue(verifyAction); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/report-verification-verdict.tool.ts b/packages/@n8n/instance-ai/src/tools/orchestration/report-verification-verdict.tool.ts index 96e49d4a78d..9b2309d4fde 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/report-verification-verdict.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/report-verification-verdict.tool.ts @@ -136,6 +136,10 @@ export function createReportVerificationVerdictTool(context: OrchestrationContex }; } + // The claim comes from the persisted run, never from the model: a + // `verified` verdict on a partially covered run must not upgrade it. + const claim = (await context.workflowTaskService.getBuildOutcome(input.workItemId)) + ?.verification?.claim; const remediation = input.remediation ?? defaultRemediationForVerdict(input); const forcedTerminalVerdict = remediation && !remediation.shouldEdit @@ -150,6 +154,7 @@ export function createReportVerificationVerdictTool(context: OrchestrationContex workflowId: input.workflowId, executionId: input.executionId, verdict: forcedTerminalVerdict ?? input.verdict, + claim, workflowInspection: input.workflowInspection, failureSignature: forcedTerminalVerdict ? (remediation?.reason ?? input.failureSignature) diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/claim.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/claim.test.ts new file mode 100644 index 00000000000..ed78ead49f5 --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/claim.test.ts @@ -0,0 +1,161 @@ +import type { VerificationAnalysis } from '../analyze-result'; +import { deriveVerificationClaim } from '../claim'; + +function makeAnalysis(overrides: Partial = {}): VerificationAnalysis { + return { + success: true, + reachedNames: new Set(['Trigger', 'Fetch', 'Send']), + reachedSimulatedNodes: [], + workflowPinnedNodeNames: [], + nodesNotReached: [], + nodeErrors: [], + ...overrides, + }; +} + +describe('deriveVerificationClaim', () => { + it('claims verified when every planned node ran for real', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis(), + plannedNodeCount: 3, + }); + + expect(claim.level).toBe('verified'); + expect(claim.publishReady).toBe(true); + expect(claim.liveTestRecommended).toBe(false); + expect(claim.reachedNodeCount).toBe(3); + }); + + it('downgrades to partial when planned nodes were never reached', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ nodesNotReached: ['Send', 'Log'] }), + plannedNodeCount: 5, + }); + + expect(claim.level).toBe('partial'); + expect(claim.publishReady).toBe(false); + expect(claim.liveTestRecommended).toBe(true); + expect(claim.nodesNotReached).toEqual(['Send', 'Log']); + }); + + it('downgrades to partial when a reached node had simulated output', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ + reachedSimulatedNodes: [{ nodeName: 'Send', reason: 'Sends a message' }], + }), + plannedNodeCount: 3, + }); + + expect(claim.level).toBe('partial'); + expect(claim.publishReady).toBe(false); + }); + + it('never claims verified for a pin-fed run', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ + reachedSimulatedNodes: [{ nodeName: 'Fetch', reason: 'Output came from pinned data' }], + workflowPinnedNodeNames: ['Fetch'], + }), + plannedNodeCount: 3, + }); + + expect(claim.level).toBe('partial'); + expect(claim.pinnedNodes).toEqual(['Fetch']); + }); + + it('reports unproven when the fix target was never reached', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ nodesNotReached: ['Send'] }), + plannedNodeCount: 4, + fixTargetNodeNames: ['Send'], + }); + + expect(claim.level).toBe('unproven'); + expect(claim.unprovenTargets).toEqual(['Send']); + expect(claim.publishReady).toBe(false); + expect(claim.liveTestRecommended).toBe(true); + }); + + it('reports unproven when the fix target had simulated output', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ + reachedSimulatedNodes: [{ nodeName: 'Send', reason: 'Sends a message' }], + }), + plannedNodeCount: 3, + fixTargetNodeNames: ['Send'], + }); + + expect(claim.level).toBe('unproven'); + expect(claim.unprovenTargets).toEqual(['Send']); + }); + + it('stays partial when the fix target was proven but coverage was incomplete', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ nodesNotReached: ['Log'] }), + plannedNodeCount: 4, + fixTargetNodeNames: ['Fetch'], + }); + + expect(claim.level).toBe('partial'); + expect(claim.unprovenTargets).toEqual([]); + }); + + it('never reports unproven when no fix target is known', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ nodesNotReached: ['Send'] }), + plannedNodeCount: 4, + }); + + expect(claim.level).toBe('partial'); + }); + + it('deduplicates repeated fix targets', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ nodesNotReached: ['Send'] }), + plannedNodeCount: 4, + fixTargetNodeNames: ['Send', 'Send'], + }); + + expect(claim.unprovenTargets).toEqual(['Send']); + }); + + it('counts reached nodes over the plan, not the whole run', () => { + // A mockable trigger is not classified, so it never appears in the plan. + // Counting every executed node against the plan length reported "2 of 1". + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ + reachedNames: new Set(['Every Morning at 9', 'Send Slack Message']), + reachedSimulatedNodes: [{ nodeName: 'Send Slack Message', reason: 'Sends a message' }], + }), + plannedNodeCount: 1, + }); + + expect(claim.plannedNodeCount).toBe(1); + expect(claim.reachedNodeCount).toBe(1); + expect(claim.reachedNodeCount).toBeLessThanOrEqual(claim.plannedNodeCount); + }); + + it('never reports a negative reached count', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ nodesNotReached: ['A', 'B', 'C'] }), + plannedNodeCount: 1, + }); + + expect(claim.reachedNodeCount).toBe(0); + }); + + it('reports failed on a failed run regardless of coverage', () => { + const claim = deriveVerificationClaim({ + analysis: makeAnalysis({ + success: false, + nodeErrors: [{ nodeName: 'Fetch', message: 'boom' }], + }), + plannedNodeCount: 3, + fixTargetNodeNames: ['Fetch'], + }); + + expect(claim.level).toBe('failed'); + expect(claim.publishReady).toBe(false); + expect(claim.liveTestRecommended).toBe(false); + }); +}); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/scripted-gate-run.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/scripted-gate-run.test.ts index 071fded9f3c..3ad92780327 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/scripted-gate-run.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/scripted-gate-run.test.ts @@ -174,4 +174,33 @@ describe('runScriptedGateVerification — merge consistency', () => { // Publish is reached only by the approve pass; the decline pass ran last. expect(analysis.simulationNote).toContain('Publish'); }); + + it('discloses a node fed by workflow pin data in any pass', async () => { + const run = vi + .fn() + .mockResolvedValueOnce({ ...approvePassResult, workflowPinnedNodeNames: ['Router'] }) + .mockResolvedValueOnce(declinePassResult); + + const { analysis } = await runScriptedGateVerification(makeArgs(run)); + + // Pins come from the run result, so they exist per pass only. Losing them + // in the merge would let a pin-fed gate run read as a live test. + expect(analysis.workflowPinnedNodeNames).toEqual(['Router']); + expect(analysis.reachedSimulatedNodes.map((node) => node.nodeName)).toContain('Router'); + expect(analysis.simulationNote).toContain('Router'); + }); + + it('does not double-count a pinned node that the plan already simulates', async () => { + const run = vi + .fn() + .mockResolvedValueOnce({ ...approvePassResult, workflowPinnedNodeNames: ['Generate'] }) + .mockResolvedValueOnce(declinePassResult); + + const { analysis } = await runScriptedGateVerification(makeArgs(run)); + + expect(analysis.workflowPinnedNodeNames).toEqual([]); + expect( + analysis.reachedSimulatedNodes.filter((node) => node.nodeName === 'Generate'), + ).toHaveLength(1); + }); }); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts index 75af1e99e74..9bda2bc7a9d 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/analyze-result.ts @@ -24,6 +24,10 @@ export type ChatModelRecoveryOptions = Pick< type ExecutionNodeError = NonNullable[number]; +/** Disclosure for a node whose output came from pin data saved on the workflow. */ +export const WORKFLOW_PIN_SIMULATION_REASON = + 'Output came from pinned data saved on the workflow — unpin it for a live test'; + const CREDENTIAL_FAILURE_KEYWORDS = [ 'credential', 'unauthorized', @@ -413,6 +417,7 @@ export interface VerificationAnalysis { success: boolean; reachedNames: Set; reachedSimulatedNodes: Array<{ nodeName: string; reason: string }>; + workflowPinnedNodeNames: string[]; nodesNotReached: string[]; remediation?: RemediationMetadata; nodesExecuted?: string[]; @@ -462,10 +467,7 @@ export function analyzeVerificationResult(args: { const plannedSimulatedNames = new Set(simulatedNodes.map((n) => n.nodeName)); const workflowPinnedNodes = (result.workflowPinnedNodeNames ?? []) .filter((name) => reachedNames.has(name) && !plannedSimulatedNames.has(name)) - .map((name) => ({ - nodeName: name, - reason: 'Output came from pinned data saved on the workflow — unpin it for a live test', - })); + .map((name) => ({ nodeName: name, reason: WORKFLOW_PIN_SIMULATION_REASON })); const reachedSimulatedNodes = [ ...simulatedNodes.filter((n) => reachedNames.has(n.nodeName)), ...workflowPinnedNodes, @@ -502,6 +504,7 @@ export function analyzeVerificationResult(args: { success, reachedNames, reachedSimulatedNodes, + workflowPinnedNodeNames: workflowPinnedNodes.map((n) => n.nodeName), nodesNotReached, remediation, nodesExecuted: namesOrDataKeys(reachedNames, result.data), diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts new file mode 100644 index 00000000000..039ee06d015 --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/claim.ts @@ -0,0 +1,70 @@ +/** + * Deterministic verification claim. + * + * The model narrates around this structure but never produces it: instruction + * following degrades over a long thread, so the strength of the claim shown to + * the user is computed from run evidence instead of asked for in a prompt. + */ + +import type { VerificationAnalysis } from './analyze-result'; +import type { + VerificationClaim, + VerificationClaimLevel, +} from '../../../workflow-loop/workflow-loop-state'; + +export interface DeriveVerificationClaimArgs { + analysis: VerificationAnalysis; + /** Planned nodes from the build outcome simulation plan. */ + plannedNodeCount: number; + /** + * Nodes this change was about: the node a previous verdict sent for repair, + * or the failing node the user reported. Empty means no target-specific + * downgrade applies and coverage alone decides the level. + */ + fixTargetNodeNames?: readonly string[]; +} + +export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): VerificationClaim { + const { analysis, plannedNodeCount, fixTargetNodeNames = [] } = args; + const nodesNotReached = [...analysis.nodesNotReached]; + const simulatedNodes = [...analysis.reachedSimulatedNodes]; + + const unproven = new Set([...nodesNotReached, ...simulatedNodes.map((n) => n.nodeName)]); + const unprovenTargets = [...new Set(fixTargetNodeNames.filter((name) => unproven.has(name)))]; + + const level = resolveLevel({ + success: analysis.success, + hasUnreached: nodesNotReached.length > 0, + hasSimulated: simulatedNodes.length > 0, + hasUnprovenTarget: unprovenTargets.length > 0, + }); + + return { + level, + plannedNodeCount, + // Counted over the simulation plan, the same set `nodesNotReached` comes + // from. `reachedNames` covers the whole run, and the plan skips mockable + // triggers, so comparing the two produced counts like "2 of 1". + reachedNodeCount: Math.max(plannedNodeCount - nodesNotReached.length, 0), + nodesNotReached, + simulatedNodes, + pinnedNodes: [...analysis.workflowPinnedNodeNames], + unprovenTargets, + publishReady: level === 'verified', + liveTestRecommended: level === 'partial' || level === 'unproven', + }; +} + +function resolveLevel(facts: { + success: boolean; + hasUnreached: boolean; + hasSimulated: boolean; + hasUnprovenTarget: boolean; +}): VerificationClaimLevel { + if (!facts.success) return 'failed'; + // A named target that never ran for real outranks overall coverage: the user + // asked about that node, so a green run elsewhere does not answer them. + if (facts.hasUnprovenTarget) return 'unproven'; + if (facts.hasUnreached || facts.hasSimulated) return 'partial'; + return 'verified'; +} diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts index ebe80364449..3b2577bde93 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/finalize-result.ts @@ -11,7 +11,10 @@ import type { } from './types'; import type { OrchestrationContext } from '../../../types'; import { createRemediation } from '../../../workflow-loop/remediation'; -import type { RemediationMetadata } from '../../../workflow-loop/workflow-loop-state'; +import type { + RemediationMetadata, + VerificationClaim, +} from '../../../workflow-loop/workflow-loop-state'; /** * Handle the no-simulation-plan case: refuse to run because destructive nodes @@ -88,11 +91,21 @@ export async function persistVerificationOutcome(args: { workflowId: string; result: ExecutionRunResult; analysis: VerificationAnalysis; + /** Deterministic verdict for this run, persisted so later turns cannot re-litigate it. */ + claim: VerificationClaim; /** Running count of verify runs for this build, used to enforce MAX_VERIFY_ATTEMPTS. */ verifyAttempts: number; }): Promise { - const { input, context, workflowTaskService, workflowId, result, analysis, verifyAttempts } = - args; + const { + input, + context, + workflowTaskService, + workflowId, + result, + analysis, + claim, + verifyAttempts, + } = args; try { const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data); await workflowTaskService.updateBuildOutcome(input.workItemId, { @@ -103,6 +116,7 @@ export async function persistVerificationOutcome(args: { executionId: result.executionId || undefined, status: result.status, failureSignature: analysis.success ? undefined : analysis.errorMessage, + claim, evidence: { nodesExecuted: executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined, diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts index 3b156ac9448..9aeaa2cf38b 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts @@ -8,6 +8,7 @@ import { analyzeVerificationResult, buildSimulationNote, + WORKFLOW_PIN_SIMULATION_REASON, type ChatModelRecoveryOptions, type VerificationAnalysis, } from './analyze-result'; @@ -154,14 +155,28 @@ function mergeAnalyses( // Rebuild the note from the union — a node simulated only in an earlier // pass must still be disclosed. - const reachedSimulatedNodes = prepared.simulatedNodes.filter((node) => + const plannedSimulated = prepared.simulatedNodes.filter((node) => reachedNames.has(node.nodeName), ); + // Pins come from the run result, so they only exist per pass. Without this + // union a pin-fed gate run would read as live (INS-1216). + const plannedSimulatedNames = new Set(plannedSimulated.map((node) => node.nodeName)); + const workflowPinnedNodeNames = [ + ...new Set(passes.flatMap((pass) => pass.analysis.workflowPinnedNodeNames)), + ].filter((name) => !plannedSimulatedNames.has(name)); + const reachedSimulatedNodes = [ + ...plannedSimulated, + ...workflowPinnedNodeNames.map((name) => ({ + nodeName: name, + reason: WORKFLOW_PIN_SIMULATION_REASON, + })), + ]; return { success: passes.every((pass) => pass.analysis.success), reachedNames, reachedSimulatedNodes, + workflowPinnedNodeNames, nodesNotReached, remediation: failing?.analysis.remediation, nodesExecuted: [...reachedNames], diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/types.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/types.ts index e6bddc791fc..c4e1048e9f0 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/types.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/types.ts @@ -1,5 +1,8 @@ import type { ExecutionNodeError, OrchestrationContext } from '../../../types'; -import type { RemediationMetadata } from '../../../workflow-loop/workflow-loop-state'; +import type { + RemediationMetadata, + VerificationClaim, +} from '../../../workflow-loop/workflow-loop-state'; export interface VerifyToolInput { workItemId?: string; @@ -11,6 +14,8 @@ export interface VerifyToolInput { includeData?: boolean; maxDataChars?: number; fixtureOverrides?: Record>>; + /** Nodes this change is about — see the tool input description. */ + fixTargetNodeNames?: string[]; /** Nodes whose override may pin zero items — the explicit opt-in for an empty-branch test. */ allowZeroItemFixtures?: string[]; } @@ -47,6 +52,7 @@ export interface VerifyBuiltWorkflowOutput { nodeErrors?: ExecutionNodeError[]; nodesNotReached?: string[]; coverageNote?: string; + claim?: VerificationClaim; data?: Record; error?: string; remediation?: RemediationMetadata; diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts index 2ff7b1ad820..03d86efdaa7 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts @@ -11,6 +11,7 @@ import { z } from 'zod'; import type { OrchestrationContext } from '../../types'; import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result'; +import { deriveVerificationClaim } from './verification/claim'; import { handleMissingSimulationPlan, persistVerificationOutcome, @@ -19,7 +20,10 @@ import { prepareVerificationRun } from './verification/prepare-run'; import { reconcileStaleCredentialPlan } from './verification/reconcile-plan'; import { resolveVerificationTarget } from './verification/resolve-target'; import { runScriptedGateVerification } from './verification/scripted-gate-run'; -import { executionNodeErrorSchema } from '../../workflow-loop/workflow-loop-state'; +import { + executionNodeErrorSchema, + verificationClaimSchema, +} from '../../workflow-loop/workflow-loop-state'; import { collectChatModelRecoveryContext } from '../workflows/chat-model-validation'; const DEFAULT_NODE_PREVIEW_CHARS = 600; @@ -81,6 +85,16 @@ export const verifyBuiltWorkflowInputSchema = z.object({ 'Optional per-run output fixtures keyed by node name. Only nodes already classified as simulated in the build outcome may be overridden. Use this for alternate deterministic scenarios, not raw trigger input. ' + 'An empty array is rejected unless the node is also listed in `allowZeroItemFixtures`.', ), + fixTargetNodeNames: z + .array(z.string()) + .optional() + .describe( + 'Node names this change is about — the node the user reported as failing, or the node ' + + 'you just repaired. The verdict shown to the user is downgraded to "changed but ' + + 'unverified" when any of these was never reached or had simulated output, so a green ' + + 'run elsewhere cannot pass as proof for them. Pass these whenever you are fixing a ' + + 'specific node rather than building from scratch.', + ), allowZeroItemFixtures: z .array(z.string()) .optional() @@ -125,6 +139,7 @@ const verifyBuiltWorkflowOutputSchema = z.object({ nodeErrors: z.array(executionNodeErrorSchema).optional(), nodesNotReached: z.array(z.string()).optional(), coverageNote: z.string().optional(), + claim: verificationClaimSchema.optional(), data: z.record(z.unknown()).optional(), error: z.string().optional(), remediation: remediationOutputSchema, @@ -238,6 +253,22 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { }; })(); + // The repair target from an earlier verdict counts even when the model + // omits it here — that is exactly the turn where it stops mentioning it. + const fixTargetNodeNames = [ + ...new Set( + [ + ...(resolvedInput.fixTargetNodeNames ?? []), + target.stateBefore?.lastFailedNodeName, + ].filter((name): name is string => name !== undefined), + ), + ]; + const claim = deriveVerificationClaim({ + analysis, + plannedNodeCount: buildOutcome.nodeSimulationPlan?.length ?? 0, + fixTargetNodeNames, + }); + await persistVerificationOutcome({ input: resolvedInput, context, @@ -245,6 +276,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { workflowId, result, analysis, + claim, verifyAttempts: (buildOutcome.verifyAttempts ?? 0) + 1, }); @@ -254,6 +286,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { resolvedWorkItemId: resolvedInput.workItemId, executionId: result.executionId || undefined, success: analysis.success, + claim, status: result.status, nodesExecuted: analysis.nodesExecuted, lastNodeExecuted: result.lastNodeExecuted, diff --git a/packages/@n8n/instance-ai/src/tools/workflows.tool.ts b/packages/@n8n/instance-ai/src/tools/workflows.tool.ts index 9b594e7c255..01535963361 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows.tool.ts @@ -24,6 +24,7 @@ import { TEMPLATABLE_PLAIN_AUTH_TYPES, } from './credentials.tool'; import { formatTimestamp } from '../utils/format-timestamp'; +import { formatClaimDisclosure } from '../workflow-loop/render-claim'; import { isSetupPanelEnabled } from './workflows/setup-items'; import { describeSetupItem, @@ -310,6 +311,14 @@ const publishBaseAction = z.object({ .describe('Publish a workflow version to production (omit versionId for latest draft)'), workflowId: z.string().describe('ID of the workflow'), versionId: z.string().optional().describe('Version ID'), + acknowledgeUnverified: z + .boolean() + .optional() + .describe( + 'Set true only after you told the user the workflow is not fully verified and they still ' + + 'asked to publish. Publishing is refused without this while the latest verification left ' + + 'nodes unreached or simulated. Never set it to skip the disclosure.', + ), }); const publishExtendedAction = publishBaseAction.extend({ @@ -1377,6 +1386,49 @@ async function resolveSetupScopeNodeNames( } } +/** + * Coverage disclosure for the workflow's latest verification, or undefined when + * it was fully verified or no claim exists. Absent evidence never blocks: a + * workflow built before this record, or outside the assistant, is unknown + * rather than unverified. + */ +async function resolveUnverifiedPublishDisclosure( + context: InstanceAiContext, + workflowId: string, +): Promise { + const workflowTaskService = context.workflowBuildContext?.workflowTaskService; + if (!workflowTaskService) return undefined; + try { + const outcome = await workflowTaskService.getLatestBuildOutcomeForWorkflow(workflowId); + const verification = outcome?.verification; + const claim = verification?.claim; + if (claim) return claim.publishReady ? undefined : formatClaimDisclosure(claim); + + // A record with no claim means verification ran and could not produce one + // — it was refused for want of a simulation plan, or it failed before it + // reached a verdict. That is a failed verification, not an unverified + // workflow, and it must not pass as an absent record. + if (verification?.attempted) { + const cause = verification.failureSignature ?? verification.evidence?.errorMessage; + return ( + 'Verification ran but produced no verdict, so nothing about this workflow is proven.' + + (cause ? ` It reported: ${cause}` : '') + ); + } + + // No record at all: the workflow is unknown, not unverified. Blocking here + // would stop publishing every workflow built before this record existed. + return undefined; + } catch (error) { + // Fail open: a storage hiccup must not block a publish the user asked for. + context.logger.warn('Failed to resolve the verification claim before publishing', { + workflowId, + error: error instanceof Error ? error.message : String(error), + }); + return undefined; + } +} + const SETUP_PANEL_ANNOUNCED_GUIDANCE = 'The setup panel next to the chat now lists what this workflow still needs (`open`); nothing is ' + 'waiting on you and no card is open. Finish your turn now: tell the user in one or two sentences ' + @@ -1754,7 +1806,23 @@ async function handlePublish( } const supportingWorkflowIds = await resolveSupportingWorkflowIds(context, input.workflowId); - const needsApproval = context.permissions?.publishWorkflow !== 'always_allow'; + const unverifiedDisclosure = await resolveUnverifiedPublishDisclosure(context, input.workflowId); + + const needsApproval = + context.permissions?.publishWorkflow !== 'always_allow' || unverifiedDisclosure !== undefined; + + if (unverifiedDisclosure && input.acknowledgeUnverified !== true) { + return { + success: false, + denied: true, + reason: 'not_verified', + verificationDisclosure: unverifiedDisclosure, + guidance: + `This workflow is not fully verified. ${unverifiedDisclosure} ` + + 'Tell the user exactly this, and offer a live end-to-end test. Publish only if they still ' + + 'ask for it, by calling publish again with `acknowledgeUnverified: true`.', + }; + } if (needsApproval && (resumeData === undefined || resumeData === null)) { const workflowName = await resolveWorkflowName(context, input.workflowId); @@ -1762,12 +1830,15 @@ async function handlePublish( supportingWorkflowIds.length > 0 ? ` and ${String(supportingWorkflowIds.length)} referenced supporting workflow(s)` : ''; + const target = input.versionId + ? `Publish version ${input.versionId} of ${workflowName} (ID: ${input.workflowId})${dependencyNote}` + : `Publish ${workflowName} (ID: ${input.workflowId})${dependencyNote}`; return await ctx.suspend({ requestId: nanoid(), - message: input.versionId - ? `Publish version ${input.versionId} of ${workflowName} (ID: ${input.workflowId})${dependencyNote}` - : `Publish ${workflowName} (ID: ${input.workflowId})${dependencyNote}`, + // The user has to read this to approve, so the disclosure lands even if + // the assistant's own message left it out. + message: unverifiedDisclosure ? `${target}\n\n${unverifiedDisclosure}` : target, severity: 'warning' as const, }); } diff --git a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/guidance.test.ts b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/guidance.test.ts index 8f463fdb2b6..80ad822c378 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/guidance.test.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/guidance.test.ts @@ -1,19 +1,111 @@ import { formatWorkflowLoopGuidance } from '../guidance'; -import type { WorkflowLoopAction } from '../workflow-loop-state'; +import type { VerificationClaim, WorkflowLoopAction } from '../workflow-loop-state'; + +function makeClaim(overrides: Partial = {}): VerificationClaim { + return { + level: 'verified', + plannedNodeCount: 3, + reachedNodeCount: 3, + nodesNotReached: [], + simulatedNodes: [], + pinnedNodes: [], + unprovenTargets: [], + publishReady: true, + liveTestRecommended: false, + ...overrides, + }; +} describe('formatWorkflowLoopGuidance', () => { // ── done ──────────────────────────────────────────────────────────────────── describe('action type "done"', () => { - it('should report completion without workflowId', () => { + it('should not claim verification when no run recorded a claim', () => { + // A `verified` verdict reported without verifying leaves no claim. This + // sentence used to say "Workflow verified successfully", which handed + // the model the exact wording with no run evidence behind it. const action: WorkflowLoopAction = { type: 'done', summary: 'All good', }; const result = formatWorkflowLoopGuidance(action); + expect(result).not.toContain('Workflow verified successfully'); + expect(result).toContain('No automatic verification evidence is recorded'); + expect(result).toContain('do NOT call the workflow verified'); + expect(result).not.toContain('Workflow ID:'); + }); + + it('should not claim success when the claim is partial', () => { + const action: WorkflowLoopAction = { + type: 'done', + summary: 'All good', + claim: makeClaim({ + level: 'partial', + reachedNodeCount: 5, + plannedNodeCount: 12, + nodesNotReached: ['Send Email', 'Log Row'], + publishReady: false, + liveTestRecommended: true, + }), + }; + const result = formatWorkflowLoopGuidance(action); + expect(result).not.toContain('Workflow verified successfully'); + expect(result).toContain('NOT fully verified'); + expect(result).toContain('2 of 12 node(s) were never reached'); + expect(result).toContain('Send Email, Log Row'); + expect(result).toContain('Do NOT offer to publish it.'); + expect(result).toContain('Offer a live end-to-end test'); + }); + + it('should name the unproven fix target when the claim is unproven', () => { + const action: WorkflowLoopAction = { + type: 'done', + summary: 'Patched', + claim: makeClaim({ + level: 'unproven', + plannedNodeCount: 4, + reachedNodeCount: 4, + unprovenTargets: ['Send Email'], + simulatedNodes: [{ nodeName: 'Send Email', reason: 'Sends a message' }], + publishReady: false, + liveTestRecommended: true, + }), + }; + const result = formatWorkflowLoopGuidance(action); + expect(result).not.toContain('Workflow verified successfully'); + expect(result).toContain('Changed but NOT verified'); + expect(result).toContain('never proven: Send Email'); + expect(result).toContain('nothing real happened at: Send Email'); + }); + + it('should keep the verified wording when the claim is verified', () => { + const action: WorkflowLoopAction = { + type: 'done', + summary: 'All good', + claim: makeClaim(), + }; + const result = formatWorkflowLoopGuidance(action); expect(result).toContain('Workflow verified successfully'); expect(result).toContain('Report completion'); - expect(result).not.toContain('Workflow ID:'); + }); + + it('should downgrade the mocked-credential guidance too', () => { + const action: WorkflowLoopAction = { + type: 'done', + summary: 'All good', + workflowId: 'wf-123', + mockedCredentialTypes: ['slackApi'], + claim: makeClaim({ + level: 'partial', + nodesNotReached: ['Send Email'], + publishReady: false, + liveTestRecommended: true, + }), + }; + const result = formatWorkflowLoopGuidance(action); + expect(result).not.toContain('Workflow verified successfully'); + expect(result).toContain('NOT fully verified'); + expect(result).toContain('workflows(action="setup")'); }); it('should include workflowId when present', () => { @@ -44,7 +136,7 @@ describe('formatWorkflowLoopGuidance', () => { }; const result = formatWorkflowLoopGuidance(action); expect(result).not.toContain('credentials(action="setup")'); - expect(result).toContain('Report completion'); + expect(result).toContain('Report the outcome'); }); it('should include credential instructions when mockedCredentialTypes has entries', () => { diff --git a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-loop-controller.test.ts b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-loop-controller.test.ts index ce912dd413c..110bd5553a3 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-loop-controller.test.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-loop-controller.test.ts @@ -451,6 +451,64 @@ describe('handleBuildOutcome', () => { // ── handleVerificationVerdict ─────────────────────────────────────────────── describe('handleVerificationVerdict', () => { + describe('deterministic claim', () => { + it('carries a downgraded claim onto the done action instead of blocking the loop', () => { + const state = makeState({ phase: 'verifying' }); + const { state: next, action } = handleVerificationVerdict( + state, + [], + makeVerdict({ + verdict: 'verified', + claim: { + level: 'partial', + plannedNodeCount: 12, + reachedNodeCount: 5, + nodesNotReached: ['Send Email'], + simulatedNodes: [], + pinnedNodes: [], + unprovenTargets: [], + publishReady: false, + liveTestRecommended: true, + }, + }), + ); + + // The loop must still settle — re-verifying replays the same coverage. + expect(next.phase).toBe('done'); + expect(next.status).toBe('completed'); + expect(action.type).toBe('done'); + if (action.type !== 'done') throw new Error('expected done'); + expect(action.claim?.level).toBe('partial'); + expect(action.claim?.publishReady).toBe(false); + }); + + it('records the repair target so a later run can tell whether it was proven', () => { + const state = makeState({ phase: 'verifying' }); + const { state: next } = handleVerificationVerdict( + state, + [], + makeVerdict({ + verdict: 'needs_patch', + failedNodeName: 'Send Email', + diagnosis: 'Wrong channel', + }), + ); + + expect(next.lastFailedNodeName).toBe('Send Email'); + }); + + it('keeps a known repair target when a later rebuild names no node', () => { + const state = makeState({ phase: 'verifying', lastFailedNodeName: 'Send Email' }); + const { state: next } = handleVerificationVerdict( + state, + [], + makeVerdict({ verdict: 'needs_rebuild', diagnosis: 'Structural repair' }), + ); + + expect(next.lastFailedNodeName).toBe('Send Email'); + }); + }); + it('ignores stale verification verdicts from a different run without resetting state', () => { const state = makeState({ runId: 'run_current', diff --git a/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts b/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts index 91cbf42d0fd..f4cced7c099 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/guidance.ts @@ -1,4 +1,5 @@ -import type { WorkflowLoopAction } from './workflow-loop-state'; +import { describeClaimCoverage, formatClaimHeadline } from './render-claim'; +import type { VerificationClaim, WorkflowLoopAction } from './workflow-loop-state'; export interface WorkflowLoopGuidanceOptions { workItemId?: string; @@ -6,6 +7,42 @@ export interface WorkflowLoopGuidanceOptions { setupPanelEnabled?: boolean; } +/** + * Lead sentence for a completed build. Derived from the claim so the guidance + * never tells the model a partially covered run was verified — that sentence + * was the strongest license for the false success claims in AIA-31. + */ +function formatClaimLead(claim: VerificationClaim | undefined): string { + if (claim?.level === 'verified') return 'Workflow verified successfully.'; + + // No claim means no verification run recorded a verdict for this build — a + // trigger-only workflow, or a verdict reported without verifying. Saying + // "verified successfully" here let a model reach that sentence with no run + // evidence at all. A live run the model inspected is still valid evidence, + // so ask it to name that rather than refusing the claim outright. + if (claim === undefined) { + return ( + 'Build complete. No automatic verification evidence is recorded for this workflow. ' + + 'If you are relying on a live run you inspected, say which execution. ' + + 'Otherwise do NOT call the workflow verified, tested, or working.' + ); + } + + // The claim is the only honest reading of the run, and nothing else tells + // the user how strong it is, so these rules carry the whole disclosure. + const rules = [ + 'Do NOT call the workflow verified, tested, working, or ready to publish.', + 'Do NOT offer to publish it.', + claim.liveTestRecommended ? 'Offer a live end-to-end test instead.' : '', + ].filter((rule) => rule !== ''); + + return [formatClaimHeadline(claim), ...describeClaimCoverage(claim), ...rules].join(' '); +} + +function isVerifiedClaim(claim: VerificationClaim | undefined): boolean { + return claim?.level === 'verified'; +} + function formatSourceFileInstruction(sourceFilePath: string | undefined): string { if (!sourceFilePath) { return 'edit the workspace source file, then call `build-workflow` with that filePath'; @@ -24,9 +61,11 @@ export function formatWorkflowLoopGuidance( case 'continue_building': return `BUILD FAILED: ${action.reason}. Fix the workflow source file: ${formatSourceFileInstruction(action.sourceFilePath)}.`; case 'done': { + const claimLead = formatClaimLead(action.claim); if (action.setupSkippedByUser) { return ( - 'Workflow verified successfully. The credentials it still needs are ones the user ' + + claimLead + + ' The credentials it still needs are ones the user ' + 'skipped earlier in this conversation, so do NOT open the setup card again. Tell them ' + 'which parts stay unconfigured and what that means when the workflow runs, and offer ' + 'to set them up whenever they want.' @@ -44,14 +83,17 @@ export function formatWorkflowLoopGuidance( ); } return ( - 'Workflow verified successfully with temporary mock data. ' + + `${claimLead} It still uses temporary mock data. ` + `Call \`workflows(action="setup")\` with workflowId "${action.workflowId ?? 'unknown'}" ` + 'to open the inline setup card in the AI Assistant panel for credentials, parameters, and triggers. ' + 'Do not tell the user to open the editor, use the canvas, or click a Setup button. ' + 'Do not call `credentials(action="setup")` or `apply-workflow-credentials` — `workflows(action="setup")` handles everything.' ); } - return `Workflow verified successfully. Report completion to the user.${action.workflowId ? ` Workflow ID: ${action.workflowId}` : ''}`; + const closing = isVerifiedClaim(action.claim) + ? 'Report completion to the user.' + : 'Report the outcome to the user.'; + return `${claimLead} ${closing}${action.workflowId ? ` Workflow ID: ${action.workflowId}` : ''}`; } case 'verify': return ( diff --git a/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts b/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts new file mode 100644 index 00000000000..629774a60ed --- /dev/null +++ b/packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts @@ -0,0 +1,59 @@ +/** + * User-facing wording for a verification claim. + * + * Every surface that discloses coverage renders from here, so the guidance the + * model reads and the publish approval the user clicks state the same facts. + * A claim at `verified` needs no disclosure. + */ + +import type { VerificationClaim } from './workflow-loop-state'; + +export function formatClaimNodeList(names: readonly string[], max = 8): string { + if (names.length <= max) return names.join(', '); + return `${names.slice(0, max).join(', ')} and ${String(names.length - max)} more`; +} + +export function formatClaimHeadline(claim: VerificationClaim): string { + switch (claim.level) { + case 'verified': + return 'Verified end to end.'; + case 'unproven': + return ( + 'Changed but NOT verified. The node(s) this change was about were never proven: ' + + `${formatClaimNodeList(claim.unprovenTargets)}.` + ); + case 'partial': + return 'Ran without errors, but NOT fully verified.'; + case 'failed': + return 'Verification did not pass.'; + } +} + +export function describeClaimCoverage(claim: VerificationClaim): string[] { + const facts: string[] = []; + + if (claim.nodesNotReached.length > 0) { + facts.push( + `${String(claim.nodesNotReached.length)} of ${String(claim.plannedNodeCount)} node(s) were ` + + `never reached, so they are UNVERIFIED: ${formatClaimNodeList(claim.nodesNotReached)}.`, + ); + } + if (claim.simulatedNodes.length > 0) { + facts.push( + 'Output was simulated, so nothing real happened at: ' + + `${formatClaimNodeList(claim.simulatedNodes.map((node) => node.nodeName))}.`, + ); + } + if (claim.pinnedNodes.length > 0) { + facts.push( + `Some output came from pin data saved on the workflow: ${formatClaimNodeList(claim.pinnedNodes)}. ` + + 'Remove those pins for a live test.', + ); + } + return facts; +} + +export function formatClaimDisclosure(claim: VerificationClaim): string | undefined { + if (claim.level === 'verified') return undefined; + return [formatClaimHeadline(claim), ...describeClaimCoverage(claim)].join(' '); +} diff --git a/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-controller.ts b/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-controller.ts index 4ca00ece77d..a474c4d9200 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-controller.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-controller.ts @@ -273,6 +273,9 @@ export function handleVerificationVerdict( switch (verdict.verdict) { case 'verified': { + // The loop still completes on a partial run — re-issuing verification + // would replay the same coverage forever. Only the claim is downgraded, + // so `done` carries an honest verdict instead of a blocked state. attempt.result = 'success'; return { state: { @@ -287,6 +290,7 @@ export function handleVerificationVerdict( type: 'done', workflowId: verdict.workflowId, summary: verdict.summary, + claim: verdict.claim, mockedCredentialTypes: normalizedState.mockedCredentialTypes, hasUnresolvedPlaceholders: normalizedState.hasUnresolvedPlaceholders, setupSkippedByUser: normalizedState.setupSkippedByUser, @@ -470,6 +474,7 @@ function escalateToRepair( status: 'active', rebuildAttempts: state.rebuildAttempts + 1, lastFailureSignature: verdict.failureSignature, + lastFailedNodeName: verdict.failedNodeName ?? state.lastFailedNodeName, lastExecutionId: verdict.executionId, lastWorkflowInspection: verdict.workflowInspection, lastRemediation: remediation, diff --git a/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts b/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts index 402d5a56aa6..8001f52f4ce 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/workflow-loop-state.ts @@ -82,6 +82,7 @@ export const workflowLoopStateSchema = z.object({ lastTaskId: z.string().optional(), lastExecutionId: z.string().optional(), lastFailureSignature: z.string().optional(), + lastFailedNodeName: z.string().optional(), lastWorkflowInspection: z.string().optional(), rebuildAttempts: z.number().int().min(0), /** Credential types that were mocked during build (persisted across phases). */ @@ -153,6 +154,42 @@ export const executionNodeErrorSchema = z.object({ message: z.string().optional(), }); +/** + * Strength of the claim a verification run supports. `verified` means every + * planned node ran for real; anything less must not be reported as verified. + * See `tools/orchestration/verification/claim.ts` for the derivation. + */ +export const verificationClaimLevelSchema = z.enum(['verified', 'partial', 'unproven', 'failed']); + +export type VerificationClaimLevel = z.infer; + +/** + * The deterministic verdict for one verification run. Backend-only: it shapes + * the tool result the model reads and gates the publish offer. It is + * deliberately not sent to the client — see INS-1308. + */ +export const verificationClaimSchema = z.object({ + level: verificationClaimLevelSchema, + /** Nodes the verification plan covers. Excludes triggers the classifier skips. */ + plannedNodeCount: z.number().int().min(0), + /** Planned nodes the run reached. Same set as `plannedNodeCount`. */ + reachedNodeCount: z.number().int().min(0), + nodesNotReached: z.array(z.string()), + /** Reached nodes whose output was mocked, so the run proves nothing about them. */ + simulatedNodes: z.array(z.object({ nodeName: z.string(), reason: z.string() })), + /** Pin-fed subset of `simulatedNodes` — these need the pin removed for a live test. */ + pinnedNodes: z.array(z.string()), + /** + * Fix targets still unreached or simulated — the reason this run cannot + * claim `verified` even when it ended without an error. + */ + unprovenTargets: z.array(z.string()), + publishReady: z.boolean(), + liveTestRecommended: z.boolean(), +}); + +export type VerificationClaim = z.infer; + /** * Structured verification evidence the builder captures when it runs * `verify-built-workflow`. Downstream checkpoint runs read this and skip @@ -164,6 +201,7 @@ export const workflowVerificationEvidenceSchema = z.object({ executionId: z.string().optional(), status: z.enum(['success', 'error', 'waiting', 'running', 'unknown']).optional(), failureSignature: z.string().optional(), + claim: verificationClaimSchema.optional(), evidence: z .object({ nodesExecuted: z.array(z.string()).optional(), @@ -454,6 +492,7 @@ export const verificationResultSchema = z.object({ workflowId: z.string(), executionId: z.string().optional(), verdict: verificationVerdictSchema, + claim: verificationClaimSchema.optional(), workflowInspection: z.string().optional(), failureSignature: z.string().optional(), failedNodeName: z.string().optional(), @@ -485,6 +524,7 @@ export type WorkflowLoopAction = type: 'done'; workflowId?: string; summary: string; + claim?: VerificationClaim; mockedCredentialTypes?: string[]; hasUnresolvedPlaceholders?: boolean; setupSkippedByUser?: boolean;