mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 13:39:44 +02:00
feat(core): Stop Instance AI reporting a simulated run as fully verified (#38060)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
eac5f9eccf
commit
efb5547368
|
|
@ -201,6 +201,14 @@ publish-readiness evidence. If the user explicitly asks to publish before a
|
|||
live execution succeeds, warn that the live path remains untested, then follow
|
||||
the requested publish flow.
|
||||
|
||||
`workflows(action="publish")` enforces this. While the latest verification left
|
||||
nodes unreached or simulated, the call is refused and returns
|
||||
`verificationDisclosure` — the coverage facts, generated from the run. Relay
|
||||
those facts to the user and offer a live end-to-end test. Publish only if they
|
||||
still ask, by calling publish again with `acknowledgeUnverified: true`. Never
|
||||
set that flag to skip the disclosure. The user also sees the same facts in the
|
||||
publish approval prompt, so a summary that contradicts them is visible to them.
|
||||
|
||||
Execution evidence can come from a run you started or a run the user started.
|
||||
If the user says they ran the workflow manually, call
|
||||
`executions(action="list", workflowId)`, identify the relevant run, and inspect
|
||||
|
|
@ -476,9 +484,24 @@ result.
|
|||
|
||||
## Claiming success
|
||||
|
||||
Do not tell the user a workflow is "fixed", "verified", "tested", "working", or
|
||||
has "no errors" unless you have a passing `verify-built-workflow`,
|
||||
`executions(action="run")`, or inspected user-run execution that exercised the
|
||||
`verify-built-workflow` returns a `claim`, and its `level` decides what you may
|
||||
say:
|
||||
|
||||
- `verified` — you may call the workflow verified, tested, or working.
|
||||
- `partial`, `unproven`, or `failed` — you may NOT. Name what is unconfirmed
|
||||
instead.
|
||||
|
||||
**`success: true` does not mean verified.** It means the run ended without an
|
||||
error, and a run with every write simulated also ends without an error. Read
|
||||
`claim.level`, not `success`.
|
||||
|
||||
Nothing else tells the user how strong the claim is. Your message is the only
|
||||
place they learn it, so name the unreached and simulated nodes yourself, and say
|
||||
what stays unconfigured and what that means when the workflow runs.
|
||||
|
||||
Without a claim, do not tell the user a workflow is "fixed", "verified",
|
||||
"tested", "working", or has "no errors" unless you have a passing
|
||||
`executions(action="run")` or an inspected user-run execution that exercised the
|
||||
path being claimed. Do not call a workflow "ready to use" or "ready to publish"
|
||||
unless a passing execution met the publish-readiness requirement above. A
|
||||
successful `build-workflow`/save, a static `workflows(action="validate")`, or
|
||||
|
|
|
|||
|
|
@ -1753,6 +1753,236 @@ describe('workflows tool', () => {
|
|||
});
|
||||
});
|
||||
|
||||
function contextWithClaim(claim: unknown, permissions?: Partial<InstanceAiPermissions>) {
|
||||
return createMockContext({
|
||||
...(permissions ? { permissions } : {}),
|
||||
workflowBuildContext: {
|
||||
threadId: 't1',
|
||||
runId: 'run-1',
|
||||
taskId: 'task-1',
|
||||
workItemId: 'wi-1',
|
||||
workflowTaskService: {
|
||||
getLatestBuildOutcomeForWorkflow: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ runId: 'run-1', verification: { claim } }),
|
||||
},
|
||||
} as never,
|
||||
});
|
||||
}
|
||||
|
||||
const partialClaim = {
|
||||
level: 'partial',
|
||||
plannedNodeCount: 12,
|
||||
reachedNodeCount: 5,
|
||||
nodesNotReached: ['Send Email', 'Log Row'],
|
||||
simulatedNodes: [{ nodeName: 'Create Event', reason: 'Creates a record' }],
|
||||
pinnedNodes: [],
|
||||
unprovenTargets: [],
|
||||
publishReady: false,
|
||||
liveTestRecommended: true,
|
||||
};
|
||||
|
||||
it('should refuse to publish an unverified workflow and disclose the coverage', async () => {
|
||||
const context = contextWithClaim(partialClaim);
|
||||
const suspend = vi.fn();
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
|
||||
suspend,
|
||||
resumeData: undefined,
|
||||
} as never);
|
||||
|
||||
expect(context.workflowService.publish).not.toHaveBeenCalled();
|
||||
// Refused before the dialog, so an `always_allow` instance is covered too.
|
||||
expect(suspend).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' });
|
||||
const { verificationDisclosure } = result as { verificationDisclosure: string };
|
||||
expect(verificationDisclosure).toContain('NOT fully verified');
|
||||
expect(verificationDisclosure).toContain('Send Email, Log Row');
|
||||
expect(verificationDisclosure).toContain('Create Event');
|
||||
});
|
||||
|
||||
it('should refuse an unverified publish even when approval is not required', async () => {
|
||||
const context = contextWithClaim(partialClaim, { publishWorkflow: 'always_allow' });
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {} as never);
|
||||
|
||||
expect(context.workflowService.publish).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' });
|
||||
});
|
||||
|
||||
it('still asks the user to approve an acknowledged unverified publish under always_allow', async () => {
|
||||
// The acknowledgement is only the model's word that the user asked, so
|
||||
// it must not be the sole gate: without the dialog an `always_allow`
|
||||
// instance would publish an unverified workflow with no disclosure
|
||||
// anywhere the user could see.
|
||||
const context = contextWithClaim(partialClaim, { publishWorkflow: 'always_allow' });
|
||||
const suspend = vi.fn();
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
await executeTool(
|
||||
tool,
|
||||
{ action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true },
|
||||
{ suspend, resumeData: undefined } as never,
|
||||
);
|
||||
|
||||
expect(context.workflowService.publish).not.toHaveBeenCalled();
|
||||
expect(suspend).toHaveBeenCalledTimes(1);
|
||||
const { message } = (suspend as Mock).mock.calls[0][0] as { message: string };
|
||||
expect(message).toContain('NOT fully verified');
|
||||
});
|
||||
|
||||
it('keeps always_allow routine for a verified publish', async () => {
|
||||
const context = contextWithClaim(
|
||||
{ ...partialClaim, level: 'verified', publishReady: true },
|
||||
{ publishWorkflow: 'always_allow' },
|
||||
);
|
||||
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
|
||||
const suspend = vi.fn();
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
|
||||
suspend,
|
||||
} as never);
|
||||
|
||||
expect(suspend).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({ success: true });
|
||||
});
|
||||
|
||||
it('should disclose the coverage in the approval prompt once acknowledged', async () => {
|
||||
const context = contextWithClaim(partialClaim);
|
||||
const suspend = vi.fn();
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
await executeTool(
|
||||
tool,
|
||||
{ action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true },
|
||||
{ suspend, resumeData: undefined } as never,
|
||||
);
|
||||
|
||||
expect(suspend).toHaveBeenCalledTimes(1);
|
||||
const { message } = (suspend as Mock).mock.calls[0][0] as { message: string };
|
||||
expect(message).toContain('Publish');
|
||||
expect(message).toContain('NOT fully verified');
|
||||
expect(message).toContain('Send Email, Log Row');
|
||||
});
|
||||
|
||||
it('should publish once acknowledged and approved', async () => {
|
||||
const context = contextWithClaim(partialClaim);
|
||||
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(
|
||||
tool,
|
||||
{ action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true },
|
||||
{ resumeData: { approved: true } } as never,
|
||||
);
|
||||
|
||||
expect(context.workflowService.publish).toHaveBeenCalledWith('wf1', {
|
||||
versionId: undefined,
|
||||
});
|
||||
expect(result).toMatchObject({ success: true, activeVersionId: 'v2' });
|
||||
});
|
||||
|
||||
it('should not gate a fully verified workflow', async () => {
|
||||
const context = contextWithClaim({ ...partialClaim, level: 'verified', publishReady: true });
|
||||
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
|
||||
const suspend = vi.fn();
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
|
||||
suspend,
|
||||
resumeData: undefined,
|
||||
} as never);
|
||||
|
||||
const { message } = (suspend as Mock).mock.calls[0][0] as { message: string };
|
||||
expect(message).not.toContain('NOT');
|
||||
});
|
||||
|
||||
it('refuses a publish when verification ran but produced no verdict', async () => {
|
||||
// `handleMissingSimulationPlan` writes an attempted record with no
|
||||
// claim. Reading that as an absent record let a failed verification
|
||||
// publish as freely as a workflow nobody ever verified.
|
||||
const context = createMockContext({
|
||||
workflowBuildContext: {
|
||||
threadId: 't1',
|
||||
runId: 'run-1',
|
||||
taskId: 'task-1',
|
||||
workItemId: 'wi-1',
|
||||
workflowTaskService: {
|
||||
getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue({
|
||||
runId: 'run-1',
|
||||
verification: {
|
||||
attempted: true,
|
||||
success: false,
|
||||
failureSignature: 'missing_simulation_plan',
|
||||
},
|
||||
}),
|
||||
},
|
||||
} as never,
|
||||
});
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
|
||||
suspend: vi.fn(),
|
||||
resumeData: undefined,
|
||||
} as never);
|
||||
|
||||
expect(context.workflowService.publish).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' });
|
||||
const { verificationDisclosure } = result as { verificationDisclosure: string };
|
||||
expect(verificationDisclosure).toContain('produced no verdict');
|
||||
expect(verificationDisclosure).toContain('missing_simulation_plan');
|
||||
});
|
||||
|
||||
it('does not gate a workflow that was never verified', async () => {
|
||||
// An absent record is unknown, not unverified. Gating it would stop
|
||||
// publishing every workflow built before the claim existed.
|
||||
const context = createMockContext({
|
||||
workflowBuildContext: {
|
||||
threadId: 't1',
|
||||
runId: 'run-1',
|
||||
taskId: 'task-1',
|
||||
workItemId: 'wi-1',
|
||||
workflowTaskService: {
|
||||
getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue({ runId: 'run-1' }),
|
||||
},
|
||||
} as never,
|
||||
});
|
||||
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
|
||||
resumeData: { approved: true },
|
||||
} as never);
|
||||
|
||||
expect(result).toMatchObject({ success: true });
|
||||
});
|
||||
|
||||
it('should fail open when the build outcome cannot be read', async () => {
|
||||
const context = createMockContext({
|
||||
workflowBuildContext: {
|
||||
threadId: 't1',
|
||||
runId: 'run-1',
|
||||
taskId: 'task-1',
|
||||
workItemId: 'wi-1',
|
||||
workflowTaskService: {
|
||||
getLatestBuildOutcomeForWorkflow: vi.fn().mockRejectedValue(new Error('storage down')),
|
||||
},
|
||||
} as never,
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() } as never,
|
||||
});
|
||||
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
|
||||
|
||||
const tool = createWorkflowsTool(context, 'full');
|
||||
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
|
||||
resumeData: { approved: true },
|
||||
} as never);
|
||||
|
||||
expect(result).toMatchObject({ success: true });
|
||||
});
|
||||
|
||||
it('should suspend for confirmation and then publish when approved', async () => {
|
||||
const context = createMockContext();
|
||||
(context.workflowService.publish as Mock).mockResolvedValue({
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ describe('report-verification-verdict tool', () => {
|
|||
expect((result as { guidance: string }).guidance).toContain('Error');
|
||||
});
|
||||
|
||||
it('returns done guidance when verdict is verified', async () => {
|
||||
it('forwards a verified verdict and reports the workflow', async () => {
|
||||
const doneAction: WorkflowLoopAction = {
|
||||
type: 'done',
|
||||
workflowId: 'wf-123',
|
||||
|
|
@ -83,10 +83,66 @@ describe('report-verification-verdict tool', () => {
|
|||
'Inspected persisted workflow wf-123; the saved graph matches the requested outcome.',
|
||||
}),
|
||||
);
|
||||
expect((result as { guidance: string }).guidance).toContain('verified successfully');
|
||||
expect((result as { guidance: string }).guidance).toContain('wf-123');
|
||||
});
|
||||
|
||||
it('forwards no claim when no run recorded one', async () => {
|
||||
// `getBuildOutcome` returns nothing, so the tool must forward `claim:
|
||||
// undefined` and let the controller decide. Asserting the guidance here
|
||||
// would prove nothing: it is rendered from the mocked action, so it stays
|
||||
// claimless even if the tool invented a verified claim. The wording for a
|
||||
// missing claim is covered in `workflow-loop/__tests__/guidance.test.ts`.
|
||||
const workflowTaskService = createWorkflowTaskService(
|
||||
vi.fn().mockResolvedValue({ type: 'done', workflowId: 'wf-123', summary: 'All good' }),
|
||||
);
|
||||
const context = createMockContext({ workflowTaskService });
|
||||
const tool = createReportVerificationVerdictTool(context);
|
||||
|
||||
await executeTool(tool, baseInput, {} as never);
|
||||
|
||||
expect(workflowTaskService.getBuildOutcome).toHaveBeenCalledWith('wi_test1234');
|
||||
const [reported] = workflowTaskService.reportVerificationVerdict.mock.calls[0] as [
|
||||
{ verdict: string; claim?: unknown },
|
||||
];
|
||||
expect(reported.claim).toBeUndefined();
|
||||
// It does not refuse the verdict: a live run the model inspected is still
|
||||
// valid evidence, so the controller settles and the guidance carries the
|
||||
// caveat.
|
||||
expect(reported.verdict).toBe('verified');
|
||||
});
|
||||
|
||||
it('keeps the verified wording when a run recorded a verified claim', async () => {
|
||||
const claim = {
|
||||
level: 'verified' as const,
|
||||
plannedNodeCount: 2,
|
||||
reachedNodeCount: 2,
|
||||
nodesNotReached: [],
|
||||
simulatedNodes: [],
|
||||
pinnedNodes: [],
|
||||
unprovenTargets: [],
|
||||
publishReady: true,
|
||||
liveTestRecommended: false,
|
||||
};
|
||||
const workflowTaskService = createWorkflowTaskService(
|
||||
vi.fn().mockResolvedValue({
|
||||
type: 'done',
|
||||
workflowId: 'wf-123',
|
||||
summary: 'All good',
|
||||
claim,
|
||||
}),
|
||||
);
|
||||
workflowTaskService.getBuildOutcome.mockResolvedValue({ verification: { claim } });
|
||||
const context = createMockContext({ workflowTaskService });
|
||||
const tool = createReportVerificationVerdictTool(context);
|
||||
|
||||
const result = await executeTool(tool, baseInput, {} as never);
|
||||
|
||||
expect(workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ claim }),
|
||||
);
|
||||
expect((result as { guidance: string }).guidance).toContain('verified successfully');
|
||||
});
|
||||
|
||||
it('returns verify guidance when action is verify', async () => {
|
||||
const verifyAction: WorkflowLoopAction = { type: 'verify', workflowId: 'wf-123' };
|
||||
const reportVerificationVerdict = vi.fn().mockResolvedValue(verifyAction);
|
||||
|
|
|
|||
|
|
@ -136,6 +136,10 @@ export function createReportVerificationVerdictTool(context: OrchestrationContex
|
|||
};
|
||||
}
|
||||
|
||||
// The claim comes from the persisted run, never from the model: a
|
||||
// `verified` verdict on a partially covered run must not upgrade it.
|
||||
const claim = (await context.workflowTaskService.getBuildOutcome(input.workItemId))
|
||||
?.verification?.claim;
|
||||
const remediation = input.remediation ?? defaultRemediationForVerdict(input);
|
||||
const forcedTerminalVerdict =
|
||||
remediation && !remediation.shouldEdit
|
||||
|
|
@ -150,6 +154,7 @@ export function createReportVerificationVerdictTool(context: OrchestrationContex
|
|||
workflowId: input.workflowId,
|
||||
executionId: input.executionId,
|
||||
verdict: forcedTerminalVerdict ?? input.verdict,
|
||||
claim,
|
||||
workflowInspection: input.workflowInspection,
|
||||
failureSignature: forcedTerminalVerdict
|
||||
? (remediation?.reason ?? input.failureSignature)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,161 @@
|
|||
import type { VerificationAnalysis } from '../analyze-result';
|
||||
import { deriveVerificationClaim } from '../claim';
|
||||
|
||||
function makeAnalysis(overrides: Partial<VerificationAnalysis> = {}): VerificationAnalysis {
|
||||
return {
|
||||
success: true,
|
||||
reachedNames: new Set(['Trigger', 'Fetch', 'Send']),
|
||||
reachedSimulatedNodes: [],
|
||||
workflowPinnedNodeNames: [],
|
||||
nodesNotReached: [],
|
||||
nodeErrors: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('deriveVerificationClaim', () => {
|
||||
it('claims verified when every planned node ran for real', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis(),
|
||||
plannedNodeCount: 3,
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('verified');
|
||||
expect(claim.publishReady).toBe(true);
|
||||
expect(claim.liveTestRecommended).toBe(false);
|
||||
expect(claim.reachedNodeCount).toBe(3);
|
||||
});
|
||||
|
||||
it('downgrades to partial when planned nodes were never reached', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({ nodesNotReached: ['Send', 'Log'] }),
|
||||
plannedNodeCount: 5,
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('partial');
|
||||
expect(claim.publishReady).toBe(false);
|
||||
expect(claim.liveTestRecommended).toBe(true);
|
||||
expect(claim.nodesNotReached).toEqual(['Send', 'Log']);
|
||||
});
|
||||
|
||||
it('downgrades to partial when a reached node had simulated output', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({
|
||||
reachedSimulatedNodes: [{ nodeName: 'Send', reason: 'Sends a message' }],
|
||||
}),
|
||||
plannedNodeCount: 3,
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('partial');
|
||||
expect(claim.publishReady).toBe(false);
|
||||
});
|
||||
|
||||
it('never claims verified for a pin-fed run', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({
|
||||
reachedSimulatedNodes: [{ nodeName: 'Fetch', reason: 'Output came from pinned data' }],
|
||||
workflowPinnedNodeNames: ['Fetch'],
|
||||
}),
|
||||
plannedNodeCount: 3,
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('partial');
|
||||
expect(claim.pinnedNodes).toEqual(['Fetch']);
|
||||
});
|
||||
|
||||
it('reports unproven when the fix target was never reached', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({ nodesNotReached: ['Send'] }),
|
||||
plannedNodeCount: 4,
|
||||
fixTargetNodeNames: ['Send'],
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('unproven');
|
||||
expect(claim.unprovenTargets).toEqual(['Send']);
|
||||
expect(claim.publishReady).toBe(false);
|
||||
expect(claim.liveTestRecommended).toBe(true);
|
||||
});
|
||||
|
||||
it('reports unproven when the fix target had simulated output', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({
|
||||
reachedSimulatedNodes: [{ nodeName: 'Send', reason: 'Sends a message' }],
|
||||
}),
|
||||
plannedNodeCount: 3,
|
||||
fixTargetNodeNames: ['Send'],
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('unproven');
|
||||
expect(claim.unprovenTargets).toEqual(['Send']);
|
||||
});
|
||||
|
||||
it('stays partial when the fix target was proven but coverage was incomplete', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({ nodesNotReached: ['Log'] }),
|
||||
plannedNodeCount: 4,
|
||||
fixTargetNodeNames: ['Fetch'],
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('partial');
|
||||
expect(claim.unprovenTargets).toEqual([]);
|
||||
});
|
||||
|
||||
it('never reports unproven when no fix target is known', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({ nodesNotReached: ['Send'] }),
|
||||
plannedNodeCount: 4,
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('partial');
|
||||
});
|
||||
|
||||
it('deduplicates repeated fix targets', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({ nodesNotReached: ['Send'] }),
|
||||
plannedNodeCount: 4,
|
||||
fixTargetNodeNames: ['Send', 'Send'],
|
||||
});
|
||||
|
||||
expect(claim.unprovenTargets).toEqual(['Send']);
|
||||
});
|
||||
|
||||
it('counts reached nodes over the plan, not the whole run', () => {
|
||||
// A mockable trigger is not classified, so it never appears in the plan.
|
||||
// Counting every executed node against the plan length reported "2 of 1".
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({
|
||||
reachedNames: new Set(['Every Morning at 9', 'Send Slack Message']),
|
||||
reachedSimulatedNodes: [{ nodeName: 'Send Slack Message', reason: 'Sends a message' }],
|
||||
}),
|
||||
plannedNodeCount: 1,
|
||||
});
|
||||
|
||||
expect(claim.plannedNodeCount).toBe(1);
|
||||
expect(claim.reachedNodeCount).toBe(1);
|
||||
expect(claim.reachedNodeCount).toBeLessThanOrEqual(claim.plannedNodeCount);
|
||||
});
|
||||
|
||||
it('never reports a negative reached count', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({ nodesNotReached: ['A', 'B', 'C'] }),
|
||||
plannedNodeCount: 1,
|
||||
});
|
||||
|
||||
expect(claim.reachedNodeCount).toBe(0);
|
||||
});
|
||||
|
||||
it('reports failed on a failed run regardless of coverage', () => {
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis: makeAnalysis({
|
||||
success: false,
|
||||
nodeErrors: [{ nodeName: 'Fetch', message: 'boom' }],
|
||||
}),
|
||||
plannedNodeCount: 3,
|
||||
fixTargetNodeNames: ['Fetch'],
|
||||
});
|
||||
|
||||
expect(claim.level).toBe('failed');
|
||||
expect(claim.publishReady).toBe(false);
|
||||
expect(claim.liveTestRecommended).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -174,4 +174,33 @@ describe('runScriptedGateVerification — merge consistency', () => {
|
|||
// Publish is reached only by the approve pass; the decline pass ran last.
|
||||
expect(analysis.simulationNote).toContain('Publish');
|
||||
});
|
||||
|
||||
it('discloses a node fed by workflow pin data in any pass', async () => {
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ...approvePassResult, workflowPinnedNodeNames: ['Router'] })
|
||||
.mockResolvedValueOnce(declinePassResult);
|
||||
|
||||
const { analysis } = await runScriptedGateVerification(makeArgs(run));
|
||||
|
||||
// Pins come from the run result, so they exist per pass only. Losing them
|
||||
// in the merge would let a pin-fed gate run read as a live test.
|
||||
expect(analysis.workflowPinnedNodeNames).toEqual(['Router']);
|
||||
expect(analysis.reachedSimulatedNodes.map((node) => node.nodeName)).toContain('Router');
|
||||
expect(analysis.simulationNote).toContain('Router');
|
||||
});
|
||||
|
||||
it('does not double-count a pinned node that the plan already simulates', async () => {
|
||||
const run = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ...approvePassResult, workflowPinnedNodeNames: ['Generate'] })
|
||||
.mockResolvedValueOnce(declinePassResult);
|
||||
|
||||
const { analysis } = await runScriptedGateVerification(makeArgs(run));
|
||||
|
||||
expect(analysis.workflowPinnedNodeNames).toEqual([]);
|
||||
expect(
|
||||
analysis.reachedSimulatedNodes.filter((node) => node.nodeName === 'Generate'),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ export type ChatModelRecoveryOptions = Pick<
|
|||
|
||||
type ExecutionNodeError = NonNullable<ExecutionRunResult['nodeErrors']>[number];
|
||||
|
||||
/** Disclosure for a node whose output came from pin data saved on the workflow. */
|
||||
export const WORKFLOW_PIN_SIMULATION_REASON =
|
||||
'Output came from pinned data saved on the workflow — unpin it for a live test';
|
||||
|
||||
const CREDENTIAL_FAILURE_KEYWORDS = [
|
||||
'credential',
|
||||
'unauthorized',
|
||||
|
|
@ -413,6 +417,7 @@ export interface VerificationAnalysis {
|
|||
success: boolean;
|
||||
reachedNames: Set<string>;
|
||||
reachedSimulatedNodes: Array<{ nodeName: string; reason: string }>;
|
||||
workflowPinnedNodeNames: string[];
|
||||
nodesNotReached: string[];
|
||||
remediation?: RemediationMetadata;
|
||||
nodesExecuted?: string[];
|
||||
|
|
@ -462,10 +467,7 @@ export function analyzeVerificationResult(args: {
|
|||
const plannedSimulatedNames = new Set(simulatedNodes.map((n) => n.nodeName));
|
||||
const workflowPinnedNodes = (result.workflowPinnedNodeNames ?? [])
|
||||
.filter((name) => reachedNames.has(name) && !plannedSimulatedNames.has(name))
|
||||
.map((name) => ({
|
||||
nodeName: name,
|
||||
reason: 'Output came from pinned data saved on the workflow — unpin it for a live test',
|
||||
}));
|
||||
.map((name) => ({ nodeName: name, reason: WORKFLOW_PIN_SIMULATION_REASON }));
|
||||
const reachedSimulatedNodes = [
|
||||
...simulatedNodes.filter((n) => reachedNames.has(n.nodeName)),
|
||||
...workflowPinnedNodes,
|
||||
|
|
@ -502,6 +504,7 @@ export function analyzeVerificationResult(args: {
|
|||
success,
|
||||
reachedNames,
|
||||
reachedSimulatedNodes,
|
||||
workflowPinnedNodeNames: workflowPinnedNodes.map((n) => n.nodeName),
|
||||
nodesNotReached,
|
||||
remediation,
|
||||
nodesExecuted: namesOrDataKeys(reachedNames, result.data),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,70 @@
|
|||
/**
|
||||
* Deterministic verification claim.
|
||||
*
|
||||
* The model narrates around this structure but never produces it: instruction
|
||||
* following degrades over a long thread, so the strength of the claim shown to
|
||||
* the user is computed from run evidence instead of asked for in a prompt.
|
||||
*/
|
||||
|
||||
import type { VerificationAnalysis } from './analyze-result';
|
||||
import type {
|
||||
VerificationClaim,
|
||||
VerificationClaimLevel,
|
||||
} from '../../../workflow-loop/workflow-loop-state';
|
||||
|
||||
export interface DeriveVerificationClaimArgs {
|
||||
analysis: VerificationAnalysis;
|
||||
/** Planned nodes from the build outcome simulation plan. */
|
||||
plannedNodeCount: number;
|
||||
/**
|
||||
* Nodes this change was about: the node a previous verdict sent for repair,
|
||||
* or the failing node the user reported. Empty means no target-specific
|
||||
* downgrade applies and coverage alone decides the level.
|
||||
*/
|
||||
fixTargetNodeNames?: readonly string[];
|
||||
}
|
||||
|
||||
export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): VerificationClaim {
|
||||
const { analysis, plannedNodeCount, fixTargetNodeNames = [] } = args;
|
||||
const nodesNotReached = [...analysis.nodesNotReached];
|
||||
const simulatedNodes = [...analysis.reachedSimulatedNodes];
|
||||
|
||||
const unproven = new Set([...nodesNotReached, ...simulatedNodes.map((n) => n.nodeName)]);
|
||||
const unprovenTargets = [...new Set(fixTargetNodeNames.filter((name) => unproven.has(name)))];
|
||||
|
||||
const level = resolveLevel({
|
||||
success: analysis.success,
|
||||
hasUnreached: nodesNotReached.length > 0,
|
||||
hasSimulated: simulatedNodes.length > 0,
|
||||
hasUnprovenTarget: unprovenTargets.length > 0,
|
||||
});
|
||||
|
||||
return {
|
||||
level,
|
||||
plannedNodeCount,
|
||||
// Counted over the simulation plan, the same set `nodesNotReached` comes
|
||||
// from. `reachedNames` covers the whole run, and the plan skips mockable
|
||||
// triggers, so comparing the two produced counts like "2 of 1".
|
||||
reachedNodeCount: Math.max(plannedNodeCount - nodesNotReached.length, 0),
|
||||
nodesNotReached,
|
||||
simulatedNodes,
|
||||
pinnedNodes: [...analysis.workflowPinnedNodeNames],
|
||||
unprovenTargets,
|
||||
publishReady: level === 'verified',
|
||||
liveTestRecommended: level === 'partial' || level === 'unproven',
|
||||
};
|
||||
}
|
||||
|
||||
function resolveLevel(facts: {
|
||||
success: boolean;
|
||||
hasUnreached: boolean;
|
||||
hasSimulated: boolean;
|
||||
hasUnprovenTarget: boolean;
|
||||
}): VerificationClaimLevel {
|
||||
if (!facts.success) return 'failed';
|
||||
// A named target that never ran for real outranks overall coverage: the user
|
||||
// asked about that node, so a green run elsewhere does not answer them.
|
||||
if (facts.hasUnprovenTarget) return 'unproven';
|
||||
if (facts.hasUnreached || facts.hasSimulated) return 'partial';
|
||||
return 'verified';
|
||||
}
|
||||
|
|
@ -11,7 +11,10 @@ import type {
|
|||
} from './types';
|
||||
import type { OrchestrationContext } from '../../../types';
|
||||
import { createRemediation } from '../../../workflow-loop/remediation';
|
||||
import type { RemediationMetadata } from '../../../workflow-loop/workflow-loop-state';
|
||||
import type {
|
||||
RemediationMetadata,
|
||||
VerificationClaim,
|
||||
} from '../../../workflow-loop/workflow-loop-state';
|
||||
|
||||
/**
|
||||
* Handle the no-simulation-plan case: refuse to run because destructive nodes
|
||||
|
|
@ -88,11 +91,21 @@ export async function persistVerificationOutcome(args: {
|
|||
workflowId: string;
|
||||
result: ExecutionRunResult;
|
||||
analysis: VerificationAnalysis;
|
||||
/** Deterministic verdict for this run, persisted so later turns cannot re-litigate it. */
|
||||
claim: VerificationClaim;
|
||||
/** Running count of verify runs for this build, used to enforce MAX_VERIFY_ATTEMPTS. */
|
||||
verifyAttempts: number;
|
||||
}): Promise<void> {
|
||||
const { input, context, workflowTaskService, workflowId, result, analysis, verifyAttempts } =
|
||||
args;
|
||||
const {
|
||||
input,
|
||||
context,
|
||||
workflowTaskService,
|
||||
workflowId,
|
||||
result,
|
||||
analysis,
|
||||
claim,
|
||||
verifyAttempts,
|
||||
} = args;
|
||||
try {
|
||||
const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data);
|
||||
await workflowTaskService.updateBuildOutcome(input.workItemId, {
|
||||
|
|
@ -103,6 +116,7 @@ export async function persistVerificationOutcome(args: {
|
|||
executionId: result.executionId || undefined,
|
||||
status: result.status,
|
||||
failureSignature: analysis.success ? undefined : analysis.errorMessage,
|
||||
claim,
|
||||
evidence: {
|
||||
nodesExecuted:
|
||||
executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import {
|
||||
analyzeVerificationResult,
|
||||
buildSimulationNote,
|
||||
WORKFLOW_PIN_SIMULATION_REASON,
|
||||
type ChatModelRecoveryOptions,
|
||||
type VerificationAnalysis,
|
||||
} from './analyze-result';
|
||||
|
|
@ -154,14 +155,28 @@ function mergeAnalyses(
|
|||
|
||||
// Rebuild the note from the union — a node simulated only in an earlier
|
||||
// pass must still be disclosed.
|
||||
const reachedSimulatedNodes = prepared.simulatedNodes.filter((node) =>
|
||||
const plannedSimulated = prepared.simulatedNodes.filter((node) =>
|
||||
reachedNames.has(node.nodeName),
|
||||
);
|
||||
// Pins come from the run result, so they only exist per pass. Without this
|
||||
// union a pin-fed gate run would read as live (INS-1216).
|
||||
const plannedSimulatedNames = new Set(plannedSimulated.map((node) => node.nodeName));
|
||||
const workflowPinnedNodeNames = [
|
||||
...new Set(passes.flatMap((pass) => pass.analysis.workflowPinnedNodeNames)),
|
||||
].filter((name) => !plannedSimulatedNames.has(name));
|
||||
const reachedSimulatedNodes = [
|
||||
...plannedSimulated,
|
||||
...workflowPinnedNodeNames.map((name) => ({
|
||||
nodeName: name,
|
||||
reason: WORKFLOW_PIN_SIMULATION_REASON,
|
||||
})),
|
||||
];
|
||||
|
||||
return {
|
||||
success: passes.every((pass) => pass.analysis.success),
|
||||
reachedNames,
|
||||
reachedSimulatedNodes,
|
||||
workflowPinnedNodeNames,
|
||||
nodesNotReached,
|
||||
remediation: failing?.analysis.remediation,
|
||||
nodesExecuted: [...reachedNames],
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import type { ExecutionNodeError, OrchestrationContext } from '../../../types';
|
||||
import type { RemediationMetadata } from '../../../workflow-loop/workflow-loop-state';
|
||||
import type {
|
||||
RemediationMetadata,
|
||||
VerificationClaim,
|
||||
} from '../../../workflow-loop/workflow-loop-state';
|
||||
|
||||
export interface VerifyToolInput {
|
||||
workItemId?: string;
|
||||
|
|
@ -11,6 +14,8 @@ export interface VerifyToolInput {
|
|||
includeData?: boolean;
|
||||
maxDataChars?: number;
|
||||
fixtureOverrides?: Record<string, Array<Record<string, unknown>>>;
|
||||
/** Nodes this change is about — see the tool input description. */
|
||||
fixTargetNodeNames?: string[];
|
||||
/** Nodes whose override may pin zero items — the explicit opt-in for an empty-branch test. */
|
||||
allowZeroItemFixtures?: string[];
|
||||
}
|
||||
|
|
@ -47,6 +52,7 @@ export interface VerifyBuiltWorkflowOutput {
|
|||
nodeErrors?: ExecutionNodeError[];
|
||||
nodesNotReached?: string[];
|
||||
coverageNote?: string;
|
||||
claim?: VerificationClaim;
|
||||
data?: Record<string, unknown>;
|
||||
error?: string;
|
||||
remediation?: RemediationMetadata;
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { z } from 'zod';
|
|||
|
||||
import type { OrchestrationContext } from '../../types';
|
||||
import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result';
|
||||
import { deriveVerificationClaim } from './verification/claim';
|
||||
import {
|
||||
handleMissingSimulationPlan,
|
||||
persistVerificationOutcome,
|
||||
|
|
@ -19,7 +20,10 @@ import { prepareVerificationRun } from './verification/prepare-run';
|
|||
import { reconcileStaleCredentialPlan } from './verification/reconcile-plan';
|
||||
import { resolveVerificationTarget } from './verification/resolve-target';
|
||||
import { runScriptedGateVerification } from './verification/scripted-gate-run';
|
||||
import { executionNodeErrorSchema } from '../../workflow-loop/workflow-loop-state';
|
||||
import {
|
||||
executionNodeErrorSchema,
|
||||
verificationClaimSchema,
|
||||
} from '../../workflow-loop/workflow-loop-state';
|
||||
import { collectChatModelRecoveryContext } from '../workflows/chat-model-validation';
|
||||
|
||||
const DEFAULT_NODE_PREVIEW_CHARS = 600;
|
||||
|
|
@ -81,6 +85,16 @@ export const verifyBuiltWorkflowInputSchema = z.object({
|
|||
'Optional per-run output fixtures keyed by node name. Only nodes already classified as simulated in the build outcome may be overridden. Use this for alternate deterministic scenarios, not raw trigger input. ' +
|
||||
'An empty array is rejected unless the node is also listed in `allowZeroItemFixtures`.',
|
||||
),
|
||||
fixTargetNodeNames: z
|
||||
.array(z.string())
|
||||
.optional()
|
||||
.describe(
|
||||
'Node names this change is about — the node the user reported as failing, or the node ' +
|
||||
'you just repaired. The verdict shown to the user is downgraded to "changed but ' +
|
||||
'unverified" when any of these was never reached or had simulated output, so a green ' +
|
||||
'run elsewhere cannot pass as proof for them. Pass these whenever you are fixing a ' +
|
||||
'specific node rather than building from scratch.',
|
||||
),
|
||||
allowZeroItemFixtures: z
|
||||
.array(z.string())
|
||||
.optional()
|
||||
|
|
@ -125,6 +139,7 @@ const verifyBuiltWorkflowOutputSchema = z.object({
|
|||
nodeErrors: z.array(executionNodeErrorSchema).optional(),
|
||||
nodesNotReached: z.array(z.string()).optional(),
|
||||
coverageNote: z.string().optional(),
|
||||
claim: verificationClaimSchema.optional(),
|
||||
data: z.record(z.unknown()).optional(),
|
||||
error: z.string().optional(),
|
||||
remediation: remediationOutputSchema,
|
||||
|
|
@ -238,6 +253,22 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
};
|
||||
})();
|
||||
|
||||
// The repair target from an earlier verdict counts even when the model
|
||||
// omits it here — that is exactly the turn where it stops mentioning it.
|
||||
const fixTargetNodeNames = [
|
||||
...new Set(
|
||||
[
|
||||
...(resolvedInput.fixTargetNodeNames ?? []),
|
||||
target.stateBefore?.lastFailedNodeName,
|
||||
].filter((name): name is string => name !== undefined),
|
||||
),
|
||||
];
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis,
|
||||
plannedNodeCount: buildOutcome.nodeSimulationPlan?.length ?? 0,
|
||||
fixTargetNodeNames,
|
||||
});
|
||||
|
||||
await persistVerificationOutcome({
|
||||
input: resolvedInput,
|
||||
context,
|
||||
|
|
@ -245,6 +276,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
workflowId,
|
||||
result,
|
||||
analysis,
|
||||
claim,
|
||||
verifyAttempts: (buildOutcome.verifyAttempts ?? 0) + 1,
|
||||
});
|
||||
|
||||
|
|
@ -254,6 +286,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
resolvedWorkItemId: resolvedInput.workItemId,
|
||||
executionId: result.executionId || undefined,
|
||||
success: analysis.success,
|
||||
claim,
|
||||
status: result.status,
|
||||
nodesExecuted: analysis.nodesExecuted,
|
||||
lastNodeExecuted: result.lastNodeExecuted,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import {
|
|||
TEMPLATABLE_PLAIN_AUTH_TYPES,
|
||||
} from './credentials.tool';
|
||||
import { formatTimestamp } from '../utils/format-timestamp';
|
||||
import { formatClaimDisclosure } from '../workflow-loop/render-claim';
|
||||
import { isSetupPanelEnabled } from './workflows/setup-items';
|
||||
import {
|
||||
describeSetupItem,
|
||||
|
|
@ -310,6 +311,14 @@ const publishBaseAction = z.object({
|
|||
.describe('Publish a workflow version to production (omit versionId for latest draft)'),
|
||||
workflowId: z.string().describe('ID of the workflow'),
|
||||
versionId: z.string().optional().describe('Version ID'),
|
||||
acknowledgeUnverified: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.describe(
|
||||
'Set true only after you told the user the workflow is not fully verified and they still ' +
|
||||
'asked to publish. Publishing is refused without this while the latest verification left ' +
|
||||
'nodes unreached or simulated. Never set it to skip the disclosure.',
|
||||
),
|
||||
});
|
||||
|
||||
const publishExtendedAction = publishBaseAction.extend({
|
||||
|
|
@ -1377,6 +1386,49 @@ async function resolveSetupScopeNodeNames(
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage disclosure for the workflow's latest verification, or undefined when
|
||||
* it was fully verified or no claim exists. Absent evidence never blocks: a
|
||||
* workflow built before this record, or outside the assistant, is unknown
|
||||
* rather than unverified.
|
||||
*/
|
||||
async function resolveUnverifiedPublishDisclosure(
|
||||
context: InstanceAiContext,
|
||||
workflowId: string,
|
||||
): Promise<string | undefined> {
|
||||
const workflowTaskService = context.workflowBuildContext?.workflowTaskService;
|
||||
if (!workflowTaskService) return undefined;
|
||||
try {
|
||||
const outcome = await workflowTaskService.getLatestBuildOutcomeForWorkflow(workflowId);
|
||||
const verification = outcome?.verification;
|
||||
const claim = verification?.claim;
|
||||
if (claim) return claim.publishReady ? undefined : formatClaimDisclosure(claim);
|
||||
|
||||
// A record with no claim means verification ran and could not produce one
|
||||
// — it was refused for want of a simulation plan, or it failed before it
|
||||
// reached a verdict. That is a failed verification, not an unverified
|
||||
// workflow, and it must not pass as an absent record.
|
||||
if (verification?.attempted) {
|
||||
const cause = verification.failureSignature ?? verification.evidence?.errorMessage;
|
||||
return (
|
||||
'Verification ran but produced no verdict, so nothing about this workflow is proven.' +
|
||||
(cause ? ` It reported: ${cause}` : '')
|
||||
);
|
||||
}
|
||||
|
||||
// No record at all: the workflow is unknown, not unverified. Blocking here
|
||||
// would stop publishing every workflow built before this record existed.
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
// Fail open: a storage hiccup must not block a publish the user asked for.
|
||||
context.logger.warn('Failed to resolve the verification claim before publishing', {
|
||||
workflowId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const SETUP_PANEL_ANNOUNCED_GUIDANCE =
|
||||
'The setup panel next to the chat now lists what this workflow still needs (`open`); nothing is ' +
|
||||
'waiting on you and no card is open. Finish your turn now: tell the user in one or two sentences ' +
|
||||
|
|
@ -1754,7 +1806,23 @@ async function handlePublish(
|
|||
}
|
||||
|
||||
const supportingWorkflowIds = await resolveSupportingWorkflowIds(context, input.workflowId);
|
||||
const needsApproval = context.permissions?.publishWorkflow !== 'always_allow';
|
||||
const unverifiedDisclosure = await resolveUnverifiedPublishDisclosure(context, input.workflowId);
|
||||
|
||||
const needsApproval =
|
||||
context.permissions?.publishWorkflow !== 'always_allow' || unverifiedDisclosure !== undefined;
|
||||
|
||||
if (unverifiedDisclosure && input.acknowledgeUnverified !== true) {
|
||||
return {
|
||||
success: false,
|
||||
denied: true,
|
||||
reason: 'not_verified',
|
||||
verificationDisclosure: unverifiedDisclosure,
|
||||
guidance:
|
||||
`This workflow is not fully verified. ${unverifiedDisclosure} ` +
|
||||
'Tell the user exactly this, and offer a live end-to-end test. Publish only if they still ' +
|
||||
'ask for it, by calling publish again with `acknowledgeUnverified: true`.',
|
||||
};
|
||||
}
|
||||
|
||||
if (needsApproval && (resumeData === undefined || resumeData === null)) {
|
||||
const workflowName = await resolveWorkflowName(context, input.workflowId);
|
||||
|
|
@ -1762,12 +1830,15 @@ async function handlePublish(
|
|||
supportingWorkflowIds.length > 0
|
||||
? ` and ${String(supportingWorkflowIds.length)} referenced supporting workflow(s)`
|
||||
: '';
|
||||
const target = input.versionId
|
||||
? `Publish version ${input.versionId} of ${workflowName} (ID: ${input.workflowId})${dependencyNote}`
|
||||
: `Publish ${workflowName} (ID: ${input.workflowId})${dependencyNote}`;
|
||||
|
||||
return await ctx.suspend({
|
||||
requestId: nanoid(),
|
||||
message: input.versionId
|
||||
? `Publish version ${input.versionId} of ${workflowName} (ID: ${input.workflowId})${dependencyNote}`
|
||||
: `Publish ${workflowName} (ID: ${input.workflowId})${dependencyNote}`,
|
||||
// The user has to read this to approve, so the disclosure lands even if
|
||||
// the assistant's own message left it out.
|
||||
message: unverifiedDisclosure ? `${target}\n\n${unverifiedDisclosure}` : target,
|
||||
severity: 'warning' as const,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,111 @@
|
|||
import { formatWorkflowLoopGuidance } from '../guidance';
|
||||
import type { WorkflowLoopAction } from '../workflow-loop-state';
|
||||
import type { VerificationClaim, WorkflowLoopAction } from '../workflow-loop-state';
|
||||
|
||||
function makeClaim(overrides: Partial<VerificationClaim> = {}): VerificationClaim {
|
||||
return {
|
||||
level: 'verified',
|
||||
plannedNodeCount: 3,
|
||||
reachedNodeCount: 3,
|
||||
nodesNotReached: [],
|
||||
simulatedNodes: [],
|
||||
pinnedNodes: [],
|
||||
unprovenTargets: [],
|
||||
publishReady: true,
|
||||
liveTestRecommended: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('formatWorkflowLoopGuidance', () => {
|
||||
// ── done ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('action type "done"', () => {
|
||||
it('should report completion without workflowId', () => {
|
||||
it('should not claim verification when no run recorded a claim', () => {
|
||||
// A `verified` verdict reported without verifying leaves no claim. This
|
||||
// sentence used to say "Workflow verified successfully", which handed
|
||||
// the model the exact wording with no run evidence behind it.
|
||||
const action: WorkflowLoopAction = {
|
||||
type: 'done',
|
||||
summary: 'All good',
|
||||
};
|
||||
const result = formatWorkflowLoopGuidance(action);
|
||||
expect(result).not.toContain('Workflow verified successfully');
|
||||
expect(result).toContain('No automatic verification evidence is recorded');
|
||||
expect(result).toContain('do NOT call the workflow verified');
|
||||
expect(result).not.toContain('Workflow ID:');
|
||||
});
|
||||
|
||||
it('should not claim success when the claim is partial', () => {
|
||||
const action: WorkflowLoopAction = {
|
||||
type: 'done',
|
||||
summary: 'All good',
|
||||
claim: makeClaim({
|
||||
level: 'partial',
|
||||
reachedNodeCount: 5,
|
||||
plannedNodeCount: 12,
|
||||
nodesNotReached: ['Send Email', 'Log Row'],
|
||||
publishReady: false,
|
||||
liveTestRecommended: true,
|
||||
}),
|
||||
};
|
||||
const result = formatWorkflowLoopGuidance(action);
|
||||
expect(result).not.toContain('Workflow verified successfully');
|
||||
expect(result).toContain('NOT fully verified');
|
||||
expect(result).toContain('2 of 12 node(s) were never reached');
|
||||
expect(result).toContain('Send Email, Log Row');
|
||||
expect(result).toContain('Do NOT offer to publish it.');
|
||||
expect(result).toContain('Offer a live end-to-end test');
|
||||
});
|
||||
|
||||
it('should name the unproven fix target when the claim is unproven', () => {
|
||||
const action: WorkflowLoopAction = {
|
||||
type: 'done',
|
||||
summary: 'Patched',
|
||||
claim: makeClaim({
|
||||
level: 'unproven',
|
||||
plannedNodeCount: 4,
|
||||
reachedNodeCount: 4,
|
||||
unprovenTargets: ['Send Email'],
|
||||
simulatedNodes: [{ nodeName: 'Send Email', reason: 'Sends a message' }],
|
||||
publishReady: false,
|
||||
liveTestRecommended: true,
|
||||
}),
|
||||
};
|
||||
const result = formatWorkflowLoopGuidance(action);
|
||||
expect(result).not.toContain('Workflow verified successfully');
|
||||
expect(result).toContain('Changed but NOT verified');
|
||||
expect(result).toContain('never proven: Send Email');
|
||||
expect(result).toContain('nothing real happened at: Send Email');
|
||||
});
|
||||
|
||||
it('should keep the verified wording when the claim is verified', () => {
|
||||
const action: WorkflowLoopAction = {
|
||||
type: 'done',
|
||||
summary: 'All good',
|
||||
claim: makeClaim(),
|
||||
};
|
||||
const result = formatWorkflowLoopGuidance(action);
|
||||
expect(result).toContain('Workflow verified successfully');
|
||||
expect(result).toContain('Report completion');
|
||||
expect(result).not.toContain('Workflow ID:');
|
||||
});
|
||||
|
||||
it('should downgrade the mocked-credential guidance too', () => {
|
||||
const action: WorkflowLoopAction = {
|
||||
type: 'done',
|
||||
summary: 'All good',
|
||||
workflowId: 'wf-123',
|
||||
mockedCredentialTypes: ['slackApi'],
|
||||
claim: makeClaim({
|
||||
level: 'partial',
|
||||
nodesNotReached: ['Send Email'],
|
||||
publishReady: false,
|
||||
liveTestRecommended: true,
|
||||
}),
|
||||
};
|
||||
const result = formatWorkflowLoopGuidance(action);
|
||||
expect(result).not.toContain('Workflow verified successfully');
|
||||
expect(result).toContain('NOT fully verified');
|
||||
expect(result).toContain('workflows(action="setup")');
|
||||
});
|
||||
|
||||
it('should include workflowId when present', () => {
|
||||
|
|
@ -44,7 +136,7 @@ describe('formatWorkflowLoopGuidance', () => {
|
|||
};
|
||||
const result = formatWorkflowLoopGuidance(action);
|
||||
expect(result).not.toContain('credentials(action="setup")');
|
||||
expect(result).toContain('Report completion');
|
||||
expect(result).toContain('Report the outcome');
|
||||
});
|
||||
|
||||
it('should include credential instructions when mockedCredentialTypes has entries', () => {
|
||||
|
|
|
|||
|
|
@ -451,6 +451,64 @@ describe('handleBuildOutcome', () => {
|
|||
// ── handleVerificationVerdict ───────────────────────────────────────────────
|
||||
|
||||
describe('handleVerificationVerdict', () => {
|
||||
describe('deterministic claim', () => {
|
||||
it('carries a downgraded claim onto the done action instead of blocking the loop', () => {
|
||||
const state = makeState({ phase: 'verifying' });
|
||||
const { state: next, action } = handleVerificationVerdict(
|
||||
state,
|
||||
[],
|
||||
makeVerdict({
|
||||
verdict: 'verified',
|
||||
claim: {
|
||||
level: 'partial',
|
||||
plannedNodeCount: 12,
|
||||
reachedNodeCount: 5,
|
||||
nodesNotReached: ['Send Email'],
|
||||
simulatedNodes: [],
|
||||
pinnedNodes: [],
|
||||
unprovenTargets: [],
|
||||
publishReady: false,
|
||||
liveTestRecommended: true,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
// The loop must still settle — re-verifying replays the same coverage.
|
||||
expect(next.phase).toBe('done');
|
||||
expect(next.status).toBe('completed');
|
||||
expect(action.type).toBe('done');
|
||||
if (action.type !== 'done') throw new Error('expected done');
|
||||
expect(action.claim?.level).toBe('partial');
|
||||
expect(action.claim?.publishReady).toBe(false);
|
||||
});
|
||||
|
||||
it('records the repair target so a later run can tell whether it was proven', () => {
|
||||
const state = makeState({ phase: 'verifying' });
|
||||
const { state: next } = handleVerificationVerdict(
|
||||
state,
|
||||
[],
|
||||
makeVerdict({
|
||||
verdict: 'needs_patch',
|
||||
failedNodeName: 'Send Email',
|
||||
diagnosis: 'Wrong channel',
|
||||
}),
|
||||
);
|
||||
|
||||
expect(next.lastFailedNodeName).toBe('Send Email');
|
||||
});
|
||||
|
||||
it('keeps a known repair target when a later rebuild names no node', () => {
|
||||
const state = makeState({ phase: 'verifying', lastFailedNodeName: 'Send Email' });
|
||||
const { state: next } = handleVerificationVerdict(
|
||||
state,
|
||||
[],
|
||||
makeVerdict({ verdict: 'needs_rebuild', diagnosis: 'Structural repair' }),
|
||||
);
|
||||
|
||||
expect(next.lastFailedNodeName).toBe('Send Email');
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores stale verification verdicts from a different run without resetting state', () => {
|
||||
const state = makeState({
|
||||
runId: 'run_current',
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { WorkflowLoopAction } from './workflow-loop-state';
|
||||
import { describeClaimCoverage, formatClaimHeadline } from './render-claim';
|
||||
import type { VerificationClaim, WorkflowLoopAction } from './workflow-loop-state';
|
||||
|
||||
export interface WorkflowLoopGuidanceOptions {
|
||||
workItemId?: string;
|
||||
|
|
@ -6,6 +7,42 @@ export interface WorkflowLoopGuidanceOptions {
|
|||
setupPanelEnabled?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lead sentence for a completed build. Derived from the claim so the guidance
|
||||
* never tells the model a partially covered run was verified — that sentence
|
||||
* was the strongest license for the false success claims in AIA-31.
|
||||
*/
|
||||
function formatClaimLead(claim: VerificationClaim | undefined): string {
|
||||
if (claim?.level === 'verified') return 'Workflow verified successfully.';
|
||||
|
||||
// No claim means no verification run recorded a verdict for this build — a
|
||||
// trigger-only workflow, or a verdict reported without verifying. Saying
|
||||
// "verified successfully" here let a model reach that sentence with no run
|
||||
// evidence at all. A live run the model inspected is still valid evidence,
|
||||
// so ask it to name that rather than refusing the claim outright.
|
||||
if (claim === undefined) {
|
||||
return (
|
||||
'Build complete. No automatic verification evidence is recorded for this workflow. ' +
|
||||
'If you are relying on a live run you inspected, say which execution. ' +
|
||||
'Otherwise do NOT call the workflow verified, tested, or working.'
|
||||
);
|
||||
}
|
||||
|
||||
// The claim is the only honest reading of the run, and nothing else tells
|
||||
// the user how strong it is, so these rules carry the whole disclosure.
|
||||
const rules = [
|
||||
'Do NOT call the workflow verified, tested, working, or ready to publish.',
|
||||
'Do NOT offer to publish it.',
|
||||
claim.liveTestRecommended ? 'Offer a live end-to-end test instead.' : '',
|
||||
].filter((rule) => rule !== '');
|
||||
|
||||
return [formatClaimHeadline(claim), ...describeClaimCoverage(claim), ...rules].join(' ');
|
||||
}
|
||||
|
||||
function isVerifiedClaim(claim: VerificationClaim | undefined): boolean {
|
||||
return claim?.level === 'verified';
|
||||
}
|
||||
|
||||
function formatSourceFileInstruction(sourceFilePath: string | undefined): string {
|
||||
if (!sourceFilePath) {
|
||||
return 'edit the workspace source file, then call `build-workflow` with that filePath';
|
||||
|
|
@ -24,9 +61,11 @@ export function formatWorkflowLoopGuidance(
|
|||
case 'continue_building':
|
||||
return `BUILD FAILED: ${action.reason}. Fix the workflow source file: ${formatSourceFileInstruction(action.sourceFilePath)}.`;
|
||||
case 'done': {
|
||||
const claimLead = formatClaimLead(action.claim);
|
||||
if (action.setupSkippedByUser) {
|
||||
return (
|
||||
'Workflow verified successfully. The credentials it still needs are ones the user ' +
|
||||
claimLead +
|
||||
' The credentials it still needs are ones the user ' +
|
||||
'skipped earlier in this conversation, so do NOT open the setup card again. Tell them ' +
|
||||
'which parts stay unconfigured and what that means when the workflow runs, and offer ' +
|
||||
'to set them up whenever they want.'
|
||||
|
|
@ -44,14 +83,17 @@ export function formatWorkflowLoopGuidance(
|
|||
);
|
||||
}
|
||||
return (
|
||||
'Workflow verified successfully with temporary mock data. ' +
|
||||
`${claimLead} It still uses temporary mock data. ` +
|
||||
`Call \`workflows(action="setup")\` with workflowId "${action.workflowId ?? 'unknown'}" ` +
|
||||
'to open the inline setup card in the AI Assistant panel for credentials, parameters, and triggers. ' +
|
||||
'Do not tell the user to open the editor, use the canvas, or click a Setup button. ' +
|
||||
'Do not call `credentials(action="setup")` or `apply-workflow-credentials` — `workflows(action="setup")` handles everything.'
|
||||
);
|
||||
}
|
||||
return `Workflow verified successfully. Report completion to the user.${action.workflowId ? ` Workflow ID: ${action.workflowId}` : ''}`;
|
||||
const closing = isVerifiedClaim(action.claim)
|
||||
? 'Report completion to the user.'
|
||||
: 'Report the outcome to the user.';
|
||||
return `${claimLead} ${closing}${action.workflowId ? ` Workflow ID: ${action.workflowId}` : ''}`;
|
||||
}
|
||||
case 'verify':
|
||||
return (
|
||||
|
|
|
|||
59
packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts
Normal file
59
packages/@n8n/instance-ai/src/workflow-loop/render-claim.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* User-facing wording for a verification claim.
|
||||
*
|
||||
* Every surface that discloses coverage renders from here, so the guidance the
|
||||
* model reads and the publish approval the user clicks state the same facts.
|
||||
* A claim at `verified` needs no disclosure.
|
||||
*/
|
||||
|
||||
import type { VerificationClaim } from './workflow-loop-state';
|
||||
|
||||
export function formatClaimNodeList(names: readonly string[], max = 8): string {
|
||||
if (names.length <= max) return names.join(', ');
|
||||
return `${names.slice(0, max).join(', ')} and ${String(names.length - max)} more`;
|
||||
}
|
||||
|
||||
export function formatClaimHeadline(claim: VerificationClaim): string {
|
||||
switch (claim.level) {
|
||||
case 'verified':
|
||||
return 'Verified end to end.';
|
||||
case 'unproven':
|
||||
return (
|
||||
'Changed but NOT verified. The node(s) this change was about were never proven: ' +
|
||||
`${formatClaimNodeList(claim.unprovenTargets)}.`
|
||||
);
|
||||
case 'partial':
|
||||
return 'Ran without errors, but NOT fully verified.';
|
||||
case 'failed':
|
||||
return 'Verification did not pass.';
|
||||
}
|
||||
}
|
||||
|
||||
export function describeClaimCoverage(claim: VerificationClaim): string[] {
|
||||
const facts: string[] = [];
|
||||
|
||||
if (claim.nodesNotReached.length > 0) {
|
||||
facts.push(
|
||||
`${String(claim.nodesNotReached.length)} of ${String(claim.plannedNodeCount)} node(s) were ` +
|
||||
`never reached, so they are UNVERIFIED: ${formatClaimNodeList(claim.nodesNotReached)}.`,
|
||||
);
|
||||
}
|
||||
if (claim.simulatedNodes.length > 0) {
|
||||
facts.push(
|
||||
'Output was simulated, so nothing real happened at: ' +
|
||||
`${formatClaimNodeList(claim.simulatedNodes.map((node) => node.nodeName))}.`,
|
||||
);
|
||||
}
|
||||
if (claim.pinnedNodes.length > 0) {
|
||||
facts.push(
|
||||
`Some output came from pin data saved on the workflow: ${formatClaimNodeList(claim.pinnedNodes)}. ` +
|
||||
'Remove those pins for a live test.',
|
||||
);
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
export function formatClaimDisclosure(claim: VerificationClaim): string | undefined {
|
||||
if (claim.level === 'verified') return undefined;
|
||||
return [formatClaimHeadline(claim), ...describeClaimCoverage(claim)].join(' ');
|
||||
}
|
||||
|
|
@ -273,6 +273,9 @@ export function handleVerificationVerdict(
|
|||
|
||||
switch (verdict.verdict) {
|
||||
case 'verified': {
|
||||
// The loop still completes on a partial run — re-issuing verification
|
||||
// would replay the same coverage forever. Only the claim is downgraded,
|
||||
// so `done` carries an honest verdict instead of a blocked state.
|
||||
attempt.result = 'success';
|
||||
return {
|
||||
state: {
|
||||
|
|
@ -287,6 +290,7 @@ export function handleVerificationVerdict(
|
|||
type: 'done',
|
||||
workflowId: verdict.workflowId,
|
||||
summary: verdict.summary,
|
||||
claim: verdict.claim,
|
||||
mockedCredentialTypes: normalizedState.mockedCredentialTypes,
|
||||
hasUnresolvedPlaceholders: normalizedState.hasUnresolvedPlaceholders,
|
||||
setupSkippedByUser: normalizedState.setupSkippedByUser,
|
||||
|
|
@ -470,6 +474,7 @@ function escalateToRepair(
|
|||
status: 'active',
|
||||
rebuildAttempts: state.rebuildAttempts + 1,
|
||||
lastFailureSignature: verdict.failureSignature,
|
||||
lastFailedNodeName: verdict.failedNodeName ?? state.lastFailedNodeName,
|
||||
lastExecutionId: verdict.executionId,
|
||||
lastWorkflowInspection: verdict.workflowInspection,
|
||||
lastRemediation: remediation,
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ export const workflowLoopStateSchema = z.object({
|
|||
lastTaskId: z.string().optional(),
|
||||
lastExecutionId: z.string().optional(),
|
||||
lastFailureSignature: z.string().optional(),
|
||||
lastFailedNodeName: z.string().optional(),
|
||||
lastWorkflowInspection: z.string().optional(),
|
||||
rebuildAttempts: z.number().int().min(0),
|
||||
/** Credential types that were mocked during build (persisted across phases). */
|
||||
|
|
@ -153,6 +154,42 @@ export const executionNodeErrorSchema = z.object({
|
|||
message: z.string().optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Strength of the claim a verification run supports. `verified` means every
|
||||
* planned node ran for real; anything less must not be reported as verified.
|
||||
* See `tools/orchestration/verification/claim.ts` for the derivation.
|
||||
*/
|
||||
export const verificationClaimLevelSchema = z.enum(['verified', 'partial', 'unproven', 'failed']);
|
||||
|
||||
export type VerificationClaimLevel = z.infer<typeof verificationClaimLevelSchema>;
|
||||
|
||||
/**
|
||||
* The deterministic verdict for one verification run. Backend-only: it shapes
|
||||
* the tool result the model reads and gates the publish offer. It is
|
||||
* deliberately not sent to the client — see INS-1308.
|
||||
*/
|
||||
export const verificationClaimSchema = z.object({
|
||||
level: verificationClaimLevelSchema,
|
||||
/** Nodes the verification plan covers. Excludes triggers the classifier skips. */
|
||||
plannedNodeCount: z.number().int().min(0),
|
||||
/** Planned nodes the run reached. Same set as `plannedNodeCount`. */
|
||||
reachedNodeCount: z.number().int().min(0),
|
||||
nodesNotReached: z.array(z.string()),
|
||||
/** Reached nodes whose output was mocked, so the run proves nothing about them. */
|
||||
simulatedNodes: z.array(z.object({ nodeName: z.string(), reason: z.string() })),
|
||||
/** Pin-fed subset of `simulatedNodes` — these need the pin removed for a live test. */
|
||||
pinnedNodes: z.array(z.string()),
|
||||
/**
|
||||
* Fix targets still unreached or simulated — the reason this run cannot
|
||||
* claim `verified` even when it ended without an error.
|
||||
*/
|
||||
unprovenTargets: z.array(z.string()),
|
||||
publishReady: z.boolean(),
|
||||
liveTestRecommended: z.boolean(),
|
||||
});
|
||||
|
||||
export type VerificationClaim = z.infer<typeof verificationClaimSchema>;
|
||||
|
||||
/**
|
||||
* Structured verification evidence the builder captures when it runs
|
||||
* `verify-built-workflow`. Downstream checkpoint runs read this and skip
|
||||
|
|
@ -164,6 +201,7 @@ export const workflowVerificationEvidenceSchema = z.object({
|
|||
executionId: z.string().optional(),
|
||||
status: z.enum(['success', 'error', 'waiting', 'running', 'unknown']).optional(),
|
||||
failureSignature: z.string().optional(),
|
||||
claim: verificationClaimSchema.optional(),
|
||||
evidence: z
|
||||
.object({
|
||||
nodesExecuted: z.array(z.string()).optional(),
|
||||
|
|
@ -454,6 +492,7 @@ export const verificationResultSchema = z.object({
|
|||
workflowId: z.string(),
|
||||
executionId: z.string().optional(),
|
||||
verdict: verificationVerdictSchema,
|
||||
claim: verificationClaimSchema.optional(),
|
||||
workflowInspection: z.string().optional(),
|
||||
failureSignature: z.string().optional(),
|
||||
failedNodeName: z.string().optional(),
|
||||
|
|
@ -485,6 +524,7 @@ export type WorkflowLoopAction =
|
|||
type: 'done';
|
||||
workflowId?: string;
|
||||
summary: string;
|
||||
claim?: VerificationClaim;
|
||||
mockedCredentialTypes?: string[];
|
||||
hasUnresolvedPlaceholders?: boolean;
|
||||
setupSkippedByUser?: boolean;
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user