feat(core): Stop Instance AI reporting a simulated run as fully verified (#38060)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaakko Husso 2026-09-09 12:25:10 +00:00 committed by GitHub
parent eac5f9eccf
commit efb5547368
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 1039 additions and 27 deletions

View File

@ -201,6 +201,14 @@ publish-readiness evidence. If the user explicitly asks to publish before a
live execution succeeds, warn that the live path remains untested, then follow
the requested publish flow.
`workflows(action="publish")` enforces this. While the latest verification left
nodes unreached or simulated, the call is refused and returns
`verificationDisclosure` — the coverage facts, generated from the run. Relay
those facts to the user and offer a live end-to-end test. Publish only if they
still ask, by calling publish again with `acknowledgeUnverified: true`. Never
set that flag to skip the disclosure. The user also sees the same facts in the
publish approval prompt, so a summary that contradicts them is visible to them.
Execution evidence can come from a run you started or a run the user started.
If the user says they ran the workflow manually, call
`executions(action="list", workflowId)`, identify the relevant run, and inspect
@ -476,9 +484,24 @@ result.
## Claiming success
Do not tell the user a workflow is "fixed", "verified", "tested", "working", or
has "no errors" unless you have a passing `verify-built-workflow`,
`executions(action="run")`, or inspected user-run execution that exercised the
`verify-built-workflow` returns a `claim`, and its `level` decides what you may
say:
- `verified` — you may call the workflow verified, tested, or working.
- `partial`, `unproven`, or `failed` — you may NOT. Name what is unconfirmed
instead.
**`success: true` does not mean verified.** It means the run ended without an
error, and a run with every write simulated also ends without an error. Read
`claim.level`, not `success`.
Nothing else tells the user how strong the claim is. Your message is the only
place they learn it, so name the unreached and simulated nodes yourself, and say
what stays unconfigured and what that means when the workflow runs.
Without a claim, do not tell the user a workflow is "fixed", "verified",
"tested", "working", or has "no errors" unless you have a passing
`executions(action="run")` or an inspected user-run execution that exercised the
path being claimed. Do not call a workflow "ready to use" or "ready to publish"
unless a passing execution met the publish-readiness requirement above. A
successful `build-workflow`/save, a static `workflows(action="validate")`, or

View File

@ -1753,6 +1753,236 @@ describe('workflows tool', () => {
});
});
function contextWithClaim(claim: unknown, permissions?: Partial<InstanceAiPermissions>) {
return createMockContext({
...(permissions ? { permissions } : {}),
workflowBuildContext: {
threadId: 't1',
runId: 'run-1',
taskId: 'task-1',
workItemId: 'wi-1',
workflowTaskService: {
getLatestBuildOutcomeForWorkflow: vi
.fn()
.mockResolvedValue({ runId: 'run-1', verification: { claim } }),
},
} as never,
});
}
const partialClaim = {
level: 'partial',
plannedNodeCount: 12,
reachedNodeCount: 5,
nodesNotReached: ['Send Email', 'Log Row'],
simulatedNodes: [{ nodeName: 'Create Event', reason: 'Creates a record' }],
pinnedNodes: [],
unprovenTargets: [],
publishReady: false,
liveTestRecommended: true,
};
it('should refuse to publish an unverified workflow and disclose the coverage', async () => {
const context = contextWithClaim(partialClaim);
const suspend = vi.fn();
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
suspend,
resumeData: undefined,
} as never);
expect(context.workflowService.publish).not.toHaveBeenCalled();
// Refused before the dialog, so an `always_allow` instance is covered too.
expect(suspend).not.toHaveBeenCalled();
expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' });
const { verificationDisclosure } = result as { verificationDisclosure: string };
expect(verificationDisclosure).toContain('NOT fully verified');
expect(verificationDisclosure).toContain('Send Email, Log Row');
expect(verificationDisclosure).toContain('Create Event');
});
it('should refuse an unverified publish even when approval is not required', async () => {
const context = contextWithClaim(partialClaim, { publishWorkflow: 'always_allow' });
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {} as never);
expect(context.workflowService.publish).not.toHaveBeenCalled();
expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' });
});
it('still asks the user to approve an acknowledged unverified publish under always_allow', async () => {
// The acknowledgement is only the model's word that the user asked, so
// it must not be the sole gate: without the dialog an `always_allow`
// instance would publish an unverified workflow with no disclosure
// anywhere the user could see.
const context = contextWithClaim(partialClaim, { publishWorkflow: 'always_allow' });
const suspend = vi.fn();
const tool = createWorkflowsTool(context, 'full');
await executeTool(
tool,
{ action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true },
{ suspend, resumeData: undefined } as never,
);
expect(context.workflowService.publish).not.toHaveBeenCalled();
expect(suspend).toHaveBeenCalledTimes(1);
const { message } = (suspend as Mock).mock.calls[0][0] as { message: string };
expect(message).toContain('NOT fully verified');
});
it('keeps always_allow routine for a verified publish', async () => {
const context = contextWithClaim(
{ ...partialClaim, level: 'verified', publishReady: true },
{ publishWorkflow: 'always_allow' },
);
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
const suspend = vi.fn();
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
suspend,
} as never);
expect(suspend).not.toHaveBeenCalled();
expect(result).toMatchObject({ success: true });
});
it('should disclose the coverage in the approval prompt once acknowledged', async () => {
const context = contextWithClaim(partialClaim);
const suspend = vi.fn();
const tool = createWorkflowsTool(context, 'full');
await executeTool(
tool,
{ action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true },
{ suspend, resumeData: undefined } as never,
);
expect(suspend).toHaveBeenCalledTimes(1);
const { message } = (suspend as Mock).mock.calls[0][0] as { message: string };
expect(message).toContain('Publish');
expect(message).toContain('NOT fully verified');
expect(message).toContain('Send Email, Log Row');
});
it('should publish once acknowledged and approved', async () => {
const context = contextWithClaim(partialClaim);
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(
tool,
{ action: 'publish', workflowId: 'wf1', acknowledgeUnverified: true },
{ resumeData: { approved: true } } as never,
);
expect(context.workflowService.publish).toHaveBeenCalledWith('wf1', {
versionId: undefined,
});
expect(result).toMatchObject({ success: true, activeVersionId: 'v2' });
});
it('should not gate a fully verified workflow', async () => {
const context = contextWithClaim({ ...partialClaim, level: 'verified', publishReady: true });
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
const suspend = vi.fn();
const tool = createWorkflowsTool(context, 'full');
await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
suspend,
resumeData: undefined,
} as never);
const { message } = (suspend as Mock).mock.calls[0][0] as { message: string };
expect(message).not.toContain('NOT');
});
it('refuses a publish when verification ran but produced no verdict', async () => {
// `handleMissingSimulationPlan` writes an attempted record with no
// claim. Reading that as an absent record let a failed verification
// publish as freely as a workflow nobody ever verified.
const context = createMockContext({
workflowBuildContext: {
threadId: 't1',
runId: 'run-1',
taskId: 'task-1',
workItemId: 'wi-1',
workflowTaskService: {
getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue({
runId: 'run-1',
verification: {
attempted: true,
success: false,
failureSignature: 'missing_simulation_plan',
},
}),
},
} as never,
});
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
suspend: vi.fn(),
resumeData: undefined,
} as never);
expect(context.workflowService.publish).not.toHaveBeenCalled();
expect(result).toMatchObject({ success: false, denied: true, reason: 'not_verified' });
const { verificationDisclosure } = result as { verificationDisclosure: string };
expect(verificationDisclosure).toContain('produced no verdict');
expect(verificationDisclosure).toContain('missing_simulation_plan');
});
it('does not gate a workflow that was never verified', async () => {
// An absent record is unknown, not unverified. Gating it would stop
// publishing every workflow built before the claim existed.
const context = createMockContext({
workflowBuildContext: {
threadId: 't1',
runId: 'run-1',
taskId: 'task-1',
workItemId: 'wi-1',
workflowTaskService: {
getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue({ runId: 'run-1' }),
},
} as never,
});
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
resumeData: { approved: true },
} as never);
expect(result).toMatchObject({ success: true });
});
it('should fail open when the build outcome cannot be read', async () => {
const context = createMockContext({
workflowBuildContext: {
threadId: 't1',
runId: 'run-1',
taskId: 'task-1',
workItemId: 'wi-1',
workflowTaskService: {
getLatestBuildOutcomeForWorkflow: vi.fn().mockRejectedValue(new Error('storage down')),
},
} as never,
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() } as never,
});
(context.workflowService.publish as Mock).mockResolvedValue({ activeVersionId: 'v2' });
const tool = createWorkflowsTool(context, 'full');
const result = await executeTool(tool, { action: 'publish', workflowId: 'wf1' }, {
resumeData: { approved: true },
} as never);
expect(result).toMatchObject({ success: true });
});
it('should suspend for confirmation and then publish when approved', async () => {
const context = createMockContext();
(context.workflowService.publish as Mock).mockResolvedValue({

View File

@ -60,7 +60,7 @@ describe('report-verification-verdict tool', () => {
expect((result as { guidance: string }).guidance).toContain('Error');
});
it('returns done guidance when verdict is verified', async () => {
it('forwards a verified verdict and reports the workflow', async () => {
const doneAction: WorkflowLoopAction = {
type: 'done',
workflowId: 'wf-123',
@ -83,10 +83,66 @@ describe('report-verification-verdict tool', () => {
'Inspected persisted workflow wf-123; the saved graph matches the requested outcome.',
}),
);
expect((result as { guidance: string }).guidance).toContain('verified successfully');
expect((result as { guidance: string }).guidance).toContain('wf-123');
});
it('forwards no claim when no run recorded one', async () => {
// `getBuildOutcome` returns nothing, so the tool must forward `claim:
// undefined` and let the controller decide. Asserting the guidance here
// would prove nothing: it is rendered from the mocked action, so it stays
// claimless even if the tool invented a verified claim. The wording for a
// missing claim is covered in `workflow-loop/__tests__/guidance.test.ts`.
const workflowTaskService = createWorkflowTaskService(
vi.fn().mockResolvedValue({ type: 'done', workflowId: 'wf-123', summary: 'All good' }),
);
const context = createMockContext({ workflowTaskService });
const tool = createReportVerificationVerdictTool(context);
await executeTool(tool, baseInput, {} as never);
expect(workflowTaskService.getBuildOutcome).toHaveBeenCalledWith('wi_test1234');
const [reported] = workflowTaskService.reportVerificationVerdict.mock.calls[0] as [
{ verdict: string; claim?: unknown },
];
expect(reported.claim).toBeUndefined();
// It does not refuse the verdict: a live run the model inspected is still
// valid evidence, so the controller settles and the guidance carries the
// caveat.
expect(reported.verdict).toBe('verified');
});
it('keeps the verified wording when a run recorded a verified claim', async () => {
const claim = {
level: 'verified' as const,
plannedNodeCount: 2,
reachedNodeCount: 2,
nodesNotReached: [],
simulatedNodes: [],
pinnedNodes: [],
unprovenTargets: [],
publishReady: true,
liveTestRecommended: false,
};
const workflowTaskService = createWorkflowTaskService(
vi.fn().mockResolvedValue({
type: 'done',
workflowId: 'wf-123',
summary: 'All good',
claim,
}),
);
workflowTaskService.getBuildOutcome.mockResolvedValue({ verification: { claim } });
const context = createMockContext({ workflowTaskService });
const tool = createReportVerificationVerdictTool(context);
const result = await executeTool(tool, baseInput, {} as never);
expect(workflowTaskService.reportVerificationVerdict).toHaveBeenCalledWith(
expect.objectContaining({ claim }),
);
expect((result as { guidance: string }).guidance).toContain('verified successfully');
});
it('returns verify guidance when action is verify', async () => {
const verifyAction: WorkflowLoopAction = { type: 'verify', workflowId: 'wf-123' };
const reportVerificationVerdict = vi.fn().mockResolvedValue(verifyAction);

View File

@ -136,6 +136,10 @@ export function createReportVerificationVerdictTool(context: OrchestrationContex
};
}
// The claim comes from the persisted run, never from the model: a
// `verified` verdict on a partially covered run must not upgrade it.
const claim = (await context.workflowTaskService.getBuildOutcome(input.workItemId))
?.verification?.claim;
const remediation = input.remediation ?? defaultRemediationForVerdict(input);
const forcedTerminalVerdict =
remediation && !remediation.shouldEdit
@ -150,6 +154,7 @@ export function createReportVerificationVerdictTool(context: OrchestrationContex
workflowId: input.workflowId,
executionId: input.executionId,
verdict: forcedTerminalVerdict ?? input.verdict,
claim,
workflowInspection: input.workflowInspection,
failureSignature: forcedTerminalVerdict
? (remediation?.reason ?? input.failureSignature)

View File

@ -0,0 +1,161 @@
import type { VerificationAnalysis } from '../analyze-result';
import { deriveVerificationClaim } from '../claim';
function makeAnalysis(overrides: Partial<VerificationAnalysis> = {}): VerificationAnalysis {
return {
success: true,
reachedNames: new Set(['Trigger', 'Fetch', 'Send']),
reachedSimulatedNodes: [],
workflowPinnedNodeNames: [],
nodesNotReached: [],
nodeErrors: [],
...overrides,
};
}
describe('deriveVerificationClaim', () => {
it('claims verified when every planned node ran for real', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis(),
plannedNodeCount: 3,
});
expect(claim.level).toBe('verified');
expect(claim.publishReady).toBe(true);
expect(claim.liveTestRecommended).toBe(false);
expect(claim.reachedNodeCount).toBe(3);
});
it('downgrades to partial when planned nodes were never reached', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({ nodesNotReached: ['Send', 'Log'] }),
plannedNodeCount: 5,
});
expect(claim.level).toBe('partial');
expect(claim.publishReady).toBe(false);
expect(claim.liveTestRecommended).toBe(true);
expect(claim.nodesNotReached).toEqual(['Send', 'Log']);
});
it('downgrades to partial when a reached node had simulated output', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({
reachedSimulatedNodes: [{ nodeName: 'Send', reason: 'Sends a message' }],
}),
plannedNodeCount: 3,
});
expect(claim.level).toBe('partial');
expect(claim.publishReady).toBe(false);
});
it('never claims verified for a pin-fed run', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({
reachedSimulatedNodes: [{ nodeName: 'Fetch', reason: 'Output came from pinned data' }],
workflowPinnedNodeNames: ['Fetch'],
}),
plannedNodeCount: 3,
});
expect(claim.level).toBe('partial');
expect(claim.pinnedNodes).toEqual(['Fetch']);
});
it('reports unproven when the fix target was never reached', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({ nodesNotReached: ['Send'] }),
plannedNodeCount: 4,
fixTargetNodeNames: ['Send'],
});
expect(claim.level).toBe('unproven');
expect(claim.unprovenTargets).toEqual(['Send']);
expect(claim.publishReady).toBe(false);
expect(claim.liveTestRecommended).toBe(true);
});
it('reports unproven when the fix target had simulated output', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({
reachedSimulatedNodes: [{ nodeName: 'Send', reason: 'Sends a message' }],
}),
plannedNodeCount: 3,
fixTargetNodeNames: ['Send'],
});
expect(claim.level).toBe('unproven');
expect(claim.unprovenTargets).toEqual(['Send']);
});
it('stays partial when the fix target was proven but coverage was incomplete', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({ nodesNotReached: ['Log'] }),
plannedNodeCount: 4,
fixTargetNodeNames: ['Fetch'],
});
expect(claim.level).toBe('partial');
expect(claim.unprovenTargets).toEqual([]);
});
it('never reports unproven when no fix target is known', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({ nodesNotReached: ['Send'] }),
plannedNodeCount: 4,
});
expect(claim.level).toBe('partial');
});
it('deduplicates repeated fix targets', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({ nodesNotReached: ['Send'] }),
plannedNodeCount: 4,
fixTargetNodeNames: ['Send', 'Send'],
});
expect(claim.unprovenTargets).toEqual(['Send']);
});
it('counts reached nodes over the plan, not the whole run', () => {
// A mockable trigger is not classified, so it never appears in the plan.
// Counting every executed node against the plan length reported "2 of 1".
const claim = deriveVerificationClaim({
analysis: makeAnalysis({
reachedNames: new Set(['Every Morning at 9', 'Send Slack Message']),
reachedSimulatedNodes: [{ nodeName: 'Send Slack Message', reason: 'Sends a message' }],
}),
plannedNodeCount: 1,
});
expect(claim.plannedNodeCount).toBe(1);
expect(claim.reachedNodeCount).toBe(1);
expect(claim.reachedNodeCount).toBeLessThanOrEqual(claim.plannedNodeCount);
});
it('never reports a negative reached count', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({ nodesNotReached: ['A', 'B', 'C'] }),
plannedNodeCount: 1,
});
expect(claim.reachedNodeCount).toBe(0);
});
it('reports failed on a failed run regardless of coverage', () => {
const claim = deriveVerificationClaim({
analysis: makeAnalysis({
success: false,
nodeErrors: [{ nodeName: 'Fetch', message: 'boom' }],
}),
plannedNodeCount: 3,
fixTargetNodeNames: ['Fetch'],
});
expect(claim.level).toBe('failed');
expect(claim.publishReady).toBe(false);
expect(claim.liveTestRecommended).toBe(false);
});
});

View File

@ -174,4 +174,33 @@ describe('runScriptedGateVerification — merge consistency', () => {
// Publish is reached only by the approve pass; the decline pass ran last.
expect(analysis.simulationNote).toContain('Publish');
});
it('discloses a node fed by workflow pin data in any pass', async () => {
const run = vi
.fn()
.mockResolvedValueOnce({ ...approvePassResult, workflowPinnedNodeNames: ['Router'] })
.mockResolvedValueOnce(declinePassResult);
const { analysis } = await runScriptedGateVerification(makeArgs(run));
// Pins come from the run result, so they exist per pass only. Losing them
// in the merge would let a pin-fed gate run read as a live test.
expect(analysis.workflowPinnedNodeNames).toEqual(['Router']);
expect(analysis.reachedSimulatedNodes.map((node) => node.nodeName)).toContain('Router');
expect(analysis.simulationNote).toContain('Router');
});
it('does not double-count a pinned node that the plan already simulates', async () => {
const run = vi
.fn()
.mockResolvedValueOnce({ ...approvePassResult, workflowPinnedNodeNames: ['Generate'] })
.mockResolvedValueOnce(declinePassResult);
const { analysis } = await runScriptedGateVerification(makeArgs(run));
expect(analysis.workflowPinnedNodeNames).toEqual([]);
expect(
analysis.reachedSimulatedNodes.filter((node) => node.nodeName === 'Generate'),
).toHaveLength(1);
});
});

View File

@ -24,6 +24,10 @@ export type ChatModelRecoveryOptions = Pick<
type ExecutionNodeError = NonNullable<ExecutionRunResult['nodeErrors']>[number];
/** Disclosure for a node whose output came from pin data saved on the workflow. */
export const WORKFLOW_PIN_SIMULATION_REASON =
'Output came from pinned data saved on the workflow — unpin it for a live test';
const CREDENTIAL_FAILURE_KEYWORDS = [
'credential',
'unauthorized',
@ -413,6 +417,7 @@ export interface VerificationAnalysis {
success: boolean;
reachedNames: Set<string>;
reachedSimulatedNodes: Array<{ nodeName: string; reason: string }>;
workflowPinnedNodeNames: string[];
nodesNotReached: string[];
remediation?: RemediationMetadata;
nodesExecuted?: string[];
@ -462,10 +467,7 @@ export function analyzeVerificationResult(args: {
const plannedSimulatedNames = new Set(simulatedNodes.map((n) => n.nodeName));
const workflowPinnedNodes = (result.workflowPinnedNodeNames ?? [])
.filter((name) => reachedNames.has(name) && !plannedSimulatedNames.has(name))
.map((name) => ({
nodeName: name,
reason: 'Output came from pinned data saved on the workflow — unpin it for a live test',
}));
.map((name) => ({ nodeName: name, reason: WORKFLOW_PIN_SIMULATION_REASON }));
const reachedSimulatedNodes = [
...simulatedNodes.filter((n) => reachedNames.has(n.nodeName)),
...workflowPinnedNodes,
@ -502,6 +504,7 @@ export function analyzeVerificationResult(args: {
success,
reachedNames,
reachedSimulatedNodes,
workflowPinnedNodeNames: workflowPinnedNodes.map((n) => n.nodeName),
nodesNotReached,
remediation,
nodesExecuted: namesOrDataKeys(reachedNames, result.data),

View File

@ -0,0 +1,70 @@
/**
* Deterministic verification claim.
*
* The model narrates around this structure but never produces it: instruction
* following degrades over a long thread, so the strength of the claim shown to
* the user is computed from run evidence instead of asked for in a prompt.
*/
import type { VerificationAnalysis } from './analyze-result';
import type {
VerificationClaim,
VerificationClaimLevel,
} from '../../../workflow-loop/workflow-loop-state';
export interface DeriveVerificationClaimArgs {
analysis: VerificationAnalysis;
/** Planned nodes from the build outcome simulation plan. */
plannedNodeCount: number;
/**
* Nodes this change was about: the node a previous verdict sent for repair,
* or the failing node the user reported. Empty means no target-specific
* downgrade applies and coverage alone decides the level.
*/
fixTargetNodeNames?: readonly string[];
}
export function deriveVerificationClaim(args: DeriveVerificationClaimArgs): VerificationClaim {
const { analysis, plannedNodeCount, fixTargetNodeNames = [] } = args;
const nodesNotReached = [...analysis.nodesNotReached];
const simulatedNodes = [...analysis.reachedSimulatedNodes];
const unproven = new Set([...nodesNotReached, ...simulatedNodes.map((n) => n.nodeName)]);
const unprovenTargets = [...new Set(fixTargetNodeNames.filter((name) => unproven.has(name)))];
const level = resolveLevel({
success: analysis.success,
hasUnreached: nodesNotReached.length > 0,
hasSimulated: simulatedNodes.length > 0,
hasUnprovenTarget: unprovenTargets.length > 0,
});
return {
level,
plannedNodeCount,
// Counted over the simulation plan, the same set `nodesNotReached` comes
// from. `reachedNames` covers the whole run, and the plan skips mockable
// triggers, so comparing the two produced counts like "2 of 1".
reachedNodeCount: Math.max(plannedNodeCount - nodesNotReached.length, 0),
nodesNotReached,
simulatedNodes,
pinnedNodes: [...analysis.workflowPinnedNodeNames],
unprovenTargets,
publishReady: level === 'verified',
liveTestRecommended: level === 'partial' || level === 'unproven',
};
}
function resolveLevel(facts: {
success: boolean;
hasUnreached: boolean;
hasSimulated: boolean;
hasUnprovenTarget: boolean;
}): VerificationClaimLevel {
if (!facts.success) return 'failed';
// A named target that never ran for real outranks overall coverage: the user
// asked about that node, so a green run elsewhere does not answer them.
if (facts.hasUnprovenTarget) return 'unproven';
if (facts.hasUnreached || facts.hasSimulated) return 'partial';
return 'verified';
}

View File

@ -11,7 +11,10 @@ import type {
} from './types';
import type { OrchestrationContext } from '../../../types';
import { createRemediation } from '../../../workflow-loop/remediation';
import type { RemediationMetadata } from '../../../workflow-loop/workflow-loop-state';
import type {
RemediationMetadata,
VerificationClaim,
} from '../../../workflow-loop/workflow-loop-state';
/**
* Handle the no-simulation-plan case: refuse to run because destructive nodes
@ -88,11 +91,21 @@ export async function persistVerificationOutcome(args: {
workflowId: string;
result: ExecutionRunResult;
analysis: VerificationAnalysis;
/** Deterministic verdict for this run, persisted so later turns cannot re-litigate it. */
claim: VerificationClaim;
/** Running count of verify runs for this build, used to enforce MAX_VERIFY_ATTEMPTS. */
verifyAttempts: number;
}): Promise<void> {
const { input, context, workflowTaskService, workflowId, result, analysis, verifyAttempts } =
args;
const {
input,
context,
workflowTaskService,
workflowId,
result,
analysis,
claim,
verifyAttempts,
} = args;
try {
const executedForEvidence = namesOrDataKeys(analysis.reachedNames, result.data);
await workflowTaskService.updateBuildOutcome(input.workItemId, {
@ -103,6 +116,7 @@ export async function persistVerificationOutcome(args: {
executionId: result.executionId || undefined,
status: result.status,
failureSignature: analysis.success ? undefined : analysis.errorMessage,
claim,
evidence: {
nodesExecuted:
executedForEvidence && executedForEvidence.length > 0 ? executedForEvidence : undefined,

View File

@ -8,6 +8,7 @@
import {
analyzeVerificationResult,
buildSimulationNote,
WORKFLOW_PIN_SIMULATION_REASON,
type ChatModelRecoveryOptions,
type VerificationAnalysis,
} from './analyze-result';
@ -154,14 +155,28 @@ function mergeAnalyses(
// Rebuild the note from the union — a node simulated only in an earlier
// pass must still be disclosed.
const reachedSimulatedNodes = prepared.simulatedNodes.filter((node) =>
const plannedSimulated = prepared.simulatedNodes.filter((node) =>
reachedNames.has(node.nodeName),
);
// Pins come from the run result, so they only exist per pass. Without this
// union a pin-fed gate run would read as live (INS-1216).
const plannedSimulatedNames = new Set(plannedSimulated.map((node) => node.nodeName));
const workflowPinnedNodeNames = [
...new Set(passes.flatMap((pass) => pass.analysis.workflowPinnedNodeNames)),
].filter((name) => !plannedSimulatedNames.has(name));
const reachedSimulatedNodes = [
...plannedSimulated,
...workflowPinnedNodeNames.map((name) => ({
nodeName: name,
reason: WORKFLOW_PIN_SIMULATION_REASON,
})),
];
return {
success: passes.every((pass) => pass.analysis.success),
reachedNames,
reachedSimulatedNodes,
workflowPinnedNodeNames,
nodesNotReached,
remediation: failing?.analysis.remediation,
nodesExecuted: [...reachedNames],

View File

@ -1,5 +1,8 @@
import type { ExecutionNodeError, OrchestrationContext } from '../../../types';
import type { RemediationMetadata } from '../../../workflow-loop/workflow-loop-state';
import type {
RemediationMetadata,
VerificationClaim,
} from '../../../workflow-loop/workflow-loop-state';
export interface VerifyToolInput {
workItemId?: string;
@ -11,6 +14,8 @@ export interface VerifyToolInput {
includeData?: boolean;
maxDataChars?: number;
fixtureOverrides?: Record<string, Array<Record<string, unknown>>>;
/** Nodes this change is about — see the tool input description. */
fixTargetNodeNames?: string[];
/** Nodes whose override may pin zero items — the explicit opt-in for an empty-branch test. */
allowZeroItemFixtures?: string[];
}
@ -47,6 +52,7 @@ export interface VerifyBuiltWorkflowOutput {
nodeErrors?: ExecutionNodeError[];
nodesNotReached?: string[];
coverageNote?: string;
claim?: VerificationClaim;
data?: Record<string, unknown>;
error?: string;
remediation?: RemediationMetadata;

View File

@ -11,6 +11,7 @@ import { z } from 'zod';
import type { OrchestrationContext } from '../../types';
import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result';
import { deriveVerificationClaim } from './verification/claim';
import {
handleMissingSimulationPlan,
persistVerificationOutcome,
@ -19,7 +20,10 @@ import { prepareVerificationRun } from './verification/prepare-run';
import { reconcileStaleCredentialPlan } from './verification/reconcile-plan';
import { resolveVerificationTarget } from './verification/resolve-target';
import { runScriptedGateVerification } from './verification/scripted-gate-run';
import { executionNodeErrorSchema } from '../../workflow-loop/workflow-loop-state';
import {
executionNodeErrorSchema,
verificationClaimSchema,
} from '../../workflow-loop/workflow-loop-state';
import { collectChatModelRecoveryContext } from '../workflows/chat-model-validation';
const DEFAULT_NODE_PREVIEW_CHARS = 600;
@ -81,6 +85,16 @@ export const verifyBuiltWorkflowInputSchema = z.object({
'Optional per-run output fixtures keyed by node name. Only nodes already classified as simulated in the build outcome may be overridden. Use this for alternate deterministic scenarios, not raw trigger input. ' +
'An empty array is rejected unless the node is also listed in `allowZeroItemFixtures`.',
),
fixTargetNodeNames: z
.array(z.string())
.optional()
.describe(
'Node names this change is about — the node the user reported as failing, or the node ' +
'you just repaired. The verdict shown to the user is downgraded to "changed but ' +
'unverified" when any of these was never reached or had simulated output, so a green ' +
'run elsewhere cannot pass as proof for them. Pass these whenever you are fixing a ' +
'specific node rather than building from scratch.',
),
allowZeroItemFixtures: z
.array(z.string())
.optional()
@ -125,6 +139,7 @@ const verifyBuiltWorkflowOutputSchema = z.object({
nodeErrors: z.array(executionNodeErrorSchema).optional(),
nodesNotReached: z.array(z.string()).optional(),
coverageNote: z.string().optional(),
claim: verificationClaimSchema.optional(),
data: z.record(z.unknown()).optional(),
error: z.string().optional(),
remediation: remediationOutputSchema,
@ -238,6 +253,22 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
};
})();
// The repair target from an earlier verdict counts even when the model
// omits it here — that is exactly the turn where it stops mentioning it.
const fixTargetNodeNames = [
...new Set(
[
...(resolvedInput.fixTargetNodeNames ?? []),
target.stateBefore?.lastFailedNodeName,
].filter((name): name is string => name !== undefined),
),
];
const claim = deriveVerificationClaim({
analysis,
plannedNodeCount: buildOutcome.nodeSimulationPlan?.length ?? 0,
fixTargetNodeNames,
});
await persistVerificationOutcome({
input: resolvedInput,
context,
@ -245,6 +276,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
workflowId,
result,
analysis,
claim,
verifyAttempts: (buildOutcome.verifyAttempts ?? 0) + 1,
});
@ -254,6 +286,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
resolvedWorkItemId: resolvedInput.workItemId,
executionId: result.executionId || undefined,
success: analysis.success,
claim,
status: result.status,
nodesExecuted: analysis.nodesExecuted,
lastNodeExecuted: result.lastNodeExecuted,

View File

@ -24,6 +24,7 @@ import {
TEMPLATABLE_PLAIN_AUTH_TYPES,
} from './credentials.tool';
import { formatTimestamp } from '../utils/format-timestamp';
import { formatClaimDisclosure } from '../workflow-loop/render-claim';
import { isSetupPanelEnabled } from './workflows/setup-items';
import {
describeSetupItem,
@ -310,6 +311,14 @@ const publishBaseAction = z.object({
.describe('Publish a workflow version to production (omit versionId for latest draft)'),
workflowId: z.string().describe('ID of the workflow'),
versionId: z.string().optional().describe('Version ID'),
acknowledgeUnverified: z
.boolean()
.optional()
.describe(
'Set true only after you told the user the workflow is not fully verified and they still ' +
'asked to publish. Publishing is refused without this while the latest verification left ' +
'nodes unreached or simulated. Never set it to skip the disclosure.',
),
});
const publishExtendedAction = publishBaseAction.extend({
@ -1377,6 +1386,49 @@ async function resolveSetupScopeNodeNames(
}
}
/**
* Coverage disclosure for the workflow's latest verification, or undefined when
* it was fully verified or no claim exists. Absent evidence never blocks: a
* workflow built before this record, or outside the assistant, is unknown
* rather than unverified.
*/
async function resolveUnverifiedPublishDisclosure(
context: InstanceAiContext,
workflowId: string,
): Promise<string | undefined> {
const workflowTaskService = context.workflowBuildContext?.workflowTaskService;
if (!workflowTaskService) return undefined;
try {
const outcome = await workflowTaskService.getLatestBuildOutcomeForWorkflow(workflowId);
const verification = outcome?.verification;
const claim = verification?.claim;
if (claim) return claim.publishReady ? undefined : formatClaimDisclosure(claim);
// A record with no claim means verification ran and could not produce one
// — it was refused for want of a simulation plan, or it failed before it
// reached a verdict. That is a failed verification, not an unverified
// workflow, and it must not pass as an absent record.
if (verification?.attempted) {
const cause = verification.failureSignature ?? verification.evidence?.errorMessage;
return (
'Verification ran but produced no verdict, so nothing about this workflow is proven.' +
(cause ? ` It reported: ${cause}` : '')
);
}
// No record at all: the workflow is unknown, not unverified. Blocking here
// would stop publishing every workflow built before this record existed.
return undefined;
} catch (error) {
// Fail open: a storage hiccup must not block a publish the user asked for.
context.logger.warn('Failed to resolve the verification claim before publishing', {
workflowId,
error: error instanceof Error ? error.message : String(error),
});
return undefined;
}
}
const SETUP_PANEL_ANNOUNCED_GUIDANCE =
'The setup panel next to the chat now lists what this workflow still needs (`open`); nothing is ' +
'waiting on you and no card is open. Finish your turn now: tell the user in one or two sentences ' +
@ -1754,7 +1806,23 @@ async function handlePublish(
}
const supportingWorkflowIds = await resolveSupportingWorkflowIds(context, input.workflowId);
const needsApproval = context.permissions?.publishWorkflow !== 'always_allow';
const unverifiedDisclosure = await resolveUnverifiedPublishDisclosure(context, input.workflowId);
const needsApproval =
context.permissions?.publishWorkflow !== 'always_allow' || unverifiedDisclosure !== undefined;
if (unverifiedDisclosure && input.acknowledgeUnverified !== true) {
return {
success: false,
denied: true,
reason: 'not_verified',
verificationDisclosure: unverifiedDisclosure,
guidance:
`This workflow is not fully verified. ${unverifiedDisclosure} ` +
'Tell the user exactly this, and offer a live end-to-end test. Publish only if they still ' +
'ask for it, by calling publish again with `acknowledgeUnverified: true`.',
};
}
if (needsApproval && (resumeData === undefined || resumeData === null)) {
const workflowName = await resolveWorkflowName(context, input.workflowId);
@ -1762,12 +1830,15 @@ async function handlePublish(
supportingWorkflowIds.length > 0
? ` and ${String(supportingWorkflowIds.length)} referenced supporting workflow(s)`
: '';
const target = input.versionId
? `Publish version ${input.versionId} of ${workflowName} (ID: ${input.workflowId})${dependencyNote}`
: `Publish ${workflowName} (ID: ${input.workflowId})${dependencyNote}`;
return await ctx.suspend({
requestId: nanoid(),
message: input.versionId
? `Publish version ${input.versionId} of ${workflowName} (ID: ${input.workflowId})${dependencyNote}`
: `Publish ${workflowName} (ID: ${input.workflowId})${dependencyNote}`,
// The user has to read this to approve, so the disclosure lands even if
// the assistant's own message left it out.
message: unverifiedDisclosure ? `${target}\n\n${unverifiedDisclosure}` : target,
severity: 'warning' as const,
});
}

View File

@ -1,19 +1,111 @@
import { formatWorkflowLoopGuidance } from '../guidance';
import type { WorkflowLoopAction } from '../workflow-loop-state';
import type { VerificationClaim, WorkflowLoopAction } from '../workflow-loop-state';
function makeClaim(overrides: Partial<VerificationClaim> = {}): VerificationClaim {
return {
level: 'verified',
plannedNodeCount: 3,
reachedNodeCount: 3,
nodesNotReached: [],
simulatedNodes: [],
pinnedNodes: [],
unprovenTargets: [],
publishReady: true,
liveTestRecommended: false,
...overrides,
};
}
describe('formatWorkflowLoopGuidance', () => {
// ── done ────────────────────────────────────────────────────────────────────
describe('action type "done"', () => {
it('should report completion without workflowId', () => {
it('should not claim verification when no run recorded a claim', () => {
// A `verified` verdict reported without verifying leaves no claim. This
// sentence used to say "Workflow verified successfully", which handed
// the model the exact wording with no run evidence behind it.
const action: WorkflowLoopAction = {
type: 'done',
summary: 'All good',
};
const result = formatWorkflowLoopGuidance(action);
expect(result).not.toContain('Workflow verified successfully');
expect(result).toContain('No automatic verification evidence is recorded');
expect(result).toContain('do NOT call the workflow verified');
expect(result).not.toContain('Workflow ID:');
});
it('should not claim success when the claim is partial', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'All good',
claim: makeClaim({
level: 'partial',
reachedNodeCount: 5,
plannedNodeCount: 12,
nodesNotReached: ['Send Email', 'Log Row'],
publishReady: false,
liveTestRecommended: true,
}),
};
const result = formatWorkflowLoopGuidance(action);
expect(result).not.toContain('Workflow verified successfully');
expect(result).toContain('NOT fully verified');
expect(result).toContain('2 of 12 node(s) were never reached');
expect(result).toContain('Send Email, Log Row');
expect(result).toContain('Do NOT offer to publish it.');
expect(result).toContain('Offer a live end-to-end test');
});
it('should name the unproven fix target when the claim is unproven', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'Patched',
claim: makeClaim({
level: 'unproven',
plannedNodeCount: 4,
reachedNodeCount: 4,
unprovenTargets: ['Send Email'],
simulatedNodes: [{ nodeName: 'Send Email', reason: 'Sends a message' }],
publishReady: false,
liveTestRecommended: true,
}),
};
const result = formatWorkflowLoopGuidance(action);
expect(result).not.toContain('Workflow verified successfully');
expect(result).toContain('Changed but NOT verified');
expect(result).toContain('never proven: Send Email');
expect(result).toContain('nothing real happened at: Send Email');
});
it('should keep the verified wording when the claim is verified', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'All good',
claim: makeClaim(),
};
const result = formatWorkflowLoopGuidance(action);
expect(result).toContain('Workflow verified successfully');
expect(result).toContain('Report completion');
expect(result).not.toContain('Workflow ID:');
});
it('should downgrade the mocked-credential guidance too', () => {
const action: WorkflowLoopAction = {
type: 'done',
summary: 'All good',
workflowId: 'wf-123',
mockedCredentialTypes: ['slackApi'],
claim: makeClaim({
level: 'partial',
nodesNotReached: ['Send Email'],
publishReady: false,
liveTestRecommended: true,
}),
};
const result = formatWorkflowLoopGuidance(action);
expect(result).not.toContain('Workflow verified successfully');
expect(result).toContain('NOT fully verified');
expect(result).toContain('workflows(action="setup")');
});
it('should include workflowId when present', () => {
@ -44,7 +136,7 @@ describe('formatWorkflowLoopGuidance', () => {
};
const result = formatWorkflowLoopGuidance(action);
expect(result).not.toContain('credentials(action="setup")');
expect(result).toContain('Report completion');
expect(result).toContain('Report the outcome');
});
it('should include credential instructions when mockedCredentialTypes has entries', () => {

View File

@ -451,6 +451,64 @@ describe('handleBuildOutcome', () => {
// ── handleVerificationVerdict ───────────────────────────────────────────────
describe('handleVerificationVerdict', () => {
describe('deterministic claim', () => {
it('carries a downgraded claim onto the done action instead of blocking the loop', () => {
const state = makeState({ phase: 'verifying' });
const { state: next, action } = handleVerificationVerdict(
state,
[],
makeVerdict({
verdict: 'verified',
claim: {
level: 'partial',
plannedNodeCount: 12,
reachedNodeCount: 5,
nodesNotReached: ['Send Email'],
simulatedNodes: [],
pinnedNodes: [],
unprovenTargets: [],
publishReady: false,
liveTestRecommended: true,
},
}),
);
// The loop must still settle — re-verifying replays the same coverage.
expect(next.phase).toBe('done');
expect(next.status).toBe('completed');
expect(action.type).toBe('done');
if (action.type !== 'done') throw new Error('expected done');
expect(action.claim?.level).toBe('partial');
expect(action.claim?.publishReady).toBe(false);
});
it('records the repair target so a later run can tell whether it was proven', () => {
const state = makeState({ phase: 'verifying' });
const { state: next } = handleVerificationVerdict(
state,
[],
makeVerdict({
verdict: 'needs_patch',
failedNodeName: 'Send Email',
diagnosis: 'Wrong channel',
}),
);
expect(next.lastFailedNodeName).toBe('Send Email');
});
it('keeps a known repair target when a later rebuild names no node', () => {
const state = makeState({ phase: 'verifying', lastFailedNodeName: 'Send Email' });
const { state: next } = handleVerificationVerdict(
state,
[],
makeVerdict({ verdict: 'needs_rebuild', diagnosis: 'Structural repair' }),
);
expect(next.lastFailedNodeName).toBe('Send Email');
});
});
it('ignores stale verification verdicts from a different run without resetting state', () => {
const state = makeState({
runId: 'run_current',

View File

@ -1,4 +1,5 @@
import type { WorkflowLoopAction } from './workflow-loop-state';
import { describeClaimCoverage, formatClaimHeadline } from './render-claim';
import type { VerificationClaim, WorkflowLoopAction } from './workflow-loop-state';
export interface WorkflowLoopGuidanceOptions {
workItemId?: string;
@ -6,6 +7,42 @@ export interface WorkflowLoopGuidanceOptions {
setupPanelEnabled?: boolean;
}
/**
* Lead sentence for a completed build. Derived from the claim so the guidance
* never tells the model a partially covered run was verified — that sentence
* was the strongest license for the false success claims in AIA-31.
*/
function formatClaimLead(claim: VerificationClaim | undefined): string {
if (claim?.level === 'verified') return 'Workflow verified successfully.';
// No claim means no verification run recorded a verdict for this build — a
// trigger-only workflow, or a verdict reported without verifying. Saying
// "verified successfully" here let a model reach that sentence with no run
// evidence at all. A live run the model inspected is still valid evidence,
// so ask it to name that rather than refusing the claim outright.
if (claim === undefined) {
return (
'Build complete. No automatic verification evidence is recorded for this workflow. ' +
'If you are relying on a live run you inspected, say which execution. ' +
'Otherwise do NOT call the workflow verified, tested, or working.'
);
}
// The claim is the only honest reading of the run, and nothing else tells
// the user how strong it is, so these rules carry the whole disclosure.
const rules = [
'Do NOT call the workflow verified, tested, working, or ready to publish.',
'Do NOT offer to publish it.',
claim.liveTestRecommended ? 'Offer a live end-to-end test instead.' : '',
].filter((rule) => rule !== '');
return [formatClaimHeadline(claim), ...describeClaimCoverage(claim), ...rules].join(' ');
}
function isVerifiedClaim(claim: VerificationClaim | undefined): boolean {
return claim?.level === 'verified';
}
function formatSourceFileInstruction(sourceFilePath: string | undefined): string {
if (!sourceFilePath) {
return 'edit the workspace source file, then call `build-workflow` with that filePath';
@ -24,9 +61,11 @@ export function formatWorkflowLoopGuidance(
case 'continue_building':
return `BUILD FAILED: ${action.reason}. Fix the workflow source file: ${formatSourceFileInstruction(action.sourceFilePath)}.`;
case 'done': {
const claimLead = formatClaimLead(action.claim);
if (action.setupSkippedByUser) {
return (
'Workflow verified successfully. The credentials it still needs are ones the user ' +
claimLead +
' The credentials it still needs are ones the user ' +
'skipped earlier in this conversation, so do NOT open the setup card again. Tell them ' +
'which parts stay unconfigured and what that means when the workflow runs, and offer ' +
'to set them up whenever they want.'
@ -44,14 +83,17 @@ export function formatWorkflowLoopGuidance(
);
}
return (
'Workflow verified successfully with temporary mock data. ' +
`${claimLead} It still uses temporary mock data. ` +
`Call \`workflows(action="setup")\` with workflowId "${action.workflowId ?? 'unknown'}" ` +
'to open the inline setup card in the AI Assistant panel for credentials, parameters, and triggers. ' +
'Do not tell the user to open the editor, use the canvas, or click a Setup button. ' +
'Do not call `credentials(action="setup")` or `apply-workflow-credentials` — `workflows(action="setup")` handles everything.'
);
}
return `Workflow verified successfully. Report completion to the user.${action.workflowId ? ` Workflow ID: ${action.workflowId}` : ''}`;
const closing = isVerifiedClaim(action.claim)
? 'Report completion to the user.'
: 'Report the outcome to the user.';
return `${claimLead} ${closing}${action.workflowId ? ` Workflow ID: ${action.workflowId}` : ''}`;
}
case 'verify':
return (

View File

@ -0,0 +1,59 @@
/**
* User-facing wording for a verification claim.
*
* Every surface that discloses coverage renders from here, so the guidance the
* model reads and the publish approval the user clicks state the same facts.
* A claim at `verified` needs no disclosure.
*/
import type { VerificationClaim } from './workflow-loop-state';
export function formatClaimNodeList(names: readonly string[], max = 8): string {
if (names.length <= max) return names.join(', ');
return `${names.slice(0, max).join(', ')} and ${String(names.length - max)} more`;
}
export function formatClaimHeadline(claim: VerificationClaim): string {
switch (claim.level) {
case 'verified':
return 'Verified end to end.';
case 'unproven':
return (
'Changed but NOT verified. The node(s) this change was about were never proven: ' +
`${formatClaimNodeList(claim.unprovenTargets)}.`
);
case 'partial':
return 'Ran without errors, but NOT fully verified.';
case 'failed':
return 'Verification did not pass.';
}
}
export function describeClaimCoverage(claim: VerificationClaim): string[] {
const facts: string[] = [];
if (claim.nodesNotReached.length > 0) {
facts.push(
`${String(claim.nodesNotReached.length)} of ${String(claim.plannedNodeCount)} node(s) were ` +
`never reached, so they are UNVERIFIED: ${formatClaimNodeList(claim.nodesNotReached)}.`,
);
}
if (claim.simulatedNodes.length > 0) {
facts.push(
'Output was simulated, so nothing real happened at: ' +
`${formatClaimNodeList(claim.simulatedNodes.map((node) => node.nodeName))}.`,
);
}
if (claim.pinnedNodes.length > 0) {
facts.push(
`Some output came from pin data saved on the workflow: ${formatClaimNodeList(claim.pinnedNodes)}. ` +
'Remove those pins for a live test.',
);
}
return facts;
}
export function formatClaimDisclosure(claim: VerificationClaim): string | undefined {
if (claim.level === 'verified') return undefined;
return [formatClaimHeadline(claim), ...describeClaimCoverage(claim)].join(' ');
}

View File

@ -273,6 +273,9 @@ export function handleVerificationVerdict(
switch (verdict.verdict) {
case 'verified': {
// The loop still completes on a partial run — re-issuing verification
// would replay the same coverage forever. Only the claim is downgraded,
// so `done` carries an honest verdict instead of a blocked state.
attempt.result = 'success';
return {
state: {
@ -287,6 +290,7 @@ export function handleVerificationVerdict(
type: 'done',
workflowId: verdict.workflowId,
summary: verdict.summary,
claim: verdict.claim,
mockedCredentialTypes: normalizedState.mockedCredentialTypes,
hasUnresolvedPlaceholders: normalizedState.hasUnresolvedPlaceholders,
setupSkippedByUser: normalizedState.setupSkippedByUser,
@ -470,6 +474,7 @@ function escalateToRepair(
status: 'active',
rebuildAttempts: state.rebuildAttempts + 1,
lastFailureSignature: verdict.failureSignature,
lastFailedNodeName: verdict.failedNodeName ?? state.lastFailedNodeName,
lastExecutionId: verdict.executionId,
lastWorkflowInspection: verdict.workflowInspection,
lastRemediation: remediation,

View File

@ -82,6 +82,7 @@ export const workflowLoopStateSchema = z.object({
lastTaskId: z.string().optional(),
lastExecutionId: z.string().optional(),
lastFailureSignature: z.string().optional(),
lastFailedNodeName: z.string().optional(),
lastWorkflowInspection: z.string().optional(),
rebuildAttempts: z.number().int().min(0),
/** Credential types that were mocked during build (persisted across phases). */
@ -153,6 +154,42 @@ export const executionNodeErrorSchema = z.object({
message: z.string().optional(),
});
/**
* Strength of the claim a verification run supports. `verified` means every
* planned node ran for real; anything less must not be reported as verified.
* See `tools/orchestration/verification/claim.ts` for the derivation.
*/
export const verificationClaimLevelSchema = z.enum(['verified', 'partial', 'unproven', 'failed']);
export type VerificationClaimLevel = z.infer<typeof verificationClaimLevelSchema>;
/**
* The deterministic verdict for one verification run. Backend-only: it shapes
* the tool result the model reads and gates the publish offer. It is
* deliberately not sent to the client — see INS-1308.
*/
export const verificationClaimSchema = z.object({
level: verificationClaimLevelSchema,
/** Nodes the verification plan covers. Excludes triggers the classifier skips. */
plannedNodeCount: z.number().int().min(0),
/** Planned nodes the run reached. Same set as `plannedNodeCount`. */
reachedNodeCount: z.number().int().min(0),
nodesNotReached: z.array(z.string()),
/** Reached nodes whose output was mocked, so the run proves nothing about them. */
simulatedNodes: z.array(z.object({ nodeName: z.string(), reason: z.string() })),
/** Pin-fed subset of `simulatedNodes` — these need the pin removed for a live test. */
pinnedNodes: z.array(z.string()),
/**
* Fix targets still unreached or simulated — the reason this run cannot
* claim `verified` even when it ended without an error.
*/
unprovenTargets: z.array(z.string()),
publishReady: z.boolean(),
liveTestRecommended: z.boolean(),
});
export type VerificationClaim = z.infer<typeof verificationClaimSchema>;
/**
* Structured verification evidence the builder captures when it runs
* `verify-built-workflow`. Downstream checkpoint runs read this and skip
@ -164,6 +201,7 @@ export const workflowVerificationEvidenceSchema = z.object({
executionId: z.string().optional(),
status: z.enum(['success', 'error', 'waiting', 'running', 'unknown']).optional(),
failureSignature: z.string().optional(),
claim: verificationClaimSchema.optional(),
evidence: z
.object({
nodesExecuted: z.array(z.string()).optional(),
@ -454,6 +492,7 @@ export const verificationResultSchema = z.object({
workflowId: z.string(),
executionId: z.string().optional(),
verdict: verificationVerdictSchema,
claim: verificationClaimSchema.optional(),
workflowInspection: z.string().optional(),
failureSignature: z.string().optional(),
failedNodeName: z.string().optional(),
@ -485,6 +524,7 @@ export type WorkflowLoopAction =
type: 'done';
workflowId?: string;
summary: string;
claim?: VerificationClaim;
mockedCredentialTypes?: string[];
hasUnresolvedPlaceholders?: boolean;
setupSkippedByUser?: boolean;