fix(core): Document the webhook input envelope and check simulated-node parameters in AI Assistant verification (no-changelog) (#37962)

This commit is contained in:
oleg 2026-09-11 08:05:32 +00:00 committed by GitHub
parent bcad21f9b6
commit f4df71c263
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 591 additions and 26 deletions

View File

@ -192,7 +192,7 @@ waiting-with-output-as-success fallback.
| Trigger | Pass | Adapter emits on `$json` |
|---|---|---|
| Form Trigger | flat field map, e.g. `{name: "Alice", email: "a@b.c"}` | `{ submittedAt, formMode: "instanceAi", name, email, ... }` — matches production. Do NOT wrap in `formFields`. |
| Webhook | body payload, e.g. `{event: "signup", userId: "..."}` | `{ headers, query, body: { event, userId, ... } }` |
| Webhook | body payload, e.g. `{event: "signup", userId: "..."}`, **or** the request envelope `{ body: {...}, query: {...}, headers: {...}, params: {...} }` when any expression reads `$json.query.*`, `$json.headers.*` or `$json.params.*` | flat payload → `{ headers: {}, query: {}, params: {}, body: { event, userId, ... } }`; envelope → passed through as-is |
| Chat Trigger | `{chatInput: "..."}` | `{ sessionId, action, chatInput }` |
| Schedule | omit | synthetic timestamp fields |
@ -211,7 +211,26 @@ record (`{ attempted, success, executionId, status, claim, evidence, verifiedAt
the build outcome so workflow-verification follow-ups and exceptional checkpoint
turns can reuse it without re-running verify.
**Returns**: `{ executionId?, success, status?, data?, error? }`
**Returns**: `{ executionId?, success, status?, data?, error?, simulationNote?, resolvedParameterWarnings?, skippedParameterChecks?, skippedParameterCheckCount? }`
**Simulated-node parameter check**: a simulated node's preview is fixture data, so
an expression that resolved to empty leaves no trace in the run. After the run the
tool replays parameter resolution (`getResolvedNodeParameters`) for every reached
simulated node and returns `resolvedParameterWarnings`, one entry per parameter
that resolved to `null`/`undefined`/`""` or threw (`{ nodeName, executionId, path, raw, issue:
'empty' | 'failed', detail? }`), with a summary appended to `simulationNote`.
For scripted gates, it checks each node against every pass that reached it.
Each warning identifies the execution used for that check.
Expressions that need live-only context (`$secrets`, `$response`, …) are excluded.
The check is advisory and does not change execution success. Suppressed parameter
values, replay failures, and missing executions produce `skippedParameterChecks`
entries (`{ nodeName, executionId?, reason }`) and a note in `simulationNote`.
The list contains at most 20 entries across all passes. `skippedParameterCheckCount`
reports the total. When entries are omitted, the note states how many are shown.
Omitted checks also leave dynamic fields unverified.
The reasons are `parameter-values-disabled`, `replay-failed`, and
`execution-unavailable`. Skipped checks expose no parameter values or replay
error details. Their dynamic fields remain unverified.
### `report-verification-verdict` *(conditional)*
@ -538,7 +557,7 @@ Default timeout: 5 minutes; max: 10 minutes. On timeout, execution is cancelled.
**Type-aware pin data**: Constructs proper pin data per trigger type:
- **Chat trigger**: `{ chatInput, sessionId, action }`
- **Form trigger**: `{ submittedAt, formMode: 'instanceAi', ...inputData }`
- **Webhook trigger**: `{ headers: {}, query: {}, body: inputData }`
- **Webhook trigger**: flat `inputData` → `{ headers: {}, query: {}, params: {}, body: inputData }`; an envelope whose keys are only `body`/`query`/`headers`/`params` is passed through, so query- and header-driven expressions can be exercised
- **Schedule trigger**: current datetime information
- **Unknown trigger**: `{ json: inputData }` (generic fallback)

View File

@ -240,6 +240,14 @@ workflow does not need to be active. Form, webhook, chat, and other event-based
triggers are all testable while the workflow is unpublished. Never publish a
workflow as a precondition for running it.
**Webhook input must carry the fields the workflow reads.** A flat `inputData`
becomes the request `body` only; `query`, `headers` and `params` stay empty. When
any expression reads `$json.query.*`, `$json.headers.*` or `$json.params.*`, pass
the request envelope `{ body: {...}, query: {...}, headers: {...}, params: {...} }` (or a
`fixtureOverrides` entry on the trigger node). Otherwise the field resolves empty,
the run still succeeds, and that field is unverified — say so instead of
reporting it as working.
Do not proactively offer, recommend, or mention publishing until a successful
execution has run every required node on the claimed path without mocked
credentials, simulated node output, fixture overrides, or temporary pin data
@ -380,6 +388,20 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries),
budget is exhausted.
- Relay `simulationNote` (nodes whose output was simulated) to the user
whenever it is present.
- Read `resolvedParameterWarnings`. A simulated node's preview is fixture
data: it never proves an expression resolved. Each warning names a
parameter that resolved to empty or threw on the real input — the usual
causes are a trigger input that lacks the field (body-only webhook input
for a `$json.query.*` expression) or a wrong expression. Fix the input
shape or the expression, re-run, and never report that field as working
while a warning stands. Each warning carries the execution ID that was
checked. Use that ID with `executions(action="get-resolved-node-parameters")`
to inspect the same input.
- Read `skippedParameterChecks`. These nodes have unchecked dynamic fields.
The list shows at most 20 checks. `skippedParameterCheckCount` includes
omitted checks, which also leave dynamic fields unverified.
State that limitation even if the run succeeded and no parameter warnings
were returned. Do not request parameter values when sharing is disabled.
3. After verification handling, if `setupRequirement.status === "required"` and
setup has not already run for this build, call `workflows(action="setup")`
with the workflowId.
@ -577,7 +599,10 @@ applies to rows or records written to an external system: never make quantitativ
claims ("22 rows written", "columns matched") that you did not read back from
the effect node's actual output (`executions(action="get-node-output")`) or from
the target system itself — a successful run status does not prove the _right
data_ was written, only that nodes ran. If you could not run the
data_ was written, only that nodes ran. Output of a simulated or pinned node is
fixture data: never quote it as what the workflow produced, and never cite it as
proof that an expression resolved — use `resolvedParameterWarnings` or
`executions(action="get-resolved-node-parameters")` for that. If you could not run the
failing path or inspect the artifact, say so plainly — "I couldn't verify X
because Y" — and name what is unconfirmed. An honest "could not verify" beats an
unverified success claim.

View File

@ -50,7 +50,9 @@ const runAction = z.object({
.describe(
'Input data passed to the workflow trigger. Works for ANY trigger type — ' +
'the system injects inputData as the trigger node output, bypassing the need for a real event. ' +
'For webhook triggers, inputData is the request body (do NOT wrap in { body: ... }). ' +
'For webhook triggers, a flat inputData is treated as the request body (placed under `body`; ' +
'`query`, `headers` and `params` stay empty). To exercise $json.query.*, $json.headers.* or ' +
'$json.params.*, pass the request envelope { body: {...}, query: {...}, headers: {...}, params: {...} } instead. ' +
'For event-based triggers (e.g. Linear, GitHub, Slack), pass inputData matching ' +
'the shape the trigger would emit (e.g. { action: "create", data: { ... } }).',
),

View File

@ -7,6 +7,7 @@ import type {
InstanceAiDataTableService,
InstanceAiWorkflowService,
OrchestrationContext,
ResolvedNodeParametersResult,
WorkflowTaskService,
} from '../../../types';
import { createRemediation, MAX_VERIFY_ATTEMPTS } from '../../../workflow-loop/remediation';
@ -32,6 +33,20 @@ type VerifyBuiltWorkflowOutput = {
}>;
simulatedNodes?: Array<{ nodeName: string; reason: string }>;
simulationNote?: string;
resolvedParameterWarnings?: Array<{
nodeName: string;
executionId?: string;
path: string;
raw: string;
issue: 'empty' | 'failed';
detail?: string;
}>;
skippedParameterChecks?: Array<{
nodeName: string;
executionId?: string;
reason: string;
}>;
skippedParameterCheckCount?: number;
lastNodeExecuted?: string;
nodesNotReached?: string[];
nodeErrors?: Array<{ nodeName: string; message?: string }>;
@ -457,6 +472,9 @@ interface VerifyToolContext {
]
) => Promise<ExecutionRunResult>
>;
getResolvedNodeParameters?: Mock<
(executionId: string, nodeName: string) => Promise<ResolvedNodeParametersResult>
>;
};
workflowService?: InstanceAiWorkflowService;
dataTableService?: InstanceAiDataTableService;
@ -496,6 +514,8 @@ function makeContext(
workflowConnections?: Record<string, unknown>;
tableRows?: Record<string, Array<Record<string, unknown>>>;
availableCredentials?: Array<{ id: string; name: string; type: string }>;
/** Replayed parameter resolution per simulated node; nodes not listed make the replay throw. */
resolvedParameters?: Record<string, ResolvedNodeParametersResult>;
} = {},
) {
const updateBuildOutcome = vi.fn(
@ -583,6 +603,13 @@ function makeContext(
error: vi.fn(),
};
const getResolvedNodeParameters = vi.fn(async (_executionId: string, nodeName: string) => {
await Promise.resolve();
const replayed = overrides.resolvedParameters?.[nodeName];
if (!replayed) throw new Error(`no run data for ${nodeName}`);
return replayed;
});
const ctx: VerifyToolContext = {
workflowTaskService: {
reportBuildOutcome: vi.fn(),
@ -602,7 +629,7 @@ function makeContext(
recordVerification: vi.fn(coordinator.recordVerification.bind(coordinator)),
} as unknown as WorkflowTaskService,
domainContext: {
executionService: { run },
executionService: { run, getResolvedNodeParameters },
workflowService,
dataTableService,
credentialService,
@ -1167,6 +1194,99 @@ describe('verify-built-workflow tool — node simulation plan', () => {
expect(result.simulationNote).toContain('no real external writes');
});
it('surfaces parameters of reached simulated nodes that resolved to empty or threw', async () => {
const { ctx } = makeContext(
makeBuildOutcome({
nodeSimulationPlan: [
simulateVerdict('Send Slack', 'Sends a message to a Slack channel'),
simulateVerdict('Send SMS', 'Sends an SMS'),
],
simulationFixtures: { 'Send Slack': [{ ok: true }], 'Send SMS': [{ sid: 'SM1' }] },
}),
{
executionId: 'exec-sim',
status: 'success',
data: {
Webhook: [{ headers: {}, query: {}, body: { CallStatus: 'no-answer' } }],
'Send Slack': [{ ok: true }],
},
},
{
resolvedParameters: {
'Send Slack': {
nodeName: 'Send Slack',
runIndex: 0,
itemIndex: 0,
parameters: { text: '={{ $json.query.caller }}' },
resolved: '{"text":""}',
failedExpressions: [
{ path: 'channel', raw: '={{ $json.body.ch.id }}', error: 'Cannot read id' },
],
emptyResolutions: [
{ path: 'text', raw: '={{ $json.query.caller }}', resolved: undefined },
],
},
},
},
);
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
expect(result.success).toBe(true);
// Only nodes the run reached are replayed; Send SMS never executed.
expect(ctx.domainContext.executionService.getResolvedNodeParameters).toHaveBeenCalledTimes(1);
expect(ctx.domainContext.executionService.getResolvedNodeParameters).toHaveBeenCalledWith(
'exec-sim',
'Send Slack',
);
expect(result.resolvedParameterWarnings).toEqual([
{
nodeName: 'Send Slack',
executionId: 'exec-sim',
path: 'channel',
raw: '={{ $json.body.ch.id }}',
issue: 'failed',
detail: 'Cannot read id',
},
{
nodeName: 'Send Slack',
executionId: 'exec-sim',
path: 'text',
raw: '={{ $json.query.caller }}',
issue: 'empty',
},
]);
expect(result.simulationNote).toContain('no real external writes');
expect(result.simulationNote).toContain('Send Slack: `channel`');
expect(result.simulationNote).toContain('`text` (={{ $json.query.caller }}) resolved to empty');
});
it('keeps the verification result when the parameter replay fails', async () => {
const { ctx } = makeContext(
makeBuildOutcome({
nodeSimulationPlan: [simulateVerdict('Send Slack', 'Sends a message to a Slack channel')],
simulationFixtures: { 'Send Slack': [{ ok: true }] },
}),
{ executionId: 'exec-sim', status: 'success', data: { 'Send Slack': [{ ok: true }] } },
);
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
expect(result.success).toBe(true);
expect(result.resolvedParameterWarnings).toBeUndefined();
expect(result.skippedParameterChecks).toEqual([
{ nodeName: 'Send Slack', executionId: 'exec-sim', reason: 'replay-failed' },
]);
expect(result.skippedParameterCheckCount).toBe(1);
expect(result.simulationNote).toContain('no real external writes');
expect(result.simulationNote).toContain('Parameter check skipped');
expect(result.simulationNote).toContain('unchecked dynamic fields');
expect(ctx.logger.debug).toHaveBeenCalledWith(
'Resolved-parameter check skipped for simulated node',
expect.objectContaining({ nodeName: 'Send Slack' }),
);
});
it('fails closed when the build outcome has no simulation plan at all', async () => {
// An undefined plan means the outcome predates classification or
// classification failed — nothing shields destructive nodes in that run.

View File

@ -0,0 +1,161 @@
import type { ResolvedNodeParametersResult } from '../../../../types';
import {
buildResolvedParameterNote,
collectResolvedParameterWarnings,
warningsFromResolution,
} from '../resolved-parameter-warnings';
function resolution(
overrides: Partial<ResolvedNodeParametersResult> = {},
): ResolvedNodeParametersResult {
return {
nodeName: 'Send SMS',
runIndex: 0,
itemIndex: 0,
parameters: {},
resolved: '{}',
failedExpressions: [],
emptyResolutions: [],
...overrides,
};
}
describe('warningsFromResolution', () => {
it('reports empty and failed expressions for the node', () => {
const warnings = warningsFromResolution(
resolution({
emptyResolutions: [{ path: 'to', raw: '={{ $json.query.caller }}', resolved: undefined }],
failedExpressions: [
{ path: 'message', raw: '={{ $json.body.text.trim() }}', error: 'Cannot read trim' },
],
}),
);
expect(warnings).toEqual([
{
nodeName: 'Send SMS',
path: 'message',
raw: '={{ $json.body.text.trim() }}',
issue: 'failed',
detail: 'Cannot read trim',
},
{ nodeName: 'Send SMS', path: 'to', raw: '={{ $json.query.caller }}', issue: 'empty' },
]);
});
it('ignores expressions that need live-only context', () => {
const warnings = warningsFromResolution(
resolution({
emptyResolutions: [
{
path: 'token',
raw: '={{ $secrets.vault.token }}',
resolved: undefined,
reason: 'unreconstructable-context',
},
],
failedExpressions: [
{
path: 'url',
raw: '={{ $response.body.next }}',
error: 'undefined',
reason: 'unreconstructable-context',
},
],
}),
);
expect(warnings).toEqual([]);
});
it('returns nothing when resolution was suppressed', () => {
const warnings = warningsFromResolution(
resolution({
parameters: null,
resolved: null,
suppressed: 'parameter-values-disabled',
emptyResolutions: [{ path: 'to', raw: '={{ $json.x }}', resolved: '' }],
}),
);
expect(warnings).toEqual([]);
});
});
describe('collectResolvedParameterWarnings', () => {
it('replays every reached simulated node and skips nodes whose replay fails', async () => {
const getResolvedNodeParameters = vi.fn(async (_executionId: string, nodeName: string) => {
await Promise.resolve();
if (nodeName === 'Broken') throw new Error('no run data');
return resolution({
nodeName,
emptyResolutions: [{ path: 'to', raw: '={{ $json.query.caller }}', resolved: undefined }],
});
});
const logger = { debug: vi.fn() };
const { warnings, skipped, skippedCount } = await collectResolvedParameterWarnings({
executionService: { getResolvedNodeParameters },
runs: [{ executionId: 'exec_1', nodeNames: ['Send SMS', 'Broken'] }],
logger,
});
expect(getResolvedNodeParameters).toHaveBeenCalledTimes(2);
expect(getResolvedNodeParameters).toHaveBeenCalledWith('exec_1', 'Send SMS');
expect(warnings).toEqual([
{
nodeName: 'Send SMS',
executionId: 'exec_1',
path: 'to',
raw: '={{ $json.query.caller }}',
issue: 'empty',
},
]);
expect(skipped).toEqual([
{ nodeName: 'Broken', executionId: 'exec_1', reason: 'replay-failed' },
]);
expect(skippedCount).toBe(1);
expect(logger.debug).toHaveBeenCalledWith(
'Resolved-parameter check skipped for simulated node',
expect.objectContaining({ nodeName: 'Broken', error: 'no run data' }),
);
});
it('does not call the service when no simulated node was reached', async () => {
const getResolvedNodeParameters = vi.fn();
const result = await collectResolvedParameterWarnings({
executionService: { getResolvedNodeParameters },
runs: [{ executionId: 'exec_1', nodeNames: [] }],
});
expect(result).toEqual({ warnings: [], skipped: [], skippedCount: 0 });
expect(getResolvedNodeParameters).not.toHaveBeenCalled();
});
});
describe('buildResolvedParameterNote', () => {
it('is undefined without warnings', () => {
expect(buildResolvedParameterNote([])).toBeUndefined();
});
it('groups warnings by node and tells the agent the fixture output is not proof', () => {
const note = buildResolvedParameterNote([
{ nodeName: 'Send SMS', path: 'to', raw: '={{ $json.query.caller }}', issue: 'empty' },
{
nodeName: 'Send SMS',
path: 'message',
raw: '={{ $json.body.text.trim() }}',
issue: 'failed',
detail: 'Cannot read trim',
},
{ nodeName: 'Post to Slack', path: 'text', raw: '={{ $json.headers.x }}', issue: 'empty' },
]);
expect(note).toContain('Send SMS: `to` (={{ $json.query.caller }}) resolved to empty');
expect(note).toContain('`message` (={{ $json.body.text.trim() }}) failed: Cannot read trim');
expect(note).toContain('Post to Slack: `text`');
expect(note).toContain('fixture data');
expect(note).toContain('{body, query, headers, params}');
});
});

View File

@ -0,0 +1,174 @@
/**
* Parameter check for simulated nodes after a verification run.
*
* A simulated node's preview is fixture data, so an expression that resolved to
* empty (or threw) leaves no trace in the run result. Replaying the node's
* parameter resolution against the saved execution exposes it.
*/
import { z } from 'zod';
import type { InstanceAiExecutionService, ResolvedNodeParametersResult } from '../../../types';
export const resolvedParameterWarningSchema = z.object({
nodeName: z.string(),
executionId: z.string().optional(),
path: z.string(),
raw: z.string(),
issue: z.enum(['empty', 'failed']),
detail: z.string().optional(),
});
export const skippedParameterCheckSchema = z.object({
nodeName: z.string(),
executionId: z.string().optional(),
reason: z.enum(['parameter-values-disabled', 'replay-failed', 'execution-unavailable']),
});
export type ResolvedParameterWarning = z.infer<typeof resolvedParameterWarningSchema>;
type SkippedParameterCheck = z.infer<typeof skippedParameterCheckSchema>;
export interface ParameterCheckRun {
executionId?: string;
nodeNames: readonly string[];
}
/** Keep the tool output compact when a node has many empty leaves. */
const MAX_WARNINGS = 20;
const MAX_SKIPPED_CHECKS = 20;
type WarningLogger = { debug(message: string, meta?: Record<string, unknown>): void };
export function warningsFromResolution(
result: ResolvedNodeParametersResult,
): ResolvedParameterWarning[] {
if (result.suppressed) return [];
// Context that only exists in a live run ($secrets, $response, …) is expected
// to be missing in a replay; it says nothing about the workflow.
const failed = result.failedExpressions
.filter((failure) => failure.reason !== 'unreconstructable-context')
.map<ResolvedParameterWarning>((failure) => ({
nodeName: result.nodeName,
path: failure.path,
raw: failure.raw,
issue: 'failed',
detail: failure.error,
}));
const empty = result.emptyResolutions
.filter((resolution) => resolution.reason !== 'unreconstructable-context')
.map<ResolvedParameterWarning>((resolution) => ({
nodeName: result.nodeName,
path: resolution.path,
raw: resolution.raw,
issue: 'empty',
}));
return [...failed, ...empty];
}
export async function collectResolvedParameterWarnings(args: {
executionService: Pick<InstanceAiExecutionService, 'getResolvedNodeParameters'>;
runs: readonly ParameterCheckRun[];
logger?: WarningLogger;
}): Promise<{
warnings: ResolvedParameterWarning[];
skipped: SkippedParameterCheck[];
skippedCount: number;
}> {
const { executionService, runs, logger } = args;
const warnings: ResolvedParameterWarning[] = [];
const skipped: SkippedParameterCheck[] = [];
let skippedCount = 0;
const recordSkippedCheck = (check: SkippedParameterCheck) => {
skippedCount++;
if (skipped.length < MAX_SKIPPED_CHECKS) skipped.push(check);
};
for (const { executionId, nodeNames } of runs) {
if (!executionId) {
for (const nodeName of nodeNames) {
recordSkippedCheck({ nodeName, reason: 'execution-unavailable' });
}
continue;
}
const settled = await Promise.allSettled(
nodeNames.map(
async (nodeName) => await executionService.getResolvedNodeParameters(executionId, nodeName),
),
);
settled.forEach((outcome, index) => {
const nodeName = nodeNames[index];
if (outcome.status === 'fulfilled') {
if (outcome.value.suppressed) {
recordSkippedCheck({ nodeName, executionId, reason: outcome.value.suppressed });
} else {
warnings.push(
...warningsFromResolution(outcome.value).map((warning) => ({
...warning,
executionId,
})),
);
}
return;
}
// Keep the run result, but disclose that its parameters were not checked.
recordSkippedCheck({ nodeName, executionId, reason: 'replay-failed' });
logger?.debug('Resolved-parameter check skipped for simulated node', {
executionId,
nodeName,
error: outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason),
});
});
}
return { warnings: warnings.slice(0, MAX_WARNINGS), skipped, skippedCount };
}
export function buildResolvedParameterNote(
warnings: readonly ResolvedParameterWarning[],
skipped: readonly SkippedParameterCheck[] = [],
skippedCount = skipped.length,
): string | undefined {
const skipReasons = {
'parameter-values-disabled': 'parameter values are disabled',
'replay-failed': 'parameter replay failed',
'execution-unavailable': 'the execution is unavailable',
};
const skippedNote =
skipped.length > 0
? 'Parameter check skipped for ' +
skipped
.map(
({ nodeName, executionId, reason }) =>
`${nodeName}${executionId ? ` (execution ${executionId})` : ''}: ${skipReasons[reason]}`,
)
.join('; ') +
(skippedCount > skipped.length
? `. Showing ${skipped.length} of ${skippedCount} skipped checks`
: '') +
'. All nodes with skipped checks have unchecked dynamic fields. Do not report those fields as verified.'
: undefined;
if (warnings.length === 0) return skippedNote;
const byNode = new Map<string, string[]>();
for (const warning of warnings) {
const entry =
warning.issue === 'failed'
? `\`${warning.path}\` (${warning.raw}) failed: ${warning.detail ?? 'expression error'}`
: `\`${warning.path}\` (${warning.raw}) resolved to empty`;
const list = byNode.get(warning.nodeName) ?? [];
list.push(entry);
byNode.set(warning.nodeName, list);
}
const summary = [...byNode.entries()]
.map(([nodeName, entries]) => `${nodeName}: ${entries.join(', ')}`)
.join('; ');
return [
`Simulated-node parameter check: ${summary}. ` +
'A simulated node’s preview is fixture data and does not prove these fields: fix the ' +
'trigger input shape (for a Webhook, pass the {body, query, headers, params} envelope) or the ' +
'expression, re-run verification, and do not report these fields as working until they resolve.',
skippedNote,
]
.filter((note): note is string => note !== undefined)
.join(' ');
}

View File

@ -13,6 +13,7 @@ import {
type VerificationAnalysis,
} from './analyze-result';
import type { PreparedVerificationRun } from './prepare-run';
import type { ParameterCheckRun } from './resolved-parameter-warnings';
import type { ExecutionRunResult } from './types';
import type { OrchestrationContext } from '../../../types';
import type {
@ -48,9 +49,11 @@ interface DecisionPass {
analysis: VerificationAnalysis;
}
export async function runScriptedGateVerification(
args: ScriptedGateRunArgs,
): Promise<{ result: ExecutionRunResult; analysis: VerificationAnalysis }> {
export async function runScriptedGateVerification(args: ScriptedGateRunArgs): Promise<{
result: ExecutionRunResult;
analysis: VerificationAnalysis;
parameterCheckRuns: ParameterCheckRun[];
}> {
const {
script,
prepared,
@ -88,7 +91,14 @@ export async function runScriptedGateVerification(
passes.push({ label: decision.label, result, analysis });
}
return { result: mergeResults(passes), analysis: mergeAnalyses(script, prepared, passes) };
return {
result: mergeResults(passes),
analysis: mergeAnalyses(script, prepared, passes),
parameterCheckRuns: passes.map(({ result, analysis }) => ({
executionId: result.executionId,
nodeNames: analysis.reachedSimulatedNodes.map((node) => node.nodeName),
})),
};
}
function mergeResults(passes: DecisionPass[]): ExecutionRunResult {

View File

@ -24,6 +24,12 @@ import {
import { prepareVerificationRun } from './verification/prepare-run';
import { reconcileStaleCredentialPlan } from './verification/reconcile-plan';
import { resolveVerificationTarget } from './verification/resolve-target';
import {
buildResolvedParameterNote,
collectResolvedParameterWarnings,
resolvedParameterWarningSchema,
skippedParameterCheckSchema,
} from './verification/resolved-parameter-warnings';
import { runScriptedGateVerification } from './verification/scripted-gate-run';
import {
executionNodeErrorSchema,
@ -47,7 +53,11 @@ export const verifyBuiltWorkflowInputSchema = z.object({
.describe(
"Input data for the workflow trigger. Shape MUST match the trigger's real-world output: " +
'Form Trigger -> flat field map like {name: "Alice", email: "a@b.c"} (do NOT wrap in formFields); ' +
'Webhook -> the body payload like {event: "signup", userId: "..."} (adapter wraps it under body); ' +
'Webhook -> a flat payload like {event: "signup", userId: "..."} is placed under `body` and leaves ' +
'`query`, `headers` and `params` EMPTY. When any expression reads $json.query.*, $json.headers.* or ' +
'$json.params.*, pass the request envelope instead: {body: {...}, query: {caller: "+1555..."}, ' +
'headers: {"x-github-event": "issues"}, params: {...}}. A flat payload cannot exercise those fields, they resolve ' +
'empty, and a simulated downstream node still looks green; ' +
'Chat Trigger -> {chatInput: "user message"}; ' +
'Schedule Trigger -> omit inputData. ' +
"If you wrap a form payload in {formFields: {...}} the adapter will reject the call; the builder's " +
@ -140,6 +150,9 @@ const verifyBuiltWorkflowOutputSchema = z.object({
.optional(),
simulatedNodes: z.array(z.object({ nodeName: z.string(), reason: z.string() })).optional(),
simulationNote: z.string().optional(),
resolvedParameterWarnings: z.array(resolvedParameterWarningSchema).optional(),
skippedParameterChecks: z.array(skippedParameterCheckSchema).optional(),
skippedParameterCheckCount: z.number().int().nonnegative().optional(),
lastNodeExecuted: z.string().optional(),
nodeErrors: z.array(executionNodeErrorSchema).optional(),
nodesNotReached: z.array(z.string()).optional(),
@ -238,7 +251,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
// A scripted gate replaces the halt with one loop-safe pass per decision;
// otherwise run the single standard pass (halted gates pin zero items).
const { result, analysis } = prepared.gateScript
const { result, analysis, parameterCheckRuns } = prepared.gateScript
? await runScriptedGateVerification({
script: prepared.gateScript,
prepared,
@ -267,20 +280,27 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
abortSignal: context.abortSignal,
},
);
const analysis = analyzeVerificationResult({
result: runResult,
buildOutcome,
simulatedNodes: prepared.simulatedNodes,
haltedGateNames: prepared.haltedGateNames,
triggerNodeName: resolvedInput.triggerNodeName,
stateBefore: target.stateBefore,
runId: context.runId,
chatModelRelatedNodeNames,
chatModelRecovery,
verificationScope,
});
return {
result: runResult,
analysis: analyzeVerificationResult({
result: runResult,
buildOutcome,
simulatedNodes: prepared.simulatedNodes,
haltedGateNames: prepared.haltedGateNames,
triggerNodeName: resolvedInput.triggerNodeName,
stateBefore: target.stateBefore,
runId: context.runId,
chatModelRelatedNodeNames,
chatModelRecovery,
verificationScope,
}),
analysis,
parameterCheckRuns: [
{
executionId: runResult.executionId,
nodeNames: analysis.reachedSimulatedNodes.map((node) => node.nodeName),
},
],
};
})();
@ -317,6 +337,29 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
claim: runClaim,
});
// A simulated node's preview is fixture data, so an expression that resolved
// to empty (e.g. `$json.query.x` on a body-only input) leaves no trace in the
// run. Replay the parameters of every reached simulated node and surface it.
const {
warnings: resolvedParameterWarnings,
skipped: skippedParameterChecks,
skippedCount: skippedParameterCheckCount,
} = await collectResolvedParameterWarnings({
executionService: target.domainContext.executionService,
runs: parameterCheckRuns,
logger: context.logger,
});
const simulationNote = [
analysis.simulationNote,
buildResolvedParameterNote(
resolvedParameterWarnings,
skippedParameterChecks,
skippedParameterCheckCount,
),
]
.filter((note): note is string => note !== undefined)
.join(' ');
const maxDataChars = resolvedInput.maxDataChars ?? DEFAULT_NODE_PREVIEW_CHARS;
const simulatedNames = new Set(analysis.reachedSimulatedNodes.map((n) => n.nodeName));
return {
@ -330,7 +373,13 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
nodePreviews: buildNodePreviews(result.data, maxDataChars, simulatedNames),
simulatedNodes:
analysis.reachedSimulatedNodes.length > 0 ? analysis.reachedSimulatedNodes : undefined,
simulationNote: analysis.simulationNote,
simulationNote: simulationNote.length > 0 ? simulationNote : undefined,
resolvedParameterWarnings:
resolvedParameterWarnings.length > 0 ? resolvedParameterWarnings : undefined,
skippedParameterChecks:
skippedParameterChecks.length > 0 ? skippedParameterChecks : undefined,
skippedParameterCheckCount:
skippedParameterCheckCount > 0 ? skippedParameterCheckCount : undefined,
nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined,
nodesNotReached: analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined,
coverageNote: analysis.coverageNote,

View File

@ -138,7 +138,12 @@ describe('WorkflowTaskCoordinator', () => {
workflowTaskService: coordinator,
logger: mock<OrchestrationContext['logger']>(),
domainContext: mock<InstanceAiContext>({
executionService: mock<InstanceAiContext['executionService']>({ run }),
executionService: mock<InstanceAiContext['executionService']>({
run,
getResolvedNodeParameters: vi
.fn()
.mockRejectedValue(new Error('No saved parameter data')),
}),
workflowService: mock<InstanceAiContext['workflowService']>({
getAsWorkflowJSON: vi.fn().mockResolvedValue({ nodes: [], connections: {} }),
}),