mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
fix(core): Document the webhook input envelope and check simulated-node parameters in AI Assistant verification (no-changelog) (#37962)
This commit is contained in:
parent
bcad21f9b6
commit
f4df71c263
|
|
@ -192,7 +192,7 @@ waiting-with-output-as-success fallback.
|
|||
| Trigger | Pass | Adapter emits on `$json` |
|
||||
|---|---|---|
|
||||
| Form Trigger | flat field map, e.g. `{name: "Alice", email: "a@b.c"}` | `{ submittedAt, formMode: "instanceAi", name, email, ... }` — matches production. Do NOT wrap in `formFields`. |
|
||||
| Webhook | body payload, e.g. `{event: "signup", userId: "..."}` | `{ headers, query, body: { event, userId, ... } }` |
|
||||
| Webhook | body payload, e.g. `{event: "signup", userId: "..."}`, **or** the request envelope `{ body: {...}, query: {...}, headers: {...}, params: {...} }` when any expression reads `$json.query.*`, `$json.headers.*` or `$json.params.*` | flat payload → `{ headers: {}, query: {}, params: {}, body: { event, userId, ... } }`; envelope → passed through as-is |
|
||||
| Chat Trigger | `{chatInput: "..."}` | `{ sessionId, action, chatInput }` |
|
||||
| Schedule | omit | synthetic timestamp fields |
|
||||
|
||||
|
|
@ -211,7 +211,26 @@ record (`{ attempted, success, executionId, status, claim, evidence, verifiedAt
|
|||
the build outcome so workflow-verification follow-ups and exceptional checkpoint
|
||||
turns can reuse it without re-running verify.
|
||||
|
||||
**Returns**: `{ executionId?, success, status?, data?, error? }`
|
||||
**Returns**: `{ executionId?, success, status?, data?, error?, simulationNote?, resolvedParameterWarnings?, skippedParameterChecks?, skippedParameterCheckCount? }`
|
||||
|
||||
**Simulated-node parameter check**: a simulated node's preview is fixture data, so
|
||||
an expression that resolved to empty leaves no trace in the run. After the run the
|
||||
tool replays parameter resolution (`getResolvedNodeParameters`) for every reached
|
||||
simulated node and returns `resolvedParameterWarnings`, one entry per parameter
|
||||
that resolved to `null`/`undefined`/`""` or threw (`{ nodeName, executionId, path, raw, issue:
|
||||
'empty' | 'failed', detail? }`), with a summary appended to `simulationNote`.
|
||||
For scripted gates, it checks each node against every pass that reached it.
|
||||
Each warning identifies the execution used for that check.
|
||||
Expressions that need live-only context (`$secrets`, `$response`, …) are excluded.
|
||||
The check is advisory and does not change execution success. Suppressed parameter
|
||||
values, replay failures, and missing executions produce `skippedParameterChecks`
|
||||
entries (`{ nodeName, executionId?, reason }`) and a note in `simulationNote`.
|
||||
The list contains at most 20 entries across all passes. `skippedParameterCheckCount`
|
||||
reports the total. When entries are omitted, the note states how many are shown.
|
||||
Omitted checks also leave dynamic fields unverified.
|
||||
The reasons are `parameter-values-disabled`, `replay-failed`, and
|
||||
`execution-unavailable`. Skipped checks expose no parameter values or replay
|
||||
error details. Their dynamic fields remain unverified.
|
||||
|
||||
### `report-verification-verdict` *(conditional)*
|
||||
|
||||
|
|
@ -538,7 +557,7 @@ Default timeout: 5 minutes; max: 10 minutes. On timeout, execution is cancelled.
|
|||
**Type-aware pin data**: Constructs proper pin data per trigger type:
|
||||
- **Chat trigger**: `{ chatInput, sessionId, action }`
|
||||
- **Form trigger**: `{ submittedAt, formMode: 'instanceAi', ...inputData }`
|
||||
- **Webhook trigger**: `{ headers: {}, query: {}, body: inputData }`
|
||||
- **Webhook trigger**: flat `inputData` → `{ headers: {}, query: {}, params: {}, body: inputData }`; an envelope whose keys are only `body`/`query`/`headers`/`params` is passed through, so query- and header-driven expressions can be exercised
|
||||
- **Schedule trigger**: current datetime information
|
||||
- **Unknown trigger**: `{ json: inputData }` (generic fallback)
|
||||
|
||||
|
|
|
|||
|
|
@ -240,6 +240,14 @@ workflow does not need to be active. Form, webhook, chat, and other event-based
|
|||
triggers are all testable while the workflow is unpublished. Never publish a
|
||||
workflow as a precondition for running it.
|
||||
|
||||
**Webhook input must carry the fields the workflow reads.** A flat `inputData`
|
||||
becomes the request `body` only; `query`, `headers` and `params` stay empty. When
|
||||
any expression reads `$json.query.*`, `$json.headers.*` or `$json.params.*`, pass
|
||||
the request envelope `{ body: {...}, query: {...}, headers: {...}, params: {...} }` (or a
|
||||
`fixtureOverrides` entry on the trigger node). Otherwise the field resolves empty,
|
||||
the run still succeeds, and that field is unverified — say so instead of
|
||||
reporting it as working.
|
||||
|
||||
Do not proactively offer, recommend, or mention publishing until a successful
|
||||
execution has run every required node on the claimed path without mocked
|
||||
credentials, simulated node output, fixture overrides, or temporary pin data
|
||||
|
|
@ -380,6 +388,20 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries),
|
|||
budget is exhausted.
|
||||
- Relay `simulationNote` (nodes whose output was simulated) to the user
|
||||
whenever it is present.
|
||||
- Read `resolvedParameterWarnings`. A simulated node's preview is fixture
|
||||
data: it never proves an expression resolved. Each warning names a
|
||||
parameter that resolved to empty or threw on the real input — the usual
|
||||
causes are a trigger input that lacks the field (body-only webhook input
|
||||
for a `$json.query.*` expression) or a wrong expression. Fix the input
|
||||
shape or the expression, re-run, and never report that field as working
|
||||
while a warning stands. Each warning carries the execution ID that was
|
||||
checked. Use that ID with `executions(action="get-resolved-node-parameters")`
|
||||
to inspect the same input.
|
||||
- Read `skippedParameterChecks`. These nodes have unchecked dynamic fields.
|
||||
The list shows at most 20 checks. `skippedParameterCheckCount` includes
|
||||
omitted checks, which also leave dynamic fields unverified.
|
||||
State that limitation even if the run succeeded and no parameter warnings
|
||||
were returned. Do not request parameter values when sharing is disabled.
|
||||
3. After verification handling, if `setupRequirement.status === "required"` and
|
||||
setup has not already run for this build, call `workflows(action="setup")`
|
||||
with the workflowId.
|
||||
|
|
@ -577,7 +599,10 @@ applies to rows or records written to an external system: never make quantitativ
|
|||
claims ("22 rows written", "columns matched") that you did not read back from
|
||||
the effect node's actual output (`executions(action="get-node-output")`) or from
|
||||
the target system itself — a successful run status does not prove the _right
|
||||
data_ was written, only that nodes ran. If you could not run the
|
||||
data_ was written, only that nodes ran. Output of a simulated or pinned node is
|
||||
fixture data: never quote it as what the workflow produced, and never cite it as
|
||||
proof that an expression resolved — use `resolvedParameterWarnings` or
|
||||
`executions(action="get-resolved-node-parameters")` for that. If you could not run the
|
||||
failing path or inspect the artifact, say so plainly — "I couldn't verify X
|
||||
because Y" — and name what is unconfirmed. An honest "could not verify" beats an
|
||||
unverified success claim.
|
||||
|
|
|
|||
|
|
@ -50,7 +50,9 @@ const runAction = z.object({
|
|||
.describe(
|
||||
'Input data passed to the workflow trigger. Works for ANY trigger type — ' +
|
||||
'the system injects inputData as the trigger node output, bypassing the need for a real event. ' +
|
||||
'For webhook triggers, inputData is the request body (do NOT wrap in { body: ... }). ' +
|
||||
'For webhook triggers, a flat inputData is treated as the request body (placed under `body`; ' +
|
||||
'`query`, `headers` and `params` stay empty). To exercise $json.query.*, $json.headers.* or ' +
|
||||
'$json.params.*, pass the request envelope { body: {...}, query: {...}, headers: {...}, params: {...} } instead. ' +
|
||||
'For event-based triggers (e.g. Linear, GitHub, Slack), pass inputData matching ' +
|
||||
'the shape the trigger would emit (e.g. { action: "create", data: { ... } }).',
|
||||
),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import type {
|
|||
InstanceAiDataTableService,
|
||||
InstanceAiWorkflowService,
|
||||
OrchestrationContext,
|
||||
ResolvedNodeParametersResult,
|
||||
WorkflowTaskService,
|
||||
} from '../../../types';
|
||||
import { createRemediation, MAX_VERIFY_ATTEMPTS } from '../../../workflow-loop/remediation';
|
||||
|
|
@ -32,6 +33,20 @@ type VerifyBuiltWorkflowOutput = {
|
|||
}>;
|
||||
simulatedNodes?: Array<{ nodeName: string; reason: string }>;
|
||||
simulationNote?: string;
|
||||
resolvedParameterWarnings?: Array<{
|
||||
nodeName: string;
|
||||
executionId?: string;
|
||||
path: string;
|
||||
raw: string;
|
||||
issue: 'empty' | 'failed';
|
||||
detail?: string;
|
||||
}>;
|
||||
skippedParameterChecks?: Array<{
|
||||
nodeName: string;
|
||||
executionId?: string;
|
||||
reason: string;
|
||||
}>;
|
||||
skippedParameterCheckCount?: number;
|
||||
lastNodeExecuted?: string;
|
||||
nodesNotReached?: string[];
|
||||
nodeErrors?: Array<{ nodeName: string; message?: string }>;
|
||||
|
|
@ -457,6 +472,9 @@ interface VerifyToolContext {
|
|||
]
|
||||
) => Promise<ExecutionRunResult>
|
||||
>;
|
||||
getResolvedNodeParameters?: Mock<
|
||||
(executionId: string, nodeName: string) => Promise<ResolvedNodeParametersResult>
|
||||
>;
|
||||
};
|
||||
workflowService?: InstanceAiWorkflowService;
|
||||
dataTableService?: InstanceAiDataTableService;
|
||||
|
|
@ -496,6 +514,8 @@ function makeContext(
|
|||
workflowConnections?: Record<string, unknown>;
|
||||
tableRows?: Record<string, Array<Record<string, unknown>>>;
|
||||
availableCredentials?: Array<{ id: string; name: string; type: string }>;
|
||||
/** Replayed parameter resolution per simulated node; nodes not listed make the replay throw. */
|
||||
resolvedParameters?: Record<string, ResolvedNodeParametersResult>;
|
||||
} = {},
|
||||
) {
|
||||
const updateBuildOutcome = vi.fn(
|
||||
|
|
@ -583,6 +603,13 @@ function makeContext(
|
|||
error: vi.fn(),
|
||||
};
|
||||
|
||||
const getResolvedNodeParameters = vi.fn(async (_executionId: string, nodeName: string) => {
|
||||
await Promise.resolve();
|
||||
const replayed = overrides.resolvedParameters?.[nodeName];
|
||||
if (!replayed) throw new Error(`no run data for ${nodeName}`);
|
||||
return replayed;
|
||||
});
|
||||
|
||||
const ctx: VerifyToolContext = {
|
||||
workflowTaskService: {
|
||||
reportBuildOutcome: vi.fn(),
|
||||
|
|
@ -602,7 +629,7 @@ function makeContext(
|
|||
recordVerification: vi.fn(coordinator.recordVerification.bind(coordinator)),
|
||||
} as unknown as WorkflowTaskService,
|
||||
domainContext: {
|
||||
executionService: { run },
|
||||
executionService: { run, getResolvedNodeParameters },
|
||||
workflowService,
|
||||
dataTableService,
|
||||
credentialService,
|
||||
|
|
@ -1167,6 +1194,99 @@ describe('verify-built-workflow tool — node simulation plan', () => {
|
|||
expect(result.simulationNote).toContain('no real external writes');
|
||||
});
|
||||
|
||||
it('surfaces parameters of reached simulated nodes that resolved to empty or threw', async () => {
|
||||
const { ctx } = makeContext(
|
||||
makeBuildOutcome({
|
||||
nodeSimulationPlan: [
|
||||
simulateVerdict('Send Slack', 'Sends a message to a Slack channel'),
|
||||
simulateVerdict('Send SMS', 'Sends an SMS'),
|
||||
],
|
||||
simulationFixtures: { 'Send Slack': [{ ok: true }], 'Send SMS': [{ sid: 'SM1' }] },
|
||||
}),
|
||||
{
|
||||
executionId: 'exec-sim',
|
||||
status: 'success',
|
||||
data: {
|
||||
Webhook: [{ headers: {}, query: {}, body: { CallStatus: 'no-answer' } }],
|
||||
'Send Slack': [{ ok: true }],
|
||||
},
|
||||
},
|
||||
{
|
||||
resolvedParameters: {
|
||||
'Send Slack': {
|
||||
nodeName: 'Send Slack',
|
||||
runIndex: 0,
|
||||
itemIndex: 0,
|
||||
parameters: { text: '={{ $json.query.caller }}' },
|
||||
resolved: '{"text":""}',
|
||||
failedExpressions: [
|
||||
{ path: 'channel', raw: '={{ $json.body.ch.id }}', error: 'Cannot read id' },
|
||||
],
|
||||
emptyResolutions: [
|
||||
{ path: 'text', raw: '={{ $json.query.caller }}', resolved: undefined },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
// Only nodes the run reached are replayed; Send SMS never executed.
|
||||
expect(ctx.domainContext.executionService.getResolvedNodeParameters).toHaveBeenCalledTimes(1);
|
||||
expect(ctx.domainContext.executionService.getResolvedNodeParameters).toHaveBeenCalledWith(
|
||||
'exec-sim',
|
||||
'Send Slack',
|
||||
);
|
||||
expect(result.resolvedParameterWarnings).toEqual([
|
||||
{
|
||||
nodeName: 'Send Slack',
|
||||
executionId: 'exec-sim',
|
||||
path: 'channel',
|
||||
raw: '={{ $json.body.ch.id }}',
|
||||
issue: 'failed',
|
||||
detail: 'Cannot read id',
|
||||
},
|
||||
{
|
||||
nodeName: 'Send Slack',
|
||||
executionId: 'exec-sim',
|
||||
path: 'text',
|
||||
raw: '={{ $json.query.caller }}',
|
||||
issue: 'empty',
|
||||
},
|
||||
]);
|
||||
expect(result.simulationNote).toContain('no real external writes');
|
||||
expect(result.simulationNote).toContain('Send Slack: `channel`');
|
||||
expect(result.simulationNote).toContain('`text` (={{ $json.query.caller }}) resolved to empty');
|
||||
});
|
||||
|
||||
it('keeps the verification result when the parameter replay fails', async () => {
|
||||
const { ctx } = makeContext(
|
||||
makeBuildOutcome({
|
||||
nodeSimulationPlan: [simulateVerdict('Send Slack', 'Sends a message to a Slack channel')],
|
||||
simulationFixtures: { 'Send Slack': [{ ok: true }] },
|
||||
}),
|
||||
{ executionId: 'exec-sim', status: 'success', data: { 'Send Slack': [{ ok: true }] } },
|
||||
);
|
||||
|
||||
const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.resolvedParameterWarnings).toBeUndefined();
|
||||
expect(result.skippedParameterChecks).toEqual([
|
||||
{ nodeName: 'Send Slack', executionId: 'exec-sim', reason: 'replay-failed' },
|
||||
]);
|
||||
expect(result.skippedParameterCheckCount).toBe(1);
|
||||
expect(result.simulationNote).toContain('no real external writes');
|
||||
expect(result.simulationNote).toContain('Parameter check skipped');
|
||||
expect(result.simulationNote).toContain('unchecked dynamic fields');
|
||||
expect(ctx.logger.debug).toHaveBeenCalledWith(
|
||||
'Resolved-parameter check skipped for simulated node',
|
||||
expect.objectContaining({ nodeName: 'Send Slack' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('fails closed when the build outcome has no simulation plan at all', async () => {
|
||||
// An undefined plan means the outcome predates classification or
|
||||
// classification failed — nothing shields destructive nodes in that run.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,161 @@
|
|||
import type { ResolvedNodeParametersResult } from '../../../../types';
|
||||
import {
|
||||
buildResolvedParameterNote,
|
||||
collectResolvedParameterWarnings,
|
||||
warningsFromResolution,
|
||||
} from '../resolved-parameter-warnings';
|
||||
|
||||
function resolution(
|
||||
overrides: Partial<ResolvedNodeParametersResult> = {},
|
||||
): ResolvedNodeParametersResult {
|
||||
return {
|
||||
nodeName: 'Send SMS',
|
||||
runIndex: 0,
|
||||
itemIndex: 0,
|
||||
parameters: {},
|
||||
resolved: '{}',
|
||||
failedExpressions: [],
|
||||
emptyResolutions: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('warningsFromResolution', () => {
|
||||
it('reports empty and failed expressions for the node', () => {
|
||||
const warnings = warningsFromResolution(
|
||||
resolution({
|
||||
emptyResolutions: [{ path: 'to', raw: '={{ $json.query.caller }}', resolved: undefined }],
|
||||
failedExpressions: [
|
||||
{ path: 'message', raw: '={{ $json.body.text.trim() }}', error: 'Cannot read trim' },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(warnings).toEqual([
|
||||
{
|
||||
nodeName: 'Send SMS',
|
||||
path: 'message',
|
||||
raw: '={{ $json.body.text.trim() }}',
|
||||
issue: 'failed',
|
||||
detail: 'Cannot read trim',
|
||||
},
|
||||
{ nodeName: 'Send SMS', path: 'to', raw: '={{ $json.query.caller }}', issue: 'empty' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('ignores expressions that need live-only context', () => {
|
||||
const warnings = warningsFromResolution(
|
||||
resolution({
|
||||
emptyResolutions: [
|
||||
{
|
||||
path: 'token',
|
||||
raw: '={{ $secrets.vault.token }}',
|
||||
resolved: undefined,
|
||||
reason: 'unreconstructable-context',
|
||||
},
|
||||
],
|
||||
failedExpressions: [
|
||||
{
|
||||
path: 'url',
|
||||
raw: '={{ $response.body.next }}',
|
||||
error: 'undefined',
|
||||
reason: 'unreconstructable-context',
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns nothing when resolution was suppressed', () => {
|
||||
const warnings = warningsFromResolution(
|
||||
resolution({
|
||||
parameters: null,
|
||||
resolved: null,
|
||||
suppressed: 'parameter-values-disabled',
|
||||
emptyResolutions: [{ path: 'to', raw: '={{ $json.x }}', resolved: '' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(warnings).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('collectResolvedParameterWarnings', () => {
|
||||
it('replays every reached simulated node and skips nodes whose replay fails', async () => {
|
||||
const getResolvedNodeParameters = vi.fn(async (_executionId: string, nodeName: string) => {
|
||||
await Promise.resolve();
|
||||
if (nodeName === 'Broken') throw new Error('no run data');
|
||||
return resolution({
|
||||
nodeName,
|
||||
emptyResolutions: [{ path: 'to', raw: '={{ $json.query.caller }}', resolved: undefined }],
|
||||
});
|
||||
});
|
||||
const logger = { debug: vi.fn() };
|
||||
|
||||
const { warnings, skipped, skippedCount } = await collectResolvedParameterWarnings({
|
||||
executionService: { getResolvedNodeParameters },
|
||||
runs: [{ executionId: 'exec_1', nodeNames: ['Send SMS', 'Broken'] }],
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(getResolvedNodeParameters).toHaveBeenCalledTimes(2);
|
||||
expect(getResolvedNodeParameters).toHaveBeenCalledWith('exec_1', 'Send SMS');
|
||||
expect(warnings).toEqual([
|
||||
{
|
||||
nodeName: 'Send SMS',
|
||||
executionId: 'exec_1',
|
||||
path: 'to',
|
||||
raw: '={{ $json.query.caller }}',
|
||||
issue: 'empty',
|
||||
},
|
||||
]);
|
||||
expect(skipped).toEqual([
|
||||
{ nodeName: 'Broken', executionId: 'exec_1', reason: 'replay-failed' },
|
||||
]);
|
||||
expect(skippedCount).toBe(1);
|
||||
expect(logger.debug).toHaveBeenCalledWith(
|
||||
'Resolved-parameter check skipped for simulated node',
|
||||
expect.objectContaining({ nodeName: 'Broken', error: 'no run data' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not call the service when no simulated node was reached', async () => {
|
||||
const getResolvedNodeParameters = vi.fn();
|
||||
|
||||
const result = await collectResolvedParameterWarnings({
|
||||
executionService: { getResolvedNodeParameters },
|
||||
runs: [{ executionId: 'exec_1', nodeNames: [] }],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ warnings: [], skipped: [], skippedCount: 0 });
|
||||
expect(getResolvedNodeParameters).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildResolvedParameterNote', () => {
|
||||
it('is undefined without warnings', () => {
|
||||
expect(buildResolvedParameterNote([])).toBeUndefined();
|
||||
});
|
||||
|
||||
it('groups warnings by node and tells the agent the fixture output is not proof', () => {
|
||||
const note = buildResolvedParameterNote([
|
||||
{ nodeName: 'Send SMS', path: 'to', raw: '={{ $json.query.caller }}', issue: 'empty' },
|
||||
{
|
||||
nodeName: 'Send SMS',
|
||||
path: 'message',
|
||||
raw: '={{ $json.body.text.trim() }}',
|
||||
issue: 'failed',
|
||||
detail: 'Cannot read trim',
|
||||
},
|
||||
{ nodeName: 'Post to Slack', path: 'text', raw: '={{ $json.headers.x }}', issue: 'empty' },
|
||||
]);
|
||||
|
||||
expect(note).toContain('Send SMS: `to` (={{ $json.query.caller }}) resolved to empty');
|
||||
expect(note).toContain('`message` (={{ $json.body.text.trim() }}) failed: Cannot read trim');
|
||||
expect(note).toContain('Post to Slack: `text`');
|
||||
expect(note).toContain('fixture data');
|
||||
expect(note).toContain('{body, query, headers, params}');
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
/**
|
||||
* Parameter check for simulated nodes after a verification run.
|
||||
*
|
||||
* A simulated node's preview is fixture data, so an expression that resolved to
|
||||
* empty (or threw) leaves no trace in the run result. Replaying the node's
|
||||
* parameter resolution against the saved execution exposes it.
|
||||
*/
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
import type { InstanceAiExecutionService, ResolvedNodeParametersResult } from '../../../types';
|
||||
|
||||
export const resolvedParameterWarningSchema = z.object({
|
||||
nodeName: z.string(),
|
||||
executionId: z.string().optional(),
|
||||
path: z.string(),
|
||||
raw: z.string(),
|
||||
issue: z.enum(['empty', 'failed']),
|
||||
detail: z.string().optional(),
|
||||
});
|
||||
|
||||
export const skippedParameterCheckSchema = z.object({
|
||||
nodeName: z.string(),
|
||||
executionId: z.string().optional(),
|
||||
reason: z.enum(['parameter-values-disabled', 'replay-failed', 'execution-unavailable']),
|
||||
});
|
||||
|
||||
export type ResolvedParameterWarning = z.infer<typeof resolvedParameterWarningSchema>;
|
||||
type SkippedParameterCheck = z.infer<typeof skippedParameterCheckSchema>;
|
||||
|
||||
export interface ParameterCheckRun {
|
||||
executionId?: string;
|
||||
nodeNames: readonly string[];
|
||||
}
|
||||
|
||||
/** Keep the tool output compact when a node has many empty leaves. */
|
||||
const MAX_WARNINGS = 20;
|
||||
const MAX_SKIPPED_CHECKS = 20;
|
||||
|
||||
type WarningLogger = { debug(message: string, meta?: Record<string, unknown>): void };
|
||||
|
||||
export function warningsFromResolution(
|
||||
result: ResolvedNodeParametersResult,
|
||||
): ResolvedParameterWarning[] {
|
||||
if (result.suppressed) return [];
|
||||
// Context that only exists in a live run ($secrets, $response, …) is expected
|
||||
// to be missing in a replay; it says nothing about the workflow.
|
||||
const failed = result.failedExpressions
|
||||
.filter((failure) => failure.reason !== 'unreconstructable-context')
|
||||
.map<ResolvedParameterWarning>((failure) => ({
|
||||
nodeName: result.nodeName,
|
||||
path: failure.path,
|
||||
raw: failure.raw,
|
||||
issue: 'failed',
|
||||
detail: failure.error,
|
||||
}));
|
||||
const empty = result.emptyResolutions
|
||||
.filter((resolution) => resolution.reason !== 'unreconstructable-context')
|
||||
.map<ResolvedParameterWarning>((resolution) => ({
|
||||
nodeName: result.nodeName,
|
||||
path: resolution.path,
|
||||
raw: resolution.raw,
|
||||
issue: 'empty',
|
||||
}));
|
||||
return [...failed, ...empty];
|
||||
}
|
||||
|
||||
export async function collectResolvedParameterWarnings(args: {
|
||||
executionService: Pick<InstanceAiExecutionService, 'getResolvedNodeParameters'>;
|
||||
runs: readonly ParameterCheckRun[];
|
||||
logger?: WarningLogger;
|
||||
}): Promise<{
|
||||
warnings: ResolvedParameterWarning[];
|
||||
skipped: SkippedParameterCheck[];
|
||||
skippedCount: number;
|
||||
}> {
|
||||
const { executionService, runs, logger } = args;
|
||||
const warnings: ResolvedParameterWarning[] = [];
|
||||
const skipped: SkippedParameterCheck[] = [];
|
||||
let skippedCount = 0;
|
||||
const recordSkippedCheck = (check: SkippedParameterCheck) => {
|
||||
skippedCount++;
|
||||
if (skipped.length < MAX_SKIPPED_CHECKS) skipped.push(check);
|
||||
};
|
||||
for (const { executionId, nodeNames } of runs) {
|
||||
if (!executionId) {
|
||||
for (const nodeName of nodeNames) {
|
||||
recordSkippedCheck({ nodeName, reason: 'execution-unavailable' });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const settled = await Promise.allSettled(
|
||||
nodeNames.map(
|
||||
async (nodeName) => await executionService.getResolvedNodeParameters(executionId, nodeName),
|
||||
),
|
||||
);
|
||||
settled.forEach((outcome, index) => {
|
||||
const nodeName = nodeNames[index];
|
||||
if (outcome.status === 'fulfilled') {
|
||||
if (outcome.value.suppressed) {
|
||||
recordSkippedCheck({ nodeName, executionId, reason: outcome.value.suppressed });
|
||||
} else {
|
||||
warnings.push(
|
||||
...warningsFromResolution(outcome.value).map((warning) => ({
|
||||
...warning,
|
||||
executionId,
|
||||
})),
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Keep the run result, but disclose that its parameters were not checked.
|
||||
recordSkippedCheck({ nodeName, executionId, reason: 'replay-failed' });
|
||||
logger?.debug('Resolved-parameter check skipped for simulated node', {
|
||||
executionId,
|
||||
nodeName,
|
||||
error: outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason),
|
||||
});
|
||||
});
|
||||
}
|
||||
return { warnings: warnings.slice(0, MAX_WARNINGS), skipped, skippedCount };
|
||||
}
|
||||
|
||||
export function buildResolvedParameterNote(
|
||||
warnings: readonly ResolvedParameterWarning[],
|
||||
skipped: readonly SkippedParameterCheck[] = [],
|
||||
skippedCount = skipped.length,
|
||||
): string | undefined {
|
||||
const skipReasons = {
|
||||
'parameter-values-disabled': 'parameter values are disabled',
|
||||
'replay-failed': 'parameter replay failed',
|
||||
'execution-unavailable': 'the execution is unavailable',
|
||||
};
|
||||
const skippedNote =
|
||||
skipped.length > 0
|
||||
? 'Parameter check skipped for ' +
|
||||
skipped
|
||||
.map(
|
||||
({ nodeName, executionId, reason }) =>
|
||||
`${nodeName}${executionId ? ` (execution ${executionId})` : ''}: ${skipReasons[reason]}`,
|
||||
)
|
||||
.join('; ') +
|
||||
(skippedCount > skipped.length
|
||||
? `. Showing ${skipped.length} of ${skippedCount} skipped checks`
|
||||
: '') +
|
||||
'. All nodes with skipped checks have unchecked dynamic fields. Do not report those fields as verified.'
|
||||
: undefined;
|
||||
if (warnings.length === 0) return skippedNote;
|
||||
|
||||
const byNode = new Map<string, string[]>();
|
||||
for (const warning of warnings) {
|
||||
const entry =
|
||||
warning.issue === 'failed'
|
||||
? `\`${warning.path}\` (${warning.raw}) failed: ${warning.detail ?? 'expression error'}`
|
||||
: `\`${warning.path}\` (${warning.raw}) resolved to empty`;
|
||||
const list = byNode.get(warning.nodeName) ?? [];
|
||||
list.push(entry);
|
||||
byNode.set(warning.nodeName, list);
|
||||
}
|
||||
const summary = [...byNode.entries()]
|
||||
.map(([nodeName, entries]) => `${nodeName}: ${entries.join(', ')}`)
|
||||
.join('; ');
|
||||
|
||||
return [
|
||||
`Simulated-node parameter check: ${summary}. ` +
|
||||
'A simulated node’s preview is fixture data and does not prove these fields: fix the ' +
|
||||
'trigger input shape (for a Webhook, pass the {body, query, headers, params} envelope) or the ' +
|
||||
'expression, re-run verification, and do not report these fields as working until they resolve.',
|
||||
skippedNote,
|
||||
]
|
||||
.filter((note): note is string => note !== undefined)
|
||||
.join(' ');
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ import {
|
|||
type VerificationAnalysis,
|
||||
} from './analyze-result';
|
||||
import type { PreparedVerificationRun } from './prepare-run';
|
||||
import type { ParameterCheckRun } from './resolved-parameter-warnings';
|
||||
import type { ExecutionRunResult } from './types';
|
||||
import type { OrchestrationContext } from '../../../types';
|
||||
import type {
|
||||
|
|
@ -48,9 +49,11 @@ interface DecisionPass {
|
|||
analysis: VerificationAnalysis;
|
||||
}
|
||||
|
||||
export async function runScriptedGateVerification(
|
||||
args: ScriptedGateRunArgs,
|
||||
): Promise<{ result: ExecutionRunResult; analysis: VerificationAnalysis }> {
|
||||
export async function runScriptedGateVerification(args: ScriptedGateRunArgs): Promise<{
|
||||
result: ExecutionRunResult;
|
||||
analysis: VerificationAnalysis;
|
||||
parameterCheckRuns: ParameterCheckRun[];
|
||||
}> {
|
||||
const {
|
||||
script,
|
||||
prepared,
|
||||
|
|
@ -88,7 +91,14 @@ export async function runScriptedGateVerification(
|
|||
passes.push({ label: decision.label, result, analysis });
|
||||
}
|
||||
|
||||
return { result: mergeResults(passes), analysis: mergeAnalyses(script, prepared, passes) };
|
||||
return {
|
||||
result: mergeResults(passes),
|
||||
analysis: mergeAnalyses(script, prepared, passes),
|
||||
parameterCheckRuns: passes.map(({ result, analysis }) => ({
|
||||
executionId: result.executionId,
|
||||
nodeNames: analysis.reachedSimulatedNodes.map((node) => node.nodeName),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
function mergeResults(passes: DecisionPass[]): ExecutionRunResult {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,12 @@ import {
|
|||
import { prepareVerificationRun } from './verification/prepare-run';
|
||||
import { reconcileStaleCredentialPlan } from './verification/reconcile-plan';
|
||||
import { resolveVerificationTarget } from './verification/resolve-target';
|
||||
import {
|
||||
buildResolvedParameterNote,
|
||||
collectResolvedParameterWarnings,
|
||||
resolvedParameterWarningSchema,
|
||||
skippedParameterCheckSchema,
|
||||
} from './verification/resolved-parameter-warnings';
|
||||
import { runScriptedGateVerification } from './verification/scripted-gate-run';
|
||||
import {
|
||||
executionNodeErrorSchema,
|
||||
|
|
@ -47,7 +53,11 @@ export const verifyBuiltWorkflowInputSchema = z.object({
|
|||
.describe(
|
||||
"Input data for the workflow trigger. Shape MUST match the trigger's real-world output: " +
|
||||
'Form Trigger -> flat field map like {name: "Alice", email: "a@b.c"} (do NOT wrap in formFields); ' +
|
||||
'Webhook -> the body payload like {event: "signup", userId: "..."} (adapter wraps it under body); ' +
|
||||
'Webhook -> a flat payload like {event: "signup", userId: "..."} is placed under `body` and leaves ' +
|
||||
'`query`, `headers` and `params` EMPTY. When any expression reads $json.query.*, $json.headers.* or ' +
|
||||
'$json.params.*, pass the request envelope instead: {body: {...}, query: {caller: "+1555..."}, ' +
|
||||
'headers: {"x-github-event": "issues"}, params: {...}}. A flat payload cannot exercise those fields, they resolve ' +
|
||||
'empty, and a simulated downstream node still looks green; ' +
|
||||
'Chat Trigger -> {chatInput: "user message"}; ' +
|
||||
'Schedule Trigger -> omit inputData. ' +
|
||||
"If you wrap a form payload in {formFields: {...}} the adapter will reject the call; the builder's " +
|
||||
|
|
@ -140,6 +150,9 @@ const verifyBuiltWorkflowOutputSchema = z.object({
|
|||
.optional(),
|
||||
simulatedNodes: z.array(z.object({ nodeName: z.string(), reason: z.string() })).optional(),
|
||||
simulationNote: z.string().optional(),
|
||||
resolvedParameterWarnings: z.array(resolvedParameterWarningSchema).optional(),
|
||||
skippedParameterChecks: z.array(skippedParameterCheckSchema).optional(),
|
||||
skippedParameterCheckCount: z.number().int().nonnegative().optional(),
|
||||
lastNodeExecuted: z.string().optional(),
|
||||
nodeErrors: z.array(executionNodeErrorSchema).optional(),
|
||||
nodesNotReached: z.array(z.string()).optional(),
|
||||
|
|
@ -238,7 +251,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
|
||||
// A scripted gate replaces the halt with one loop-safe pass per decision;
|
||||
// otherwise run the single standard pass (halted gates pin zero items).
|
||||
const { result, analysis } = prepared.gateScript
|
||||
const { result, analysis, parameterCheckRuns } = prepared.gateScript
|
||||
? await runScriptedGateVerification({
|
||||
script: prepared.gateScript,
|
||||
prepared,
|
||||
|
|
@ -267,20 +280,27 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
abortSignal: context.abortSignal,
|
||||
},
|
||||
);
|
||||
const analysis = analyzeVerificationResult({
|
||||
result: runResult,
|
||||
buildOutcome,
|
||||
simulatedNodes: prepared.simulatedNodes,
|
||||
haltedGateNames: prepared.haltedGateNames,
|
||||
triggerNodeName: resolvedInput.triggerNodeName,
|
||||
stateBefore: target.stateBefore,
|
||||
runId: context.runId,
|
||||
chatModelRelatedNodeNames,
|
||||
chatModelRecovery,
|
||||
verificationScope,
|
||||
});
|
||||
return {
|
||||
result: runResult,
|
||||
analysis: analyzeVerificationResult({
|
||||
result: runResult,
|
||||
buildOutcome,
|
||||
simulatedNodes: prepared.simulatedNodes,
|
||||
haltedGateNames: prepared.haltedGateNames,
|
||||
triggerNodeName: resolvedInput.triggerNodeName,
|
||||
stateBefore: target.stateBefore,
|
||||
runId: context.runId,
|
||||
chatModelRelatedNodeNames,
|
||||
chatModelRecovery,
|
||||
verificationScope,
|
||||
}),
|
||||
analysis,
|
||||
parameterCheckRuns: [
|
||||
{
|
||||
executionId: runResult.executionId,
|
||||
nodeNames: analysis.reachedSimulatedNodes.map((node) => node.nodeName),
|
||||
},
|
||||
],
|
||||
};
|
||||
})();
|
||||
|
||||
|
|
@ -317,6 +337,29 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
claim: runClaim,
|
||||
});
|
||||
|
||||
// A simulated node's preview is fixture data, so an expression that resolved
|
||||
// to empty (e.g. `$json.query.x` on a body-only input) leaves no trace in the
|
||||
// run. Replay the parameters of every reached simulated node and surface it.
|
||||
const {
|
||||
warnings: resolvedParameterWarnings,
|
||||
skipped: skippedParameterChecks,
|
||||
skippedCount: skippedParameterCheckCount,
|
||||
} = await collectResolvedParameterWarnings({
|
||||
executionService: target.domainContext.executionService,
|
||||
runs: parameterCheckRuns,
|
||||
logger: context.logger,
|
||||
});
|
||||
const simulationNote = [
|
||||
analysis.simulationNote,
|
||||
buildResolvedParameterNote(
|
||||
resolvedParameterWarnings,
|
||||
skippedParameterChecks,
|
||||
skippedParameterCheckCount,
|
||||
),
|
||||
]
|
||||
.filter((note): note is string => note !== undefined)
|
||||
.join(' ');
|
||||
|
||||
const maxDataChars = resolvedInput.maxDataChars ?? DEFAULT_NODE_PREVIEW_CHARS;
|
||||
const simulatedNames = new Set(analysis.reachedSimulatedNodes.map((n) => n.nodeName));
|
||||
return {
|
||||
|
|
@ -330,7 +373,13 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) {
|
|||
nodePreviews: buildNodePreviews(result.data, maxDataChars, simulatedNames),
|
||||
simulatedNodes:
|
||||
analysis.reachedSimulatedNodes.length > 0 ? analysis.reachedSimulatedNodes : undefined,
|
||||
simulationNote: analysis.simulationNote,
|
||||
simulationNote: simulationNote.length > 0 ? simulationNote : undefined,
|
||||
resolvedParameterWarnings:
|
||||
resolvedParameterWarnings.length > 0 ? resolvedParameterWarnings : undefined,
|
||||
skippedParameterChecks:
|
||||
skippedParameterChecks.length > 0 ? skippedParameterChecks : undefined,
|
||||
skippedParameterCheckCount:
|
||||
skippedParameterCheckCount > 0 ? skippedParameterCheckCount : undefined,
|
||||
nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined,
|
||||
nodesNotReached: analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined,
|
||||
coverageNote: analysis.coverageNote,
|
||||
|
|
|
|||
|
|
@ -138,7 +138,12 @@ describe('WorkflowTaskCoordinator', () => {
|
|||
workflowTaskService: coordinator,
|
||||
logger: mock<OrchestrationContext['logger']>(),
|
||||
domainContext: mock<InstanceAiContext>({
|
||||
executionService: mock<InstanceAiContext['executionService']>({ run }),
|
||||
executionService: mock<InstanceAiContext['executionService']>({
|
||||
run,
|
||||
getResolvedNodeParameters: vi
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('No saved parameter data')),
|
||||
}),
|
||||
workflowService: mock<InstanceAiContext['workflowService']>({
|
||||
getAsWorkflowJSON: vi.fn().mockResolvedValue({ nodes: [], connections: {} }),
|
||||
}),
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user