diff --git a/packages/@n8n/instance-ai/docs/tools.md b/packages/@n8n/instance-ai/docs/tools.md index 80c26e846f1..0f2a6807e1b 100644 --- a/packages/@n8n/instance-ai/docs/tools.md +++ b/packages/@n8n/instance-ai/docs/tools.md @@ -192,7 +192,7 @@ waiting-with-output-as-success fallback. | Trigger | Pass | Adapter emits on `$json` | |---|---|---| | Form Trigger | flat field map, e.g. `{name: "Alice", email: "a@b.c"}` | `{ submittedAt, formMode: "instanceAi", name, email, ... }` — matches production. Do NOT wrap in `formFields`. | -| Webhook | body payload, e.g. `{event: "signup", userId: "..."}` | `{ headers, query, body: { event, userId, ... } }` | +| Webhook | body payload, e.g. `{event: "signup", userId: "..."}`, **or** the request envelope `{ body: {...}, query: {...}, headers: {...}, params: {...} }` when any expression reads `$json.query.*`, `$json.headers.*` or `$json.params.*` | flat payload → `{ headers: {}, query: {}, params: {}, body: { event, userId, ... } }`; envelope → passed through as-is | | Chat Trigger | `{chatInput: "..."}` | `{ sessionId, action, chatInput }` | | Schedule | omit | synthetic timestamp fields | @@ -211,7 +211,26 @@ record (`{ attempted, success, executionId, status, claim, evidence, verifiedAt the build outcome so workflow-verification follow-ups and exceptional checkpoint turns can reuse it without re-running verify. -**Returns**: `{ executionId?, success, status?, data?, error? }` +**Returns**: `{ executionId?, success, status?, data?, error?, simulationNote?, resolvedParameterWarnings?, skippedParameterChecks?, skippedParameterCheckCount? }` + +**Simulated-node parameter check**: a simulated node's preview is fixture data, so +an expression that resolved to empty leaves no trace in the run. After the run the +tool replays parameter resolution (`getResolvedNodeParameters`) for every reached +simulated node and returns `resolvedParameterWarnings`, one entry per parameter +that resolved to `null`/`undefined`/`""` or threw (`{ nodeName, executionId, path, raw, issue: +'empty' | 'failed', detail? }`), with a summary appended to `simulationNote`. +For scripted gates, it checks each node against every pass that reached it. +Each warning identifies the execution used for that check. +Expressions that need live-only context (`$secrets`, `$response`, …) are excluded. +The check is advisory and does not change execution success. Suppressed parameter +values, replay failures, and missing executions produce `skippedParameterChecks` +entries (`{ nodeName, executionId?, reason }`) and a note in `simulationNote`. +The list contains at most 20 entries across all passes. `skippedParameterCheckCount` +reports the total. When entries are omitted, the note states how many are shown. +Omitted checks also leave dynamic fields unverified. +The reasons are `parameter-values-disabled`, `replay-failed`, and +`execution-unavailable`. Skipped checks expose no parameter values or replay +error details. Their dynamic fields remain unverified. ### `report-verification-verdict` *(conditional)* @@ -538,7 +557,7 @@ Default timeout: 5 minutes; max: 10 minutes. On timeout, execution is cancelled. **Type-aware pin data**: Constructs proper pin data per trigger type: - **Chat trigger**: `{ chatInput, sessionId, action }` - **Form trigger**: `{ submittedAt, formMode: 'instanceAi', ...inputData }` -- **Webhook trigger**: `{ headers: {}, query: {}, body: inputData }` +- **Webhook trigger**: flat `inputData` → `{ headers: {}, query: {}, params: {}, body: inputData }`; an envelope whose keys are only `body`/`query`/`headers`/`params` is passed through, so query- and header-driven expressions can be exercised - **Schedule trigger**: current datetime information - **Unknown trigger**: `{ json: inputData }` (generic fallback) diff --git a/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md b/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md index c78565b3706..7836342bd23 100644 --- a/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md +++ b/packages/@n8n/instance-ai/skills/post-build-flow/SKILL.md @@ -240,6 +240,14 @@ workflow does not need to be active. Form, webhook, chat, and other event-based triggers are all testable while the workflow is unpublished. Never publish a workflow as a precondition for running it. +**Webhook input must carry the fields the workflow reads.** A flat `inputData` +becomes the request `body` only; `query`, `headers` and `params` stay empty. When +any expression reads `$json.query.*`, `$json.headers.*` or `$json.params.*`, pass +the request envelope `{ body: {...}, query: {...}, headers: {...}, params: {...} }` (or a +`fixtureOverrides` entry on the trigger node). Otherwise the field resolves empty, +the run still succeeds, and that field is unverified — say so instead of +reporting it as working. + Do not proactively offer, recommend, or mention publishing until a successful execution has run every required node on the claimed path without mocked credentials, simulated node output, fixture overrides, or temporary pin data @@ -380,6 +388,20 @@ For a workflow with more than one trigger (`triggerNodes` has multiple entries), budget is exhausted. - Relay `simulationNote` (nodes whose output was simulated) to the user whenever it is present. + - Read `resolvedParameterWarnings`. A simulated node's preview is fixture + data: it never proves an expression resolved. Each warning names a + parameter that resolved to empty or threw on the real input — the usual + causes are a trigger input that lacks the field (body-only webhook input + for a `$json.query.*` expression) or a wrong expression. Fix the input + shape or the expression, re-run, and never report that field as working + while a warning stands. Each warning carries the execution ID that was + checked. Use that ID with `executions(action="get-resolved-node-parameters")` + to inspect the same input. + - Read `skippedParameterChecks`. These nodes have unchecked dynamic fields. + The list shows at most 20 checks. `skippedParameterCheckCount` includes + omitted checks, which also leave dynamic fields unverified. + State that limitation even if the run succeeded and no parameter warnings + were returned. Do not request parameter values when sharing is disabled. 3. After verification handling, if `setupRequirement.status === "required"` and setup has not already run for this build, call `workflows(action="setup")` with the workflowId. @@ -577,7 +599,10 @@ applies to rows or records written to an external system: never make quantitativ claims ("22 rows written", "columns matched") that you did not read back from the effect node's actual output (`executions(action="get-node-output")`) or from the target system itself — a successful run status does not prove the _right -data_ was written, only that nodes ran. If you could not run the +data_ was written, only that nodes ran. Output of a simulated or pinned node is +fixture data: never quote it as what the workflow produced, and never cite it as +proof that an expression resolved — use `resolvedParameterWarnings` or +`executions(action="get-resolved-node-parameters")` for that. If you could not run the failing path or inspect the artifact, say so plainly — "I couldn't verify X because Y" — and name what is unconfirmed. An honest "could not verify" beats an unverified success claim. diff --git a/packages/@n8n/instance-ai/src/tools/executions.tool.ts b/packages/@n8n/instance-ai/src/tools/executions.tool.ts index 330f50a370f..927c6cc9719 100644 --- a/packages/@n8n/instance-ai/src/tools/executions.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/executions.tool.ts @@ -50,7 +50,9 @@ const runAction = z.object({ .describe( 'Input data passed to the workflow trigger. Works for ANY trigger type — ' + 'the system injects inputData as the trigger node output, bypassing the need for a real event. ' + - 'For webhook triggers, inputData is the request body (do NOT wrap in { body: ... }). ' + + 'For webhook triggers, a flat inputData is treated as the request body (placed under `body`; ' + + '`query`, `headers` and `params` stay empty). To exercise $json.query.*, $json.headers.* or ' + + '$json.params.*, pass the request envelope { body: {...}, query: {...}, headers: {...}, params: {...} } instead. ' + 'For event-based triggers (e.g. Linear, GitHub, Slack), pass inputData matching ' + 'the shape the trigger would emit (e.g. { action: "create", data: { ... } }).', ), diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts index d7ae163eaf9..3cf3c83ef26 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts @@ -7,6 +7,7 @@ import type { InstanceAiDataTableService, InstanceAiWorkflowService, OrchestrationContext, + ResolvedNodeParametersResult, WorkflowTaskService, } from '../../../types'; import { createRemediation, MAX_VERIFY_ATTEMPTS } from '../../../workflow-loop/remediation'; @@ -32,6 +33,20 @@ type VerifyBuiltWorkflowOutput = { }>; simulatedNodes?: Array<{ nodeName: string; reason: string }>; simulationNote?: string; + resolvedParameterWarnings?: Array<{ + nodeName: string; + executionId?: string; + path: string; + raw: string; + issue: 'empty' | 'failed'; + detail?: string; + }>; + skippedParameterChecks?: Array<{ + nodeName: string; + executionId?: string; + reason: string; + }>; + skippedParameterCheckCount?: number; lastNodeExecuted?: string; nodesNotReached?: string[]; nodeErrors?: Array<{ nodeName: string; message?: string }>; @@ -457,6 +472,9 @@ interface VerifyToolContext { ] ) => Promise >; + getResolvedNodeParameters?: Mock< + (executionId: string, nodeName: string) => Promise + >; }; workflowService?: InstanceAiWorkflowService; dataTableService?: InstanceAiDataTableService; @@ -496,6 +514,8 @@ function makeContext( workflowConnections?: Record; tableRows?: Record>>; availableCredentials?: Array<{ id: string; name: string; type: string }>; + /** Replayed parameter resolution per simulated node; nodes not listed make the replay throw. */ + resolvedParameters?: Record; } = {}, ) { const updateBuildOutcome = vi.fn( @@ -583,6 +603,13 @@ function makeContext( error: vi.fn(), }; + const getResolvedNodeParameters = vi.fn(async (_executionId: string, nodeName: string) => { + await Promise.resolve(); + const replayed = overrides.resolvedParameters?.[nodeName]; + if (!replayed) throw new Error(`no run data for ${nodeName}`); + return replayed; + }); + const ctx: VerifyToolContext = { workflowTaskService: { reportBuildOutcome: vi.fn(), @@ -602,7 +629,7 @@ function makeContext( recordVerification: vi.fn(coordinator.recordVerification.bind(coordinator)), } as unknown as WorkflowTaskService, domainContext: { - executionService: { run }, + executionService: { run, getResolvedNodeParameters }, workflowService, dataTableService, credentialService, @@ -1167,6 +1194,99 @@ describe('verify-built-workflow tool — node simulation plan', () => { expect(result.simulationNote).toContain('no real external writes'); }); + it('surfaces parameters of reached simulated nodes that resolved to empty or threw', async () => { + const { ctx } = makeContext( + makeBuildOutcome({ + nodeSimulationPlan: [ + simulateVerdict('Send Slack', 'Sends a message to a Slack channel'), + simulateVerdict('Send SMS', 'Sends an SMS'), + ], + simulationFixtures: { 'Send Slack': [{ ok: true }], 'Send SMS': [{ sid: 'SM1' }] }, + }), + { + executionId: 'exec-sim', + status: 'success', + data: { + Webhook: [{ headers: {}, query: {}, body: { CallStatus: 'no-answer' } }], + 'Send Slack': [{ ok: true }], + }, + }, + { + resolvedParameters: { + 'Send Slack': { + nodeName: 'Send Slack', + runIndex: 0, + itemIndex: 0, + parameters: { text: '={{ $json.query.caller }}' }, + resolved: '{"text":""}', + failedExpressions: [ + { path: 'channel', raw: '={{ $json.body.ch.id }}', error: 'Cannot read id' }, + ], + emptyResolutions: [ + { path: 'text', raw: '={{ $json.query.caller }}', resolved: undefined }, + ], + }, + }, + }, + ); + + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + + expect(result.success).toBe(true); + // Only nodes the run reached are replayed; Send SMS never executed. + expect(ctx.domainContext.executionService.getResolvedNodeParameters).toHaveBeenCalledTimes(1); + expect(ctx.domainContext.executionService.getResolvedNodeParameters).toHaveBeenCalledWith( + 'exec-sim', + 'Send Slack', + ); + expect(result.resolvedParameterWarnings).toEqual([ + { + nodeName: 'Send Slack', + executionId: 'exec-sim', + path: 'channel', + raw: '={{ $json.body.ch.id }}', + issue: 'failed', + detail: 'Cannot read id', + }, + { + nodeName: 'Send Slack', + executionId: 'exec-sim', + path: 'text', + raw: '={{ $json.query.caller }}', + issue: 'empty', + }, + ]); + expect(result.simulationNote).toContain('no real external writes'); + expect(result.simulationNote).toContain('Send Slack: `channel`'); + expect(result.simulationNote).toContain('`text` (={{ $json.query.caller }}) resolved to empty'); + }); + + it('keeps the verification result when the parameter replay fails', async () => { + const { ctx } = makeContext( + makeBuildOutcome({ + nodeSimulationPlan: [simulateVerdict('Send Slack', 'Sends a message to a Slack channel')], + simulationFixtures: { 'Send Slack': [{ ok: true }] }, + }), + { executionId: 'exec-sim', status: 'success', data: { 'Send Slack': [{ ok: true }] } }, + ); + + const result = await runTool(ctx, { workItemId: 'wi-1', workflowId: 'wf-1' }); + + expect(result.success).toBe(true); + expect(result.resolvedParameterWarnings).toBeUndefined(); + expect(result.skippedParameterChecks).toEqual([ + { nodeName: 'Send Slack', executionId: 'exec-sim', reason: 'replay-failed' }, + ]); + expect(result.skippedParameterCheckCount).toBe(1); + expect(result.simulationNote).toContain('no real external writes'); + expect(result.simulationNote).toContain('Parameter check skipped'); + expect(result.simulationNote).toContain('unchecked dynamic fields'); + expect(ctx.logger.debug).toHaveBeenCalledWith( + 'Resolved-parameter check skipped for simulated node', + expect.objectContaining({ nodeName: 'Send Slack' }), + ); + }); + it('fails closed when the build outcome has no simulation plan at all', async () => { // An undefined plan means the outcome predates classification or // classification failed — nothing shields destructive nodes in that run. diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/resolved-parameter-warnings.test.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/resolved-parameter-warnings.test.ts new file mode 100644 index 00000000000..9a9925bc98b --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/__tests__/resolved-parameter-warnings.test.ts @@ -0,0 +1,161 @@ +import type { ResolvedNodeParametersResult } from '../../../../types'; +import { + buildResolvedParameterNote, + collectResolvedParameterWarnings, + warningsFromResolution, +} from '../resolved-parameter-warnings'; + +function resolution( + overrides: Partial = {}, +): ResolvedNodeParametersResult { + return { + nodeName: 'Send SMS', + runIndex: 0, + itemIndex: 0, + parameters: {}, + resolved: '{}', + failedExpressions: [], + emptyResolutions: [], + ...overrides, + }; +} + +describe('warningsFromResolution', () => { + it('reports empty and failed expressions for the node', () => { + const warnings = warningsFromResolution( + resolution({ + emptyResolutions: [{ path: 'to', raw: '={{ $json.query.caller }}', resolved: undefined }], + failedExpressions: [ + { path: 'message', raw: '={{ $json.body.text.trim() }}', error: 'Cannot read trim' }, + ], + }), + ); + + expect(warnings).toEqual([ + { + nodeName: 'Send SMS', + path: 'message', + raw: '={{ $json.body.text.trim() }}', + issue: 'failed', + detail: 'Cannot read trim', + }, + { nodeName: 'Send SMS', path: 'to', raw: '={{ $json.query.caller }}', issue: 'empty' }, + ]); + }); + + it('ignores expressions that need live-only context', () => { + const warnings = warningsFromResolution( + resolution({ + emptyResolutions: [ + { + path: 'token', + raw: '={{ $secrets.vault.token }}', + resolved: undefined, + reason: 'unreconstructable-context', + }, + ], + failedExpressions: [ + { + path: 'url', + raw: '={{ $response.body.next }}', + error: 'undefined', + reason: 'unreconstructable-context', + }, + ], + }), + ); + + expect(warnings).toEqual([]); + }); + + it('returns nothing when resolution was suppressed', () => { + const warnings = warningsFromResolution( + resolution({ + parameters: null, + resolved: null, + suppressed: 'parameter-values-disabled', + emptyResolutions: [{ path: 'to', raw: '={{ $json.x }}', resolved: '' }], + }), + ); + + expect(warnings).toEqual([]); + }); +}); + +describe('collectResolvedParameterWarnings', () => { + it('replays every reached simulated node and skips nodes whose replay fails', async () => { + const getResolvedNodeParameters = vi.fn(async (_executionId: string, nodeName: string) => { + await Promise.resolve(); + if (nodeName === 'Broken') throw new Error('no run data'); + return resolution({ + nodeName, + emptyResolutions: [{ path: 'to', raw: '={{ $json.query.caller }}', resolved: undefined }], + }); + }); + const logger = { debug: vi.fn() }; + + const { warnings, skipped, skippedCount } = await collectResolvedParameterWarnings({ + executionService: { getResolvedNodeParameters }, + runs: [{ executionId: 'exec_1', nodeNames: ['Send SMS', 'Broken'] }], + logger, + }); + + expect(getResolvedNodeParameters).toHaveBeenCalledTimes(2); + expect(getResolvedNodeParameters).toHaveBeenCalledWith('exec_1', 'Send SMS'); + expect(warnings).toEqual([ + { + nodeName: 'Send SMS', + executionId: 'exec_1', + path: 'to', + raw: '={{ $json.query.caller }}', + issue: 'empty', + }, + ]); + expect(skipped).toEqual([ + { nodeName: 'Broken', executionId: 'exec_1', reason: 'replay-failed' }, + ]); + expect(skippedCount).toBe(1); + expect(logger.debug).toHaveBeenCalledWith( + 'Resolved-parameter check skipped for simulated node', + expect.objectContaining({ nodeName: 'Broken', error: 'no run data' }), + ); + }); + + it('does not call the service when no simulated node was reached', async () => { + const getResolvedNodeParameters = vi.fn(); + + const result = await collectResolvedParameterWarnings({ + executionService: { getResolvedNodeParameters }, + runs: [{ executionId: 'exec_1', nodeNames: [] }], + }); + + expect(result).toEqual({ warnings: [], skipped: [], skippedCount: 0 }); + expect(getResolvedNodeParameters).not.toHaveBeenCalled(); + }); +}); + +describe('buildResolvedParameterNote', () => { + it('is undefined without warnings', () => { + expect(buildResolvedParameterNote([])).toBeUndefined(); + }); + + it('groups warnings by node and tells the agent the fixture output is not proof', () => { + const note = buildResolvedParameterNote([ + { nodeName: 'Send SMS', path: 'to', raw: '={{ $json.query.caller }}', issue: 'empty' }, + { + nodeName: 'Send SMS', + path: 'message', + raw: '={{ $json.body.text.trim() }}', + issue: 'failed', + detail: 'Cannot read trim', + }, + { nodeName: 'Post to Slack', path: 'text', raw: '={{ $json.headers.x }}', issue: 'empty' }, + ]); + + expect(note).toContain('Send SMS: `to` (={{ $json.query.caller }}) resolved to empty'); + expect(note).toContain('`message` (={{ $json.body.text.trim() }}) failed: Cannot read trim'); + expect(note).toContain('Post to Slack: `text`'); + expect(note).toContain('fixture data'); + expect(note).toContain('{body, query, headers, params}'); + }); +}); diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/resolved-parameter-warnings.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/resolved-parameter-warnings.ts new file mode 100644 index 00000000000..ac5edc996d3 --- /dev/null +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/resolved-parameter-warnings.ts @@ -0,0 +1,174 @@ +/** + * Parameter check for simulated nodes after a verification run. + * + * A simulated node's preview is fixture data, so an expression that resolved to + * empty (or threw) leaves no trace in the run result. Replaying the node's + * parameter resolution against the saved execution exposes it. + */ + +import { z } from 'zod'; + +import type { InstanceAiExecutionService, ResolvedNodeParametersResult } from '../../../types'; + +export const resolvedParameterWarningSchema = z.object({ + nodeName: z.string(), + executionId: z.string().optional(), + path: z.string(), + raw: z.string(), + issue: z.enum(['empty', 'failed']), + detail: z.string().optional(), +}); + +export const skippedParameterCheckSchema = z.object({ + nodeName: z.string(), + executionId: z.string().optional(), + reason: z.enum(['parameter-values-disabled', 'replay-failed', 'execution-unavailable']), +}); + +export type ResolvedParameterWarning = z.infer; +type SkippedParameterCheck = z.infer; + +export interface ParameterCheckRun { + executionId?: string; + nodeNames: readonly string[]; +} + +/** Keep the tool output compact when a node has many empty leaves. */ +const MAX_WARNINGS = 20; +const MAX_SKIPPED_CHECKS = 20; + +type WarningLogger = { debug(message: string, meta?: Record): void }; + +export function warningsFromResolution( + result: ResolvedNodeParametersResult, +): ResolvedParameterWarning[] { + if (result.suppressed) return []; + // Context that only exists in a live run ($secrets, $response, …) is expected + // to be missing in a replay; it says nothing about the workflow. + const failed = result.failedExpressions + .filter((failure) => failure.reason !== 'unreconstructable-context') + .map((failure) => ({ + nodeName: result.nodeName, + path: failure.path, + raw: failure.raw, + issue: 'failed', + detail: failure.error, + })); + const empty = result.emptyResolutions + .filter((resolution) => resolution.reason !== 'unreconstructable-context') + .map((resolution) => ({ + nodeName: result.nodeName, + path: resolution.path, + raw: resolution.raw, + issue: 'empty', + })); + return [...failed, ...empty]; +} + +export async function collectResolvedParameterWarnings(args: { + executionService: Pick; + runs: readonly ParameterCheckRun[]; + logger?: WarningLogger; +}): Promise<{ + warnings: ResolvedParameterWarning[]; + skipped: SkippedParameterCheck[]; + skippedCount: number; +}> { + const { executionService, runs, logger } = args; + const warnings: ResolvedParameterWarning[] = []; + const skipped: SkippedParameterCheck[] = []; + let skippedCount = 0; + const recordSkippedCheck = (check: SkippedParameterCheck) => { + skippedCount++; + if (skipped.length < MAX_SKIPPED_CHECKS) skipped.push(check); + }; + for (const { executionId, nodeNames } of runs) { + if (!executionId) { + for (const nodeName of nodeNames) { + recordSkippedCheck({ nodeName, reason: 'execution-unavailable' }); + } + continue; + } + + const settled = await Promise.allSettled( + nodeNames.map( + async (nodeName) => await executionService.getResolvedNodeParameters(executionId, nodeName), + ), + ); + settled.forEach((outcome, index) => { + const nodeName = nodeNames[index]; + if (outcome.status === 'fulfilled') { + if (outcome.value.suppressed) { + recordSkippedCheck({ nodeName, executionId, reason: outcome.value.suppressed }); + } else { + warnings.push( + ...warningsFromResolution(outcome.value).map((warning) => ({ + ...warning, + executionId, + })), + ); + } + return; + } + // Keep the run result, but disclose that its parameters were not checked. + recordSkippedCheck({ nodeName, executionId, reason: 'replay-failed' }); + logger?.debug('Resolved-parameter check skipped for simulated node', { + executionId, + nodeName, + error: outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason), + }); + }); + } + return { warnings: warnings.slice(0, MAX_WARNINGS), skipped, skippedCount }; +} + +export function buildResolvedParameterNote( + warnings: readonly ResolvedParameterWarning[], + skipped: readonly SkippedParameterCheck[] = [], + skippedCount = skipped.length, +): string | undefined { + const skipReasons = { + 'parameter-values-disabled': 'parameter values are disabled', + 'replay-failed': 'parameter replay failed', + 'execution-unavailable': 'the execution is unavailable', + }; + const skippedNote = + skipped.length > 0 + ? 'Parameter check skipped for ' + + skipped + .map( + ({ nodeName, executionId, reason }) => + `${nodeName}${executionId ? ` (execution ${executionId})` : ''}: ${skipReasons[reason]}`, + ) + .join('; ') + + (skippedCount > skipped.length + ? `. Showing ${skipped.length} of ${skippedCount} skipped checks` + : '') + + '. All nodes with skipped checks have unchecked dynamic fields. Do not report those fields as verified.' + : undefined; + if (warnings.length === 0) return skippedNote; + + const byNode = new Map(); + for (const warning of warnings) { + const entry = + warning.issue === 'failed' + ? `\`${warning.path}\` (${warning.raw}) failed: ${warning.detail ?? 'expression error'}` + : `\`${warning.path}\` (${warning.raw}) resolved to empty`; + const list = byNode.get(warning.nodeName) ?? []; + list.push(entry); + byNode.set(warning.nodeName, list); + } + const summary = [...byNode.entries()] + .map(([nodeName, entries]) => `${nodeName}: ${entries.join(', ')}`) + .join('; '); + + return [ + `Simulated-node parameter check: ${summary}. ` + + 'A simulated node’s preview is fixture data and does not prove these fields: fix the ' + + 'trigger input shape (for a Webhook, pass the {body, query, headers, params} envelope) or the ' + + 'expression, re-run verification, and do not report these fields as working until they resolve.', + skippedNote, + ] + .filter((note): note is string => note !== undefined) + .join(' '); +} diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts index fc56e7e4b3f..2b752e461c1 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verification/scripted-gate-run.ts @@ -13,6 +13,7 @@ import { type VerificationAnalysis, } from './analyze-result'; import type { PreparedVerificationRun } from './prepare-run'; +import type { ParameterCheckRun } from './resolved-parameter-warnings'; import type { ExecutionRunResult } from './types'; import type { OrchestrationContext } from '../../../types'; import type { @@ -48,9 +49,11 @@ interface DecisionPass { analysis: VerificationAnalysis; } -export async function runScriptedGateVerification( - args: ScriptedGateRunArgs, -): Promise<{ result: ExecutionRunResult; analysis: VerificationAnalysis }> { +export async function runScriptedGateVerification(args: ScriptedGateRunArgs): Promise<{ + result: ExecutionRunResult; + analysis: VerificationAnalysis; + parameterCheckRuns: ParameterCheckRun[]; +}> { const { script, prepared, @@ -88,7 +91,14 @@ export async function runScriptedGateVerification( passes.push({ label: decision.label, result, analysis }); } - return { result: mergeResults(passes), analysis: mergeAnalyses(script, prepared, passes) }; + return { + result: mergeResults(passes), + analysis: mergeAnalyses(script, prepared, passes), + parameterCheckRuns: passes.map(({ result, analysis }) => ({ + executionId: result.executionId, + nodeNames: analysis.reachedSimulatedNodes.map((node) => node.nodeName), + })), + }; } function mergeResults(passes: DecisionPass[]): ExecutionRunResult { diff --git a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts index 205b59c1816..d2cdfc46fbd 100644 --- a/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/orchestration/verify-built-workflow.tool.ts @@ -24,6 +24,12 @@ import { import { prepareVerificationRun } from './verification/prepare-run'; import { reconcileStaleCredentialPlan } from './verification/reconcile-plan'; import { resolveVerificationTarget } from './verification/resolve-target'; +import { + buildResolvedParameterNote, + collectResolvedParameterWarnings, + resolvedParameterWarningSchema, + skippedParameterCheckSchema, +} from './verification/resolved-parameter-warnings'; import { runScriptedGateVerification } from './verification/scripted-gate-run'; import { executionNodeErrorSchema, @@ -47,7 +53,11 @@ export const verifyBuiltWorkflowInputSchema = z.object({ .describe( "Input data for the workflow trigger. Shape MUST match the trigger's real-world output: " + 'Form Trigger -> flat field map like {name: "Alice", email: "a@b.c"} (do NOT wrap in formFields); ' + - 'Webhook -> the body payload like {event: "signup", userId: "..."} (adapter wraps it under body); ' + + 'Webhook -> a flat payload like {event: "signup", userId: "..."} is placed under `body` and leaves ' + + '`query`, `headers` and `params` EMPTY. When any expression reads $json.query.*, $json.headers.* or ' + + '$json.params.*, pass the request envelope instead: {body: {...}, query: {caller: "+1555..."}, ' + + 'headers: {"x-github-event": "issues"}, params: {...}}. A flat payload cannot exercise those fields, they resolve ' + + 'empty, and a simulated downstream node still looks green; ' + 'Chat Trigger -> {chatInput: "user message"}; ' + 'Schedule Trigger -> omit inputData. ' + "If you wrap a form payload in {formFields: {...}} the adapter will reject the call; the builder's " + @@ -140,6 +150,9 @@ const verifyBuiltWorkflowOutputSchema = z.object({ .optional(), simulatedNodes: z.array(z.object({ nodeName: z.string(), reason: z.string() })).optional(), simulationNote: z.string().optional(), + resolvedParameterWarnings: z.array(resolvedParameterWarningSchema).optional(), + skippedParameterChecks: z.array(skippedParameterCheckSchema).optional(), + skippedParameterCheckCount: z.number().int().nonnegative().optional(), lastNodeExecuted: z.string().optional(), nodeErrors: z.array(executionNodeErrorSchema).optional(), nodesNotReached: z.array(z.string()).optional(), @@ -238,7 +251,7 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { // A scripted gate replaces the halt with one loop-safe pass per decision; // otherwise run the single standard pass (halted gates pin zero items). - const { result, analysis } = prepared.gateScript + const { result, analysis, parameterCheckRuns } = prepared.gateScript ? await runScriptedGateVerification({ script: prepared.gateScript, prepared, @@ -267,20 +280,27 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { abortSignal: context.abortSignal, }, ); + const analysis = analyzeVerificationResult({ + result: runResult, + buildOutcome, + simulatedNodes: prepared.simulatedNodes, + haltedGateNames: prepared.haltedGateNames, + triggerNodeName: resolvedInput.triggerNodeName, + stateBefore: target.stateBefore, + runId: context.runId, + chatModelRelatedNodeNames, + chatModelRecovery, + verificationScope, + }); return { result: runResult, - analysis: analyzeVerificationResult({ - result: runResult, - buildOutcome, - simulatedNodes: prepared.simulatedNodes, - haltedGateNames: prepared.haltedGateNames, - triggerNodeName: resolvedInput.triggerNodeName, - stateBefore: target.stateBefore, - runId: context.runId, - chatModelRelatedNodeNames, - chatModelRecovery, - verificationScope, - }), + analysis, + parameterCheckRuns: [ + { + executionId: runResult.executionId, + nodeNames: analysis.reachedSimulatedNodes.map((node) => node.nodeName), + }, + ], }; })(); @@ -317,6 +337,29 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { claim: runClaim, }); + // A simulated node's preview is fixture data, so an expression that resolved + // to empty (e.g. `$json.query.x` on a body-only input) leaves no trace in the + // run. Replay the parameters of every reached simulated node and surface it. + const { + warnings: resolvedParameterWarnings, + skipped: skippedParameterChecks, + skippedCount: skippedParameterCheckCount, + } = await collectResolvedParameterWarnings({ + executionService: target.domainContext.executionService, + runs: parameterCheckRuns, + logger: context.logger, + }); + const simulationNote = [ + analysis.simulationNote, + buildResolvedParameterNote( + resolvedParameterWarnings, + skippedParameterChecks, + skippedParameterCheckCount, + ), + ] + .filter((note): note is string => note !== undefined) + .join(' '); + const maxDataChars = resolvedInput.maxDataChars ?? DEFAULT_NODE_PREVIEW_CHARS; const simulatedNames = new Set(analysis.reachedSimulatedNodes.map((n) => n.nodeName)); return { @@ -330,7 +373,13 @@ export function createVerifyBuiltWorkflowTool(context: OrchestrationContext) { nodePreviews: buildNodePreviews(result.data, maxDataChars, simulatedNames), simulatedNodes: analysis.reachedSimulatedNodes.length > 0 ? analysis.reachedSimulatedNodes : undefined, - simulationNote: analysis.simulationNote, + simulationNote: simulationNote.length > 0 ? simulationNote : undefined, + resolvedParameterWarnings: + resolvedParameterWarnings.length > 0 ? resolvedParameterWarnings : undefined, + skippedParameterChecks: + skippedParameterChecks.length > 0 ? skippedParameterChecks : undefined, + skippedParameterCheckCount: + skippedParameterCheckCount > 0 ? skippedParameterCheckCount : undefined, nodeErrors: analysis.nodeErrors.length > 0 ? analysis.nodeErrors : undefined, nodesNotReached: analysis.nodesNotReached.length > 0 ? analysis.nodesNotReached : undefined, coverageNote: analysis.coverageNote, diff --git a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts index 69fd2ceb907..c1ed6f5084d 100644 --- a/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts +++ b/packages/@n8n/instance-ai/src/workflow-loop/__tests__/workflow-task-service.test.ts @@ -138,7 +138,12 @@ describe('WorkflowTaskCoordinator', () => { workflowTaskService: coordinator, logger: mock(), domainContext: mock({ - executionService: mock({ run }), + executionService: mock({ + run, + getResolvedNodeParameters: vi + .fn() + .mockRejectedValue(new Error('No saved parameter data')), + }), workflowService: mock({ getAsWorkflowJSON: vi.fn().mockResolvedValue({ nodes: [], connections: {} }), }),