github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-10-17 05:29:44 +02:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity Graph

fix(ai-builder): Honor neverError in eval mocks, seed agent tables, filter pinned reads and report start refusals (no-changelog) (#39337)

Browse Source 
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
José Braulio González Valido 2026-09-25 07:47:28 +00:00 committed by GitHub
parent 1a4a91256d
commit e3fe95d228
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 1524 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
packages/@n8n/instance-ai/evaluations/__tests__/agent-execution-seed.test.ts Normal file
View File

@ -0,0 +1,122 @@
import type { InstanceAiEvalAgentExecutionResult } from '@n8n/api-types';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { N8nClient } from '../clients/n8n-client';
import { executeAgentScenario } from '../harness/agent-execution';
import type { EvalLogger } from '../harness/logger';
import { reseedScenarioTables } from '../harness/seed-tables';
vi.mock('../harness/seed-tables', async (importOriginal) => ({
...(await importOriginal<typeof import('../harness/seed-tables')>()),
reseedScenarioTables: vi.fn(),
}));
vi.mock('../harness/scenario-execution', async (importOriginal) => ({
...(await importOriginal<typeof import('../harness/scenario-execution')>()),
writeScenarioVerificationSnapshot: vi.fn(),
}));
vi.mock('../checklist/verifier', () => ({
verifyChecklist: vi.fn().mockResolvedValue({
results: [{ id: 1, pass: true, reasoning: 'ok' }],
attempts: [],
}),
}));
const silentLogger: EvalLogger = {
info: () => {},
verbose: () => {},
success: () => {},
warn: () => {},
error: () => {},
isVerbose: false,
};
const scenario = {
name: 'known-customer',
description: 'A known customer is looked up.',
dataSetup: 'The Customers table contains Ada.',
successCriteria: 'The agent finds Ada.',
seedDataTables: [
{
id: 'seed-table-1',
name: 'Customers',
columns: [{ name: 'email', type: 'string' as const }],
rows: [{ email: 'ada@example.com' }],
},
],
};
const agentRun = {
runId: 'run-1',
success: true,
errors: [],
finalText: 'Found Ada.',
model: 'openai/gpt-5-mini',
finishReason: 'stop',
toolCalls: [],
modelTurns: [],
seed: {
openingMessage: 'From ada@example.com: help',
globalContext: '',
toolHints: {},
warnings: [],
},
skippedFeatures: [],
} as unknown as InstanceAiEvalAgentExecutionResult;
function clientRecording(calls: string[]): N8nClient {
return {
getPersonalProjectId: vi.fn().mockResolvedValue('project-1'),
executeAgentWithLlmMock: vi.fn().mockImplementation(async () => {
calls.push('execute');
return agentRun;
}),
} as unknown as N8nClient;
}
describe('executeAgentScenario', () => {
beforeEach(() => {
vi.mocked(reseedScenarioTables).mockReset();
});
it('seeds the scenario rows before the agent runs when a seed context is given', async () => {
const calls: string[] = [];
vi.mocked(reseedScenarioTables).mockImplementation(async () => {
calls.push('reseed');
});
const client = clientRecording(calls);
const seedContext = { threadId: 'thread-1', tableIdsByName: { Customers: 'dt-real-1' } };
const result = await executeAgentScenario(
client,
'agent-1',
scenario,
'AGENT CONTEXT',
silentLogger,
10_000,
'case-a',
undefined,
undefined,
seedContext,
);
expect(reseedScenarioTables).toHaveBeenCalledWith(
client,
scenario,
'thread-1',
{ Customers: 'dt-real-1' },
silentLogger,
);
expect(calls).toEqual(['reseed', 'execute']);
expect(result.success).toBe(true);
});
it('leaves the tables alone when no seed context is given', async () => {
const calls: string[] = [];
const client = clientRecording(calls);
await executeAgentScenario(client, 'agent-1', scenario, 'AGENT CONTEXT', silentLogger, 10_000);
expect(reseedScenarioTables).not.toHaveBeenCalled();
expect(calls).toEqual(['execute']);
});
});

88
packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts
View File

@ -22,14 +22,6 @@ vi.mock('../harness/cleanup', async (importOriginal) => {
};
});
vi.mock('../harness/seed-tables', async (importOriginal) => {
const actual = await importOriginal<typeof import('../harness/seed-tables')>();
return {
...actual,
warnAgentSeedDataTablesIgnored: vi.fn(),
};
});
const silentLogger: EvalLogger = {
info: () => {},
verbose: () => {},
@ -584,6 +576,86 @@ describe('seed-table scenarios (TRUST-311 parity)', () => {
});
});
const seededAgentBuild = (overrides: Partial<BuildResult> = {}) =>
okBuild({
workflowId: undefined,
workflowJsons: [],
transcript: [] as never,
artifactRefs: [{ type: 'agent', id: 'agent-1' }] as never,
...overrides,
});
const runnableAgentContext = () =>
new Map([
[
'0:case-a',
Promise.resolve({
rendered: 'AGENT CONTEXT',
artifact: {
agentId: 'agent-1',
config: { name: 'Support agent', model: 'openai/gpt-5-mini' },
skills: {},
},
}),
],
]);
it('passes the build seed context to an agent scenario', async () => {
const lane = makeLane();
vi.mocked(lane.tracedExecuteAgent).mockResolvedValue({
success: true,
score: 1,
reasoning: 'agent did it',
agentEvalResult: { errors: [] },
} as never);
const orchestrator = makeOrchestrator({
build: seededAgentBuild({
threadId: 'thread-1',
seededScenarioTableIdsByName: { Jobs: 'dt-real-1' },
}),
lane,
buildDurationMs: 1,
});
const pipeline = createCasePipeline(
makeDeps(orchestrator, {
testCaseByFileSlug: new Map([['case-a', seededCase(['happy-path'])]]),
agentContextByKey: runnableAgentContext(),
}),
);
const output = await pipeline.runRow(rowInputs('happy-path'));
expect(output).toMatchObject({ passed: true, agentId: 'agent-1' });
expect(lane.tracedExecuteAgent).toHaveBeenCalledWith(
expect.objectContaining({
scenario: expect.objectContaining({
seedDataTables: [expect.objectContaining({ name: 'Jobs' })],
}) as unknown,
seedContext: { threadId: 'thread-1', tableIdsByName: { Jobs: 'dt-real-1' } },
}),
);
});
it('refuses to run a seeded agent scenario when the build carries no seeded-table mapping', async () => {
const lane = makeLane();
const orchestrator = makeOrchestrator({ build: seededAgentBuild(), lane, buildDurationMs: 1 });
const pipeline = createCasePipeline(
makeDeps(orchestrator, {
testCaseByFileSlug: new Map([['case-a', seededCase(['happy-path'])]]),
agentContextByKey: runnableAgentContext(),
}),
);
const output = await pipeline.runRow(rowInputs('happy-path'));
expect(lane.tracedExecuteAgent).not.toHaveBeenCalled();
expect(output).toMatchObject({
passed: false,
agentId: 'agent-1',
failureCategory: 'framework_issue',
reasoning: expect.stringContaining('no seeded-table mapping') as unknown,
});
});
it('serializes rows of a seeded case so table reseeding cannot interleave', async () => {
const lane = makeLane();
const started: string[] = [];

27
packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts
View File

@ -435,6 +435,33 @@ describe('comparableDiff (post-write verification)', () => {
).toEqual([]);
});
const table = {
id: 'seed-table-1',
name: 'Queue',
columns: [{ name: 'message', type: 'string' as const }],
};
const scenario = { name: 'empty-queue', description: 'd', dataSetup: 's', successCriteria: 'ok' };
it('reads a stored seed table without `rows` as the disk table that declares `rows: []`', () => {
const written = item('c', {
executionScenarios: [{ ...scenario, seedDataTables: [{ ...table, rows: [] }] }],
}).testCase;
const stored = body({ executionScenarios: [{ ...scenario, seedDataTables: [table] }] });
expect(comparableDiff(stored, written)).toEqual([]);
});
it('names `executionScenarios` when the stored table lost the declared rows', () => {
const written = item('c', {
executionScenarios: [
{ ...scenario, seedDataTables: [{ ...table, rows: [{ message: 'hello' }] }] },
],
}).testCase;
const stored = body({ executionScenarios: [{ ...scenario, seedDataTables: [table] }] });
expect(comparableDiff(stored, written)).toEqual(['executionScenarios']);
});
it('names `seed` when a pre-#113 server dropped it', () => {
const written = item('c', { seed: inlineSeed() }).testCase;
expect(comparableDiff(body(), written)).toEqual(['seed']);

30
packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts
View File

@ -85,6 +85,36 @@ describe('diskCaseToLangTracerCreate', () => {
expect('description' in body).toBe(false);
});
it('forwards typed scenario seed tables so lang-tracer stores their rows', () => {
const seedDataTables = [
{
id: 'seed-table-1',
name: 'Customers',
columns: [{ name: 'email', type: 'string' as const }],
rows: [{ email: 'ada@example.com' }],
},
];
const body = diskCaseToLangTracerCreate(
diskCase({
executionScenarios: [
{
name: 'seeded',
description: 'd',
dataSetup: 's',
successCriteria: 'ok',
seedDataTables,
},
{ name: 'plain', description: 'd', dataSetup: 's', successCriteria: 'ok' },
],
}),
'c',
{ suiteId: 1, setKind: 'regression', synthetic: true },
);
expect(body.scenarios?.[0].seedDataTables).toEqual(seedDataTables);
expect('seedDataTables' in (body.scenarios?.[1] ?? {})).toBe(false);
});
it('preserves a scenario `requires` field when present', () => {
const body = diskCaseToLangTracerCreate(
diskCase({

15
packages/@n8n/instance-ai/evaluations/harness/agent-execution.ts
View File

@ -14,6 +14,7 @@ import { agentHandler } from './artifacts/agent-handler';
import { attributionForScenario, type EvalAttribution } from './attribution';
import type { EvalLogger } from './logger';
import { writeScenarioVerificationSnapshot, type VerificationArtifact } from './scenario-execution';
import { reseedScenarioTables, type ScenarioSeedContext } from './seed-tables';
import {
throwIfServerBudgetStop,
isTransientExecutionAbort,
@ -99,7 +100,8 @@ export async function fetchAgentScenarioContext(
/**
* Execute one scenario against a built first-class Agent and verify the
* result — the agent-artifact counterpart of runScenario. The agent reasons
* with its real model; its tools' outbound HTTP is served by the mock layer.
* with its real model; its tools' outbound HTTP is served by the mock layer and
* its Data Table tools read the real table, seeded with the scenario's rows first.
*/
export async function executeAgentScenario(
client: N8nClient,
@ -111,7 +113,18 @@ export async function executeAgentScenario(
testCaseName?: string,
buildTrace?: BuildTrace,
outputDir?: string,
seedContext?: ScenarioSeedContext,
): Promise<ExecutionScenarioResult> {
if (seedContext) {
await reseedScenarioTables(
client,
scenario,
seedContext.threadId,
seedContext.tableIdsByName,
logger,
);
}
const execStart = Date.now();
const projectId = await client.getPersonalProjectId();
let evalResult = await client.executeAgentWithLlmMock(

13
packages/@n8n/instance-ai/evaluations/harness/seed-tables.ts
View File

@ -198,16 +198,3 @@ function sameSeedTableShape(
JSON.stringify({ columns: b.columns, rows: b.rows })
);
}
/** Agent scenarios don't seed data-table rows (tables exist but stay empty) — shared warning for both orchestration paths. */
export function warnAgentSeedDataTablesIgnored(
logger: EvalLogger,
scenarioName: string,
seedDataTables: unknown[] | undefined,
): void {
if ((seedDataTables?.length ?? 0) > 0) {
logger.warn(
` [${scenarioName}] seedDataTables are not seeded on the agent execution path — tables exist but stay empty`,
);
}
}

20
packages/@n8n/instance-ai/evaluations/langtracer/push.ts
View File

@ -3,6 +3,8 @@
// seeding). Pure — no network — so the create/update/unchanged partitioning is
// unit-testable against in-memory suite state.
import { isRecord } from '@n8n/utils/is-record';
import type { LangTracerUpdateCaseBody } from './client';
import { normalizeExportedCase } from './normalize';
import { unsupportedPushReason, type LangTracerCreateCaseBody } from './to-exported';
@ -139,6 +141,10 @@ function projectComparable(src: unknown): Record<string, unknown> {
out[key] = seedWithoutMessageIds(value);
continue;
}
if (key === 'executionScenarios') {
out[key] = scenariosWithoutEmptyRows(value);
continue;
}
out[key] = value;
}
return out;
@ -168,6 +174,20 @@ function seedWithoutMessageIds(value: unknown): unknown {
return seed;
}
/** lang-tracer stores a seed table's empty `rows` as absent, so `[]` must compare equal to it. */
function scenariosWithoutEmptyRows(value: unknown): unknown {
if (!Array.isArray(value)) return value;
return value.map((scenario: unknown) => {
if (!isRecord(scenario) || !Array.isArray(scenario.seedDataTables)) return scenario;
const seedDataTables = scenario.seedDataTables.map((table: unknown) => {
if (!isRecord(table) || !Array.isArray(table.rows) || table.rows.length > 0) return table;
const { rows, ...rest } = table;
return rest;
});
return { ...scenario, seedDataTables };
});
}
/** Stable JSON with sorted object keys, so field/scenario ordering never affects equality. */
function canonicalize(value: unknown): string {
return JSON.stringify(sortKeysDeep(value));

5
packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts
View File

@ -3,7 +3,7 @@
// network call so the disk→API key-renaming contract is unit-testable without a server.
import type { CaseSeed, EvalTestCaseInput } from '../harness/schema';
import type { TestCaseCredential } from '../types';
import type { ExecutionScenario, TestCaseCredential } from '../types';
/** One scenario in the create-case payload (`executionScenarios` renamed to `scenarios`). */
export interface LangTracerScenario {
@ -12,6 +12,7 @@ export interface LangTracerScenario {
dataSetup?: string;
successCriteria?: string;
requires?: string;
seedDataTables?: ExecutionScenario['seedDataTables'];
}
/** Body for `POST /api/v1/cases`. Disk keys are renamed (`complexity`→`evalComplexity`,
@ -169,11 +170,13 @@ function mapScenario(scenario: {
dataSetup?: string;
successCriteria?: string;
requires?: string;
seedDataTables?: ExecutionScenario['seedDataTables'];
}): LangTracerScenario {
const mapped: LangTracerScenario = { name: scenario.name };
if (scenario.description !== undefined) mapped.description = scenario.description;
if (scenario.dataSetup !== undefined) mapped.dataSetup = scenario.dataSetup;
if (scenario.successCriteria !== undefined) mapped.successCriteria = scenario.successCriteria;
if (scenario.requires !== undefined) mapped.requires = scenario.requires;
if (scenario.seedDataTables !== undefined) mapped.seedDataTables = scenario.seedDataTables;
return mapped;
}

1
packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts
View File

@ -163,6 +163,7 @@ export interface LaneState {
buildTrace?: BuildResult['buildTrace'];
timeoutMs: number;
testCaseName?: string;
seedContext?: ScenarioSeedContext;
}) => Promise<Awaited<ReturnType<typeof executeAgentScenario>>>;
}

205
packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts
View File

@ -23,11 +23,7 @@ import {
import { buildFailedOnInfra } from '../harness/build-workflow';
import { cleanupBuild, effectiveTimeoutMs } from '../harness/cleanup';
import type { EvalLogger } from '../harness/logger';
import {
scenariosRequireSerialSeeding,
warnAgentSeedDataTablesIgnored,
type ScenarioSeedContext,
} from '../harness/seed-tables';
import { scenariosRequireSerialSeeding, type ScenarioSeedContext } from '../harness/seed-tables';
import {
classifyScenarioExecutionError,
extractErrorMessage,
@ -38,6 +34,10 @@ import {
import { BUILD_ONLY_SCENARIO_NAME, type DatasetExampleInputs } from '../langsmith/dataset-sync';
import type { BuildExpectationResult, ExecutionScenario, WorkflowTestCase } from '../types';
const NO_SEED_MAPPING_REASON =
'Scenario declares seedDataTables but the build provided no seeded-table mapping ' +
'(MCP/prebuilt builds do not seed data tables) — refusing to run without the declared rows';
/** Row inputs: the per-scenario fields (dataset example shape) plus the
* iteration tag the drivers stamp when expanding for N iterations. */
export type ScenarioRowInputs = DatasetExampleInputs & { _iteration?: number };
@ -304,18 +304,14 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline {
});
}
const seedContext: ScenarioSeedContext | undefined =
build.threadId && build.seededScenarioTableIdsByName
? { threadId: build.threadId, tableIdsByName: build.seededScenarioTableIdsByName }
: undefined;
// Agent scenario path — real model, mocked tool HTTP. Mirrors the
// workflow branch below (retry loop, framework_issue guard, output shape).
if (agentRunnable && agentRef) {
// Dataset rows don't carry seedDataTables — check the authored scenario.
warnAgentSeedDataTablesIgnored(
logger,
scenario.name,
testCaseByFileSlug
.get(inputs.testCaseFile)
?.executionScenarios?.find((s) => s.name === scenario.name)?.seedDataTables,
);
const agentExecStart = Date.now();
const capturedAgent = await agentContextByKey.get(cacheKey);
const agentContext = capturedAgent?.rendered ?? '(agent configuration could not be fetched)';
const agentArtifactFields = capturedAgent?.artifact
@ -347,78 +343,107 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline {
planRejections: build.proxyDecisionStats?.rejection ?? 0,
});
}
let agentResult;
for (let attempt = 1; ; attempt++) {
try {
agentResult = await builtOnLane.tracedExecuteAgent({
agentId: agentRef.id,
scenario,
agentContext,
buildTrace: build.buildTrace,
timeoutMs: effectiveTimeoutMs(
testCaseByFileSlug.get(inputs.testCaseFile)?.complexity,
args.timeoutMs,
),
testCaseName: inputs.testCaseFile,
});
break;
} catch (error: unknown) {
const errorMessage = extractErrorMessage(error);
if (shouldRetryScenarioExecution(errorMessage, attempt)) {
logger.warn(
` [${scenario.name}] agent execution attempt ${attempt}/${MAX_EXEC_ATTEMPTS} failed (${errorMessage}); retrying`,
);
await new Promise((resolve) => setTimeout(resolve, 500 * attempt));
continue;
}
logger.error(` ERROR [${scenario.name}]: ${errorMessage}`);
return await attachExpectations({
buildSuccess: true,
agentId: agentRef.id,
agentContext,
...agentArtifactFields,
passed: false,
score: 0,
reasoning: `Agent scenario execution error: ${errorMessage}`,
failureCategory: 'framework_issue',
attribution: 'framework_issue',
execErrors: [errorMessage],
buildDurationMs,
...buildSpendFields,
execDurationMs: Date.now() - agentExecStart,
nodeCount: 0,
threadId: build.threadId,
buildTrace: build.buildTrace,
planRejections: build.proxyDecisionStats?.rejection ?? 0,
});
}
if ((scenario.seedDataTables?.length ?? 0) > 0 && !seedContext) {
logger.error(` ERROR [${scenario.name}]: ${NO_SEED_MAPPING_REASON}`);
return await attachExpectations({
buildSuccess: true,
agentId: agentRef.id,
agentContext,
...agentArtifactFields,
passed: false,
score: 0,
reasoning: NO_SEED_MAPPING_REASON,
failureCategory: 'framework_issue',
attribution: 'framework_issue',
execErrors: [NO_SEED_MAPPING_REASON],
buildDurationMs,
...buildSpendFields,
execDurationMs: 0,
nodeCount: 0,
threadId: build.threadId,
buildTrace: build.buildTrace,
planRejections: build.proxyDecisionStats?.rejection ?? 0,
});
}
const runAgentScenario = async (): Promise<TargetOutput> => {
const agentExecStart = Date.now();
let agentResult;
for (let attempt = 1; ; attempt++) {
try {
agentResult = await builtOnLane.tracedExecuteAgent({
agentId: agentRef.id,
scenario,
agentContext,
buildTrace: build.buildTrace,
timeoutMs: effectiveTimeoutMs(
testCaseByFileSlug.get(inputs.testCaseFile)?.complexity,
args.timeoutMs,
),
testCaseName: inputs.testCaseFile,
seedContext,
});
break;
} catch (error: unknown) {
const errorMessage = extractErrorMessage(error);
if (shouldRetryScenarioExecution(errorMessage, attempt)) {
logger.warn(
` [${scenario.name}] agent execution attempt ${attempt}/${MAX_EXEC_ATTEMPTS} failed (${errorMessage}); retrying`,
);
await new Promise((resolve) => setTimeout(resolve, 500 * attempt));
continue;
}
logger.error(` ERROR [${scenario.name}]: ${errorMessage}`);
return await attachExpectations({
buildSuccess: true,
agentId: agentRef.id,
agentContext,
...agentArtifactFields,
passed: false,
score: 0,
reasoning: `Agent scenario execution error: ${errorMessage}`,
failureCategory: 'framework_issue',
attribution: 'framework_issue',
execErrors: [errorMessage],
buildDurationMs,
...buildSpendFields,
execDurationMs: Date.now() - agentExecStart,
nodeCount: 0,
threadId: build.threadId,
buildTrace: build.buildTrace,
planRejections: build.proxyDecisionStats?.rejection ?? 0,
});
}
}
const agentFailureCategory = agentResult.success ? undefined : agentResult.failureCategory;
const agentRootCause = agentResult.success ? undefined : agentResult.rootCause;
const agentAttribution = agentResult.success ? undefined : agentResult.attribution;
return await attachExpectations({
buildSuccess: true,
agentId: agentRef.id,
agentContext,
...agentArtifactFields,
agentEvalResult: agentResult.agentEvalResult,
passed: agentResult.success,
score: agentResult.score,
reasoning: agentResult.reasoning,
failureCategory: agentFailureCategory,
attribution: agentAttribution,
rootCause: agentRootCause,
...(agentResult.incomplete ? { incomplete: true } : {}),
execErrors: agentResult.agentEvalResult?.errors ?? [],
buildDurationMs,
...buildSpendFields,
execDurationMs: Date.now() - agentExecStart,
nodeCount: 0,
threadId: build.threadId,
buildTrace: build.buildTrace,
planRejections: build.proxyDecisionStats?.rejection ?? 0,
});
const agentFailureCategory = agentResult.success ? undefined : agentResult.failureCategory;
const agentRootCause = agentResult.success ? undefined : agentResult.rootCause;
const agentAttribution = agentResult.success ? undefined : agentResult.attribution;
return await attachExpectations({
buildSuccess: true,
agentId: agentRef.id,
agentContext,
...agentArtifactFields,
agentEvalResult: agentResult.agentEvalResult,
passed: agentResult.success,
score: agentResult.score,
reasoning: agentResult.reasoning,
failureCategory: agentFailureCategory,
attribution: agentAttribution,
rootCause: agentRootCause,
...(agentResult.incomplete ? { incomplete: true } : {}),
execErrors: agentResult.agentEvalResult?.errors ?? [],
buildDurationMs,
...buildSpendFields,
execDurationMs: Date.now() - agentExecStart,
nodeCount: 0,
threadId: build.threadId,
buildTrace: build.buildTrace,
planRejections: build.proxyDecisionStats?.rejection ?? 0,
});
};
return scenariosRequireSerialSeeding(authoredScenarios)
? await withSerialSeeding(cacheKey, runAgentScenario)
: await runAgentScenario();
}
if (!build.workflowId) {
// agentRunnable without an agentRef is unreachable; this narrows for TS
@ -428,18 +453,8 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline {
// Captured as a const so the narrowing survives into the closure below.
const workflowId = build.workflowId;
// Mirrors the retired direct loop (TRUST-311): a seeded row resets + seeds
const seedContext: ScenarioSeedContext | undefined =
build.threadId && build.seededScenarioTableIdsByName
? { threadId: build.threadId, tableIdsByName: build.seededScenarioTableIdsByName }
: undefined;
// A scenario that declares seed tables must not run without them (MCP and
// prebuilt builds never seed data tables) — executing anyway would grade the
// workflow against empty tables and report the miss as a builder failure.
if ((scenario.seedDataTables?.length ?? 0) > 0 && !seedContext) {
const reason =
'Scenario declares seedDataTables but the build provided no seeded-table mapping ' +
'(MCP/prebuilt builds do not seed data tables) — refusing to run without the declared rows';
const reason = NO_SEED_MAPPING_REASON;
logger.error(` ERROR [${scenario.name}]: ${reason}`);
return await attachExpectations({
buildSuccess: true,

2
packages/@n8n/instance-ai/evaluations/run/eval-session.ts
View File

@ -211,6 +211,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession {
buildTrace?: BuildResult['buildTrace'];
timeoutMs: number;
testCaseName?: string;
seedContext?: ScenarioSeedContext;
}) =>
await executeAgentScenario(
lane.client,
@ -222,6 +223,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession {
execArgs.testCaseName,
execArgs.buildTrace,
args.outputDir,
execArgs.seedContext,
),
),
};

2
packages/@n8n/workflow-sdk/src/codegen/emit-instance-ai.test.ts
View File

@ -287,7 +287,9 @@ describe('emit-instance-ai', () => {
'findEnvelopeKey',
'findOutputParserTargets',
'isAiRootNodeType',
'literalParameter',
'parsePinDataResponse',
'readDataTableReadParameters',
'repairStructuredOutput',
'workflowToMermaid',
// Display-options matching

2
packages/@n8n/workflow-sdk/src/mock-data/context.ts
View File

@ -1,6 +1,7 @@
import { DATA_TABLE_SYSTEM_COLUMNS } from 'n8n-workflow';
import { findEnvelopeKey } from './ai-root-shapes';
import { readDataTableReadParameters } from './data-table-read';
import type {
DataTableColumnInfo,
DeclaredFieldContract,
@ -58,6 +59,7 @@ export function buildSchemaContexts(
schema,
outputParser,
dataTableColumns: columns,
dataTableRead: readDataTableReadParameters(node),
declaredFields: buildDeclaredFieldContract(node.type, schema, outputParser, columns),
};
});

87
packages/@n8n/workflow-sdk/src/mock-data/data-table-read.ts Normal file
View File

@ -0,0 +1,87 @@
import { z } from 'zod';
// Mirrors the Data Table node's `get` parameters; absent keys mean the node defaults.
const ROWS_LIMIT_DEFAULT = 50;
const conditionSchema = z.object({
keyName: z.string().min(1),
condition: z.string().default('eq'),
keyValue: z.unknown().optional(),
});
// An expression-filled field reads as its widest setting, so the rest of the filter stays usable.
const readParametersSchema = z.object({
resource: z.string().default('row'),
operation: z.string().default('insert'),
matchType: z.enum(['anyCondition', 'allConditions']).catch('anyCondition'),
filters: z
.object({ conditions: z.array(conditionSchema).default([]) })
.default({ conditions: [] }),
returnAll: z.boolean().default(false).catch(true),
limit: z.unknown(),
orderBy: z.boolean().default(false).catch(false),
orderByColumn: z.string().default('createdAt'),
orderByDirection: z.enum(['ASC', 'DESC']).default('DESC').catch('DESC'),
});
export type DataTableReadCondition = z.infer<typeof conditionSchema>;
export interface DataTableReadParameters {
matchType: 'anyCondition' | 'allConditions';
conditions: DataTableReadCondition[];
returnAll: boolean;
/** Absent when the node's limit is an expression. */
limit?: number;
sortBy?: [column: string, direction: 'ASC' | 'DESC'];
}
/** A `=` prefix puts a parameter in expression mode; without `{{ }}` the rest is a literal. */
export function literalParameter(value: unknown): { literal: unknown; isExpression: boolean } {
if (typeof value !== 'string' || !value.startsWith('=')) {
return { literal: value, isExpression: false };
}
return value.includes('{{')
? { literal: undefined, isExpression: true }
: { literal: value.slice(1), isExpression: false };
}
export function readDataTableReadParameters(node: {
type: string;
parameters?: unknown;
}): DataTableReadParameters | undefined {
if (node.type !== 'n8n-nodes-base.dataTable') return undefined;
const parsed = readParametersSchema.safeParse(node.parameters ?? {});
if (!parsed.success || parsed.data.resource !== 'row' || parsed.data.operation !== 'get') {
return undefined;
}
const { matchType, filters, returnAll, limit, orderBy, orderByColumn, orderByDirection } =
parsed.data;
const read: DataTableReadParameters = { matchType, conditions: filters.conditions, returnAll };
if (limit === undefined) read.limit = ROWS_LIMIT_DEFAULT;
else if (typeof limit === 'number' && Number.isInteger(limit) && limit > 0) read.limit = limit;
const column = literalParameter(orderByColumn);
if (orderBy && typeof column.literal === 'string' && column.literal !== '') {
read.sortBy = [column.literal, orderByDirection];
}
return read;
}
export function describeDataTableRead(read: DataTableReadParameters): string {
const joiner = read.matchType === 'allConditions' ? ' AND ' : ' OR ';
const filter =
read.conditions.length === 0
? 'no filter (every row)'
: read.conditions
.map(
(c) =>
`\`${c.keyName}\` ${c.condition}${c.keyValue === undefined ? '' : ` ${JSON.stringify(c.keyValue)}`}`,
)
.join(joiner);
const order = read.sortBy ? `; ordered by \`${read.sortBy[0]}\` ${read.sortBy[1]}` : '';
const cap = read.returnAll
? ''
: read.limit === undefined
? '; the row limit is an expression'
: `; at most ${String(read.limit)} row(s)`;
return `${filter}${order}${cap}`;
}

6
packages/@n8n/workflow-sdk/src/mock-data/index.ts
View File

@ -23,6 +23,12 @@ export {
findOutputParserTargets,
collectDownstreamConsumers,
} from './context';
export {
literalParameter,
readDataTableReadParameters,
type DataTableReadCondition,
type DataTableReadParameters,
} from './data-table-read';
export { workflowToMermaid } from './mermaid';
export { buildDateAnchors } from './date-anchors';
export {

109
packages/@n8n/workflow-sdk/src/mock-data/mock-data.test.ts
View File

@ -1,5 +1,6 @@
import { buildSchemaContexts, findOutputParserTargets } from './context';
import { buildDateAnchors } from './date-anchors';
import { describeDataTableRead, readDataTableReadParameters } from './data-table-read';
import { workflowToMermaid } from './mermaid';
import { parsePinDataResponse, repairStructuredOutput } from './parse';
import { buildNodeSchemaSection, buildPinDataUserPrompt } from './prompt';
@ -111,6 +112,114 @@ describe('buildSchemaContexts', () => {
});
});
describe('Data Table read parameters', () => {
const lookup = {
name: 'Zone Lookup',
type: 'n8n-nodes-base.dataTable',
typeVersion: 1.1,
parameters: {
resource: 'row',
operation: 'get',
matchType: 'allConditions',
filters: {
conditions: [{ keyName: 'country', condition: 'eq', keyValue: '={{ $json.country }}' }],
},
returnAll: false,
limit: 1,
},
} as unknown as NodeJSON;
it('derives the filter, match type and limit of a get node into its context', () => {
expect(buildSchemaContexts([lookup])[0].dataTableRead).toEqual({
matchType: 'allConditions',
conditions: [{ keyName: 'country', condition: 'eq', keyValue: '={{ $json.country }}' }],
returnAll: false,
limit: 1,
});
});
it('leaves other nodes and Data Table writes without read parameters', () => {
const insert = { ...lookup, parameters: { resource: 'row', operation: 'insert' } };
expect(buildSchemaContexts([insert])[0].dataTableRead).toBeUndefined();
expect(buildSchemaContexts([workflow.nodes[3]])[0].dataTableRead).toBeUndefined();
});
it('tells the generator to pin the rows the node returns, resolving expression values', () => {
const prompt = buildPinDataUserPrompt(
{ nodes: [lookup], connections: {} } as unknown as WorkflowJSON,
buildSchemaContexts([lookup]),
{ dateAnchors: 'anchors' },
);
expect(prompt).toContain(
'OUTPUTS only the table rows its filter selects — `country` eq "={{ $json.country }}"; at most 1 row(s)',
);
expect(prompt).toContain('resolve it from the Test Scenario');
expect(prompt).toContain('never the whole table');
});
it('reads an expression-filled match type, returnAll or limit as the widest setting', () => {
const dynamic = {
...lookup,
parameters: {
...lookup.parameters,
matchType: '={{ $json.mode }}',
returnAll: '={{ $json.all }}',
limit: '={{ $json.n }}',
},
};
expect(readDataTableReadParameters(dynamic)).toEqual({
matchType: 'anyCondition',
conditions: [{ keyName: 'country', condition: 'eq', keyValue: '={{ $json.country }}' }],
returnAll: true,
});
});
it('reads the order the node applies and leaves an expression-filled limit open', () => {
const ordered = {
...lookup,
parameters: {
...lookup.parameters,
orderBy: true,
orderByColumn: 'employees',
limit: '={{ $json.n }}',
},
};
expect(readDataTableReadParameters(ordered)).toMatchObject({ sortBy: ['employees', 'DESC'] });
expect(readDataTableReadParameters(ordered)?.limit).toBeUndefined();
expect(
describeDataTableRead({
matchType: 'anyCondition',
conditions: [],
returnAll: false,
sortBy: ['createdAt', 'DESC'],
}),
).toBe('no filter (every row); ordered by `createdAt` DESC; the row limit is an expression');
});
it('keeps the read description when the real table columns are known', () => {
const section = buildNodeSchemaSection({
nodeName: 'Zone Lookup',
nodeType: 'n8n-nodes-base.dataTable',
typeVersion: 1.1,
dataTableColumns: [{ name: 'country', type: 'string' }],
dataTableRead: {
matchType: 'allConditions',
conditions: [{ keyName: 'country', condition: 'eq', keyValue: 'FR' }],
returnAll: false,
limit: 1,
},
}).join('\n');
expect(section).toContain(
'OUTPUTS only the table rows its filter selects — `country` eq "FR"; at most 1 row(s)',
);
expect(section).toContain('REAL Data Table columns');
});
});
describe('buildPinDataUserPrompt', () => {
const anchors = buildDateAnchors(new Date('2026-01-15T12:00:00Z'));

9
packages/@n8n/workflow-sdk/src/mock-data/prompt.ts
View File

@ -1,3 +1,4 @@
import { describeDataTableRead } from './data-table-read';
import { describeAiRootShape, isAiRootNodeType } from './ai-root-shapes';
import { collectDownstreamConsumers, resolveEnvelopeKey } from './context';
import { workflowToMermaid } from './mermaid';
@ -61,6 +62,14 @@ export function buildNodeSchemaSection(ctx: NodeSchemaContext): string[] {
}
}
if (ctx.dataTableRead) {
lines.push(
`- This node OUTPUTS only the table rows its filter selects — ${describeDataTableRead(ctx.dataTableRead)}. ` +
'A value written as `={{ … }}` is an expression over the item this node receives: resolve it from the Test Scenario and the data description (for example the trigger payload) and keep only the rows that match. ' +
'Pin the rows the node RETURNS, never the whole table; an empty array is the right answer when nothing matches.',
);
}
// Real table columns are authoritative and supersede the static `__schema__`
// (which only knows the system columns).
if (ctx.dataTableColumns && ctx.dataTableColumns.length > 0) {

4
packages/@n8n/workflow-sdk/src/mock-data/types.ts
View File

@ -1,3 +1,5 @@
import type { DataTableReadParameters } from './data-table-read';
/**
* Shared types for LLM mock/pin-data generation.
*
@ -65,6 +67,8 @@ export interface NodeSchemaContext {
outputParser?: OutputParserContext;
/** Real Data Table columns for dataTable reads — rendered in the prompt as the authoritative row shape. */
dataTableColumns?: DataTableColumnInfo[];
/** The filter a Data Table `get` node applies; the pinned rows are its output, not the table. */
dataTableRead?: DataTableReadParameters;
declaredFields?: DeclaredFieldContract;
}

204
packages/cli/src/modules/instance-ai/eval/__tests__/data-table-pin-filter.test.ts Normal file
View File

@ -0,0 +1,204 @@
import type { IDataObject, INode, INodeExecutionData } from 'n8n-workflow';
import { applyDataTableReadParameters } from '../data-table-pin-filter';
function readNode(parameters: Record<string, unknown>): INode {
return {
id: 'node-1',
name: 'Read Rows',
type: 'n8n-nodes-base.dataTable',
typeVersion: 1.1,
position: [0, 0],
parameters: { resource: 'row', operation: 'get', ...parameters },
};
}
function rows(...values: IDataObject[]): INodeExecutionData[] {
return values.map((json) => ({ json }));
}
const filter = (conditions: Array<Record<string, unknown>>, matchType = 'anyCondition') => ({
matchType,
filters: { conditions },
});
const seeded = rows(
{ country: 'FR', zone: 'eu', employees: 40 },
{ country: 'DE', zone: 'eu', employees: 12 },
{ country: 'US', zone: 'row', employees: 300 },
);
describe('applyDataTableReadParameters', () => {
it('keeps only the rows an eq condition matches', () => {
const node = readNode(filter([{ keyName: 'country', condition: 'eq', keyValue: 'US' }]));
const result = applyDataTableReadParameters(node, seeded);
expect(result.items.map((item) => item.json.country)).toEqual(['US']);
expect(result.warnings).toEqual([]);
});
it('trims the rows to the limit unless returnAll is set', () => {
expect(applyDataTableReadParameters(readNode({ limit: 1 }), seeded).items).toHaveLength(1);
expect(
applyDataTableReadParameters(readNode({ returnAll: true, limit: 1 }), seeded).items,
).toHaveLength(3);
});
it('leaves the rows untouched when the node has no conditions', () => {
const result = applyDataTableReadParameters(readNode({}), seeded);
expect(result.items).toEqual(seeded);
expect(result.warnings).toEqual([]);
});
it('keeps an empty pin empty', () => {
const node = readNode(filter([{ keyName: 'country', condition: 'eq', keyValue: 'US' }]));
expect(applyDataTableReadParameters(node, []).items).toEqual([]);
});
it('leaves the rows untouched and reports an unknown condition', () => {
const node = readNode(filter([{ keyName: 'country', condition: 'startsWith', keyValue: 'U' }]));
const result = applyDataTableReadParameters(node, seeded);
expect(result.items).toEqual(seeded);
expect(result.warnings).toEqual([
'Pinned Data Table read "Read Rows": unknown condition "startsWith" on "country"; rows were not filtered by it',
]);
expect(result.flags).toEqual([]);
});
it('leaves the rows untouched and reports a condition whose value is an expression', () => {
const node = readNode({
...filter([{ keyName: 'country', condition: 'eq', keyValue: '={{ $json.country }}' }]),
limit: 1,
});
const result = applyDataTableReadParameters(node, seeded);
expect(result.items).toEqual(seeded);
expect(result.warnings[0]).toContain('uses an expression');
expect(result.flags).toEqual(result.warnings);
});
it('does not flag an unevaluable condition when the rows fit the limit', () => {
const node = readNode({
...filter([{ keyName: 'country', condition: 'eq', keyValue: '={{ $json.country }}' }]),
limit: 1,
});
const result = applyDataTableReadParameters(node, seeded.slice(0, 1));
expect(result.items).toEqual(seeded.slice(0, 1));
expect(result.warnings).toHaveLength(1);
expect(result.flags).toEqual([]);
});
it('distinguishes anyCondition from allConditions', () => {
const conditions = [
{ keyName: 'zone', condition: 'eq', keyValue: 'eu' },
{ keyName: 'employees', condition: 'gt', keyValue: 20 },
];
const anyMatch = applyDataTableReadParameters(
readNode(filter(conditions, 'anyCondition')),
seeded,
);
const allMatch = applyDataTableReadParameters(
readNode(filter(conditions, 'allConditions')),
seeded,
);
expect(anyMatch.items.map((item) => item.json.country)).toEqual(['FR', 'DE', 'US']);
expect(allMatch.items.map((item) => item.json.country)).toEqual(['FR']);
});
it('applies the evaluable conditions of an allConditions filter and skips the limit when one is an expression', () => {
const node = readNode({
...filter(
[
{ keyName: 'zone', condition: 'eq', keyValue: 'eu' },
{ keyName: 'country', condition: 'eq', keyValue: '={{ $json.country }}' },
],
'allConditions',
),
limit: 1,
});
const result = applyDataTableReadParameters(node, seeded);
expect(result.items.map((item) => item.json.country)).toEqual(['FR', 'DE']);
});
it('matches like, ilike, empty and numeric comparisons the way the node does', () => {
const table = rows(
{ name: 'Harbor Cafe', status: '', amount: '120.50' },
{ name: 'harbor books', status: 'open', amount: 80 },
{ name: 'Cedar', status: null, amount: 200 },
);
const names = (conditions: Array<Record<string, unknown>>) =>
applyDataTableReadParameters(readNode(filter(conditions)), table).items.map(
(item) => item.json.name,
);
expect(names([{ keyName: 'name', condition: 'like', keyValue: 'Harbor' }])).toEqual([
'Harbor Cafe',
]);
expect(names([{ keyName: 'name', condition: 'ilike', keyValue: 'harbor' }])).toEqual([
'Harbor Cafe',
'harbor books',
]);
expect(names([{ keyName: 'status', condition: 'isEmpty' }])).toEqual(['Harbor Cafe', 'Cedar']);
expect(names([{ keyName: 'status', condition: 'like', keyValue: '%' }])).toEqual([
'Harbor Cafe',
'harbor books',
]);
expect(names([{ keyName: 'amount', condition: 'gte', keyValue: '120.5' }])).toEqual([
'Harbor Cafe',
'Cedar',
]);
});
it('resolves a literal written in expression mode and skips a column name that is an expression', () => {
const literal = applyDataTableReadParameters(
readNode(filter([{ keyName: '=country', condition: 'eq', keyValue: '=US' }])),
seeded,
);
expect(literal.items.map((item) => item.json.country)).toEqual(['US']);
expect(literal.warnings).toEqual([]);
const dynamicColumn = applyDataTableReadParameters(
readNode(filter([{ keyName: '={{ $json.column }}', condition: 'eq', keyValue: 'US' }])),
seeded,
);
expect(dynamicColumn.items).toEqual(seeded);
expect(dynamicColumn.warnings[0]).toContain('uses an expression');
});
it('orders the rows the way the node does before applying the limit', () => {
const ascending = applyDataTableReadParameters(
readNode({ orderBy: true, orderByColumn: 'employees', orderByDirection: 'ASC', limit: 2 }),
seeded,
);
expect(ascending.items.map((item) => item.json.country)).toEqual(['DE', 'FR']);
const descending = applyDataTableReadParameters(
readNode({ orderBy: true, orderByColumn: 'employees' }),
seeded,
);
expect(descending.items.map((item) => item.json.country)).toEqual(['US', 'FR', 'DE']);
});
it('keeps every row without a flag when the limit or returnAll is an expression', () => {
const openLimit = applyDataTableReadParameters(readNode({ limit: '={{ $json.n }}' }), seeded);
expect(openLimit.items).toEqual(seeded);
expect(openLimit.flags).toEqual([]);
expect(
applyDataTableReadParameters(readNode({ returnAll: '={{ $json.all }}', limit: 1 }), seeded)
.items,
).toEqual(seeded);
});
});

186
packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts
View File

@ -5,13 +5,14 @@ import type { User } from '@n8n/db';
import { mock } from 'vitest-mock-extended';
import type { BinaryDataService } from 'n8n-core';
import type {
IConnections,
INode,
IRunExecutionData,
IRun,
IWorkflowBase,
INodeTypeDescription,
} from 'n8n-workflow';
import { TimeoutExecutionCancelledError, UserError } from 'n8n-workflow';
import { NodeConnectionTypes, TimeoutExecutionCancelledError, UserError } from 'n8n-workflow';
import type { ActiveExecutions } from '@/active-executions';
import type { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
@ -71,7 +72,8 @@ const mockRestoreNoProxy = vi.fn();
vi.mock('@n8n/backend-network/proxy', () => ({
ensureHostsBypassProxy: vi.fn(() => mockRestoreNoProxy),
}));
vi.mock('@n8n/workflow-sdk', () => ({
vi.mock('@n8n/workflow-sdk', async (importOriginal) => ({
...(await importOriginal<typeof import('@n8n/workflow-sdk')>()),
normalizePinData: vi.fn((pd: unknown) => pd),
}));
@ -81,11 +83,16 @@ vi.mock('n8n-workflow', async () => {
const actual = await vi.importActual<typeof import('n8n-workflow')>('n8n-workflow');
class MockWorkflow {
nodes: Record<string, unknown>;
connections: IConnections;
getStartNode = mockGetStartNode;
constructor(options: { nodes?: Array<{ name: string }> }) {
constructor(options: { nodes?: Array<{ name: string }>; connections?: IConnections }) {
// Key the entity's node objects by name (same references, so the SUT's
// in-place parameter patching propagates to the executed workflow).
this.nodes = Object.fromEntries((options?.nodes ?? []).map((node) => [node.name, node]));
this.connections = options?.connections ?? {};
}
getChildNodes(nodeName: string) {
return actual.getChildNodes(this.connections, nodeName);
}
}
return {
@ -423,6 +430,179 @@ describe('EvalExecutionService', () => {
expect(runArg.pinData?.Webhook).toEqual([]);
});
describe('trigger content', () => {
function triggerCapableStart() {
nodeTypes.getByNameAndVersion.mockReturnValue({
description: { properties: [] } as unknown as INodeTypeDescription,
webhook: vi.fn(),
} as never);
}
it('returns a framework error and does not run when Phase 1 leaves a trigger start node without content', async () => {
triggerCapableStart();
const empty = makeEmptyHints();
empty.triggerContent = {};
empty.warnings = ['Phase 1 attempt 2/2: empty triggerContent'];
generateMockHintsMock.mockResolvedValue(empty);
const result = await service.executeWithLlmMock('wf-1', makeUser());
expect(result.success).toBe(false);
expect(result.errors[0]).toMatch(
/^FRAMEWORK ISSUE: .*no trigger content for start node "Webhook" \(Phase 1 attempt 2\/2: empty triggerContent\)/,
);
expect(workflowRunner.run).not.toHaveBeenCalled();
});
it('keeps the zero-item pin when the scenario says the trigger emits nothing', async () => {
triggerCapableStart();
const hints = makeEmptyHints();
hints.triggerContent = {};
hints.triggerEmitsNoItems = true;
generateMockHintsMock.mockResolvedValue(hints);
await service.executeWithLlmMock('wf-1', makeUser());
const runArg = workflowRunner.run.mock.calls[0][0] as unknown as {
pinData?: Record<string, unknown[]>;
};
expect(runArg.pinData?.Webhook).toEqual([]);
});
it('runs as usual when the trigger content is present', async () => {
triggerCapableStart();
await service.executeWithLlmMock('wf-1', makeUser());
expect(workflowRunner.run).toHaveBeenCalledTimes(1);
});
});
describe('engine readiness', () => {
// The patcher leaves an invalid id-mode locator alone, so the readiness check rejects it.
const idLocatorNode = (): INode =>
({
id: 'node-2',
name: 'HTTP Request',
type: 'n8n-nodes-base.httpRequest',
typeVersion: 1,
position: [200, 0],
parameters: { project: { __rl: true, mode: 'id', value: 'IT' } },
}) as INode;
const startToLocator: IConnections = {
Webhook: { main: [[{ node: 'HTTP Request', type: NodeConnectionTypes.Main, index: 0 }]] },
};
function mockIdLocatorNodeType() {
nodeTypes.getByNameAndVersion.mockImplementation((nodeType) => {
if (nodeType !== 'n8n-nodes-base.httpRequest') {
return { description: { properties: [] } as unknown as INodeTypeDescription } as never;
}
return {
description: {
properties: [
{
displayName: 'Project',
name: 'project',
type: 'resourceLocator',
default: { mode: 'list', value: '' },
required: true,
modes: [
{
displayName: 'ID',
name: 'id',
type: 'string',
validation: [
{
type: 'regex',
properties: {
regex: '^[0-9]+$',
errorMessage: 'Not a valid Jira Project ID',
},
},
],
},
],
},
],
} as unknown as INodeTypeDescription,
} as never;
});
}
it('refuses to start when a node downstream of the start node still has a parameter issue', async () => {
workflowFinderService.findWorkflowForUser.mockResolvedValue(
makeWorkflowEntity({
nodes: [makeStartNode(), idLocatorNode()],
connections: startToLocator,
}) as never,
);
mockIdLocatorNodeType();
const result = await service.executeWithLlmMock('wf-1', makeUser());
expect(workflowRunner.run).not.toHaveBeenCalled();
expect(result.success).toBe(false);
expect(result.errors[0]).toMatch(
/^Execution failed: n8n refused to start the workflow: .*'HTTP Request' node has issues:\n- Not a valid Jira Project ID/,
);
expect(result.nodeResults['HTTP Request']?.configIssues).toBeDefined();
expect(result.nodeResults.Webhook?.executionMode).not.toBe('pinned');
});
it('runs when the node with the parameter issue is not downstream of the start node', async () => {
workflowFinderService.findWorkflowForUser.mockResolvedValue(
makeWorkflowEntity({
nodes: [makeStartNode(), idLocatorNode()],
connections: {},
}) as never,
);
mockIdLocatorNodeType();
await service.executeWithLlmMock('wf-1', makeUser());
expect(workflowRunner.run).toHaveBeenCalledTimes(1);
});
});
it("applies a pinned Data Table read's literal conditions and limit to the generated rows", async () => {
const readNode = {
id: 'node-3',
name: 'Read Pending Tasks',
type: 'n8n-nodes-base.dataTable',
typeVersion: 1.1,
position: [400, 0],
parameters: {
operation: 'get',
matchType: 'allConditions',
filters: { conditions: [{ keyName: 'status', condition: 'eq', keyValue: 'pending' }] },
returnAll: false,
limit: 1,
},
} as INode;
workflowFinderService.findWorkflowForUser.mockResolvedValue(
makeWorkflowEntity({ nodes: [makeStartNode(), readNode] }) as never,
);
identifyNodesForPinDataMock.mockReturnValue([readNode]);
emitsDataTableRowsMock.mockReturnValue(true);
generatePinDataMock.mockResolvedValue({
'Read Pending Tasks': [
{ json: { task: 'T-1', status: 'done' } },
{ json: { task: 'T-2', status: 'pending' } },
{ json: { task: 'T-3', status: 'pending' } },
],
});
await service.executeWithLlmMock('wf-1', makeUser());
const runArg = workflowRunner.run.mock.calls[0][0] as unknown as {
pinData?: Record<string, unknown[]>;
};
expect(runArg.pinData?.['Read Pending Tasks']).toEqual([
{ json: { task: 'T-2', status: 'pending' } },
]);
});
it('routes through WorkflowRunner with evaluation mode + pin data + user', async () => {
const hints = makeEmptyHints();
hints.triggerContent = { body: { email: 'jane@example.com' } };

35
packages/cli/src/modules/instance-ai/eval/__tests__/workflow-analysis.test.ts
View File

@ -16,6 +16,7 @@ import {
detectBinaryDependencies,
emitsDataTableRows,
generateMockHints,
TRIGGER_CONTENT_CORRECTION,
identifyNodesForHints,
identifyNodesForPinData,
isDataTableRead,
@ -1135,6 +1136,40 @@ describe('generateMockHints', () => {
]);
});
it('names the empty trigger content in the retry prompt', async () => {
const generate = mockAgentResponses(
JSON.stringify({ globalContext: '', triggerContent: {}, nodeHints: { Slack: 'foo' } }),
JSON.stringify({
globalContext: '',
triggerContent: { timestamp: '2024-01-01T00:00:00Z' },
nodeHints: { Slack: 'foo' },
}),
);
await generateMockHints({ workflow, nodeNames: ['Schedule', 'Slack'] });
expect(generate.mock.calls[0][0]).not.toContain('## Correction required');
expect(generate.mock.calls[1][0]).toContain('## Correction required');
expect(generate.mock.calls[1][0]).toContain(TRIGGER_CONTENT_CORRECTION);
});
it('names the failure reason in the retry prompt when the first attempt threw', async () => {
const generate = mockAgentResponses(
new Error('Unexpected end of JSON input'),
JSON.stringify({
globalContext: '',
triggerContent: { timestamp: '2024-01-01T00:00:00Z' },
nodeHints: { Slack: 'foo' },
}),
);
await generateMockHints({ workflow, nodeNames: ['Schedule', 'Slack'] });
expect(generate.mock.calls[1][0]).toContain(
'The previous answer was unusable: Unexpected end of JSON input',
);
});
it('should return emptyResult with both warnings when every attempt fails', async () => {
const generate = mockAgentResponses(
JSON.stringify({ globalContext: '', triggerContent: {}, nodeHints: { Slack: 'foo' } }),

200
packages/cli/src/modules/instance-ai/eval/data-table-pin-filter.ts Normal file
View File

@ -0,0 +1,200 @@
import {
literalParameter,
readDataTableReadParameters,
type DataTableReadCondition,
type DataTableReadParameters,
} from '@n8n/workflow-sdk';
import type { INode, INodeExecutionData } from 'n8n-workflow';
type Row = INodeExecutionData['json'];
type RowPredicate = (row: Row) => boolean;
export interface PinnedReadFilterResult {
items: INodeExecutionData[];
/** Conditions the harness could not apply, for the server log. */
warnings: string[];
/** The warnings the judge sees: only when the pin holds more rows than the node's limit. */
flags: string[];
}
// A condition the harness cannot evaluate counts as a match and lifts the limit, so no row the real node might return is dropped.
export function applyDataTableReadParameters(
node: INode,
items: INodeExecutionData[],
): PinnedReadFilterResult {
const read = readDataTableReadParameters(node);
if (!read) {
return {
items,
warnings: [
`Pinned Data Table read "${node.name}": its filter parameters could not be read; rows were left as generated`,
],
flags: [],
};
}
const { matchType, conditions, returnAll, limit, sortBy } = read;
const warnings: string[] = [];
const evaluators = conditions.map((condition) =>
conditionPredicate(condition, node.name, warnings),
);
const predicates = evaluators.filter((p): p is RowPredicate => p !== undefined);
const allEvaluable = predicates.length === evaluators.length;
let kept = items;
if (predicates.length > 0) {
kept = items.filter((item) =>
matchType === 'allConditions'
? predicates.every((p) => p(item.json))
: !allEvaluable || predicates.some((p) => p(item.json)),
);
}
if (sortBy) kept = sortRows(kept, sortBy);
if (!returnAll && limit !== undefined && allEvaluable) kept = kept.slice(0, limit);
const flags =
!allEvaluable && !returnAll && limit !== undefined && kept.length > limit ? warnings : [];
return { items: kept, warnings, flags };
}
function conditionPredicate(
condition: DataTableReadCondition,
nodeName: string,
warnings: string[],
): RowPredicate | undefined {
const { condition: operator } = condition;
const key = literalParameter(condition.keyName);
const value = literalParameter(condition.keyValue);
if (key.isExpression || value.isExpression) {
warnings.push(
`Pinned Data Table read "${nodeName}": the condition on "${condition.keyName}" uses an expression that cannot be evaluated before the run; rows were not filtered by it`,
);
return undefined;
}
const keyName = typeof key.literal === 'string' ? key.literal : condition.keyName;
const keyValue = value.literal;
switch (operator) {
case 'isEmpty':
return (row) => isEmptyCell(row[keyName]);
case 'isNotEmpty':
return (row) => !isEmptyCell(row[keyName]);
case 'isTrue':
return (row) => row[keyName] === true || row[keyName] === 'true';
case 'isFalse':
return (row) => row[keyName] === false || row[keyName] === 'false';
case 'eq':
return (row) => cellEquals(row[keyName], keyValue);
case 'neq':
return (row) => !cellEquals(row[keyName], keyValue);
case 'gt':
case 'gte':
case 'lt':
case 'lte':
return (row) => compareCells(row[keyName], keyValue, operator);
case 'like':
return (row) => matchesLike(row[keyName], keyValue, true);
case 'ilike':
return (row) => matchesLike(row[keyName], keyValue, false);
default:
warnings.push(
`Pinned Data Table read "${nodeName}": unknown condition "${operator}" on "${keyName}"; rows were not filtered by it`,
);
return undefined;
}
}
function sortRows(
items: INodeExecutionData[],
[column, direction]: NonNullable<DataTableReadParameters['sortBy']>,
): INodeExecutionData[] {
const sign = direction === 'ASC' ? 1 : -1;
return [...items].sort((a, b) => {
if (compareCells(a.json[column], b.json[column], 'lt')) return -sign;
if (compareCells(a.json[column], b.json[column], 'gt')) return sign;
return 0;
});
}
function isEmptyCell(value: unknown): boolean {
return value === null || value === undefined || value === '';
}
function asText(value: unknown): string | undefined {
if (typeof value === 'string') return value;
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
return undefined;
}
function asNumber(value: unknown): number | undefined {
if (typeof value === 'number') return Number.isFinite(value) ? value : undefined;
if (typeof value === 'string' && value.trim() !== '') {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
return undefined;
}
function asDateMs(value: unknown): number | undefined {
if (value instanceof Date) return value.getTime();
if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}/.test(value)) return undefined;
const ms = new Date(value).getTime();
return Number.isNaN(ms) ? undefined : ms;
}
function cellEquals(cell: unknown, value: unknown): boolean {
const cellNull = cell === null || cell === undefined;
const valueNull = value === null || value === undefined;
if (cellNull || valueNull) return cellNull && valueNull;
if (typeof cell === 'boolean' || typeof value === 'boolean') {
return asText(cell) === asText(value);
}
const [a, b] = [asNumber(cell), asNumber(value)];
if (a !== undefined && b !== undefined) return a === b;
const [dateA, dateB] = [asDateMs(cell), asDateMs(value)];
if (dateA !== undefined && dateB !== undefined) return dateA === dateB;
const [textA, textB] = [asText(cell), asText(value)];
return textA !== undefined && textB !== undefined && textA === textB;
}
function compareCells(
cell: unknown,
value: unknown,
operator: 'gt' | 'gte' | 'lt' | 'lte',
): boolean {
if (isEmptyCell(cell) || value === null || value === undefined) return false;
const [a, b] = [asNumber(cell), asNumber(value)];
const [dateA, dateB] = [asDateMs(cell), asDateMs(value)];
const [textA, textB] = [asText(cell), asText(value)];
let left: number | string;
let right: number | string;
if (a !== undefined && b !== undefined) [left, right] = [a, b];
else if (dateA !== undefined && dateB !== undefined) [left, right] = [dateA, dateB];
else if (textA !== undefined && textB !== undefined) [left, right] = [textA, textB];
else return false;
switch (operator) {
case 'gt':
return left > right;
case 'gte':
return left >= right;
case 'lt':
return left < right;
case 'lte':
return left <= right;
}
}
// The node wraps a value without `%` in `%…%`; only `%` is a wildcard.
function matchesLike(cell: unknown, value: unknown, caseSensitive: boolean): boolean {
const text = asText(cell);
if (text === undefined || typeof value !== 'string') return false;
const pattern = value.includes('%') ? value : `%${value}%`;
const haystack = caseSensitive ? text : text.toLowerCase();
const parts = (caseSensitive ? pattern : pattern.toLowerCase()).split('%');
if (!haystack.startsWith(parts[0])) return false;
let cursor = parts[0].length;
for (const part of parts.slice(1, -1)) {
const index = haystack.indexOf(part, cursor);
if (index === -1) return false;
cursor = index + part.length;
}
const tail = parts[parts.length - 1];
return haystack.length - cursor >= tail.length && haystack.endsWith(tail);
}

74
packages/cli/src/modules/instance-ai/eval/execution.service.ts
View File

@ -17,6 +17,7 @@ import {
type EvalLlmMockHandler,
type EvalMockHttpResponse,
synthesizeBinaryFixture,
WorkflowHasIssuesError,
} from 'n8n-core';
import {
type IBinaryData,
@ -38,6 +39,7 @@ import {
TimeoutExecutionCancelledError,
UserError,
Workflow,
type IWorkflowIssues,
} from 'n8n-workflow';
import { randomUUID } from 'node:crypto';
@ -62,6 +64,7 @@ import {
isOpenAiResponsesUrl,
normalizeOpenAiResponsesMockResponse,
} from './openai-responses-envelope';
import { applyDataTableReadParameters } from './data-table-pin-filter';
import { generatePinData } from './pin-data-generator';
import {
buildVendorLlmRouting,
@ -256,6 +259,14 @@ export class EvalExecutionService {
}),
);
// A trigger pinned without content runs with no items and blames every downstream miss on the builder.
const triggerStart = this.triggerStartNode(workflowEntity, hints);
if (triggerStart && lacksTriggerContent(hints)) {
throw new Error(
`FRAMEWORK ISSUE: Phase 1 produced no trigger content for start node "${triggerStart.name}" (${hints.warnings.join('; ') || 'no details'}); the scenario cannot run without a trigger event`,
);
}
if (!hints.globalContext && nodeNames.length > 0) {
this.logger.warn(
'[EvalMock] Phase 1 hint generation returned empty — mock responses will lack cross-node consistency',
@ -279,6 +290,7 @@ export class EvalExecutionService {
bypassNodeNames,
hints.globalContext,
timings,
hints.warnings,
scenarioHints,
);
this.logger.debug(
@ -301,6 +313,7 @@ export class EvalExecutionService {
bypassNodeNames: string[],
globalContext: string,
timings: EvalTimings,
warnings: string[],
scenarioHints?: string,
): Promise<IPinData> {
if (bypassNodeNames.length === 0) return {};
@ -344,6 +357,15 @@ export class EvalExecutionService {
}
}
const bypassSet = new Set(bypassNodeNames);
for (const node of workflowEntity.nodes) {
if (!bypassSet.has(node.name) || !emitsDataTableRows(node)) continue;
const filtered = applyDataTableReadParameters(node, normalized[node.name]);
normalized[node.name] = filtered.items;
for (const warning of filtered.warnings) this.logger.warn(`[EvalMock] ${warning}`);
warnings.push(...filtered.flags);
}
return normalized;
} catch (error) {
const errorMsg = error instanceof Error ? error.message : String(error);
@ -503,6 +525,20 @@ export class EvalExecutionService {
// Genuinely unresolved misconfigurations are still recorded and still fail.
this.patchParameterIssuesForEval(workflow, pinDataNodeNames);
this.checkNodeConfig(workflow, nodeResults, pinDataNodeNames);
// The engine drops a refused execution before the runner can await it, so report the refusal first.
const blockingIssues = this.issuesBlockingRun(workflow, startNode, pinDataNodeNames);
if (blockingIssues) {
const reason = new WorkflowHasIssuesError(blockingIssues, workflow.nodes).message;
this.logger.warn(`[EvalMock] Workflow cannot start: ${reason}`);
return this.buildPartialFailureResult(
randomUUID(),
new Error(`n8n refused to start the workflow: ${reason}`),
nodeResults,
hints,
undefined,
);
}
const executionData = this.buildExecutionData(startNode, pinData);
// Mark the trigger node as pinned (it gets its output from pin data, not execution).
@ -651,6 +687,40 @@ export class EvalExecutionService {
return workflow.getStartNode() ?? this.findWebhookNode(workflow);
}
private triggerStartNode(workflowEntity: IWorkflowBase, hints: MockHints): INode | undefined {
const hinted = hints.startNodeName
? workflowEntity.nodes.find((node) => node.name === hints.startNodeName)
: undefined;
return (
this.asTriggerNode(hinted) ??
this.asTriggerNode(this.findStartNode(this.buildWorkflow(workflowEntity)))
);
}
// Same rule as `WorkflowExecute.checkReadyForExecution`.
private issuesBlockingRun(
workflow: Workflow,
startNode: INode,
pinDataNodeNames: string[],
): IWorkflowIssues | null {
const issues: IWorkflowIssues = {};
for (const nodeName of [...workflow.getChildNodes(startNode.name), startNode.name]) {
const node = workflow.nodes[nodeName];
if (!node || node.disabled) continue;
const nodeType = this.nodeTypes.getByNameAndVersion(node.type, node.typeVersion);
const nodeIssues = nodeType
? NodeHelpers.getNodeParametersIssues(
nodeType.description.properties,
node,
nodeType.description,
pinDataNodeNames,
)
: { typeUnknown: true };
if (nodeIssues) issues[node.name] = nodeIssues;
}
return Object.keys(issues).length > 0 ? issues : null;
}
/** Accept a Phase-1 start-node hint only when it names a real, enabled trigger-capable node. */
private asTriggerNode(node: INode | undefined): INode | undefined {
if (!node || node.disabled) return undefined;
@ -1160,6 +1230,10 @@ export class EvalExecutionService {
}
}
function lacksTriggerContent(hints: MockHints): boolean {
return !hints.triggerEmitsNoItems && Object.keys(hints.triggerContent).length === 0;
}
/** `warnings` is the channel the verification artifact renders as FRAMEWORK ISSUE flags. */
function withInterceptionGaps(
hints: MockHints,

15
packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts
View File

@ -592,6 +592,9 @@ export interface GenerateMockHintsOptions {
scenarioHints?: string;
}
export const TRIGGER_CONTENT_CORRECTION =
'The previous answer left "triggerContent" empty. The Test Scenario describes the event that fires the workflow\'s trigger or start node, so "triggerContent" must carry that event as the node\'s output object and must not be {}. Set "triggerEmitsNoItems": true only when the scenario says the trigger has nothing to emit.';
const SYSTEM_PROMPT = `You are a test data planner for n8n workflow automation. Your job is to create a consistent data context, trigger output data, and per-node hints that will guide an API mock server to generate realistic, coherent responses across all nodes in a workflow.
RULES:
@ -703,11 +706,15 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
if (nodeNames.length === 0) return emptyResult;
const userPrompt = buildUserPrompt(workflow, nodeNames, scenarioHints);
const basePrompt = buildUserPrompt(workflow, nodeNames, scenarioHints);
const warnings: string[] = [];
let lastReason = '';
for (let attempt = 1; attempt <= MAX_HINT_ATTEMPTS; attempt++) {
let reason = '';
const userPrompt = lastReason
? `${basePrompt}\n\n## Correction required\n\n${correctionFor(lastReason)}`
: basePrompt;
try {
const agent = createEvalAgent('eval-hint-generator', {
instructions: SYSTEM_PROMPT,
@ -776,6 +783,7 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
}
warnings.push(`Phase 1 attempt ${attempt}/${MAX_HINT_ATTEMPTS}: ${reason}`);
lastReason = reason;
if (attempt < MAX_HINT_ATTEMPTS) {
Container.get(Logger).warn(
`[EvalMock] Phase 1 attempt ${attempt}/${MAX_HINT_ATTEMPTS} unusable (${reason}) — retrying`,
@ -788,3 +796,8 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
);
return { ...emptyResult, warnings };
}
function correctionFor(reason: string): string {
if (reason === 'empty triggerContent') return TRIGGER_CONTENT_CORRECTION;
return `The previous answer was unusable: ${reason}. Return only the JSON object described under "Expected Output".`;
}

103
packages/core/src/execution-engine/__tests__/eval-mock-helpers.test.ts
View File

@ -340,7 +340,17 @@ describe('eval-mock-helpers', () => {
expect(result.headers).toEqual({ 'x-custom': 'value' });
expect(result.statusCode).toBe(201);
expect(result.statusMessage).toBe('OK');
expect(result.statusMessage).toBe('Created');
});
it('should carry the standard reason phrase of the status code', () => {
const result = serializeMockToHttpResponse({
body: { error: 'not found' },
headers: {},
statusCode: 404,
});
expect(result.statusMessage).toBe('Not Found');
});
it('should pass through a Buffer body without double-encoding', () => {
@ -494,6 +504,30 @@ describe('eval-mock-helpers', () => {
// normalizeLegacyRequest
// -----------------------------------------------------------------------
describe('normalizeLegacyRequest', () => {
it('maps the legacy `simple: false` opt-out to ignoreHttpStatusErrors', () => {
const fromString = normalizeLegacyRequest('https://api.example.com/data', {
method: 'GET',
simple: false,
});
const fromObject = normalizeLegacyRequest({
uri: 'https://api.example.com/data',
method: 'GET',
simple: false,
resolveWithFullResponse: true,
});
expect(fromString.ignoreHttpStatusErrors).toBe(true);
expect(fromObject.ignoreHttpStatusErrors).toBe(true);
});
it('leaves ignoreHttpStatusErrors unset when the legacy request did not opt out', () => {
const defaulted = normalizeLegacyRequest({ uri: 'https://api.example.com', method: 'GET' });
const explicit = normalizeLegacyRequest('https://api.example.com', { simple: true });
expect('ignoreHttpStatusErrors' in defaulted).toBe(false);
expect('ignoreHttpStatusErrors' in explicit).toBe(false);
});
it('should convert string URI + options object into IHttpRequestOptions', () => {
const options: IRequestOptions = {
method: 'POST',
@ -625,6 +659,73 @@ describe('eval-mock-helpers', () => {
expect(result).toEqual({ data: 'mocked' });
});
const notFoundResponse: EvalMockHttpResponse = {
body: { error: 'not found' },
headers: { 'content-type': 'application/json' },
statusCode: 404,
};
it('hands a 404 back as a full response when a legacy request set simple:false', async () => {
const handler: EvalLlmMockHandler = vi.fn().mockResolvedValue(notFoundResponse);
const result = await callEvalMockHandler(
handler,
normalizeLegacyRequest({
uri: requestOptions.url,
method: 'GET',
simple: false,
resolveWithFullResponse: true,
}),
node,
true,
'legacy',
);
expect(result).toMatchObject({
statusCode: 404,
statusMessage: 'Not Found',
body: { error: 'not found' },
});
});
it('hands a 404 body back when ignoreHttpStatusErrors is set without a full response', async () => {
const handler: EvalLlmMockHandler = vi.fn().mockResolvedValue(notFoundResponse);
const result = await callEvalMockHandler(
handler,
{ ...requestOptions, ignoreHttpStatusErrors: true },
node,
);
expect(result).toEqual({ error: 'not found' });
});
it('honours the except list of an ignoreHttpStatusErrors config', async () => {
const options: IHttpRequestOptions = {
...requestOptions,
ignoreHttpStatusErrors: { ignore: true, except: [404] },
};
const notFound: EvalLlmMockHandler = vi.fn().mockResolvedValue(notFoundResponse);
const serverError: EvalLlmMockHandler = vi
.fn()
.mockResolvedValue({ ...notFoundResponse, statusCode: 500 });
await expect(callEvalMockHandler(notFound, options, node)).rejects.toThrow(
'Request failed with status code 404',
);
await expect(callEvalMockHandler(serverError, options, node, true)).resolves.toMatchObject({
statusCode: 500,
});
});
it('still throws for status >= 400 when ignoreHttpStatusErrors is false', async () => {
const handler: EvalLlmMockHandler = vi.fn().mockResolvedValue(notFoundResponse);
await expect(
callEvalMockHandler(handler, { ...requestOptions, ignoreHttpStatusErrors: false }, node),
).rejects.toThrow('Request failed with status code 404');
});
it('should return serialized response when returnFullResponse is true', async () => {
const handler: EvalLlmMockHandler = vi.fn().mockResolvedValue(successResponse);

25
packages/core/src/execution-engine/eval-mock-helpers.ts
View File

@ -9,6 +9,7 @@
import type { IHttpRequestOptions, INode, INodeProperties, IRequestOptions } from 'n8n-workflow';
import { generateKeyPairSync } from 'node:crypto';
import { STATUS_CODES } from 'node:http';
import { Readable } from 'node:stream';
import type { EvalLlmMockHandler, EvalMockHttpResponse } from './index';
@ -122,7 +123,7 @@ export function serializeMockToHttpResponse(
const common = {
headers: mock.headers,
statusCode: mock.statusCode,
statusMessage: 'OK',
statusMessage: STATUS_CODES[mock.statusCode] ?? 'OK',
};
const bytes = () =>
@ -164,7 +165,7 @@ export function serializeMockToHttpResponse(
* slots: `body`, `formData` (multipart), and `form` (URL-encoded) — fold them
* all into `body` so the mock layer sees the payload regardless of transport
* encoding (the binary redactor reduces multipart to part metadata before the
* LLM).
* LLM). `simple: false` becomes `ignoreHttpStatusErrors`.
*/
export function normalizeLegacyRequest(
uriOrObject: string | IRequestOptions,
@ -177,6 +178,7 @@ export function normalizeLegacyRequest(
headers: options?.headers,
body: (options?.body ?? options?.formData ?? options?.form) as IHttpRequestOptions['body'],
qs: options?.qs,
...(options?.simple === false ? { ignoreHttpStatusErrors: true } : {}),
};
}
return {
@ -187,6 +189,7 @@ export function normalizeLegacyRequest(
uriOrObject.formData ??
uriOrObject.form) as IHttpRequestOptions['body'],
qs: uriOrObject.qs,
...(uriOrObject.simple === false ? { ignoreHttpStatusErrors: true } : {}),
};
}
@ -195,7 +198,8 @@ export function normalizeLegacyRequest(
* When `returnFullResponse` is true, serializes to `{ body: Buffer, headers, statusCode }`
* matching the shape that nodes expect from real HTTP responses.
* For error responses (status >= 400), throws an error matching the HTTP library's
* error shape so nodes handle it identically to real HTTP failures.
* error shape so nodes handle it identically to real HTTP failures, unless the
* request set `ignoreHttpStatusErrors`.
* Returns `undefined` if the handler did not produce a response.
*/
export async function callEvalMockHandler(
@ -208,13 +212,26 @@ export async function callEvalMockHandler(
const response = await handler(requestOptions, node);
if (!response) return undefined;
if (response.statusCode >= 400) {
if (
response.statusCode >= 400 &&
!ignoresStatusError(requestOptions.ignoreHttpStatusErrors, response.statusCode)
) {
throwHttpError(response, httpLibrary);
}
return returnFullResponse ? serializeMockToHttpResponse(response, requestOptions) : response.body;
}
// Same rule as @n8n/backend-network: `true` ignores every status, the config form all but `except`.
function ignoresStatusError(
option: IHttpRequestOptions['ignoreHttpStatusErrors'],
statusCode: number,
): boolean {
if (option === true) return true;
if (typeof option === 'object' && option.ignore) return !option.except.includes(statusCode);
return false;
}
/**
* Throw an error matching what the real HTTP library would throw,
* so node error handling (retries, continueOnFail, NodeApiError) works identically.

62
packages/core/src/execution-engine/node-execution-context/utils/request-helpers/__tests__/factory-eval-mock.test.ts Normal file
View File

@ -0,0 +1,62 @@
import type { INode, IWorkflowExecuteAdditionalData, Workflow } from 'n8n-workflow';
import { mock, mockDeep } from 'vitest-mock-extended';
import type { EvalLlmMockHandler } from '@/execution-engine';
import { getRequestHelperFunctions } from '../factory';
describe('request helpers under the eval mock', () => {
const workflow = mock<Workflow>();
const node = mock<INode>({ name: 'HTTP Request', type: 'n8n-nodes-base.httpRequest' });
const notFound = {
statusCode: 404,
headers: { 'content-type': 'application/json' },
body: { error: 'not found' },
};
const url = 'https://api.example.com/items';
function helpersUnderMock(handler: EvalLlmMockHandler) {
const additionalData = mockDeep<IWorkflowExecuteAdditionalData>();
additionalData.evalLlmMockHandler = handler;
additionalData.otel = undefined;
return getRequestHelperFunctions(workflow, node, additionalData);
}
test('legacy request with simple:false resolves with the mocked 404 as a full response', async () => {
const helpers = helpersUnderMock(vi.fn().mockResolvedValue(notFound));
await expect(
helpers.request({ uri: url, method: 'GET', simple: false, resolveWithFullResponse: true }),
).resolves.toMatchObject({ statusCode: 404, body: { error: 'not found' } });
});
test('legacy request without simple:false rejects with the legacy 404 error shape', async () => {
const helpers = helpersUnderMock(vi.fn().mockResolvedValue(notFound));
await expect(
helpers.request({ uri: url, method: 'GET', resolveWithFullResponse: true }),
).rejects.toMatchObject({ statusCode: 404, response: { body: { error: 'not found' } } });
});
test('httpRequest with ignoreHttpStatusErrors resolves with the mocked 404 as a full response', async () => {
const helpers = helpersUnderMock(vi.fn().mockResolvedValue(notFound));
await expect(
helpers.httpRequest({
url,
method: 'GET',
ignoreHttpStatusErrors: true,
returnFullResponse: true,
}),
).resolves.toMatchObject({ statusCode: 404, body: { error: 'not found' } });
});
test('httpRequest without ignoreHttpStatusErrors rejects with the axios 404 error shape', async () => {
const helpers = helpersUnderMock(vi.fn().mockResolvedValue(notFound));
await expect(helpers.httpRequest({ url, method: 'GET' })).rejects.toMatchObject({
isAxiosError: true,
response: { status: 404 },
});
});
});