feat(core): Run seeded workflows before the graded turn in Instance AI evals (no-changelog) (#37516)

packages/@n8n/instance-ai/evaluations/README.md

@@ -783,6 +783,37 @@ between the two repos. They read asymmetrically — `inline` names where the see
 lives, `replay` names what the harness does with it — which is a cost we took
 knowingly: renaming `replay` would break a lang-tracer API contract.
 
+#### `seed.priorRuns` — execution history the agent must look up
+
+An `inline` seed can also **run** its workflows before the graded turn. Each run creates a
+real execution record, so the case can ask about "the last run" and the honest answer
+requires the agent to read it.
+
+```jsonc
+"seed": {
+  "mode": "inline",
+  "workflows": [{ "id": "dS8xQ2mV6bTn4Kp1", "name": "Daily Sync", "nodes": [], "connections": {} }],
+  "priorRuns": [
+    { "workflow": "dS8xQ2mV6bTn4Kp1", "hints": "the HTTP Request node returns 500" }
+  ]
+}
+```
+
+- **A failing run is the point.** `hints` steers the mock layer exactly as
+  `executionScenarios[].dataSetup` does, so a case can stage one specific failure and then
+  say only "it broke again". Grade the behaviour that follows: did the agent check the
+  record, or did it ask the user?
+- **A prior run that fails does not fail the build.** Outcomes go to the log, named by the
+  workflow the case declared.
+- **`workflow` is the seed workflow's `id`**, the same key `conversation[0].attach.workflow`
+  uses. The schema rejects an id the seed does not declare, so a typo fails at load rather
+  than mid-build.
+- **A run that never happens is not a staged failure.** If no execution record lands, the
+  case is reported as a framework issue rather than scored — it would otherwise be graded
+  against history the instance does not have.
+- Runs execute sequentially in declared order, before the live turn, on a 120s budget —
+  tighter than a scenario execution, which gets the case's build budget (900s by default).
+
 #### `mode: "replay"` — reproduce a real conversation (no repo content)
 
 The case carries only a **thread id**. At run time the harness pulls that thread's runs from LangSmith, reconstructs the message log (user/assistant text + resolved tool-call blocks, deduped across suspend/resume), and splits at the **last user message**: everything before it is restored as the seed, that last message is sent live. The seed workflow is compiled from the build/patch tool's captured SDK code **as of the seed boundary**, so it matches what the live turn first saw.

packages/@n8n/instance-ai/evaluations/__tests__/build-orchestrator.test.ts

@@ -387,6 +387,33 @@ describe('createBuildOrchestrator', () => {
 		expect(verdicts?.[0].reason).toContain('no agent output');
 	});
 
+	// The row is short-circuited in `case-pipeline`, but expectations are counted
+	// separately and only a verdict's OWN `incomplete` excludes it — the row's flag never
+	// reaches them. A priorRuns case is usually expectation-only, so without this bail-out
+	// the single graded unit still lands in the builder's baseline as a red.
+	it('does not judge expectations when prior-run staging never landed', async () => {
+		vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true }));
+		const building = vi
+			.fn()
+			.mockResolvedValue(okBuild({ priorRunFailed: 'sEeDeDwF1234567a: fetch failed' }));
+		const deps = makeDeps([makeLane(1, building)], {
+			testCaseByFileSlug: new Map([
+				['case-a', baseCase({ processExpectations: ['reads the failed execution'] })],
+			]),
+		});
+		const orchestrator = createBuildOrchestrator(deps);
+
+		await orchestrator.getOrBuild(0, 'case-a');
+
+		expect(vi.mocked(verifyBuildExpectations)).not.toHaveBeenCalled();
+		const verdicts = await deps.buildExpectationsByKey.get('0:case-a');
+		expect(verdicts).toHaveLength(1);
+		expect(verdicts?.[0].expectation).toBe('reads the failed execution');
+		// `incomplete` is the ONLY thing that keeps a verdict out of the pass rate.
+		expect(verdicts?.[0].incomplete).toBe(true);
+		expect(verdicts?.[0].reason).toContain('premise is missing');
+	});
+
 	it('serves prebuilt workflows by fetching them, never invoking the builder', async () => {
 		const tracedBuild = vi.fn().mockResolvedValue(okBuild());
 		const lane = makeLane(1, tracedBuild);

packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts

@@ -115,6 +115,82 @@ describe('buildWorkflow scenario-seed data table lifecycle', () => {
 		expect(build.createdDataTableIds).toContain('scenario-dt-1');
 	});
 
+	// A staged prior run that produced no execution record must reach the caller even
+	// when the BUILD SUCCEEDED — that is the only path where the case still gets graded,
+	// and `buildFailedOnInfra` cannot catch it because it returns false for a success.
+	it('carries priorRunFailed on a SUCCESSFUL build when a prior run never ran', async () => {
+		const client = makeClient({
+			restoreThread: vi.fn().mockResolvedValue({
+				restored: 0,
+				workflowIds: ['seeded-wf-1'],
+				dataTableIds: ['scenario-dt-1'],
+				agentIds: [],
+			}),
+			// Rejected before the runner: an error result with an id no execution exists under.
+			executeWithLlmMock: vi.fn().mockResolvedValue({
+				executionId: 'fabricated-uuid',
+				success: false,
+				nodeResults: {},
+				errors: ['No trigger or start node found in the workflow'],
+				hints: {},
+				mockedCredentials: [],
+			}),
+			getExecution: vi.fn().mockRejectedValue(new Error('404 not found')),
+			// Seeding evicts same-named leftovers before restoring.
+			listWorkflows: vi.fn().mockResolvedValue([]),
+		});
+
+		const build = await buildWorkflow({
+			client,
+			...baseConfig,
+			seed: {
+				mode: 'inline' as const,
+				messages: [],
+				workflows: [{ id: 'sEeDeDwF1234567a', name: 'Daily Sync', nodes: [], connections: {} }],
+				dataTables: [],
+				agents: [],
+				projects: [],
+				priorRuns: [{ workflow: 'sEeDeDwF1234567a' }],
+			},
+		});
+
+		expect(build.success).toBe(true);
+		expect(build.priorRunFailed).toContain('sEeDeDwF1234567a');
+	});
+
+	// A throw from staging is an authoring/harness fault. Without `seedingFailed` the
+	// outer catch returns a plain failed build and the case is scored `build_failure` /
+	// `builder_issue` — a builder red for something the builder had no part in.
+	it('flags seedingFailed when a prior run names an id the seed never created', async () => {
+		const client = makeClient({
+			restoreThread: vi.fn().mockResolvedValue({
+				restored: 0,
+				workflowIds: [],
+				dataTableIds: ['scenario-dt-1'],
+				agentIds: [],
+			}),
+			listWorkflows: vi.fn().mockResolvedValue([]),
+		});
+
+		const build = await buildWorkflow({
+			client,
+			...baseConfig,
+			seed: {
+				mode: 'inline' as const,
+				messages: [],
+				workflows: [{ id: 'sEeDeDwF1234567a', name: 'Daily Sync', nodes: [], connections: {} }],
+				dataTables: [],
+				agents: [],
+				projects: [],
+				// Not the declared id — `executePriorRuns` throws.
+				priorRuns: [{ workflow: 'nOtDeClArEd1234a' }],
+			},
+		});
+
+		expect(build.success).toBe(false);
+		expect(build.seedingFailed).toBe(true);
+	});
+
 	it('returns the built workflow, both tables, and the name→id map on success', async () => {
 		const client = makeClient();
 

packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts

@@ -166,6 +166,35 @@ describe('createCasePipeline', () => {
 		expect(orchestrator.buildCache.size).toBe(0);
 	});
 
+	// A staged prior run that produced no execution record leaves the case grading
+	// against history the instance does not have. The biting case is a build that
+	// SUCCEEDED: `buildFailedOnInfra` returns false there, so without this branch the
+	// row would be scored as an agent failure on a premise that never existed.
+	it('routes a successful build with a failed prior run to framework_issue', async () => {
+		const lane = makeLane();
+		const cached: CachedBuild = {
+			build: { ...okBuild(), priorRunFailed: 'seedWf1: fetch failed' },
+			lane,
+			buildDurationMs: 42,
+		};
+		const orchestrator = makeOrchestrator(cached);
+		const pipeline = createCasePipeline(makeDeps(orchestrator));
+
+		const output = await pipeline.runRow(rowInputs('happy-path'));
+
+		expect(output).toMatchObject({
+			buildSuccess: true,
+			passed: false,
+			// Not graded rather than failed — stays out of the builder's baseline.
+			incomplete: true,
+			failureCategory: 'framework_issue',
+			attribution: 'framework_issue',
+		});
+		expect(String(output.reasoning)).toContain('premise is missing');
+		// The scenario must never run: its result would describe absent history.
+		expect(vi.mocked(lane.tracedExecute)).not.toHaveBeenCalled();
+	});
+
 	it('keeps everything when --keep-workflows is set', async () => {
 		const lane = makeLane();
 		vi.mocked(lane.tracedExecute).mockResolvedValue({

packages/@n8n/instance-ai/evaluations/__tests__/prior-runs.test.ts

@@ -0,0 +1,414 @@
+import type { InstanceAiEvalExecutionResult } from '@n8n/api-types';
+import { mock } from 'vitest-mock-extended';
+
+import type { N8nClient } from '../clients/n8n-client';
+import { remapSeedArtifactIds, type ConversationSeed } from '../harness/conversation-seed';
+import type { EvalLogger } from '../harness/logger';
+import { executePriorRuns, STAGING_TIMEOUT_MS } from '../harness/prior-runs';
+import { EvalTestCaseSchema } from '../harness/schema';
+import { MAX_EXEC_ATTEMPTS } from '../harness/transient-error';
+
+function execResult(
+	overrides: Partial<InstanceAiEvalExecutionResult> = {},
+): InstanceAiEvalExecutionResult {
+	return {
+		executionId: 'exec-1',
+		success: true,
+		nodeResults: {},
+		errors: [],
+		hints: {},
+		mockedCredentials: [],
+		...overrides,
+	} as InstanceAiEvalExecutionResult;
+}
+
+function logger() {
+	return mock<EvalLogger>();
+}
+
+const noSleep = async () => {};
+const seeded = new Map([['dS8xQ2mV6bTn4Kp1', { id: 'wf-1', name: 'Daily Sync' }]]);
+
+describe('executePriorRuns', () => {
+	it('runs each declared workflow and reports what happened', async () => {
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(execResult());
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1', hints: 'the HTTP node returns 500' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(outcomes).toEqual([
+			{
+				workflow: 'dS8xQ2mV6bTn4Kp1',
+				workflowId: 'wf-1',
+				ran: true,
+				executionId: 'exec-1',
+				success: true,
+				errors: [],
+			},
+		]);
+		// `hints` reaches the mock layer the same way `dataSetup` does — that is how a
+		// case stages one specific failure.
+		// Staging gets its own tight budget, not the 15-minute build budget.
+		expect(client.executeWithLlmMock).toHaveBeenCalledWith(
+			'wf-1',
+			'the HTTP node returns 500',
+			STAGING_TIMEOUT_MS,
+		);
+	});
+
+	it('does NOT fail the build when a prior run fails', async () => {
+		// The staged failure is the whole point: the case says only "it broke again" and
+		// the agent has to go and find out how.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(
+			execResult({ success: false, errors: ['HTTP Request: 500'] }),
+		);
+		client.getExecution.mockResolvedValue({ id: 'exec-1' } as never);
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(outcomes[0].success).toBe(false);
+		expect(outcomes[0].errors).toEqual(['HTTP Request: 500']);
+	});
+
+	it('records the failure in the log, so an author can see the staged history', async () => {
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(
+			execResult({ success: false, errors: ['HTTP Request: 500'] }),
+		);
+		client.getExecution.mockResolvedValue({ id: 'exec-1' } as never);
+		const log = logger();
+
+		await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: log,
+			sleep: noSleep,
+		});
+
+		expect(log.info).toHaveBeenCalledWith(expect.stringContaining('Daily Sync'));
+		expect(log.info).toHaveBeenCalledWith(expect.stringContaining('failed (HTTP Request: 500)'));
+	});
+
+	it('runs sequentially, in declared order', async () => {
+		// A case can stage a sequence, and a later run may depend on state an earlier one
+		// left behind.
+		const order: string[] = [];
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockImplementation(async (workflowId: string) => {
+			order.push(workflowId);
+			return await Promise.resolve(execResult());
+		});
+
+		await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'fIrStSeEd1234567' }, { workflow: 'sEcOnDsEeD123456' }],
+			seedWorkflows: new Map([
+				['fIrStSeEd1234567', { id: 'wf-a', name: 'First' }],
+				['sEcOnDsEeD123456', { id: 'wf-b', name: 'Second' }],
+			]),
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(order).toEqual(['wf-a', 'wf-b']);
+	});
+
+	it('retries a transient DB abort rather than recording it as the staged failure', async () => {
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock
+			.mockResolvedValueOnce(
+				execResult({
+					success: false,
+					errors: ['SQLITE_CONSTRAINT: FOREIGN KEY constraint failed'],
+				}),
+			)
+			.mockResolvedValueOnce(execResult({ success: true }));
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(client.executeWithLlmMock).toHaveBeenCalledTimes(2);
+		expect(outcomes[0].success).toBe(true);
+	});
+
+	it('retries a transient network throw, so a blip does not void the premise', async () => {
+		// The graded turn would otherwise run against history that never landed.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock
+			.mockRejectedValueOnce(new Error('fetch failed'))
+			.mockResolvedValueOnce(execResult());
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(client.executeWithLlmMock).toHaveBeenCalledTimes(2);
+		expect(outcomes[0].success).toBe(true);
+	});
+
+	it('does not announce a retry it will not make on the last attempt', async () => {
+		// The in-band abort branch has no cap of its own, so without a guard the final
+		// attempt would log "retrying" and sleep before giving up.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(
+			execResult({ success: false, errors: ['SQLITE_CONSTRAINT: FOREIGN KEY constraint failed'] }),
+		);
+		const log = logger();
+		const delays: number[] = [];
+
+		await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: log,
+			sleep: async (ms) => {
+				delays.push(ms);
+				await Promise.resolve();
+			},
+		});
+
+		expect(client.executeWithLlmMock).toHaveBeenCalledTimes(MAX_EXEC_ATTEMPTS);
+		expect(delays).toHaveLength(MAX_EXEC_ATTEMPTS - 1);
+		const retryLogs = log.warn.mock.calls.filter(([line]) => String(line).includes('retrying'));
+		expect(retryLogs).toHaveLength(MAX_EXEC_ATTEMPTS - 1);
+	});
+
+	it('records a NON-retryable error immediately instead of killing the build', async () => {
+		// Throwing here would report an instance problem as a builder problem.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockRejectedValue(new Error('workflow is not runnable'));
+		const log = logger();
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: log,
+			sleep: noSleep,
+		});
+
+		expect(client.executeWithLlmMock).toHaveBeenCalledTimes(1);
+		expect(outcomes[0].success).toBe(false);
+		expect(outcomes[0].errors).toEqual(['workflow is not runnable']);
+		expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('could not complete'));
+	});
+
+	it('gives up after the retry budget and reports the real error', async () => {
+		// Not a synthetic "exhausted" string: the log has to name what went wrong.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockRejectedValue(new Error('ECONNRESET'));
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(client.executeWithLlmMock).toHaveBeenCalledTimes(MAX_EXEC_ATTEMPTS);
+		expect(outcomes[0].success).toBe(false);
+		expect(outcomes[0].errors).toEqual(['ECONNRESET']);
+	});
+
+	it('does not record a server budget stop as the staged failure', async () => {
+		// The stop comes back in-band. Recorded as-is, HARNESS text would become the
+		// workflow's failure reason in the execution record the graded agent then reads.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(
+			execResult({
+				success: false,
+				errors: ['Workflow exceeded its 120s eval budget and was stopped'],
+			}),
+		);
+
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		// Thrown, then classified as a timeout — so it never reaches the staged-failure
+		// return, and the case is left with no execution record rather than a fake one.
+		expect(outcomes[0].ran).toBe(false);
+		expect(outcomes[0].errors.join(' ')).toContain('aborted due to timeout');
+	});
+
+	it('separates "failed as staged" from "never ran"', async () => {
+		// `success: false` alone cannot tell these apart, and only the second is a reason
+		// to distrust the grade.
+		const staged = mock<N8nClient>();
+		staged.executeWithLlmMock.mockResolvedValue(
+			execResult({ success: false, errors: ['HTTP Request: 500'] }),
+		);
+		staged.getExecution.mockResolvedValue({ id: 'exec-1' } as never);
+		const [asStaged] = await executePriorRuns({
+			client: staged,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+		expect(asStaged).toMatchObject({ ran: true, executionId: 'exec-1', success: false });
+
+		const broken = mock<N8nClient>();
+		broken.executeWithLlmMock.mockRejectedValue(new Error('workflow is not runnable'));
+		const [neverRan] = await executePriorRuns({
+			client: broken,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+		expect(neverRan.ran).toBe(false);
+		expect(neverRan.executionId).toBeUndefined();
+	});
+
+	it('does not trust a fabricated executionId from a pre-run rejection', async () => {
+		// The eval service rejects some requests BEFORE it calls the workflow runner —
+		// unknown workflow, no trigger node — and returns an error result carrying a
+		// freshly minted UUID no execution was ever stored under. Trusting it would report
+		// a premise that does not exist as staged history.
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(
+			execResult({
+				success: false,
+				executionId: '3f7c1e90-0000-4000-8000-000000000000',
+				errors: ['No trigger or start node found in the workflow'],
+			}),
+		);
+		client.getExecution.mockRejectedValue(new Error('404 not found'));
+
+		const [outcome] = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(outcome.ran).toBe(false);
+	});
+
+	it('confirms the record for a genuinely staged failure', async () => {
+		const client = mock<N8nClient>();
+		client.executeWithLlmMock.mockResolvedValue(
+			execResult({ success: false, executionId: '42', errors: ['HTTP Request: 500'] }),
+		);
+		client.getExecution.mockResolvedValue({
+			id: '42',
+			workflowId: 'wf-1',
+			status: 'error',
+		} as never);
+
+		const [outcome] = await executePriorRuns({
+			client,
+			priorRuns: [{ workflow: 'dS8xQ2mV6bTn4Kp1' }],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(outcome).toMatchObject({ ran: true, executionId: '42', success: false });
+	});
+
+	it('throws when the seed never created the named workflow', async () => {
+		// The schema cross-checks names at load, so reaching here means the premise
+		// silently never happened. That is worth stopping for.
+		const client = mock<N8nClient>();
+
+		await expect(
+			executePriorRuns({
+				client,
+				priorRuns: [{ workflow: 'nEvErSeEdEd12345' }],
+				seedWorkflows: seeded,
+				logger: logger(),
+				sleep: noSleep,
+			}),
+		).rejects.toThrow('which the seed did not create');
+		expect(client.executeWithLlmMock).not.toHaveBeenCalled();
+	});
+
+	it('does nothing when a case declares none', async () => {
+		const client = mock<N8nClient>();
+		const outcomes = await executePriorRuns({
+			client,
+			priorRuns: [],
+			seedWorkflows: seeded,
+			logger: logger(),
+			sleep: noSleep,
+		});
+
+		expect(outcomes).toEqual([]);
+		expect(client.executeWithLlmMock).not.toHaveBeenCalled();
+	});
+});
+
+describe('priorRuns schema validation', () => {
+	function caseWith(priorRuns: Array<{ workflow: string; hints?: string }>) {
+		return {
+			conversation: [{ role: 'user', text: 'it broke again, fix it' }],
+			complexity: 'simple',
+			tags: [],
+			processExpectations: ['the agent reads the failed execution instead of asking'],
+			seed: {
+				mode: 'inline',
+				messages: [],
+				workflows: [{ id: 'dS8xQ2mV6bTn4Kp1', name: 'Daily Sync', nodes: [], connections: {} }],
+				priorRuns,
+			},
+		};
+	}
+
+	it('accepts a prior run naming a seed workflow id', () => {
+		const parsed = EvalTestCaseSchema.safeParse(caseWith([{ workflow: 'dS8xQ2mV6bTn4Kp1' }]));
+		expect(parsed.success).toBe(true);
+	});
+
+	it('accepts the seed the README documents, through the REAL remap', () => {
+		// Every seed goes through `remapSeedArtifactIds` at build time, and it refuses ids
+		// shorter than 8 characters. A documented example that throws there is worse than
+		// no example, and schema parsing alone would not catch it.
+		const parsed = EvalTestCaseSchema.safeParse(caseWith([{ workflow: 'dS8xQ2mV6bTn4Kp1' }]));
+		expect(parsed.success).toBe(true);
+		if (parsed.success && parsed.data.seed?.mode === 'inline') {
+			expect(() => remapSeedArtifactIds(parsed.data.seed as ConversationSeed)).not.toThrow();
+		}
+	});
+
+	it('rejects a prior run naming an id the seed does not declare', () => {
+		// Catching the typo at load beats a mid-build failure, which reads like an
+		// infrastructure fault rather than an authoring mistake.
+		const parsed = EvalTestCaseSchema.safeParse(caseWith([{ workflow: 'nOtDeClArEd12345' }]));
+		expect(parsed.success).toBe(false);
+		if (!parsed.success) {
+			expect(JSON.stringify(parsed.error.issues)).toContain('which this seed does not declare');
+			expect(JSON.stringify(parsed.error.issues)).toContain('dS8xQ2mV6bTn4Kp1');
+		}
+	});
+});

packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts

@@ -43,6 +43,7 @@ import {
 import { loadProviderFixtures } from './fixture-server';
 import { reconstructSeedFromThread } from './langsmith-seed';
 import type { EvalLogger } from './logger';
+import { executePriorRuns } from './prior-runs';
 import { redactSecretsInTextDeep } from './redact';
 import type { CaseSeed } from './schema';
 import {
@@ -262,6 +263,11 @@ export interface BuildResult {
 	/** Transport-level failure (network error, or the lane unreachable right
 	 *  after failing — e.g. timed out against a dead lane). Routed to `framework_issue`. */
 	transportFailure?: boolean;
+	/** Set when a `seed.priorRuns` staging run produced no execution record, so the
+	 *  history the case grades against does not exist. Unlike the other infra flags this
+	 *  one applies even when the BUILD SUCCEEDED, which is exactly the case that would
+	 *  otherwise be scored as an agent failure. */
+	priorRunFailed?: string;
 	/** Evidence that the MODEL PROVIDER, not the builder, failed this build (a
 	 *  5xx/429 upstream of the n8n instance). Set only after the retry budget is
 	 *  spent. Routed to `framework_issue` with `PROVIDER_OUTAGE_ROOT_CAUSE`, so an
@@ -518,6 +524,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 	let builtDataTableIds: string[] = [];
 	let seededTranscript: TranscriptTurn[] = [];
 	let seedingFailed = false;
+	let priorRunFailed: string | undefined;
 	// Seed-declared workflow id -> the workflow as actually restored (fresh id and
 	// name). Lets an authored `attach` reference survive the per-run remap.
 	let seedWorkflowsBySeedId = new Map<string, { id: string; name: string }>();
@@ -815,6 +822,39 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 					`Seeding failed: ${error instanceof Error ? error.message : String(error)}`,
 				);
 			}
+			// Run AFTER the seeding try/catch, so a prior-run problem is not reported as
+			// "Seeding failed" — the artifacts did land, it is the pre-turn history that did
+			// not. Before the live turn, so the agent's first look already sees it.
+			if (config.seed?.mode === 'inline' && config.seed.priorRuns?.length) {
+				// A throw here (an id the seed never created) is an authoring/harness fault.
+				// Without the flag the outer catch returns a plain failed build and the case
+				// is recorded as `build_failure` / `builder_issue` — a builder red for
+				// something the builder had no part in.
+				try {
+					const outcomes = await executePriorRuns({
+						client,
+						priorRuns: config.seed.priorRuns,
+						// Already maps authored seed id → the restored workflow, and it is built
+						// from `remapped`, which the server pins its ids to.
+						seedWorkflows: seedWorkflowsBySeedId,
+						logger,
+						laneTag: config.laneTag,
+					});
+					// A staged run that never produced an execution record leaves the case's
+					// premise missing, so the graded turn answers a question the instance cannot
+					// support. Recorded rather than thrown: the build itself is fine, and the
+					// case is routed to infra instead of scored.
+					const missing = outcomes.filter((outcome) => !outcome.ran);
+					if (missing.length > 0) {
+						priorRunFailed = missing
+							.map((outcome) => `${outcome.workflow}: ${outcome.errors.join('; ') || 'unknown'}`)
+							.join(' | ');
+					}
+				} catch (error: unknown) {
+					seedingFailed = true;
+					throw error;
+				}
+			}
 		}
 
 		// TRUST-311 follow-up: create the case's execution-scenario data tables EMPTY
@@ -1041,6 +1081,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 					transcript,
 					credentialViewPinned,
 					seedingFailed,
+					...(priorRunFailed ? { priorRunFailed } : {}),
 					credentialSetup: await credentialSetupFacts(),
 				};
 			}
@@ -1061,6 +1102,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 				transcript,
 				credentialViewPinned,
 				seedingFailed,
+				...(priorRunFailed ? { priorRunFailed } : {}),
 				credentialSetup: await credentialSetupFacts(),
 			};
 		}
@@ -1086,6 +1128,11 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 		// per-scenario rows are seeded in runScenario via seededScenarioTableIdsByName.
 		return {
 			success: true,
+			// Carried on the SUCCESS path too. A staged run that never landed is the one
+			// infra signal that outlives a healthy build, and that is exactly the case
+			// `case-pipeline` has to catch — the graded turn answered a question the
+			// instance cannot support.
+			...(priorRunFailed ? { priorRunFailed } : {}),
 			workflowId: outcome.workflowsCreated[0].id,
 			workflowJsons: outcome.workflowJsons,
 			buildTrace,
@@ -1121,6 +1168,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 			threadId,
 			credentialViewPinned,
 			seedingFailed,
+			...(priorRunFailed ? { priorRunFailed } : {}),
 			laneBootFailed,
 			credentialSetup: await credentialSetupFacts(),
 		};

packages/@n8n/instance-ai/evaluations/harness/prior-runs.ts

@@ -0,0 +1,189 @@
+import { sleep } from '@n8n/utils/sleep';
+
+import type { EvalLogger } from './logger';
+import {
+	extractErrorMessage,
+	isTransientExecutionAbort,
+	MAX_EXEC_ATTEMPTS,
+	shouldRetryScenarioExecution,
+	throwIfServerBudgetStop,
+} from './transient-error';
+import type { N8nClient } from '../clients/n8n-client';
+import type { SeedPriorRun } from '../types';
+
+/**
+ * Staging is not the graded turn, so it gets its own budget. A scenario execution is
+ * given `effectiveTimeoutMs(complexity, args.timeoutMs)` — 900s by default, 1350s for a
+ * complex case — and a prior run may retry, which would let scene-setting outspend the
+ * thing under test. Passed explicitly rather than left to the client default, which
+ * happens to match today but is not staging's to inherit.
+ */
+export const STAGING_TIMEOUT_MS = 120_000;
+
+/** What one pre-turn run did, so the caller can see what history the agent was given. */
+export interface PriorRunOutcome {
+	/** The seed id the case named. */
+	workflow: string;
+	workflowId: string;
+	/**
+	 * Whether an execution record exists. This is the field that separates "failed
+	 * exactly as the case staged" from "never ran, so the premise is missing" —
+	 * `success: false` alone cannot tell those apart, and only the second is a reason
+	 * to distrust the grade.
+	 */
+	ran: boolean;
+	/** Present whenever the run reached the instance. Proof the record landed. */
+	executionId?: string;
+	success: boolean;
+	errors: string[];
+}
+
+export interface PriorRunsOptions {
+	client: N8nClient;
+	priorRuns: SeedPriorRun[];
+	/** Authored seed id → the restored workflow, as `seedWorkflowsBySeedId` holds it. */
+	seedWorkflows: Map<string, { id: string; name: string }>;
+	logger: EvalLogger;
+	laneTag?: string;
+	/** Injectable for tests. */
+	sleep?: (ms: number) => Promise<void>;
+}
+
+/**
+ * Run seeded workflows BEFORE the graded turn, so the instance carries real execution
+ * history the agent can look up.
+ *
+ * A failing run is the point rather than a problem. `hints` steers the mock layer the
+ * same way `executionScenarios[].dataSetup` does, so a case can establish "the 06:00 run
+ * died on the HTTP node" and then ask only "it broke again" — the agent has to go and
+ * find out how.
+ *
+ * A failed prior run therefore never fails the build. A prior run that never RAN is a
+ * different matter: the caller reads `ran` and routes the case to infra, because the
+ * premise it was graded against does not exist.
+ *
+ * Runs sequentially in declared order: a case can stage a sequence (a run that succeeds,
+ * then one that fails) and later runs may depend on state the earlier ones left behind.
+ */
+export async function executePriorRuns(options: PriorRunsOptions): Promise<PriorRunOutcome[]> {
+	const { client, priorRuns, seedWorkflows, logger, laneTag } = options;
+	const delay = options.sleep ?? sleep;
+	const outcomes: PriorRunOutcome[] = [];
+
+	for (const priorRun of priorRuns) {
+		const restored = seedWorkflows.get(priorRun.workflow);
+		if (!restored) {
+			// The schema cross-checks these ids at load, so reaching here means the seed
+			// did not create what it declared. Failing loudly beats grading a case whose
+			// premise silently never happened.
+			throw new Error(
+				`Prior run names seed workflow id "${priorRun.workflow}", which the seed did not create. Created: ${[...seedWorkflows.keys()].join(', ') || '(none)'}`,
+			);
+		}
+
+		const label = `${restored.name} (${priorRun.workflow})`;
+		const outcome = await runOnce(client, priorRun, restored.id, label, logger, delay);
+		outcomes.push(outcome);
+		logger.info(
+			`  Prior run "${label}": ${
+				outcome.ran
+					? outcome.success
+						? 'succeeded'
+						: `failed (${outcome.errors.join('; ') || 'no error detail'})`
+					: `NEVER RAN — no execution record (${outcome.errors.join('; ') || 'no error detail'})`
+			}${laneTag ?? ''}`,
+		);
+	}
+
+	return outcomes;
+}
+
+/**
+ * Whether an execution record actually exists for this result.
+ *
+ * The result SHAPE cannot answer it. The eval service rejects some requests before it
+ * ever calls the workflow runner — an unknown workflow, no trigger node — and returns an
+ * error result carrying a freshly minted UUID that no execution was ever stored under.
+ * Trusting `executionId` there would report a premise that does not exist as staged
+ * history, which is the reading this whole flag exists to prevent.
+ *
+ * Only checked on a failed result: a run that succeeded necessarily executed.
+ */
+async function executionRecordExists(
+	client: N8nClient,
+	result: { success: boolean; executionId: string },
+): Promise<boolean> {
+	if (result.success) return true;
+	if (!result.executionId) return false;
+	try {
+		const execution = await client.getExecution(result.executionId);
+		return execution.id === result.executionId;
+	} catch {
+		// A miss is the answer, not an error: the record is not there.
+		return false;
+	}
+}
+
+async function runOnce(
+	client: N8nClient,
+	priorRun: SeedPriorRun,
+	workflowId: string,
+	label: string,
+	logger: EvalLogger,
+	delay: (ms: number) => Promise<void>,
+): Promise<PriorRunOutcome> {
+	const base = { workflow: priorRun.workflow, workflowId };
+	let lastErrors: string[] = [];
+
+	for (let attempt = 1; attempt <= MAX_EXEC_ATTEMPTS; attempt++) {
+		let retryReason: string;
+		try {
+			const result = await client.executeWithLlmMock(
+				workflowId,
+				priorRun.hints,
+				STAGING_TIMEOUT_MS,
+			);
+			// A run the server stopped for exceeding its budget comes back in-band. Recording
+			// it as a staged failure would put HARNESS text in the execution record the graded
+			// agent then reads as the workflow's own failure reason.
+			throwIfServerBudgetStop(result);
+			// A DB write race aborts the run before any node executes and reports in-band.
+			// That is not the failure the case is staging, so retry it rather than record it.
+			if (result.success || !isTransientExecutionAbort(result.errors)) {
+				return {
+					...base,
+					ran: await executionRecordExists(client, result),
+					executionId: result.executionId,
+					success: result.success,
+					errors: result.errors,
+				};
+			}
+			lastErrors = result.errors;
+			retryReason = `transient DB abort (${result.errors.join('; ') || 'no error detail'})`;
+		} catch (error: unknown) {
+			const message = extractErrorMessage(error);
+			// Infrastructure, not the staged failure. Retried on the same terms a scenario
+			// execution gets, because a blip here silently voids the case's premise: the
+			// graded turn would then run against history that never landed.
+			if (!shouldRetryScenarioExecution(message, attempt)) {
+				logger.warn(`    Prior run "${label}" could not complete: ${message}`);
+				return { ...base, ran: false, success: false, errors: [message] };
+			}
+			lastErrors = [message];
+			retryReason = message;
+		}
+		// Only the throw branch caps itself, via `shouldRetryScenarioExecution`. The
+		// in-band abort branch falls through to here, so the last attempt would otherwise
+		// announce a retry it will not make and sleep before giving up.
+		if (attempt < MAX_EXEC_ATTEMPTS) {
+			logger.warn(
+				`    Prior run "${label}" ${retryReason} (attempt ${String(attempt)}/${String(MAX_EXEC_ATTEMPTS)}); retrying`,
+			);
+			await delay(500 * attempt);
+		}
+	}
+
+	// Every attempt hit a retryable fault, so no execution record landed. Reports the last
+	// real errors rather than a synthetic message, so the log names what went wrong.
+	return { ...base, ran: false, success: false, errors: lastErrors };
+}

packages/@n8n/instance-ai/evaluations/harness/schema.ts

@@ -105,6 +105,26 @@ export const CaseSeedSchema = z.discriminatedUnion('mode', [
 	ConversationSeedSchema.extend({
 		mode: z.literal('inline'),
 		messages: inlineSeedMessagesSchema.default([]),
+		/**
+		 * Workflows to RUN before the graded turn, creating real execution records the
+		 * agent can look up. `workflow` is the seed workflow's **id**, the same key
+		 * `conversation[0].attach.workflow` uses — one rule for pointing at a seeded
+		 * workflow, and it stays unambiguous without depending on seed names being unique.
+		 *
+		 * A failing prior run is the point rather than a problem: `hints` steers the mock
+		 * layer, so a case can establish "the 06:00 run died on the HTTP node" and then ask
+		 * only "it broke again". A prior run that fails does NOT fail the build.
+		 */
+		priorRuns: z
+			.array(
+				z
+					.object({
+						workflow: z.string().min(1),
+						hints: z.string().min(1).max(2000).optional(),
+					})
+					.strict(),
+			)
+			.optional(),
 	}).strict(),
 	/** Reproduce a real conversation from its LangSmith trace at run time (seed =
 	 *  before the live turn, live = that turn). Commits only the thread id;
@@ -294,6 +314,20 @@ export const EvalTestCaseSchema = evalTestCaseObjectSchema
 		// the issue list, which would otherwise backslash-escape them and break substring/regex
 		// matching against the raw error message in callers and tests.
 		//
+		// A prior run needs a workflow to run. Catching the typo at authoring time beats a
+		// mid-build failure, which reads like an infrastructure fault rather than a typo.
+		if (c.seed?.mode === 'inline' && c.seed.priorRuns?.length) {
+			const declared = new Set(c.seed.workflows.map((workflow) => workflow.id));
+			for (const [index, priorRun] of c.seed.priorRuns.entries()) {
+				if (!declared.has(priorRun.workflow)) {
+					ctx.addIssue({
+						code: z.ZodIssueCode.custom,
+						path: ['seed', 'priorRuns', index, 'workflow'],
+						message: `priorRuns names workflow id "${priorRun.workflow}", which this seed does not declare. Seeded workflow ids: ${[...declared].join(', ') || '(none)'}`,
+					});
+				}
+			}
+		}
 		// A case needs at least one gradable unit. Execution scenarios grade the built workflow;
 		// process/outcome expectations grade the conversation, the workflow, and any non-workflow
 		// artifact (agent, config-eval) rendered into the judge context.

packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts

@@ -13,6 +13,7 @@ import { sleep } from '@n8n/utils/sleep';
 
 import type { LaneAllocator } from './lane-allocator';
 import { provisionCaseBuildUser, type LaneUserPool } from './lane-users';
+import { collectExpectations } from '../build-expectations/collect';
 import { selectAuthorExpectations } from '../build-expectations/select';
 import { allFailVerdicts, verifyBuildExpectations } from '../build-expectations/verifier';
 import type { CliArgs } from '../cli/args';
@@ -426,6 +427,23 @@ export function createBuildOrchestrator(deps: BuildOrchestratorDeps): BuildOrche
 			searchableBuildText(build);
 		const testCase = testCaseByFileSlug.get(fileSlug);
 		if (!testCase) return;
+		// Staging never landed, so the case has no premise to be judged against. The row
+		// itself is short-circuited in `case-pipeline`, but expectations are counted
+		// separately and only a verdict's OWN `incomplete` excludes it — the row's flag
+		// does not reach them. A priorRuns case is usually expectation-only, so without
+		// this the single graded unit still lands in the builder's baseline as a red.
+		if (build.priorRunFailed) {
+			buildExpectationsByKey.set(
+				key,
+				Promise.resolve(
+					allFailVerdicts(
+						collectExpectations(testCase),
+						`not judged — prior run staging did not land, so the case premise is missing: ${build.priorRunFailed}`,
+					),
+				),
+			);
+			return;
+		}
 		// Deterministic credential-setup verdicts, started EAGERLY: per-build
 		// cleanup deletes artifacts later, and a credential read that lost that
 		// race would report "not created" for a run that did create one.

packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts

@@ -202,6 +202,35 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline {
 			return verdicts && verdicts.length > 0 ? { ...output, expectationResults: verdicts } : output;
 		};
 
+		// A staged prior run that never produced an execution record leaves the case
+		// grading against history the instance does not have. Checked BEFORE every other
+		// branch, and independently of `build.success`, because the biting case is a build
+		// that succeeded: `buildFailedOnInfra` returns false there, so the case would
+		// otherwise be scored as an agent failure on a premise that never existed.
+		if (build.priorRunFailed) {
+			return await attachExpectations({
+				buildSuccess: build.success,
+				passed: false,
+				// Not graded rather than failed — the same treatment an ungraded expectation
+				// gets, so this stays out of the pass rate instead of landing in the
+				// builder's baseline as a red.
+				incomplete: true,
+				score: 0,
+				reasoning: `Prior run staging did not land, so the case premise is missing: ${build.priorRunFailed}`,
+				failureCategory: 'framework_issue',
+				attribution: 'framework_issue',
+				execErrors: [build.priorRunFailed],
+				buildDurationMs,
+				...buildSpendFields,
+				execDurationMs: 0,
+				nodeCount: build.workflowJsons[0]?.nodes.length ?? 0,
+				threadId: build.threadId,
+				workflowChecks: build.workflowChecks,
+				workflowJson: build.workflowJsons[0],
+				buildTrace: build.buildTrace,
+			});
+		}
+
 		// Build-only case — the build plus its expectation judging (in getOrBuild) is the
 		// whole test; skip execution. The sentinel's outcome IS the expectation verdicts,
 		// so LangSmith pass metrics stay truthful for scenario-less cases. Checked before

packages/@n8n/instance-ai/evaluations/types.ts

@@ -311,6 +311,26 @@ export interface ExecutionScenarioResult {
 	incomplete?: boolean;
 }
 
+/**
+ * A seeded workflow to run BEFORE the graded turn.
+ *
+ * Creates a real execution record in the instance, so a case can ask about "the last
+ * run" and the honest answer requires the agent to go and read it. Without this,
+ * execution history is unreachable as a premise: the harness only ever executes a
+ * workflow *after* a build.
+ */
+export interface SeedPriorRun {
+	/** Seeded workflow to run, by the `id` the seed declares — the same key
+	 *  `conversation[0].attach.workflow` uses. */
+	workflow: string;
+	/**
+	 * Steers the mock layer, exactly as `executionScenarios[].dataSetup` does. This is
+	 * how a prior run is made to fail in a specific way, which is the interesting case:
+	 * the user reports "it broke again" and the agent has to find out how.
+	 */
+	hints?: string;
+}
+
 /** Verdict for one author-written build expectation. Scored as a unit in the
  *  pass rate alongside execution scenarios. */
 export interface BuildExpectationResult {