fix(ai-builder): Reconstruct file-based seeded workflows and judge the whole conversation (no-changelog) (#33048)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts

@@ -220,6 +220,37 @@ describe('transcriptPrefixFromSeed', () => {
 		]);
 	});
 
+	it('renders a seeded confirmation block (not ask-user/setup) as a confirmation step', () => {
+		const turns = transcriptPrefixFromSeed([
+			{
+				id: 'a1',
+				type: 'llm',
+				role: 'assistant',
+				content: [
+					{
+						type: 'tool-call',
+						toolName: 'workflows',
+						state: 'resolved',
+						// Resume block re-states the request in input, decision in output.
+						input: { resumeReason: 'resource-decision', message: 'Which credential?' },
+						output: { approved: false, feedback: 'use the prod one' },
+					},
+				],
+				createdAt: '2026-01-01T00:00:00Z',
+			},
+		]);
+		expect(turns[0].steps).toEqual([
+			{
+				kind: 'confirmation',
+				toolName: 'workflows',
+				resumeReason: 'resource-decision',
+				approved: false,
+				message: 'Which credential?',
+				feedback: 'use the prod one',
+			},
+		]);
+	});
+
 	it('renders a seeded setup outcome (completedNodes/skippedNodes) as a setup-wizard step', () => {
 		const turns = transcriptPrefixFromSeed([
 			{

packages/@n8n/instance-ai/evaluations/__tests__/conversation-text.test.ts

@@ -1,6 +1,7 @@
 import type { ConversationTurn, TranscriptTurn } from '../types';
 import {
 	conversationUserTurnsAsText,
+	lastAgentText,
 	perTurnToolCallCounts,
 	transcriptAsText,
 	userTurnsAsText,
@@ -161,3 +162,16 @@ describe('perTurnToolCallCounts', () => {
 		);
 	});
 });
+
+describe('lastAgentText', () => {
+	it('returns the most recent turn agent narration, skipping trailing turns with none', () => {
+		expect(lastAgentText([])).toBe('');
+		const transcript: TranscriptTurn[] = [
+			{ steps: [{ kind: 'agent-text', text: 'first' }], seeded: true },
+			{ steps: [{ kind: 'agent-text', text: 'latest' }], seeded: true },
+			// Live turn that produced no narration (the no-op fallback case).
+			{ userMessage: 'and now?', steps: [] },
+		];
+		expect(lastAgentText(transcript)).toBe('latest');
+	});
+});

packages/@n8n/instance-ai/evaluations/__tests__/event-parser.test.ts

@@ -2,8 +2,10 @@ import {
 	buildConversationMetrics,
 	buildMetrics,
 	extractOutcomeFromEvents,
+	mergeSeededConversationMetrics,
+	seededTurnCounters,
 } from '../outcome/event-parser';
-import type { CapturedEvent } from '../types';
+import type { CapturedEvent, ConversationMetrics, TranscriptTurn } from '../types';
 
 // ---------------------------------------------------------------------------
 // extractOutcomeFromEvents
@@ -578,3 +580,131 @@ describe('buildConversationMetrics', () => {
 		expect(result.perTurn[1].runFinishStatus).toBe('cancelled');
 	});
 });
+
+// ---------------------------------------------------------------------------
+// seededTurnCounters / mergeSeededConversationMetrics
+// ---------------------------------------------------------------------------
+
+const seededTurn = (steps: TranscriptTurn['steps']): TranscriptTurn => ({ steps, seeded: true });
+
+describe('seededTurnCounters', () => {
+	it('counts tool-call steps and their errors', () => {
+		const [counter] = seededTurnCounters([
+			seededTurn([
+				{ kind: 'tool-call', toolName: 'a' },
+				{ kind: 'tool-call', toolName: 'b', error: 'boom' },
+			]),
+		]);
+		expect(counter).toMatchObject({ toolCallCount: 2, toolErrorCount: 1 });
+	});
+
+	it('counts a plan step as a tool call', () => {
+		const [counter] = seededTurnCounters([seededTurn([{ kind: 'plan', tasks: [] }])]);
+		expect(counter.toolCallCount).toBe(1);
+	});
+
+	it('dual-counts ask-user as a tool call and a questions confirmation (mirrors live)', () => {
+		const [counter] = seededTurnCounters([seededTurn([{ kind: 'ask-user', questions: [] }])]);
+		expect(counter.toolCallCount).toBe(1);
+		expect(counter.confirmationAskedTotal).toBe(1);
+		expect(counter.confirmationAskedByKind).toEqual({ questions: 1 });
+	});
+
+	it('counts setup-card as a tool call and a setup confirmation', () => {
+		const [counter] = seededTurnCounters([
+			seededTurn([{ kind: 'setup-card', requests: [], outcome: 'pending' }]),
+		]);
+		expect(counter.toolCallCount).toBe(1);
+		expect(counter.confirmationAskedByKind).toEqual({ setup: 1 });
+	});
+
+	it('counts a confirmation as a tool call, keyed by its resumeReason', () => {
+		const [counter] = seededTurnCounters([
+			seededTurn([{ kind: 'confirmation', toolName: 'workflows', resumeReason: 'plan-review' }]),
+		]);
+		expect(counter.toolCallCount).toBe(1);
+		expect(counter.confirmationAskedTotal).toBe(1);
+		expect(counter.confirmationAskedByKind).toEqual({ 'plan-review': 1 });
+	});
+
+	it('counts setup-wizard as a tool call, ignores agent-text, leaves non-derivable counters at default', () => {
+		const [counter] = seededTurnCounters([
+			seededTurn([
+				{ kind: 'agent-text', text: 'hello' },
+				{ kind: 'setup-wizard', completedNodes: [], skippedNodes: [] },
+			]),
+		]);
+		expect(counter.toolCallCount).toBe(1); // setup-wizard is a tool call; agent-text is not
+		expect(counter.confirmationAskedTotal).toBe(0);
+		expect(counter.replanAfterErrorCount).toBe(0);
+		expect(counter.repeatQuestionCount).toBe(0);
+		expect(counter.runFinishStatus).toBeUndefined();
+	});
+
+	it('returns one counter per turn (numbered) and [] for no turns', () => {
+		expect(seededTurnCounters([])).toEqual([]);
+		expect(seededTurnCounters([seededTurn([]), seededTurn([])]).map((c) => c.turn)).toEqual([1, 2]);
+	});
+});
+
+describe('mergeSeededConversationMetrics', () => {
+	const live: ConversationMetrics = {
+		turnCount: 1,
+		perTurn: [
+			{
+				turn: 1,
+				toolCallCount: 3,
+				toolErrorCount: 0,
+				confirmationAskedTotal: 2,
+				confirmationAskedByKind: { questions: 2 },
+				replanAfterErrorCount: 0,
+				repeatQuestionCount: 0,
+				runFinishStatus: 'completed',
+			},
+		],
+		confirmationAskedTotal: 2,
+		confirmationAskedByKind: { questions: 2 },
+		reachedRunFinishCleanly: true,
+	};
+
+	it('prepends seeded turns, renumbers, sums aggregates, and preserves the live finish status', () => {
+		const seeded: TranscriptTurn[] = [
+			seededTurn([{ kind: 'ask-user', questions: [] }]),
+			seededTurn([{ kind: 'setup-card', requests: [], outcome: 'pending' }]),
+		];
+		const merged = mergeSeededConversationMetrics(seeded, live);
+		expect(merged.perTurn.map((c) => c.turn)).toEqual([1, 2, 3]);
+		expect(merged.turnCount).toBe(3); // 2 seeded + live turnCount 1
+		expect(merged.confirmationAskedTotal).toBe(4); // seeded 2 + live 2
+		expect(merged.confirmationAskedByKind).toEqual({ questions: 3, setup: 1 });
+		expect(merged.reachedRunFinishCleanly).toBe(true);
+		// The live turn is last, unchanged apart from its new turn number.
+		expect(merged.perTurn[2]).toMatchObject({
+			turn: 3,
+			toolCallCount: 3,
+			runFinishStatus: 'completed',
+		});
+	});
+
+	it('preserves a false live finish status regardless of seeded content', () => {
+		const merged = mergeSeededConversationMetrics(
+			[seededTurn([{ kind: 'agent-text', text: 'x' }])],
+			{
+				...live,
+				reachedRunFinishCleanly: false,
+			},
+		);
+		expect(merged.reachedRunFinishCleanly).toBe(false);
+	});
+
+	it('an empty seeded prefix yields metrics deep-equal to the live metrics (no live regression)', () => {
+		expect(mergeSeededConversationMetrics([], live)).toEqual(live);
+	});
+
+	it('does not mutate the passed-in live metrics', () => {
+		mergeSeededConversationMetrics([seededTurn([{ kind: 'ask-user', questions: [] }])], live);
+		expect(live.perTurn[0].turn).toBe(1);
+		expect(live.confirmationAskedByKind).toEqual({ questions: 2 });
+		expect(live.turnCount).toBe(1);
+	});
+});

packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts

@@ -46,6 +46,25 @@ function turn(id: string, sec: number, message: string): FakeRun {
 	return { id, run_type: 'chain', name: 'turn', start_time: t(sec), inputs: { message } };
 }
 
+/** A `tool` run (workspace edit, build, get-as-code) attached to root r1. */
+function tool(
+	id: string,
+	sec: number,
+	name: string,
+	inputs: Record<string, unknown>,
+	outputs: Record<string, unknown> = { success: true },
+): FakeRun {
+	return {
+		id,
+		run_type: 'tool',
+		name,
+		start_time: t(sec),
+		inputs,
+		outputs,
+		extra: { metadata: { langsmith_root_run_id: 'r1' } },
+	};
+}
+
 describe('reconstructSeedFromThread', () => {
 	it('splits at the last user turn: seed = before, liveTurn = last', async () => {
 		const runs: FakeRun[] = [
@@ -237,6 +256,28 @@ describe('reconstructSeedFromThread', () => {
 		);
 	});
 
+	it('throws when builds succeeded but no source was recoverable (shape drift)', async () => {
+		// A recognised filePath build (post-#32545) whose workspace file was never
+		// captured and has no get-as-code fallback → nothing to reconstruct from.
+		const build: FakeRun = {
+			id: 'tool1',
+			run_type: 'tool',
+			name: 'build-workflow',
+			start_time: t(5),
+			inputs: { filePath: 'src/workflows/main.workflow.ts' },
+			outputs: { success: true, workflowId: 'WF1' },
+			extra: { metadata: { langsmith_root_run_id: 'r1' } },
+		};
+		const runs: FakeRun[] = [
+			{ ...turn('r1', 1, 'Build it'), outputs: { response: 'Done.' } },
+			build,
+			turn('r2', 30, 'Change'),
+		];
+		await expect(reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs))).rejects.toThrow(
+			/built in the trace but reconstruction recovered 0/,
+		);
+	});
+
 	it('does not false-positive on a read-only tool that returns a workflowId', async () => {
 		// get-workflow returns a workflowId but takes no `code` — not build-like,
 		// so the drift detector stays quiet and the seed simply has no workflow.
@@ -382,6 +423,127 @@ describe('reconstructSeedFromThread', () => {
 	});
 });
 
+describe('reconstructSeedFromThread — filesystem-based builds (post-#32545)', () => {
+	const FILE = 'src/workflows/main.workflow.ts';
+
+	it('reconstructs a filePath build by replaying workspace file edits', async () => {
+		const runs: FakeRun[] = [
+			{ ...turn('r1', 1, 'Build it'), outputs: { response: 'Building…' } },
+			tool('w1', 2, 'workspace_write_file', { path: FILE, content: 'CODE_V1' }),
+			tool('e1', 3, 'workspace_str_replace_file', {
+				path: FILE,
+				old_str: 'CODE_V1',
+				new_str: 'CODE_V2',
+			}),
+			tool(
+				'b1',
+				4,
+				'build-workflow',
+				{ filePath: FILE, name: 'Main' },
+				{
+					success: true,
+					workflowId: 'WF1',
+					workflowName: 'Main',
+				},
+			),
+			turn('r2', 30, 'change'),
+		];
+		const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs));
+
+		expect(result.seed.workflows).toHaveLength(1);
+		expect(result.seed.workflows[0]).toMatchObject({ id: 'WF1', name: 'Main' });
+		// The edited file content (V2), not the initial write (V1), reaches the compiler.
+		expect(result.seed.workflows[0].nodes[0]).toMatchObject({ __code: 'CODE_V2' });
+	});
+
+	it('skips a failed edit so the replay matches the unchanged sandbox file', async () => {
+		const runs: FakeRun[] = [
+			{ ...turn('r1', 1, 'Build it'), outputs: { response: '…' } },
+			tool('w1', 2, 'workspace_write_file', { path: FILE, content: 'GOOD' }),
+			// A failed str-replace (e.g. non-unique anchor) left the real file unchanged.
+			tool(
+				'e1',
+				3,
+				'workspace_str_replace_file',
+				{ path: FILE, old_str: 'GOOD', new_str: 'BAD' },
+				{ success: false },
+			),
+			tool('b1', 4, 'build-workflow', { filePath: FILE }, { success: true, workflowId: 'WF1' }),
+			turn('r2', 30, 'change'),
+		];
+		const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs));
+		expect(result.seed.workflows[0].nodes[0]).toMatchObject({ __code: 'GOOD' });
+	});
+
+	it('falls back to a get-as-code capture when a successful edit cannot be replayed', async () => {
+		const runs: FakeRun[] = [
+			{ ...turn('r1', 1, 'Build it'), outputs: { response: '…' } },
+			// Authoritative source captured by get-as-code.
+			tool(
+				'g1',
+				2,
+				'workflows[get-as-code]',
+				{ action: 'get-as-code', workflowId: 'WF1' },
+				{
+					workflowId: 'WF1',
+					name: 'Main',
+					code: 'FROM_GET_AS_CODE',
+				},
+			),
+			tool('w1', 3, 'workspace_write_file', { path: FILE, content: 'CODE_V1' }),
+			// A *successful* edit whose anchor is absent in our replay → divergence
+			// (mimics an untracked shell edit having changed the file first).
+			tool('e1', 4, 'workspace_str_replace_file', {
+				path: FILE,
+				old_str: 'NOT_IN_REPLAY',
+				new_str: 'X',
+			}),
+			tool('b1', 5, 'build-workflow', { filePath: FILE }, { success: true, workflowId: 'WF1' }),
+			turn('r2', 30, 'change'),
+		];
+		const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs));
+		expect(result.seed.workflows).toHaveLength(1);
+		expect(result.seed.workflows[0].nodes[0]).toMatchObject({ __code: 'FROM_GET_AS_CODE' });
+	});
+
+	it('emits a workflow only for built files, ignoring other written files', async () => {
+		const runs: FakeRun[] = [
+			{ ...turn('r1', 1, 'Build it'), outputs: { response: '…' } },
+			tool('kb', 2, 'workspace_write_file', {
+				path: 'knowledge-base/notes.md',
+				content: 'scratch',
+			}),
+			tool('w1', 3, 'workspace_write_file', { path: FILE, content: 'REAL' }),
+			tool('b1', 4, 'build-workflow', { filePath: FILE }, { success: true, workflowId: 'WF1' }),
+			turn('r2', 30, 'change'),
+		];
+		const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs));
+		// Only the built file becomes a workflow; the scratch write is ignored.
+		expect(result.seed.workflows).toHaveLength(1);
+		expect(result.seed.workflows[0].nodes[0]).toMatchObject({ __code: 'REAL' });
+	});
+
+	it('applies a batch str-replace atomically: any missing anchor leaves the file untouched', async () => {
+		const runs: FakeRun[] = [
+			{ ...turn('r1', 1, 'Build it'), outputs: { response: '…' } },
+			tool('w1', 2, 'workspace_write_file', { path: FILE, content: 'CODE_V1' }),
+			// One anchor matches, one is absent. The real tool is atomic (all-or-nothing),
+			// so the file must stay CODE_V1 — never a half-applied 'GOOD'.
+			tool('e1', 3, 'workspace_batch_str_replace_file', {
+				path: FILE,
+				replacements: [
+					{ old_str: 'CODE_V1', new_str: 'GOOD' },
+					{ old_str: 'NOT_PRESENT', new_str: 'Y' },
+				],
+			}),
+			tool('b1', 4, 'build-workflow', { filePath: FILE }, { success: true, workflowId: 'WF1' }),
+			turn('r2', 30, 'change'),
+		];
+		const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs));
+		expect(result.seed.workflows[0].nodes[0]).toMatchObject({ __code: 'CODE_V1' });
+	});
+});
+
 describe('reconstructSeedFromThread — workspace auto-discovery', () => {
 	const seedableRuns: FakeRun[] = [
 		{ ...turn('r1', 1, 'Build it'), outputs: { response: 'Built.' } },

packages/@n8n/instance-ai/evaluations/__tests__/workspace-tool-contract.test.ts

@@ -0,0 +1,35 @@
+import { createWorkspaceTools } from '@n8n/agents';
+
+import { IGNORED_WORKSPACE_TOOLS, REPLAYED_WORKSPACE_TOOL_ARGS } from '../harness/langsmith-seed';
+
+// The seed reconstructor replays @n8n/agents workspace file ops by matching tool
+// names + arg keys (langsmith-seed.ts). @n8n/agents doesn't export those, so pin
+// the duplicated names/keys against the live factory here — a rename or new tool
+// there fails this test instead of silently turning a replayed mutation into a no-op.
+describe('workspace tool contract (@n8n/agents drift guard)', () => {
+	const liveByName = new Map(
+		createWorkspaceTools({ filesystem: {} } as unknown as Parameters<
+			typeof createWorkspaceTools
+		>[0]).map((tool) => [tool.name, tool] as const),
+	);
+
+	it('classifies every live filesystem tool as replayed or intentionally ignored', () => {
+		const classified = [...Object.keys(REPLAYED_WORKSPACE_TOOL_ARGS), ...IGNORED_WORKSPACE_TOOLS];
+		expect([...liveByName.keys()].sort()).toEqual(classified.sort());
+	});
+
+	it('reads only arg keys the live tool schema still defines', () => {
+		for (const [name, args] of Object.entries(REPLAYED_WORKSPACE_TOOL_ARGS)) {
+			const schema = liveByName.get(name)?.inputSchema;
+			// Duck-type the Zod object shape — cross-package `instanceof` is unreliable.
+			const liveKeys =
+				schema && typeof schema === 'object' && 'shape' in schema
+					? Object.keys((schema as { shape: Record<string, unknown> }).shape)
+					: [];
+			expect(liveKeys, `${name} exposes no Zod object schema`).not.toHaveLength(0);
+			for (const key of args) {
+				expect(liveKeys, `${name} no longer defines arg "${key}"`).toContain(key);
+			}
+		}
+	});
+});

packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts

@@ -8,6 +8,7 @@ import { randomUUID } from 'node:crypto';
 import { readFileSync } from 'node:fs';
 import { z } from 'zod';
 
+import { DOMAIN_TOOL_IDS, ORCHESTRATION_TOOL_IDS } from '../../src/tools/tool-ids';
 import {
 	extractAskUserAnswers,
 	extractAskUserQuestions,
@@ -181,7 +182,7 @@ type SeedStepInterpreter = (call: SeedToolCall) => TranscriptStep | null;
 
 const interpretAskUser: SeedStepInterpreter = (call) => {
 	const questions = call.input?.questions;
-	if (call.toolName !== 'ask-user' || !Array.isArray(questions)) return null;
+	if (call.toolName !== DOMAIN_TOOL_IDS.ASK_USER || !Array.isArray(questions)) return null;
 	const parsed = extractAskUserQuestions(questions);
 	if (parsed.length === 0) return null;
 	// The kept (resume) block carries the user's answers in its output.
@@ -193,7 +194,7 @@ const interpretAskUser: SeedStepInterpreter = (call) => {
 
 const interpretPlan: SeedStepInterpreter = (call) => {
 	const tasks = call.input?.tasks;
-	if (call.toolName !== 'create-tasks' || !Array.isArray(tasks)) return null;
+	if (call.toolName !== ORCHESTRATION_TOOL_IDS.CREATE_TASKS || !Array.isArray(tasks)) return null;
 	const parsed = extractPlanTasks(tasks);
 	return parsed.length > 0 ? { kind: 'plan', tasks: parsed } : null;
 };
@@ -219,11 +220,36 @@ const interpretSetupCard: SeedStepInterpreter = (call) => {
 	return requests.length > 0 ? { kind: 'setup-card', requests, outcome: 'pending' } : null;
 };
 
+// A HITL confirmation other than ask-user/setup-card (plan-review, resource decision, …):
+// the request is in the resume block's input, the decision in its output.
+const interpretConfirmation: SeedStepInterpreter = (call) => {
+	const reasonRaw = call.input?.resumeReason ?? call.input?.inputType;
+	const resumeReason = typeof reasonRaw === 'string' ? reasonRaw : undefined;
+	if (!resumeReason || resumeReason === 'questions' || Array.isArray(call.input?.setupRequests)) {
+		return null;
+	}
+	const toolNameRaw = call.input?.toolName;
+	const messageRaw = call.input?.message;
+	const approvedRaw = call.output?.approved;
+	const feedbackRaw = call.output?.feedback;
+	return {
+		kind: 'confirmation',
+		toolName: typeof toolNameRaw === 'string' ? toolNameRaw : call.toolName,
+		resumeReason,
+		approved: typeof approvedRaw === 'boolean' ? approvedRaw : undefined,
+		// Plan-review prompts are boilerplate; the plan renders separately.
+		message:
+			resumeReason === 'plan-review' || typeof messageRaw !== 'string' ? undefined : messageRaw,
+		feedback: typeof feedbackRaw === 'string' ? feedbackRaw : undefined,
+	};
+};
+
 const SEED_STEP_INTERPRETERS: SeedStepInterpreter[] = [
 	interpretAskUser,
 	interpretPlan,
 	interpretSetupWizard,
 	interpretSetupCard,
+	interpretConfirmation,
 ];
 
 /** Map a seeded tool-call block to a transcript step (special interpreters above,

packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts

@@ -1,7 +1,7 @@
 // Reconstruct a conversation seed from a LangSmith trace at run time: pull the
 // thread's runs, rebuild the message log, split at the last user turn (before =
-// seed, last = live), and compile the seed workflow from the build tool's
-// captured SDK code at the boundary. Transient: traces retain ~14 days.
+// seed, last = live), and reconstruct each seed workflow from its source at the
+// build boundary. Transient: traces retain ~14 days.
 
 import { isRecord } from '@n8n/utils';
 import { Client } from 'langsmith';
@@ -9,12 +9,54 @@ import type { Run } from 'langsmith/schemas';
 
 import type { ConversationSeed } from './conversation-seed';
 import { parseSeedWorkflowCode } from './parse-seed-workflow';
+import { DOMAIN_TOOL_IDS } from '../../src/tools/tool-ids';
 
 /** Default project that instance-ai conversations are traced to (same name in
  *  every workspace). Override per case with `seedThread.project` if it differs. */
 const DEFAULT_SOURCE_PROJECT = 'instance-ai';
 
-const WORKFLOW_BUILD_TOOLS = new Set(['build-workflow', 'patch-workflow', 'submit-workflow']);
+// Reference the live tool-id so a rename there follows here (or breaks the import).
+// patch/submit-workflow were removed in #32545 but stay for older traces.
+const WORKFLOW_BUILD_TOOLS = new Set<string>([
+	DOMAIN_TOOL_IDS.BUILD_WORKFLOW,
+	'patch-workflow',
+	'submit-workflow',
+]);
+
+// Workspace tools (@n8n/agents) whose ops mutate file content we replay. The names
+// live here only; a contract test pins them — and the arg keys below — against the
+// live tool set + Zod schema, so a rename in @n8n/agents fails CI loudly instead of
+// silently no-op'ing a mutation (the same drift class #32545 caused).
+const WORKSPACE_TOOL = {
+	WRITE: 'workspace_write_file',
+	APPEND: 'workspace_append_file',
+	STR_REPLACE: 'workspace_str_replace_file',
+	BATCH_STR_REPLACE: 'workspace_batch_str_replace_file',
+	MOVE: 'workspace_move_file',
+	COPY: 'workspace_copy_file',
+	DELETE: 'workspace_delete_file',
+} as const;
+
+/** Input keys the replay reads per tool — pinned against the live Zod schema. */
+export const REPLAYED_WORKSPACE_TOOL_ARGS: Record<string, readonly string[]> = {
+	[WORKSPACE_TOOL.WRITE]: ['path', 'content'],
+	[WORKSPACE_TOOL.APPEND]: ['path', 'content'],
+	[WORKSPACE_TOOL.STR_REPLACE]: ['path', 'old_str', 'new_str'],
+	[WORKSPACE_TOOL.BATCH_STR_REPLACE]: ['path', 'replacements'],
+	[WORKSPACE_TOOL.MOVE]: ['src', 'dest'],
+	[WORKSPACE_TOOL.COPY]: ['src', 'dest'],
+	[WORKSPACE_TOOL.DELETE]: ['path'],
+};
+
+/** Live filesystem tools we deliberately don't replay (reads + dir ops) — listed so
+ *  the contract test can prove every live filesystem tool is classified. */
+export const IGNORED_WORKSPACE_TOOLS: readonly string[] = [
+	'workspace_read_file',
+	'workspace_list_files',
+	'workspace_file_stat',
+	'workspace_mkdir',
+	'workspace_rmdir',
+];
 
 // The source thread may live in a different workspace than the eval writes to
 // (e.g. seed from prod, trace to staging). A PAT spans workspaces, so we
@@ -346,51 +388,210 @@ function buildSeedMessages(
 	return messages;
 }
 
-/** Compile the seed's workflows from the build tool's captured SDK code — latest
- *  successful build per workflow id before the boundary. */
+/** Replay one content-mutating workspace tool onto the reconstructed file map.
+ *  Returns false when an edit can't be applied faithfully (a str-replace whose
+ *  anchor is absent) — the reconstruction has diverged from the real sandbox. */
+function applyFileMutation(files: Map<string, string>, tool: Run): boolean {
+	const input = isRecord(tool.inputs) ? tool.inputs : {};
+	// A failed edit left the real file unchanged — treat as a no-op (no divergence).
+	if (isRecord(tool.outputs) && tool.outputs.success === false) return true;
+	if (tool.name === WORKSPACE_TOOL.WRITE) {
+		const path = asString(input.path);
+		const content = asString(input.content);
+		if (path && content !== undefined) files.set(path, content);
+		return true;
+	}
+	if (tool.name === WORKSPACE_TOOL.APPEND) {
+		const path = asString(input.path);
+		if (path) files.set(path, (files.get(path) ?? '') + (asString(input.content) ?? ''));
+		return true;
+	}
+	if (tool.name === WORKSPACE_TOOL.STR_REPLACE) {
+		// The tool requires one exact, unique match; mirror it (first occurrence).
+		const path = asString(input.path);
+		const current = path !== undefined ? files.get(path) : undefined;
+		const oldStr = asString(input.old_str);
+		const newStr = asString(input.new_str);
+		if (path === undefined || current === undefined || !oldStr || newStr === undefined) return true;
+		if (!current.includes(oldStr)) return false;
+		files.set(path, current.replace(oldStr, newStr));
+		return true;
+	}
+	if (tool.name === WORKSPACE_TOOL.BATCH_STR_REPLACE) {
+		// The real tool is atomic — it validates every anchor up front and applies
+		// all or nothing (@n8n/ai-utilities TextEditorDocument.executeBatch). Mirror
+		// that, like single str-replace: any missing anchor → file untouched, diverged.
+		const path = asString(input.path);
+		const current = path !== undefined ? files.get(path) : undefined;
+		if (path === undefined || current === undefined) return true;
+		const replacements = Array.isArray(input.replacements) ? input.replacements : [];
+		let next = current;
+		for (const replacement of replacements) {
+			if (!isRecord(replacement)) continue;
+			const oldStr = asString(replacement.old_str);
+			const newStr = asString(replacement.new_str);
+			if (!oldStr || newStr === undefined) continue;
+			if (!next.includes(oldStr)) return false;
+			next = next.replace(oldStr, newStr);
+		}
+		files.set(path, next);
+		return true;
+	}
+	if (tool.name === WORKSPACE_TOOL.MOVE || tool.name === WORKSPACE_TOOL.COPY) {
+		const src = asString(input.src);
+		const dest = asString(input.dest);
+		const content = src !== undefined ? files.get(src) : undefined;
+		if (src !== undefined && dest !== undefined && content !== undefined) {
+			files.set(dest, content);
+			if (tool.name === WORKSPACE_TOOL.MOVE) files.delete(src);
+		}
+		return true;
+	}
+	if (tool.name === WORKSPACE_TOOL.DELETE) {
+		const path = asString(input.path);
+		if (path) files.delete(path);
+		return true;
+	}
+	return true; // read/list/grep/etc. — no content change
+}
+
+/** Reconstruct the seed's workflows: the latest successful build per workflow id
+ *  before the boundary. Post-#32545 the builder builds from a workspace file
+ *  (`build-workflow {filePath}`, no inline code), so the source is that file
+ *  replayed from the workspace ops; inline `code` and `get-as-code` are fallbacks.
+ *  Only files an actual build references become workflows. */
 function buildSeedWorkflows(
 	toolRuns: Run[],
 	boundaryMs: number,
 	threadId: string,
 ): ConversationSeed['workflows'] {
-	const latestBuildByWorkflowId = new Map<string, Run>();
-	// Name-independent build signal (code in + workflowId out): detects a renamed
-	// build tool instead of silently dropping the workflow.
-	let sawBuildLikeRun = false;
-	for (const tool of toolRuns) {
-		if (new Date(tool.start_time).getTime() >= boundaryMs) continue;
-		const out = (tool.outputs ?? {}) as Record<string, unknown>;
-		const workflowId = asString(out.workflowId);
-		const buildLike = out.success === true && !!workflowId && !!asString(tool.inputs?.code);
-		if (buildLike) sawBuildLikeRun = true;
-		if (!WORKFLOW_BUILD_TOOLS.has(tool.name)) continue;
-		if (!buildLike || !workflowId) continue;
-		// Sorted ascending, so a later run overwrites — last successful build wins.
-		latestBuildByWorkflowId.set(workflowId, tool);
-	}
+	const files = new Map<string, string>();
+	const divergedPaths = new Set<string>();
+	const getAsCodeByWorkflowId = new Map<string, string>();
+	// workflowId -> reconstructed source + name at its latest successful build.
+	const builtByWorkflowId = new Map<string, { code: string; diverged: boolean; name?: string }>();
+	// Workflow ids with a build-shaped run (source in, success + workflowId out),
+	// name-independent — drives the drift tripwire below.
+	const buildSignalIds = new Set<string>();
 
-	if (latestBuildByWorkflowId.size === 0 && sawBuildLikeRun) {
-		throw new Error(
-			`Thread ${threadId}: a tool run built a workflow (SDK code in, workflowId out) but its name isn't in the known build set {${[...WORKFLOW_BUILD_TOOLS].join(', ')}} — the build tool was likely renamed; update WORKFLOW_BUILD_TOOLS.`,
-		);
+	for (const tool of toolRuns) {
+		if (new Date(tool.start_time ?? 0).getTime() >= boundaryMs) continue;
+
+		if (tool.name.startsWith('workspace_')) {
+			const path = asString((isRecord(tool.inputs) ? tool.inputs : {}).path);
+			const applied = applyFileMutation(files, tool);
+			// A full write replaces the file, healing any earlier divergence.
+			if (tool.name === WORKSPACE_TOOL.WRITE) {
+				if (path !== undefined) divergedPaths.delete(path);
+			} else if (!applied && path !== undefined) {
+				divergedPaths.add(path);
+			}
+			continue;
+		}
+
+		const out = isRecord(tool.outputs) ? tool.outputs : {};
+		if (tool.name.includes('get-as-code')) {
+			const code = asString(out.code);
+			const workflowId = asString(out.workflowId);
+			if (code && workflowId) getAsCodeByWorkflowId.set(workflowId, code);
+		}
+
+		const workflowId = asString(out.workflowId);
+		if (out.success !== true || workflowId === undefined) continue;
+		const input = isRecord(tool.inputs) ? tool.inputs : {};
+		const filePath = asString(input.filePath);
+		const inlineCode = asString(input.code);
+		if (filePath === undefined && inlineCode === undefined) continue;
+		// A build happened for this id (even if the tool's name isn't recognised).
+		buildSignalIds.add(workflowId);
+		if (!WORKFLOW_BUILD_TOOLS.has(tool.name)) continue;
+		// Current builder: the workspace file's content at this build. Legacy: inline code.
+		const code = filePath !== undefined ? (files.get(filePath) ?? '') : (inlineCode ?? '');
+		const diverged = filePath !== undefined ? divergedPaths.has(filePath) : false;
+		// Sorted ascending → the latest successful build wins.
+		builtByWorkflowId.set(workflowId, { code, diverged, name: asString(out.workflowName) });
 	}
 
 	const workflows: ConversationSeed['workflows'] = [];
-	for (const [workflowId, build] of latestBuildByWorkflowId) {
-		const code = unredactCode(asString(build.inputs?.code) ?? '');
-		const parsed = parseSeedWorkflowCode(code);
-		const out = (build.outputs ?? {}) as Record<string, unknown>;
+	const degraded: string[] = [];
+	const skipped: string[] = [];
+	for (const [workflowId, built] of builtByWorkflowId) {
+		const getAsCode = getAsCodeByWorkflowId.get(workflowId) ?? '';
+		// Resolve in descending order of trust: clean file replay, then a
+		// `get-as-code` capture, then a diverged replay as a last resort.
+		const candidates: Array<[string, string]> = [
+			['replay', built.diverged ? '' : built.code],
+			['get-as-code', getAsCode],
+			['diverged-replay', built.diverged ? built.code : ''],
+		];
+		let resolved: ReturnType<typeof tryParseSeedWorkflow>;
+		let via = '';
+		for (const [label, code] of candidates) {
+			if (code === '') continue;
+			resolved = tryParseSeedWorkflow(unredactCode(code));
+			if (resolved) {
+				via = label;
+				break;
+			}
+		}
+		if (!resolved) {
+			skipped.push(workflowId);
+			continue;
+		}
+		if (via !== 'replay') degraded.push(`${workflowId}→${via}`);
 		workflows.push({
 			id: workflowId,
-			name: asString(out.workflowName) ?? parsed.workflow.name ?? 'workflow',
-			nodes: (parsed.workflow.nodes ?? []) as Array<Record<string, unknown>>,
-			connections: (parsed.workflow.connections ?? {}) as Record<string, unknown>,
+			name: built.name ?? resolved.name ?? 'workflow',
+			nodes: resolved.nodes,
+			connections: resolved.connections,
 		});
 	}
+	if (degraded.length > 0) {
+		console.warn(
+			`[seed] Thread ${threadId}: ${degraded.length} workflow(s) reconstructed via a fallback source (file replay diverged — likely an untracked shell edit); verify before trusting: ${degraded.join(', ')}`,
+		);
+	}
+	if (skipped.length > 0) {
+		console.warn(
+			`[seed] Thread ${threadId}: ${skipped.length} built workflow(s) could not be reconstructed and were skipped: ${skipped.join(', ')}`,
+		);
+	}
+
+	// Builds happened but we recovered nothing → throw rather than silently seed 0
+	// workflows (reported as a framework_issue; the message names the likely cause).
+	if (buildSignalIds.size > 0 && workflows.length === 0) {
+		throw new Error(
+			`Thread ${threadId}: ${buildSignalIds.size} workflow(s) were built in the trace but reconstruction recovered 0 — the build tool was likely renamed or its input/output shape changed (e.g. inline-code → filePath). Update reconstruction (WORKFLOW_BUILD_TOOLS / source extraction in buildSeedWorkflows).`,
+		);
+	}
+	if (workflows.length < buildSignalIds.size) {
+		console.warn(
+			`[seed] Thread ${threadId}: reconstructed ${workflows.length}/${buildSignalIds.size} built workflow(s) — partial; check for trace-shape drift if unexpected.`,
+		);
+	}
 
 	return workflows;
 }
 
+/** Parse reconstructed SDK code into workflow JSON, returning undefined instead
+ *  of throwing so a caller can fall back to another source. */
+function tryParseSeedWorkflow(
+	code: string,
+):
+	| { name?: string; nodes: Array<Record<string, unknown>>; connections: Record<string, unknown> }
+	| undefined {
+	try {
+		const { workflow } = parseSeedWorkflowCode(code);
+		return {
+			name: workflow.name,
+			nodes: (workflow.nodes ?? []) as unknown as Array<Record<string, unknown>>,
+			connections: (workflow.connections ?? {}) as Record<string, unknown>,
+		};
+	} catch {
+		return undefined;
+	}
+}
+
 type DataTableColumnType = 'string' | 'number' | 'boolean' | 'date';
 const DATA_TABLE_COLUMN_TYPES = new Set<string>(['string', 'number', 'boolean', 'date']);
 function isDataTableColumnType(value: string): value is DataTableColumnType {
@@ -405,7 +606,7 @@ function buildSeedDataTables(toolRuns: Run[], boundaryMs: number): ConversationS
 	const created = new Map<string, ConversationSeed['dataTables'][number]>();
 
 	for (const tool of toolRuns) {
-		if (new Date(tool.start_time).getTime() >= boundaryMs) continue;
+		if (new Date(tool.start_time ?? 0).getTime() >= boundaryMs) continue;
 		const out = isRecord(tool.outputs) ? tool.outputs : {};
 
 		// A `create`: output carries the new table's id, name and columns.

packages/@n8n/instance-ai/evaluations/harness/runner.ts

@@ -40,7 +40,11 @@ import { allFailVerdicts, verifyBuildExpectations } from '../build-expectations/
 import { type VerifierAttemptDebug, verifyChecklist } from '../checklist/verifier';
 import { N8nApiError, type N8nClient, type WorkflowResponse } from '../clients/n8n-client';
 import { createDeclaredCredentials } from '../credentials/seeder';
-import { buildConversationMetrics, extractOutcomeFromEvents } from '../outcome/event-parser';
+import {
+	buildConversationMetrics,
+	extractOutcomeFromEvents,
+	mergeSeededConversationMetrics,
+} from '../outcome/event-parser';
 import { buildTranscriptFromEvents } from '../outcome/transcript-from-events';
 import { buildAgentOutcome, extractWorkflowIdsFromMessages } from '../outcome/workflow-discovery';
 import type {
@@ -61,6 +65,7 @@ import type {
 import {
 	conversationUserTurnsAsText,
 	failedBuildsPerTurn,
+	lastAgentText,
 	userTurnsAsText,
 } from '../utils/conversation-text';
 import { UserProxyLlm, type ProxyDecisionStats } from '../utils/user-proxy';
@@ -621,7 +626,10 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 		abortController.abort();
 		await ssePromise.catch(() => {});
 
-		const conversationMetrics = buildConversationMetrics(events);
+		const conversationMetrics = mergeSeededConversationMetrics(
+			seededTranscript,
+			buildConversationMetrics(events),
+		);
 		const transcript = [
 			...seededTranscript,
 			...buildTranscriptFromEvents({
@@ -647,7 +655,8 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
 			...new Set([...eventOutcome.workflowIds, ...messageWorkflowIds, ...restoredWorkflowIds]),
 		];
 		const buildTrace: BuildTrace = {
-			finalText: eventOutcome.finalText,
+			finalText:
+				eventOutcome.finalText.length > 0 ? eventOutcome.finalText : lastAgentText(transcript),
 			toolCalls: eventOutcome.toolCalls,
 			agentActivities: eventOutcome.agentActivities,
 		};

packages/@n8n/instance-ai/evaluations/outcome/event-parser.ts

@@ -11,6 +11,7 @@ import type {
 	ConversationMetrics,
 	EventOutcome,
 	InstanceAiMetrics,
+	TranscriptTurn,
 	TurnCounter,
 } from '../types';
 import { getNestedRecord as getRecord, getString } from '../utils/safe-extract';
@@ -401,6 +402,82 @@ export function buildConversationMetrics(events: CapturedEvent[]): ConversationM
 	};
 }
 
+/** Per-turn counters for the SEEDED prefix. Seeded turns emit no SSE events, so we
+ *  count tool calls + confirmations from step kinds (mirrors buildConversationMetrics).
+ *  Best-effort: replanAfterError / repeatQuestion / runFinishStatus aren't recoverable. */
+export function seededTurnCounters(seededTurns: TranscriptTurn[]): TurnCounter[] {
+	return seededTurns.map((turn, i) => {
+		const counter: TurnCounter = {
+			turn: i + 1,
+			toolCallCount: 0,
+			toolErrorCount: 0,
+			confirmationAskedTotal: 0,
+			confirmationAskedByKind: {},
+			replanAfterErrorCount: 0,
+			repeatQuestionCount: 0,
+		};
+		for (const step of turn.steps) {
+			// Every non-narration step is a tool call (+1); in a live run the HITL
+			// ones also emit a confirmation-request, counted below.
+			if (step.kind === 'agent-text') continue;
+			counter.toolCallCount++;
+			switch (step.kind) {
+				case 'tool-call':
+					if (step.error !== undefined) counter.toolErrorCount++;
+					break;
+				case 'ask-user':
+					counter.confirmationAskedTotal++;
+					counter.confirmationAskedByKind.questions =
+						(counter.confirmationAskedByKind.questions ?? 0) + 1;
+					break;
+				case 'setup-card':
+					counter.confirmationAskedTotal++;
+					counter.confirmationAskedByKind.setup = (counter.confirmationAskedByKind.setup ?? 0) + 1;
+					break;
+				case 'confirmation': {
+					counter.confirmationAskedTotal++;
+					const kind = step.resumeReason;
+					counter.confirmationAskedByKind[kind] = (counter.confirmationAskedByKind[kind] ?? 0) + 1;
+					break;
+				}
+				default:
+					break; // plan, setup-wizard — tool call only
+			}
+		}
+		return counter;
+	});
+}
+
+/** Prepend the seeded prefix's counters to live metrics so a seedThread case's
+ *  metrics span the whole conversation (matching the unified transcript). Live
+ *  `reachedRunFinishCleanly` is preserved (it describes the evaluated run); an
+ *  empty prefix returns metrics deep-equal to the live ones. */
+export function mergeSeededConversationMetrics(
+	seededTurns: TranscriptTurn[],
+	liveMetrics: ConversationMetrics,
+): ConversationMetrics {
+	const seededPerTurn = seededTurnCounters(seededTurns);
+	const perTurn = [...seededPerTurn, ...liveMetrics.perTurn].map((counter, i) => ({
+		...counter,
+		turn: i + 1,
+	}));
+	const confirmationAskedByKind = { ...liveMetrics.confirmationAskedByKind };
+	let confirmationAskedTotal = liveMetrics.confirmationAskedTotal;
+	for (const counter of seededPerTurn) {
+		confirmationAskedTotal += counter.confirmationAskedTotal;
+		for (const [kind, count] of Object.entries(counter.confirmationAskedByKind)) {
+			confirmationAskedByKind[kind] = (confirmationAskedByKind[kind] ?? 0) + count;
+		}
+	}
+	return {
+		turnCount: seededPerTurn.length + liveMetrics.turnCount,
+		perTurn,
+		confirmationAskedTotal,
+		confirmationAskedByKind,
+		reachedRunFinishCleanly: liveMetrics.reachedRunFinishCleanly,
+	};
+}
+
 /** Split events into turns. Each turn begins at a `run-start` event; events
  *  before the first `run-start` form a leading pseudo-turn (unusual but handled). */
 export function splitEventsIntoTurns(events: CapturedEvent[]): CapturedEvent[][] {

packages/@n8n/instance-ai/evaluations/report/workflow-report.ts

@@ -777,7 +777,13 @@ function renderConversationTranscript(transcript: TranscriptTurn[] | undefined):
 }
 
 function renderTranscriptTurn(turn: TranscriptTurn, turnNum: number): string {
-	const parts: string[] = [`<div class="transcript-turn-header">Turn ${String(turnNum)}</div>`];
+	// Judged as part of the whole conversation; marked subtly only for human readers.
+	const seededTag = turn.seeded
+		? ' <span class="transcript-seeded" title="restored prior context — not part of the evaluated run">seeded</span>'
+		: '';
+	const parts: string[] = [
+		`<div class="transcript-turn-header">Turn ${String(turnNum)}${seededTag}</div>`,
+	];
 	if (turn.userMessage) {
 		parts.push(
 			`<div class="transcript-line transcript-user"><span class="transcript-icon">👤</span><span class="transcript-text">${escapeHtml(turn.userMessage)}</span></div>`,
@@ -787,7 +793,7 @@ function renderTranscriptTurn(turn: TranscriptTurn, turnNum: number): string {
 		const block = renderStep(step);
 		if (block) parts.push(block);
 	}
-	return `<div class="transcript-turn">${parts.join('')}</div>`;
+	return `<div class="transcript-turn${turn.seeded ? ' seeded' : ''}">${parts.join('')}</div>`;
 }
 
 function renderStep(step: TranscriptStep): string | null {
@@ -1484,6 +1490,8 @@ export function generateWorkflowReport(results: WorkflowTestCaseResult[]): strin
 	.transcript-turn { padding: 8px 0; border-bottom: 1px dashed var(--border-light); }
 	.transcript-turn:last-child { border-bottom: none; }
 	.transcript-turn-header { font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text-muted); margin-bottom: 6px; }
+	.transcript-turn.seeded { border-left: 2px solid var(--border-light); padding-left: 10px; }
+	.transcript-seeded { text-transform: none; letter-spacing: 0; color: var(--text-muted); border: 1px solid var(--border-light); border-radius: 3px; padding: 0 5px; margin-left: 6px; }
 	.transcript-line { display: flex; gap: 8px; padding: 4px 0; align-items: flex-start; font-size: 13px; line-height: 1.5; }
 	.transcript-icon { width: 18px; text-align: center; flex-shrink: 0; }
 	.transcript-text { color: var(--text-primary); white-space: pre-wrap; }

packages/@n8n/instance-ai/evaluations/system-prompts/build-expectations-verify.ts

@@ -6,12 +6,13 @@ These expectations are NOT about whether the workflow executes correctly — the
 
 1. **Workflow structure**: the final built workflow — every node, its config, and the connections JSON. If no workflow was built (the build failed), this says "(no workflow built)".
 2. **Conversation transcript**: the full multi-turn conversation, both sides, turn by turn. Each turn shows the user message, the agent's text, and compact summaries of tool interactions — plans, questions asked to the user (with answers), setup cards the agent showed (the values it asked the user to provide — credentials and/or parameters — and whether the user filled or dismissed them), setup-wizard configured/skipped nodes, resumed confirmations (approved/rejected), and the names of tools called.
-3. **Conversation metrics (ground truth)**: deterministic counters extracted from the event stream — turn count, how many confirmations were asked and of which kind, whether the run finished cleanly. Treat these as authoritative. Do NOT recount turns or confirmations from the raw transcript when a metric already states the number.
+3. **Conversation metrics (ground truth)**: deterministic counters spanning the whole conversation (restored/seeded context plus the live evaluated turn) — turn count, per-turn tool-call and confirmation counts, and how many confirmations were asked and of which kind. The reachedRunFinishCleanly flag and any per-turn runFinishStatus describe ONLY the evaluated (live) run; seeded turns carry no finish status, so a blank status on a seeded turn does not indicate an unfinished run. Treat these as authoritative. Do NOT recount turns or confirmations from the raw transcript when a metric already states the number.
 4. **Expectations**: a numbered list (indices start at 0). Judge each one.
 
 ## How to judge
 
 - Judge **each expectation independently** and literally. Read what it actually asserts.
+- The unit of evaluation is the **whole conversation** — every turn (both sides) and the tool interactions within them, not just the latest turn. Follow each expectation's own specifics: when it calls out a particular turn, moment, or ordering, hold it to that.
 - **Process / temporal claims** ("asked X before building", "pushed back on the plan", "asked N questions") → judge from the transcript and the metrics. Order matters: "asked before building" means the question appears in the transcript prior to the workflow being created/finalized.
 - **Setup cards** are one of the ways the builder asks the user for node configuration — it surfaces a card listing the credentials and/or parameters it needs the user to fill. A value appearing in a setup card means the builder asked the user to provide that value (the user may fill or dismiss it); treat it the same as the builder asking for that value directly.
 - **Outcome claims** ("the final workflow contains a Switch routing to Slack", "the follow-up change is reflected") → judge from the workflow structure. If no workflow was built, any expectation that requires the final workflow **fails**.

packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts

@@ -57,12 +57,8 @@ export function conversationUserTurnsAsText(conversation: ConversationTurn[] | u
 export function transcriptAsText(transcript: TranscriptTurn[]): string {
 	return transcript
 		.map((turn, i) => {
-			// Seeded turns are restored prior context — they predate the evaluated
-			// run, and judges must not score them as live behaviour.
-			const seededSuffix = turn.seeded
-				? ' (seeded prior context — predates the evaluated run)'
-				: '';
-			const lines: string[] = [`### Turn ${String(i + 1)}${seededSuffix}`];
+			// No seeded label: the judge evaluates the whole conversation as one.
+			const lines: string[] = [`### Turn ${String(i + 1)}`];
 			if (turn.userMessage) lines.push(`User: ${turn.userMessage}`);
 			for (const step of turn.steps) {
 				const line = describeStep(step);
@@ -78,6 +74,16 @@ export function agentTextOf(turn: TranscriptTurn): string {
 	return turn.steps.flatMap((s) => (s.kind === 'agent-text' ? [s.text] : [])).join('');
 }
 
+/** The most recent turn's agent narration — a finalText fallback for seeded
+ *  conversations whose live turn produced no text-delta events. */
+export function lastAgentText(transcript: TranscriptTurn[]): string {
+	for (let i = transcript.length - 1; i >= 0; i--) {
+		const text = agentTextOf(transcript[i]);
+		if (text.length > 0) return text;
+	}
+	return '';
+}
+
 /** Tool id the builder calls to create or modify the workflow graph. */
 export const BUILD_WORKFLOW_TOOL_NAME = 'build-workflow';