fix(ai-builder): Read eval pin data when the model explains it after the JSON (no-changelog) (#40682)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.test.ts

@@ -416,6 +416,24 @@ describe('generateEvalPinData', () => {
 		expect(result.Slack).toHaveLength(1);
 	});
 
+	it('should read fenced JSON that the model explains afterwards', async () => {
+		const workflow: SimpleWorkflow = {
+			name: 'Test',
+			nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })],
+			connections: {},
+		};
+
+		const llmResponse =
+			'```json\n{"Slack": [{"json": {"ok": true}}]}\n```\n\n**Slack:** the post succeeds, so one item.';
+
+		const result = await generateEvalPinData(workflow, {
+			llm: createMockLLM(llmResponse),
+			nodeTypes,
+		});
+
+		expect(result.Slack).toHaveLength(1);
+	});
+
 	it('should wrap raw objects in { json: ... } format', async () => {
 		const workflow: SimpleWorkflow = {
 			name: 'Test',

packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.ts

@@ -14,6 +14,7 @@
 
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { HumanMessage, SystemMessage } from '@langchain/core/messages';
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import type { NodeSchemaContext, OutputSchemaLookup, WorkflowJSON } from '@n8n/workflow-sdk';
 import {
 	buildDateAnchors,
@@ -195,7 +196,8 @@ export async function generateEvalPinData(
 		const responseText =
 			typeof response.content === 'string' ? response.content : JSON.stringify(response.content);
 
-		const pinData = parsePinDataResponse(responseText, expectedNodeNames);
+		// The model sometimes explains its answer after the JSON block.
+		const pinData = parsePinDataResponse(extractJsonCandidate(responseText), expectedNodeNames);
 
 		// Envelope repair for parser-target roots; the shared helper derives the
 		// envelope key from each root's with-parser `__schema__` variant

packages/cli/src/modules/instance-ai/eval/__tests__/pin-data-generator.test.ts

@@ -61,6 +61,16 @@ describe('generatePinData', () => {
 		expect(result).toEqual({ 'Get Posted Keys': [] });
 	});
 
+	it('reads the fenced JSON when the model explains it afterwards', async () => {
+		respondWith(
+			'```json\n{ "Get Posted Keys": [] }\n```\n\n**Get Posted Keys (`[]`):** every key is already stored, so the read returns no rows.',
+		);
+
+		const result = await generatePinData({ workflow, nodeNames: ['Get Posted Keys'] });
+
+		expect(result).toEqual({ 'Get Posted Keys': [] });
+	});
+
 	it('wraps raw items and passes json-wrapped items through', async () => {
 		respondWith(
 			JSON.stringify({

packages/cli/src/modules/instance-ai/eval/pin-data-generator.ts

@@ -11,6 +11,7 @@
  * LLM call (`createEvalAgent`) and the `__schema__` lookup wiring.
  */
 
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import { createEvalAgent, extractText } from '@n8n/instance-ai';
 import type {
 	WorkflowJSON,
@@ -109,7 +110,8 @@ export async function generatePinData(options: GeneratePinDataOptions): Promise<
 			abortSignal: AbortSignal.timeout(PIN_DATA_LLM_TIMEOUT_MS),
 		});
 
-		const responseText = extractText(result);
+		// The model sometimes explains its answer after the JSON block.
+		const responseText = extractJsonCandidate(extractText(result));
 		const pinData = parsePinDataResponse(responseText, expectedNodeNames);
 
 		const missing = expectedNodeNames.filter((name) => !(name in pinData));