diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.test.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.test.ts index 3d34d5eb8de..1de2ec03ea8 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.test.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.test.ts @@ -416,6 +416,24 @@ describe('generateEvalPinData', () => { expect(result.Slack).toHaveLength(1); }); + it('should read fenced JSON that the model explains afterwards', async () => { + const workflow: SimpleWorkflow = { + name: 'Test', + nodes: [makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' })], + connections: {}, + }; + + const llmResponse = + '```json\n{"Slack": [{"json": {"ok": true}}]}\n```\n\n**Slack:** the post succeeds, so one item.'; + + const result = await generateEvalPinData(workflow, { + llm: createMockLLM(llmResponse), + nodeTypes, + }); + + expect(result.Slack).toHaveLength(1); + }); + it('should wrap raw objects in { json: ... } format', async () => { const workflow: SimpleWorkflow = { name: 'Test', diff --git a/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.ts b/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.ts index ca43d746afb..61740f55887 100644 --- a/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.ts +++ b/packages/@n8n/ai-workflow-builder.ee/evaluations/support/pin-data-generator.ts @@ -14,6 +14,7 @@ import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; import { HumanMessage, SystemMessage } from '@langchain/core/messages'; +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import type { NodeSchemaContext, OutputSchemaLookup, WorkflowJSON } from '@n8n/workflow-sdk'; import { buildDateAnchors, @@ -195,7 +196,8 @@ export async function generateEvalPinData( const responseText = typeof response.content === 'string' ? response.content : JSON.stringify(response.content); - const pinData = parsePinDataResponse(responseText, expectedNodeNames); + // The model sometimes explains its answer after the JSON block. + const pinData = parsePinDataResponse(extractJsonCandidate(responseText), expectedNodeNames); // Envelope repair for parser-target roots; the shared helper derives the // envelope key from each root's with-parser `__schema__` variant diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/pin-data-generator.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/pin-data-generator.test.ts index f9ede10c4db..8a62bf26e22 100644 --- a/packages/cli/src/modules/instance-ai/eval/__tests__/pin-data-generator.test.ts +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/pin-data-generator.test.ts @@ -61,6 +61,16 @@ describe('generatePinData', () => { expect(result).toEqual({ 'Get Posted Keys': [] }); }); + it('reads the fenced JSON when the model explains it afterwards', async () => { + respondWith( + '```json\n{ "Get Posted Keys": [] }\n```\n\n**Get Posted Keys (`[]`):** every key is already stored, so the read returns no rows.', + ); + + const result = await generatePinData({ workflow, nodeNames: ['Get Posted Keys'] }); + + expect(result).toEqual({ 'Get Posted Keys': [] }); + }); + it('wraps raw items and passes json-wrapped items through', async () => { respondWith( JSON.stringify({ diff --git a/packages/cli/src/modules/instance-ai/eval/pin-data-generator.ts b/packages/cli/src/modules/instance-ai/eval/pin-data-generator.ts index 9844fc58906..8e3bb0425d5 100644 --- a/packages/cli/src/modules/instance-ai/eval/pin-data-generator.ts +++ b/packages/cli/src/modules/instance-ai/eval/pin-data-generator.ts @@ -11,6 +11,7 @@ * LLM call (`createEvalAgent`) and the `__schema__` lookup wiring. */ +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import { createEvalAgent, extractText } from '@n8n/instance-ai'; import type { WorkflowJSON, @@ -109,7 +110,8 @@ export async function generatePinData(options: GeneratePinDataOptions): Promise< abortSignal: AbortSignal.timeout(PIN_DATA_LLM_TIMEOUT_MS), }); - const responseText = extractText(result); + // The model sometimes explains its answer after the JSON block. + const responseText = extractJsonCandidate(extractText(result)); const pinData = parsePinDataResponse(responseText, expectedNodeNames); const missing = expectedNodeNames.filter((name) => !(name in pinData));