From 4f3000394feacc901d7a07fac78fd50eb0c4890f Mon Sep 17 00:00:00 2001 From: Jose Date: Sat, 25 Jul 2026 16:07:53 +0100 Subject: [PATCH] fix(ai-builder): Snapshot eval ledger bodies and normalize Gmail list envelopes (no-changelog) Two judge-evidence/mock-fidelity gaps found while grading the TRUST-344 validation run: - The HTTP-path ledger stored the served body BY REFERENCE. Nodes that mutate the response in place (the OpenAI node's json_schema output mode parses output[].content[].text into an object) retroactively rewrite the ledger, so judges see a body the mock never sent and misattribute builder-side failures to mock_issue. Ledger entries are now JSON-round-trip snapshots (Buffers pass by reference). - Gmail messages.list answered as a bare array (a recurring model slip) yields zero messages in the node. Deterministic backstop wraps a bare array into the { messages, resultSizeEstimate } envelope, with a matching submit_response violation nudge. Co-Authored-By: Claude Fable 5 --- .claude/settings.json | 2 +- .../eval/__tests__/execution.service.test.ts | 30 +++++++++++- .../eval/__tests__/provider-shapes.test.ts | 48 +++++++++++++++++++ .../instance-ai/eval/execution.service.ts | 17 ++++++- .../instance-ai/eval/provider-shapes.ts | 30 ++++++++++++ 5 files changed, 124 insertions(+), 3 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index 355cd61d289..d77b6e72a61 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -12,7 +12,7 @@ "Bash(popd)", "Bash(pushd:*)", "Bash(mkdir -p .claude/plans)", - "Write(.claude/plans/*)" + "Edit(.claude/plans/*)" ] }, "hooks": { diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts index 23dce50273a..98b92b0f0ed 100644 --- a/packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts @@ -94,7 +94,7 @@ vi.mock('n8n-workflow', async () => { // Import SUT and mocked modules (after vi.mock calls) // --------------------------------------------------------------------------- -import { EvalExecutionService } from '../execution.service'; +import { EvalExecutionService, snapshotLedgerBody } from '../execution.service'; import { createLlmMockHandler } from '../mock-handler'; import { generatePinData } from '../pin-data-generator'; import { @@ -1473,3 +1473,31 @@ describe('EvalExecutionService', () => { }); }); }); + +describe('snapshotLedgerBody', () => { + it('detaches the ledger entry from later in-place mutations by node code', () => { + const served = { output: [{ content: [{ type: 'output_text', text: '{"a":1}' }] }] }; + const snapshot = snapshotLedgerBody(served) as typeof served; + + // e.g. the OpenAI node's json_schema output mode parses text in place + served.output[0].content[0].text = { a: 1 } as never; + + expect(snapshot.output[0].content[0].text).toBe('{"a":1}'); + }); + + it('passes Buffers and nullish bodies through by reference', () => { + const buffer = Buffer.from('bytes'); + expect(snapshotLedgerBody(buffer)).toBe(buffer); + expect(snapshotLedgerBody(undefined)).toBeUndefined(); + expect(snapshotLedgerBody(null)).toBeNull(); + }); + + it('detaches circular bodies without throwing', () => { + const circular: Record = { tag: 'original' }; + circular.self = circular; + const snapshot = snapshotLedgerBody(circular) as Record; + circular.tag = 'mutated'; + expect(snapshot).not.toBe(circular); + expect(snapshot.tag).toBe('original'); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/provider-shapes.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/provider-shapes.test.ts index 6b14416473e..8685415d8cf 100644 --- a/packages/cli/src/modules/instance-ai/eval/__tests__/provider-shapes.test.ts +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/provider-shapes.test.ts @@ -249,3 +249,51 @@ describe('findProviderShapeViolation', () => { expect(findProviderShapeViolation(googleDocs, { replies: [{}] })).toBeUndefined(); }); }); + +describe('Gmail messages.list', () => { + const gmailList = info({ + hostname: 'www.googleapis.com', + pathname: '/gmail/v1/users/me/messages', + method: 'GET', + }); + + it('wraps a bare array into the messages envelope', () => { + const spec: Spec = { + type: 'json', + body: [{ id: 'msg_1', threadId: 'thr_1' }], + }; + applyProviderShapeNormalizers(gmailList, spec); + expect(spec.body).toEqual({ + messages: [{ id: 'msg_1', threadId: 'thr_1' }], + resultSizeEstimate: 1, + }); + }); + + it('leaves a correct envelope untouched', () => { + const spec: Spec = { + type: 'json', + body: { messages: [{ id: 'msg_1' }], resultSizeEstimate: 1 }, + }; + const before = structuredClone(spec.body); + applyProviderShapeNormalizers(gmailList, spec); + expect(spec.body).toEqual(before); + }); + + it('does not touch the single-message GET', () => { + const single = info({ + hostname: 'www.googleapis.com', + pathname: '/gmail/v1/users/me/messages/msg_1', + method: 'GET', + }); + const spec: Spec = { type: 'json', body: [{ anything: true }] }; + applyProviderShapeNormalizers(single, spec); + expect(Array.isArray(spec.body)).toBe(true); + }); + + it('reports a violation for a bare array and passes the envelope', () => { + expect(findProviderShapeViolation(gmailList, [{ id: 'msg_1' }])).toContain('bare array'); + expect( + findProviderShapeViolation(gmailList, { messages: [], resultSizeEstimate: 0 }), + ).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/execution.service.ts b/packages/cli/src/modules/instance-ai/eval/execution.service.ts index a5dbf78f4ce..19fe15f047c 100644 --- a/packages/cli/src/modules/instance-ai/eval/execution.service.ts +++ b/packages/cli/src/modules/instance-ai/eval/execution.service.ts @@ -32,6 +32,7 @@ import { type IWorkflowExecuteAdditionalData, type IWorkflowExecutionDataProcess, createRunExecutionData, + deepCopy, fileTypeFromMimeType, NodeHelpers, UserError, @@ -812,7 +813,7 @@ export class EvalExecutionService { method: requestOptions.method ?? 'GET', nodeType: node.type, requestBody: requestOptions.body, - mockResponse: response?.body, + mockResponse: snapshotLedgerBody(response?.body), }); this.logger.debug( @@ -1001,6 +1002,20 @@ export class EvalExecutionService { } } +/** + * Ledger entries must be immutable snapshots. The served body object is handed + * to node code, and some nodes mutate it in place — e.g. the OpenAI node's + * json_schema output mode parses `output[].content[].text` into an object — + * so an aliased ledger entry rewrites history and the judge blames the mock + * for a body it never sent. `deepCopy` matches the artifact's JSON semantics; + * Buffers pass by reference (response bytes are not mutated in place, and + * copying megabyte buffers element-wise would be wasteful). + */ +export function snapshotLedgerBody(body: unknown): unknown { + if (body === undefined || body === null || Buffer.isBuffer(body)) return body; + return deepCopy(body); +} + /** Synthesize a structurally valid binary entry (real bytes, base64-inlined). */ function synthesizeBinaryEntry(contentType: string, filename: string): IBinaryData { const bytes = synthesizeBinaryFixture(contentType, filename); diff --git a/packages/cli/src/modules/instance-ai/eval/provider-shapes.ts b/packages/cli/src/modules/instance-ai/eval/provider-shapes.ts index 23dc49718fc..709daa22211 100644 --- a/packages/cli/src/modules/instance-ai/eval/provider-shapes.ts +++ b/packages/cli/src/modules/instance-ai/eval/provider-shapes.ts @@ -80,6 +80,14 @@ function isGoogleDocsBatchUpdate(host: string, path: string): boolean { ); } +function isGmailMessagesList(host: string, path: string, method: string): boolean { + return ( + method.toUpperCase() === 'GET' && + host.endsWith('googleapis.com') && + /\/gmail\/v1\/users\/[^/]+\/messages\/?$/.test(path) + ); +} + // --------------------------------------------------------------------------- // Tolerant text extraction (shared by the Gemini normalizer) // --------------------------------------------------------------------------- @@ -234,6 +242,21 @@ function resolveHubspotVid(body: Record): number { return DEFAULT_HUBSPOT_VID; } +// --------------------------------------------------------------------------- +// Gmail — messages.list +// --------------------------------------------------------------------------- + +/** + * The Gmail node's list operation reads `responseData.messages`. A recurring + * model slip (despite quirk guidance) is answering with a bare ARRAY of + * message objects — the node then sees zero messages and the workflow's list + * branch silently does nothing. Wrap a bare array into the real envelope. + */ +function normalizeGmailMessagesList(spec: NormalizableSpec): void { + if (!Array.isArray(spec.body)) return; + spec.body = { messages: spec.body, resultSizeEstimate: spec.body.length }; +} + // --------------------------------------------------------------------------- // Google Docs — /documents/{id}:batchUpdate // --------------------------------------------------------------------------- @@ -273,6 +296,7 @@ export function applyProviderShapeNormalizers( if (redditKind) return normalizeReddit(spec, redditKind); if (isHubspotUpsert(host, path)) return normalizeHubspotUpsert(spec); if (isGoogleDocsBatchUpdate(host, path)) return normalizeGoogleDocsBatchUpdate(spec); + if (isGmailMessagesList(host, path, info.method)) return normalizeGmailMessagesList(spec); } /** @@ -293,10 +317,16 @@ export function findProviderShapeViolation( if (redditKind) return redditViolation(body, redditKind); if (isHubspotUpsert(host, path)) return hubspotViolation(body); if (isGoogleDocsBatchUpdate(host, path)) return googleDocsViolation(body); + if (isGmailMessagesList(host, path, info.method)) return gmailMessagesListViolation(body); return undefined; } +function gmailMessagesListViolation(body: unknown): string | undefined { + if (!Array.isArray(body)) return undefined; + return 'Invalid: Gmail messages.list returns an OBJECT envelope `{ "messages": [{ "id", "threadId" }, ...], "resultSizeEstimate": }` — the node reads response.messages, so a bare array yields zero messages. Resubmit wrapped in that envelope.'; +} + function openAiImagesViolation(body: unknown): string | undefined { const data = isPlainObject(body) ? body.data : undefined; if (!Array.isArray(data) || data.length === 0) {