Merge 4f3000394f into 4f7c725b83

.claude/settings.json

@@ -12,7 +12,7 @@
 			"Bash(popd)",
 			"Bash(pushd:*)",
 			"Bash(mkdir -p .claude/plans)",
-			"Write(.claude/plans/*)"
+			"Edit(.claude/plans/*)"
 		]
 	},
 	"hooks": {

packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts

@@ -94,7 +94,7 @@ vi.mock('n8n-workflow', async () => {
 // Import SUT and mocked modules (after vi.mock calls)
 // ---------------------------------------------------------------------------
 
-import { EvalExecutionService } from '../execution.service';
+import { EvalExecutionService, snapshotLedgerBody } from '../execution.service';
 import { createLlmMockHandler } from '../mock-handler';
 import { generatePinData } from '../pin-data-generator';
 import {
@@ -1473,3 +1473,31 @@ describe('EvalExecutionService', () => {
 		});
 	});
 });
+
+describe('snapshotLedgerBody', () => {
+	it('detaches the ledger entry from later in-place mutations by node code', () => {
+		const served = { output: [{ content: [{ type: 'output_text', text: '{"a":1}' }] }] };
+		const snapshot = snapshotLedgerBody(served) as typeof served;
+
+		// e.g. the OpenAI node's json_schema output mode parses text in place
+		served.output[0].content[0].text = { a: 1 } as never;
+
+		expect(snapshot.output[0].content[0].text).toBe('{"a":1}');
+	});
+
+	it('passes Buffers and nullish bodies through by reference', () => {
+		const buffer = Buffer.from('bytes');
+		expect(snapshotLedgerBody(buffer)).toBe(buffer);
+		expect(snapshotLedgerBody(undefined)).toBeUndefined();
+		expect(snapshotLedgerBody(null)).toBeNull();
+	});
+
+	it('detaches circular bodies without throwing', () => {
+		const circular: Record<string, unknown> = { tag: 'original' };
+		circular.self = circular;
+		const snapshot = snapshotLedgerBody(circular) as Record<string, unknown>;
+		circular.tag = 'mutated';
+		expect(snapshot).not.toBe(circular);
+		expect(snapshot.tag).toBe('original');
+	});
+});

packages/cli/src/modules/instance-ai/eval/__tests__/provider-shapes.test.ts

@@ -249,3 +249,51 @@ describe('findProviderShapeViolation', () => {
 		expect(findProviderShapeViolation(googleDocs, { replies: [{}] })).toBeUndefined();
 	});
 });
+
+describe('Gmail messages.list', () => {
+	const gmailList = info({
+		hostname: 'www.googleapis.com',
+		pathname: '/gmail/v1/users/me/messages',
+		method: 'GET',
+	});
+
+	it('wraps a bare array into the messages envelope', () => {
+		const spec: Spec = {
+			type: 'json',
+			body: [{ id: 'msg_1', threadId: 'thr_1' }],
+		};
+		applyProviderShapeNormalizers(gmailList, spec);
+		expect(spec.body).toEqual({
+			messages: [{ id: 'msg_1', threadId: 'thr_1' }],
+			resultSizeEstimate: 1,
+		});
+	});
+
+	it('leaves a correct envelope untouched', () => {
+		const spec: Spec = {
+			type: 'json',
+			body: { messages: [{ id: 'msg_1' }], resultSizeEstimate: 1 },
+		};
+		const before = structuredClone(spec.body);
+		applyProviderShapeNormalizers(gmailList, spec);
+		expect(spec.body).toEqual(before);
+	});
+
+	it('does not touch the single-message GET', () => {
+		const single = info({
+			hostname: 'www.googleapis.com',
+			pathname: '/gmail/v1/users/me/messages/msg_1',
+			method: 'GET',
+		});
+		const spec: Spec = { type: 'json', body: [{ anything: true }] };
+		applyProviderShapeNormalizers(single, spec);
+		expect(Array.isArray(spec.body)).toBe(true);
+	});
+
+	it('reports a violation for a bare array and passes the envelope', () => {
+		expect(findProviderShapeViolation(gmailList, [{ id: 'msg_1' }])).toContain('bare array');
+		expect(
+			findProviderShapeViolation(gmailList, { messages: [], resultSizeEstimate: 0 }),
+		).toBeUndefined();
+	});
+});

packages/cli/src/modules/instance-ai/eval/execution.service.ts

@@ -32,6 +32,7 @@ import {
 	type IWorkflowExecuteAdditionalData,
 	type IWorkflowExecutionDataProcess,
 	createRunExecutionData,
+	deepCopy,
 	fileTypeFromMimeType,
 	NodeHelpers,
 	UserError,
@@ -812,7 +813,7 @@ export class EvalExecutionService {
 				method: requestOptions.method ?? 'GET',
 				nodeType: node.type,
 				requestBody: requestOptions.body,
-				mockResponse: response?.body,
+				mockResponse: snapshotLedgerBody(response?.body),
 			});
 
 			this.logger.debug(
@@ -1001,6 +1002,20 @@ export class EvalExecutionService {
 	}
 }
 
+/**
+ * Ledger entries must be immutable snapshots. The served body object is handed
+ * to node code, and some nodes mutate it in place — e.g. the OpenAI node's
+ * json_schema output mode parses `output[].content[].text` into an object —
+ * so an aliased ledger entry rewrites history and the judge blames the mock
+ * for a body it never sent. `deepCopy` matches the artifact's JSON semantics;
+ * Buffers pass by reference (response bytes are not mutated in place, and
+ * copying megabyte buffers element-wise would be wasteful).
+ */
+export function snapshotLedgerBody(body: unknown): unknown {
+	if (body === undefined || body === null || Buffer.isBuffer(body)) return body;
+	return deepCopy(body);
+}
+
 /** Synthesize a structurally valid binary entry (real bytes, base64-inlined). */
 function synthesizeBinaryEntry(contentType: string, filename: string): IBinaryData {
 	const bytes = synthesizeBinaryFixture(contentType, filename);

packages/cli/src/modules/instance-ai/eval/provider-shapes.ts

@@ -80,6 +80,14 @@ function isGoogleDocsBatchUpdate(host: string, path: string): boolean {
 	);
 }
 
+function isGmailMessagesList(host: string, path: string, method: string): boolean {
+	return (
+		method.toUpperCase() === 'GET' &&
+		host.endsWith('googleapis.com') &&
+		/\/gmail\/v1\/users\/[^/]+\/messages\/?$/.test(path)
+	);
+}
+
 // ---------------------------------------------------------------------------
 // Tolerant text extraction (shared by the Gemini normalizer)
 // ---------------------------------------------------------------------------
@@ -234,6 +242,21 @@ function resolveHubspotVid(body: Record<string, unknown>): number {
 	return DEFAULT_HUBSPOT_VID;
 }
 
+// ---------------------------------------------------------------------------
+// Gmail — messages.list
+// ---------------------------------------------------------------------------
+
+/**
+ * The Gmail node's list operation reads `responseData.messages`. A recurring
+ * model slip (despite quirk guidance) is answering with a bare ARRAY of
+ * message objects — the node then sees zero messages and the workflow's list
+ * branch silently does nothing. Wrap a bare array into the real envelope.
+ */
+function normalizeGmailMessagesList(spec: NormalizableSpec): void {
+	if (!Array.isArray(spec.body)) return;
+	spec.body = { messages: spec.body, resultSizeEstimate: spec.body.length };
+}
+
 // ---------------------------------------------------------------------------
 // Google Docs — /documents/{id}:batchUpdate
 // ---------------------------------------------------------------------------
@@ -273,6 +296,7 @@ export function applyProviderShapeNormalizers(
 	if (redditKind) return normalizeReddit(spec, redditKind);
 	if (isHubspotUpsert(host, path)) return normalizeHubspotUpsert(spec);
 	if (isGoogleDocsBatchUpdate(host, path)) return normalizeGoogleDocsBatchUpdate(spec);
+	if (isGmailMessagesList(host, path, info.method)) return normalizeGmailMessagesList(spec);
 }
 
 /**
@@ -293,10 +317,16 @@ export function findProviderShapeViolation(
 	if (redditKind) return redditViolation(body, redditKind);
 	if (isHubspotUpsert(host, path)) return hubspotViolation(body);
 	if (isGoogleDocsBatchUpdate(host, path)) return googleDocsViolation(body);
+	if (isGmailMessagesList(host, path, info.method)) return gmailMessagesListViolation(body);
 
 	return undefined;
 }
 
+function gmailMessagesListViolation(body: unknown): string | undefined {
+	if (!Array.isArray(body)) return undefined;
+	return 'Invalid: Gmail messages.list returns an OBJECT envelope `{ "messages": [{ "id", "threadId" }, ...], "resultSizeEstimate": <n> }` — the node reads response.messages, so a bare array yields zero messages. Resubmit wrapped in that envelope.';
+}
+
 function openAiImagesViolation(body: unknown): string | undefined {
 	const data = isPlainObject(body) ? body.data : undefined;
 	if (!Array.isArray(data) || data.length === 0) {