feat(ai-builder): Move eval mocks to Sonnet 5.5 (no-changelog) (#40721)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.github/workflows/test-evals-instance-ai.yml

@@ -346,7 +346,7 @@ jobs:
           # ANTHROPIC_API_KEY is the provider-native key resolveEvalModelConfig prefers
           # when N8N_INSTANCE_AI_EVAL_MODEL differs from N8N_INSTANCE_AI_MODEL.
           MODEL_ARGS+=(
-            -e N8N_INSTANCE_AI_EVAL_MODEL=anthropic/claude-sonnet-4-6
+            -e N8N_INSTANCE_AI_EVAL_MODEL=anthropic/claude-sonnet-5-5
             -e ANTHROPIC_API_KEY="$EVALS_ANTHROPIC_KEY"
           )
 

packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts

@@ -481,6 +481,38 @@ describe('EvalExecutionService', () => {
 
 				expect(workflowRunner.run).toHaveBeenCalledTimes(1);
 			});
+
+			it('pins one empty item for a Manual Trigger without trigger content', async () => {
+				nodeTypes.getByNameAndVersion.mockReturnValue({
+					description: { properties: [] } as unknown as INodeTypeDescription,
+					trigger: vi.fn(),
+				} as never);
+				const manualTrigger = {
+					id: 'node-1',
+					name: 'Run',
+					type: 'n8n-nodes-base.manualTrigger',
+					typeVersion: 1,
+					position: [0, 0],
+					parameters: {},
+				} as INode;
+				mockGetStartNode.mockReturnValue(manualTrigger);
+				workflowFinderService.findWorkflowForUser.mockResolvedValue(
+					makeWorkflowEntity({ nodes: [manualTrigger] }) as never,
+				);
+				const hints = makeEmptyHints();
+				hints.triggerContent = {};
+				generateMockHintsMock.mockResolvedValue(hints);
+
+				await service.executeWithLlmMock('wf-1', makeUser());
+
+				expect(generateMockHintsMock).toHaveBeenCalledWith(
+					expect.objectContaining({ defaultStartNodeName: 'Run' }),
+				);
+				const runArg = workflowRunner.run.mock.calls[0][0] as unknown as {
+					pinData?: Record<string, unknown[]>;
+				};
+				expect(runArg.pinData?.Run).toEqual([{ json: {} }]);
+			});
 		});
 
 		describe('engine readiness', () => {

packages/cli/src/modules/instance-ai/eval/__tests__/workflow-analysis.test.ts

@@ -1261,6 +1261,56 @@ describe('generateMockHints', () => {
 		},
 	);
 
+	describe('Manual Trigger start', () => {
+		const manualWorkflow = makeWorkflow([
+			makeNode({ name: 'Run', type: 'n8n-nodes-base.manualTrigger' }),
+			makeNode({ name: 'Schedule', type: 'n8n-nodes-base.scheduleTrigger' }),
+			makeNode({ name: 'Slack', type: 'n8n-nodes-base.slack' }),
+		]);
+
+		it('accepts empty triggerContent without a retry, and keeps the hints', async () => {
+			const generate = mockAgentResponses(
+				JSON.stringify({ globalContext: 'ctx', triggerContent: {}, nodeHints: { Slack: 'foo' } }),
+			);
+
+			const result = await generateMockHints({
+				workflow: manualWorkflow,
+				nodeNames: ['Run', 'Slack'],
+				defaultStartNodeName: 'Run',
+			});
+
+			expect(generate).toHaveBeenCalledTimes(1);
+			expect(result.triggerContent).toEqual({});
+			expect(result.globalContext).toBe('ctx');
+			expect(result.warnings).toEqual([]);
+		});
+
+		it('still retries empty triggerContent when the hints name another trigger', async () => {
+			const generate = mockAgentResponses(
+				JSON.stringify({
+					triggerContent: {},
+					startNodeName: 'Schedule',
+					nodeHints: { Slack: 'foo' },
+				}),
+				JSON.stringify({
+					triggerContent: { timestamp: '2024-01-01T00:00:00Z' },
+					startNodeName: 'Schedule',
+					nodeHints: { Slack: 'foo' },
+				}),
+			);
+
+			const result = await generateMockHints({
+				workflow: manualWorkflow,
+				nodeNames: ['Schedule', 'Slack'],
+				defaultStartNodeName: 'Run',
+			});
+
+			expect(generate).toHaveBeenCalledTimes(2);
+			expect(result.startNodeName).toBe('Schedule');
+			expect(result.triggerContent).toEqual({ timestamp: '2024-01-01T00:00:00Z' });
+		});
+	});
+
 	it('should not call the agent when there are no hint-eligible nodes', async () => {
 		const generate = mockAgentResponses('should never be called');
 

packages/cli/src/modules/instance-ai/eval/execution.service.ts

@@ -35,6 +35,7 @@ import {
 	type IWorkflowExecutionDataProcess,
 	createRunExecutionData,
 	fileTypeFromMimeType,
+	MANUAL_TRIGGER_NODE_TYPE,
 	NodeHelpers,
 	TimeoutExecutionCancelledError,
 	UserError,
@@ -278,12 +279,18 @@ export class EvalExecutionService {
 					workflow: workflowEntity,
 					nodeNames,
 					scenarioHints,
+					defaultStartNodeName: this.findStartNode(this.buildWorkflow(workflowEntity))?.name,
 				}),
 		);
 
 		// A trigger pinned without content runs with no items and blames every downstream miss on the builder.
+		// A Manual Trigger is the exception: it emits one empty item (see buildTriggerPinData).
 		const triggerStart = this.triggerStartNode(workflowEntity, hints);
-		if (triggerStart && lacksTriggerContent(hints)) {
+		if (
+			triggerStart &&
+			triggerStart.type !== MANUAL_TRIGGER_NODE_TYPE &&
+			lacksTriggerContent(hints)
+		) {
 			throw new Error(
 				`FRAMEWORK ISSUE: Phase 1 produced no trigger content for start node "${triggerStart.name}" (${hints.warnings.join('; ') || 'no details'}); the scenario cannot run without a trigger event`,
 			);
@@ -914,7 +921,12 @@ export class EvalExecutionService {
 		// which is the point of a "no new items" scenario. No pin at all would instead
 		// start the trigger with one injected empty item.
 		if (triggerEmitsNoItems) return { [startNode.name]: [] };
-		if (Object.keys(triggerContent).length === 0 && !binaryRequirement) return {};
+		if (Object.keys(triggerContent).length === 0 && !binaryRequirement) {
+			// A Manual Trigger's real output is one empty item.
+			return startNode.type === MANUAL_TRIGGER_NODE_TYPE
+				? { [startNode.name]: [{ json: {} }] }
+				: {};
+		}
 
 		// Mirror any LLM-embedded binary map as real item-level binary; json stays
 		// untouched so $json.binary.* references keep resolving.

packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts

@@ -11,6 +11,7 @@ import {
 	type IPinData,
 	type IWorkflowBase,
 	jsonParse,
+	MANUAL_TRIGGER_NODE_TYPE,
 	mapConnectionsByDestination,
 	UserError,
 } from 'n8n-workflow';
@@ -592,6 +593,8 @@ export interface GenerateMockHintsOptions {
 	workflow: IWorkflowBase;
 	nodeNames: string[];
 	scenarioHints?: string;
+	/** The node the run starts at when the hints name no start node. */
+	defaultStartNodeName?: string;
 }
 
 export const TRIGGER_CONTENT_CORRECTION =
@@ -606,8 +609,8 @@ RULES:
    - For webhook triggers: include { headers: {}, query: {}, body: { ...fields } } since downstream nodes reference $json.body.fieldName
    - For service-specific triggers (Gmail Trigger, Slack Trigger, etc.): match the service's real event/message output format
    - For schedule triggers: include timestamp fields
-   - For manual triggers: include the fields that downstream nodes reference
-   - CRITICAL: triggerContent must NEVER be an empty object ({}). Even for scenarios that test empty payloads ("empty submission", "no data", "missing fields"), emit the trigger envelope with empty *nested* fields — an empty webhook is { headers: {}, query: {}, body: {} }, a schedule with no context is { timestamp: "..." }. The workflow cannot execute without trigger output. The one exception is a polling or event trigger that the scenario says has NOTHING to emit ("no new emails", "no new rows", "no results"): then set "triggerEmitsNoItems": true and omit triggerContent — the harness pins the trigger to zero items so downstream nodes do not run.
+   - For a Manual Trigger: it emits one empty item, so return {} and never invent fields for it, even when downstream nodes reference trigger fields
+   - CRITICAL: apart from a Manual Trigger, triggerContent must NEVER be an empty object ({}). Even for scenarios that test empty payloads ("empty submission", "no data", "missing fields"), emit the trigger envelope with empty *nested* fields — an empty webhook is { headers: {}, query: {}, body: {} }, a schedule with no context is { timestamp: "..." }. The workflow cannot execute without trigger output. The one exception is a polling or event trigger that the scenario says has NOTHING to emit ("no new emails", "no new rows", "no results"): then set "triggerEmitsNoItems": true and omit triggerContent — the harness pins the trigger to zero items so downstream nodes do not run.
    - CRITICAL: check what downstream nodes reference (e.g., $json.body.email, $json.subject, $json.text) and ensure those paths exist in triggerContent
    - CRITICAL: when the workflow has MULTIPLE trigger nodes, pick the ONE the Test Scenario targets (the trigger whose firing the scenario describes, e.g. "The weekly Schedule Trigger fires") and return its exact node name in a "startNodeName" field. triggerContent must be THAT trigger's output.
    - CRITICAL: triggerContent must NEVER contain binary file CONTENT — no base64 blobs, no fake file-bytes placeholders. When the trigger carries a file (form upload, email attachment, incoming media), declare it with a METADATA-ONLY binary map instead: "binary": { "<propertyKey>": { "mimeType": "<real MIME>", "fileName": "<name.ext>" } } — the harness synthesizes real file bytes from that metadata and attaches them at the item level. The MIME type and file name MUST match the scenario: an image/png upload scenario needs mimeType "image/png" and a .png fileName, never a generic application/octet-stream. Use "data" as the propertyKey unless downstream nodes reference a different binary property name.
@@ -695,6 +698,12 @@ const MAX_HINT_ATTEMPTS = 2;
  */
 const HINT_LLM_TIMEOUT_MS = 300_000;
 
+/** A Manual Trigger emits one empty item, so `{}` is its faithful trigger content. */
+function isManualTriggerNode(workflow: IWorkflowBase, nodeName?: string): boolean {
+	const node = workflow.nodes.find((n) => n.name === nodeName);
+	return node?.type === MANUAL_TRIGGER_NODE_TYPE && !node.disabled;
+}
+
 /** One LLM call → globalContext + triggerContent + per-node hints. Retried once on structural issues. */
 export async function generateMockHints(options: GenerateMockHintsOptions): Promise<MockHints> {
 	const { workflow, nodeNames, scenarioHints } = options;
@@ -759,7 +768,15 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 				// The model answers a bool as a word often enough to read both spellings.
 				const triggerEmitsNoItems =
 					parsed.triggerEmitsNoItems === true || parsed.triggerEmitsNoItems === 'true';
-				if (Object.keys(triggerContent).length === 0 && !triggerEmitsNoItems) {
+				const startNodeName =
+					typeof parsed.startNodeName === 'string' && parsed.startNodeName.length > 0
+						? parsed.startNodeName
+						: undefined;
+				if (
+					Object.keys(triggerContent).length === 0 &&
+					!triggerEmitsNoItems &&
+					!isManualTriggerNode(workflow, startNodeName ?? options.defaultStartNodeName)
+				) {
 					reason = 'empty triggerContent';
 				} else {
 					// Coerce nodeHints values to strings — LLM may return objects instead of strings
@@ -771,9 +788,7 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 						globalContext,
 						nodeHints,
 						triggerContent: triggerContent as Record<string, unknown>,
-						...(typeof parsed.startNodeName === 'string' && parsed.startNodeName.length > 0
-							? { startNodeName: parsed.startNodeName }
-							: {}),
+						...(startNodeName ? { startNodeName } : {}),
 						...(triggerEmitsNoItems ? { triggerEmitsNoItems: true } : {}),
 						warnings,
 						bypassPinData: {},