fix(core): Let a mocked trigger emit zero items in Instance AI evals (no-changelog) (#39113)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts

@@ -409,6 +409,20 @@ describe('EvalExecutionService', () => {
 			expect(result.executionId).toBe(DB_EXECUTION_ID);
 		});
 
+		it('pins the trigger to zero items when the scenario says it emits nothing', async () => {
+			const hints = makeEmptyHints();
+			hints.triggerContent = {};
+			hints.triggerEmitsNoItems = true;
+			generateMockHintsMock.mockResolvedValue(hints);
+
+			await service.executeWithLlmMock('wf-1', makeUser());
+
+			const runArg = workflowRunner.run.mock.calls[0][0] as unknown as {
+				pinData?: Record<string, unknown[]>;
+			};
+			expect(runArg.pinData?.Webhook).toEqual([]);
+		});
+
 		it('routes through WorkflowRunner with evaluation mode + pin data + user', async () => {
 			const hints = makeEmptyHints();
 			hints.triggerContent = { body: { email: 'jane@example.com' } };

packages/cli/src/modules/instance-ai/eval/__tests__/workflow-analysis.test.ts

@@ -1184,6 +1184,26 @@ describe('generateMockHints', () => {
 		expect(result.warnings).toEqual([expect.stringContaining('invalid nodeHints')]);
 	});
 
+	it.each([true, 'true'])(
+		'accepts empty triggerContent when triggerEmitsNoItems is %j, and forwards the flag',
+		async (flag) => {
+			const generate = mockAgentResponses(
+				JSON.stringify({
+					globalContext: '',
+					nodeHints: { Slack: 'foo' },
+					triggerEmitsNoItems: flag,
+				}),
+			);
+
+			const result = await generateMockHints({ workflow, nodeNames: ['Schedule', 'Slack'] });
+
+			expect(generate).toHaveBeenCalledTimes(1);
+			expect(result.triggerContent).toEqual({});
+			expect(result.triggerEmitsNoItems).toBe(true);
+			expect(result.warnings).toEqual([]);
+		},
+	);
+
 	it('should not call the agent when there are no hint-eligible nodes', async () => {
 		const generate = mockAgentResponses('should never be called');
 

packages/cli/src/modules/instance-ai/eval/execution.service.ts

@@ -490,6 +490,7 @@ export class EvalExecutionService {
 			startNode,
 			hints.triggerContent,
 			binaryRequirement,
+			hints.triggerEmitsNoItems,
 		);
 		const pinData: IPinData = { ...triggerPinData, ...hints.bypassPinData };
 		const pinDataNodeNames = Object.keys(pinData);
@@ -761,7 +762,12 @@ export class EvalExecutionService {
 		startNode: INode,
 		triggerContent: Record<string, unknown>,
 		binaryRequirement?: TriggerBinaryRequirement,
+		triggerEmitsNoItems = false,
 	): IPinData {
+		// A pinned empty array is "this node emitted nothing": downstream nodes stay idle,
+		// which is the point of a "no new items" scenario. No pin at all would instead
+		// start the trigger with one injected empty item.
+		if (triggerEmitsNoItems) return { [startNode.name]: [] };
 		if (Object.keys(triggerContent).length === 0 && !binaryRequirement) return {};
 
 		// Mirror any LLM-embedded binary map as real item-level binary; json stays

packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts

@@ -578,6 +578,8 @@ export interface MockHints {
 	triggerContent: Record<string, unknown>;
 	/** For multi-trigger workflows: the trigger node the scenario targets (Phase-1 LLM's pick). */
 	startNodeName?: string;
+	/** The scenario says the trigger has nothing to emit; the harness pins it to zero items. */
+	triggerEmitsNoItems?: boolean;
 	/** Errors encountered during hint generation or mock execution */
 	warnings: string[];
 	/** Pin data for nodes that bypass the HTTP mock layer (AI roots, protocol nodes) */
@@ -600,7 +602,7 @@ RULES:
    - For service-specific triggers (Gmail Trigger, Slack Trigger, etc.): match the service's real event/message output format
    - For schedule triggers: include timestamp fields
    - For manual triggers: include the fields that downstream nodes reference
-   - CRITICAL: triggerContent must NEVER be an empty object ({}). Even for scenarios that test empty payloads ("empty submission", "no data", "missing fields"), emit the trigger envelope with empty *nested* fields — an empty webhook is { headers: {}, query: {}, body: {} }, a schedule with no context is { timestamp: "..." }. The workflow cannot execute without trigger output.
+   - CRITICAL: triggerContent must NEVER be an empty object ({}). Even for scenarios that test empty payloads ("empty submission", "no data", "missing fields"), emit the trigger envelope with empty *nested* fields — an empty webhook is { headers: {}, query: {}, body: {} }, a schedule with no context is { timestamp: "..." }. The workflow cannot execute without trigger output. The one exception is a polling or event trigger that the scenario says has NOTHING to emit ("no new emails", "no new rows", "no results"): then set "triggerEmitsNoItems": true and omit triggerContent — the harness pins the trigger to zero items so downstream nodes do not run.
    - CRITICAL: check what downstream nodes reference (e.g., $json.body.email, $json.subject, $json.text) and ensure those paths exist in triggerContent
    - CRITICAL: when the workflow has MULTIPLE trigger nodes, pick the ONE the Test Scenario targets (the trigger whose firing the scenario describes, e.g. "The weekly Schedule Trigger fires") and return its exact node name in a "startNodeName" field. triggerContent must be THAT trigger's output.
    - CRITICAL: triggerContent must NEVER contain binary file CONTENT — no base64 blobs, no fake file-bytes placeholders. When the trigger carries a file (form upload, email attachment, incoming media), declare it with a METADATA-ONLY binary map instead: "binary": { "<propertyKey>": { "mimeType": "<real MIME>", "fileName": "<name.ext>" } } — the harness synthesizes real file bytes from that metadata and attaches them at the item level. The MIME type and file name MUST match the scenario: an image/png upload scenario needs mimeType "image/png" and a .png fileName, never a generic application/octet-stream. Use "data" as the propertyKey unless downstream nodes reference a different binary property name.
@@ -657,6 +659,9 @@ function buildUserPrompt(
 		'  "startNodeName": "exact trigger node name the scenario targets (only when the workflow has multiple triggers)",',
 	);
 	sections.push('  "triggerContent": { "...exact output the trigger node would produce..." },');
+	sections.push(
+		'  "triggerEmitsNoItems": "true only when the scenario says the trigger emits nothing; then omit triggerContent",',
+	);
 	sections.push('  "nodeHints": {');
 	for (let i = 0; i < Math.min(nodeNames.length, 3); i++) {
 		const comma = i < Math.min(nodeNames.length, 3) - 1 ? ',' : '';
@@ -742,7 +747,10 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 					!Array.isArray(parsed.triggerContent)
 						? parsed.triggerContent
 						: {};
-				if (Object.keys(triggerContent).length === 0) {
+				// The model answers a bool as a word often enough to read both spellings.
+				const triggerEmitsNoItems =
+					parsed.triggerEmitsNoItems === true || parsed.triggerEmitsNoItems === 'true';
+				if (Object.keys(triggerContent).length === 0 && !triggerEmitsNoItems) {
 					reason = 'empty triggerContent';
 				} else {
 					// Coerce nodeHints values to strings — LLM may return objects instead of strings
@@ -757,6 +765,7 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 						...(typeof parsed.startNodeName === 'string' && parsed.startNodeName.length > 0
 							? { startNodeName: parsed.startNodeName }
 							: {}),
+						...(triggerEmitsNoItems ? { triggerEmitsNoItems: true } : {}),
 						warnings,
 						bypassPinData: {},
 					};