fix(ai-builder): Validate only the selected case files in the eval push CLI (no-changelog) (#39248)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

.agents/skills/create-instance-ai-eval/SKILL.md

@@ -847,6 +847,13 @@ pnpm exec dotenvx run -f .env.eval -- pnpm eval:langtracer-push --suite baseline
   `<slugs...>` (exact file slugs), `--changed` (new/untracked + staged + modified
   `data/{workflows,agents}/*.json`, ideal right after authoring an uncommitted case),
   `--filter`/`--tier` (with `--exclude` as a modifier).
+- **Validation is selective.** Exact slugs and `--changed` read only the named
+  files, so an unrelated invalid file in `data/workflows` (a case authored on a
+  newer branch, a half-written draft) never blocks your push. `--filter` parses
+  only the files whose slug matches; `--tier` reads the tier from inside each
+  file, so it parses them all. An invalid file either one parses prints
+  `⚠ skipped invalid case file …` and the push continues without it. A file you
+  named still fails the push when it is invalid.
 - **Multiple positional slugs? Skip pnpm — call the script directly.** `pnpm
   eval:langtracer-push … slugA slugB` forwards the slugs as one joined argument
   (`"slugA slugB"`), so no case file matches and nothing is pushed. Either use a

packages/@n8n/instance-ai/evaluations/README.md

@@ -198,6 +198,10 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai \
 
 In langtracer mode, `--dataset` / `--baseline-prefix` default to a suite-scoped, eval-tagged name (`instance-ai-langtracer-<suite>`) so runs don't touch the shared `instance-ai-workflow-evals` cohort and re-runs of a suite upsert one stable dataset. `--filter` / `--exclude` / `--tier` still narrow within the suite. The MCP manifest builder (`eval:build-mcp-manifest`) accepts the same `--source langtracer --suite` flags.
 
+#### Pushing cases to a suite (`eval:langtracer-push`)
+
+`pnpm eval:langtracer-push --suite <slug|id> <slugs...> [--dry-run]` upserts on-disk cases into a suite: create missing, update changed, leave unchanged. Select with exact slugs, `--changed` (git-new or modified case files), `--filter <csv>` or `--tier <name>`. Validation is selective: exact slugs and `--changed` read only the named files, so an unrelated invalid file in `data/workflows` (a case authored on a newer branch) never blocks a push. `--filter` parses only the files whose slug matches; `--tier` reads the tier from inside each file, so it parses them all. An invalid file either one parses is reported as a warning and skipped. A file you named still fails the push when it is invalid.
+
 ### Outputs
 
 Every run produces:

packages/@n8n/instance-ai/evaluations/__tests__/load-eval-cases.test.ts

@@ -0,0 +1,55 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { vi } from 'vitest';
+
+import { loadEvalCasesFromDir } from '../utils/load-eval-cases';
+
+const validCase = {
+	conversation: [{ role: 'user', text: 'Build a webhook that posts to Slack' }],
+	complexity: 'simple',
+	tags: [],
+	processExpectations: ['the agent builds a webhook workflow'],
+};
+
+// A sibling with a key the schema does not know yet (a case authored on a newer
+// branch) used to fail every push from master.
+describe('loadEvalCasesFromDir', () => {
+	let dir: string;
+
+	beforeEach(() => {
+		dir = mkdtempSync(join(tmpdir(), 'eval-cases-'));
+		writeFileSync(join(dir, 'good-case.json'), JSON.stringify(validCase));
+		writeFileSync(
+			join(dir, 'newer-schema-case.json'),
+			JSON.stringify({ ...validCase, keyFromANewerSchema: true }),
+		);
+	});
+
+	afterEach(() => {
+		rmSync(dir, { recursive: true, force: true });
+	});
+
+	it('parses only the selected slugs, so an invalid sibling never blocks them', () => {
+		const loaded = loadEvalCasesFromDir(dir, undefined, undefined, undefined, {
+			slugs: new Set(['good-case']),
+		});
+
+		expect(loaded.map((c) => c.fileSlug)).toEqual(['good-case']);
+	});
+
+	it('reports and skips an invalid file when asked to, instead of throwing', () => {
+		const onInvalid = vi.fn();
+
+		const loaded = loadEvalCasesFromDir(dir, undefined, undefined, undefined, { onInvalid });
+
+		expect(loaded.map((c) => c.fileSlug)).toEqual(['good-case']);
+		expect(onInvalid).toHaveBeenCalledTimes(1);
+		expect(onInvalid.mock.calls[0][0]).toContain('newer-schema-case.json');
+		expect(onInvalid.mock.calls[0][1].message).toContain('keyFromANewerSchema');
+	});
+
+	it('still throws on an invalid file by default, so the runner never grades a partial corpus', () => {
+		expect(() => loadEvalCasesFromDir(dir)).toThrow(/newer-schema-case\.json/);
+	});
+});

packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts

@@ -16,6 +16,7 @@ import { LangTracerClient } from '../langtracer/client';
 import { resolveLangTracerConfig } from '../langtracer/config';
 import { comparableDiff, planPush, toUpdatePatch } from '../langtracer/push';
 import { diskCaseToLangTracerCreate } from '../langtracer/to-exported';
+import type { LoadEvalCasesOptions } from '../utils/load-eval-cases';
 
 interface CliArgs {
 	suite: string;
@@ -41,6 +42,10 @@ Selectors (at least one required — no accidental push-all):
   --tier <name>         Cases whose datasets include <name>
   --exclude <csv>       Substring exclude (modifier, not a selector on its own)
 
+Exact slugs and --changed read only the named files, so an unrelated invalid file
+never blocks the push. --filter and --tier report an invalid file they match as a
+warning and skip it.
+
 Options:
   --suite <slug|id>     Target suite (required)
   --set-kind <kind>     regression (default) | capability_gap
@@ -175,12 +180,22 @@ async function main() {
 		throw new Error(`suite "${args.suite}" not found. Available: ${known || '(none)'}.`);
 	}
 
-	// Select disk cases: loader applies --filter/--exclude, --tier narrows by the
-	// case's datasets (mirrors data/source.ts); then narrow to the exact slugs
-	// from positional args + --changed (if either was given).
+	// Select disk cases: exact slugs (positional + --changed) are read before
+	// loading, so unrelated files are never parsed. --filter/--exclude apply in the
+	// loader by file name; --tier reads each file's datasets, so it parses every
+	// file. Either way an invalid file is reported and skipped rather than failing
+	// the push.
+	const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]);
+	const loadOptions: LoadEvalCasesOptions =
+		exactSlugs.size > 0
+			? { slugs: exactSlugs }
+			: {
+					onInvalid: (file, error) =>
+						console.warn(`⚠ skipped invalid case file ${basename(file)}: ${error.message}`),
+				};
 	const loaded = [
-		...loadWorkflowTestCasesWithFiles(args.filter, args.exclude),
-		...loadAgentEvalTestCasesWithFiles(args.filter, args.exclude),
+		...loadWorkflowTestCasesWithFiles(args.filter, args.exclude, loadOptions),
+		...loadAgentEvalTestCasesWithFiles(args.filter, args.exclude, loadOptions),
 	];
 	const dupes = loaded.filter((c, i) => loaded.findIndex((o) => o.fileSlug === c.fileSlug) !== i);
 	if (dupes.length > 0) {
@@ -190,7 +205,6 @@ async function main() {
 	}
 	const tier = args.tier;
 	const all = tier ? loaded.filter((c) => c.testCase.datasets.includes(tier)) : loaded;
-	const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]);
 	const selected = exactSlugs.size > 0 ? all.filter((c) => exactSlugs.has(c.fileSlug)) : all;
 
 	const missing = [...exactSlugs].filter((s) => !all.some((c) => c.fileSlug === s));

packages/@n8n/instance-ai/evaluations/data/agents/index.ts

@@ -6,11 +6,16 @@
 // it, push it, and delete the local file. Requires the agents module.
 // ---------------------------------------------------------------------------
 
-import { loadEvalCasesFromDir, type WorkflowTestCaseWithFile } from '../../utils/load-eval-cases';
+import {
+	loadEvalCasesFromDir,
+	type LoadEvalCasesOptions,
+	type WorkflowTestCaseWithFile,
+} from '../../utils/load-eval-cases';
 
 export function loadAgentEvalTestCasesWithFiles(
 	filter?: string,
 	exclude?: string,
+	options?: LoadEvalCasesOptions,
 ): WorkflowTestCaseWithFile[] {
-	return loadEvalCasesFromDir(__dirname, filter, exclude);
+	return loadEvalCasesFromDir(__dirname, filter, exclude, undefined, options);
 }

packages/@n8n/instance-ai/evaluations/data/workflows/index.ts

@@ -1,4 +1,8 @@
-import { loadEvalCasesFromDir, type WorkflowTestCaseWithFile } from '../../utils/load-eval-cases';
+import {
+	loadEvalCasesFromDir,
+	type LoadEvalCasesOptions,
+	type WorkflowTestCaseWithFile,
+} from '../../utils/load-eval-cases';
 
 export type { WorkflowTestCaseWithFile } from '../../utils/load-eval-cases';
 
@@ -6,6 +10,7 @@ export type { WorkflowTestCaseWithFile } from '../../utils/load-eval-cases';
 export function loadWorkflowTestCasesWithFiles(
 	filter?: string,
 	exclude?: string,
+	options?: LoadEvalCasesOptions,
 ): WorkflowTestCaseWithFile[] {
-	return loadEvalCasesFromDir(__dirname, filter, exclude);
+	return loadEvalCasesFromDir(__dirname, filter, exclude, undefined, options);
 }

packages/@n8n/instance-ai/evaluations/utils/load-eval-cases.ts

@@ -11,6 +11,13 @@ export interface WorkflowTestCaseWithFile {
 	fileSlug: string;
 }
 
+export interface LoadEvalCasesOptions {
+	/** Parse only these file slugs; other files in the directory are never read. */
+	slugs?: ReadonlySet<string>;
+	/** Report an invalid file and skip it instead of throwing. */
+	onInvalid?: (filePath: string, error: Error) => void;
+}
+
 function parseTestCaseFile(filePath: string): WorkflowTestCase {
 	const content = readFileSync(filePath, 'utf-8');
 
@@ -40,12 +47,26 @@ export function loadEvalCasesFromDir(
 	filter?: string,
 	exclude?: string,
 	transform?: (testCase: WorkflowTestCase) => WorkflowTestCase,
+	options: LoadEvalCasesOptions = {},
 ): WorkflowTestCaseWithFile[] {
-	return getJsonFiles(dataDir, filter, exclude).map((f) => {
-		const testCase = parseTestCaseFile(f);
-		return {
+	const { slugs, onInvalid } = options;
+	const files = getJsonFiles(dataDir, filter, exclude).filter(
+		(f) => slugs === undefined || slugs.has(basename(f, '.json')),
+	);
+	const loaded: WorkflowTestCaseWithFile[] = [];
+	for (const f of files) {
+		let testCase: WorkflowTestCase;
+		try {
+			testCase = parseTestCaseFile(f);
+		} catch (error) {
+			if (!onInvalid) throw error;
+			onInvalid(f, error instanceof Error ? error : new Error(String(error)));
+			continue;
+		}
+		loaded.push({
 			testCase: transform ? transform(testCase) : testCase,
 			fileSlug: basename(f, '.json'),
-		};
-	});
+		});
+	}
+	return loaded;
 }