diff --git a/.agents/skills/create-instance-ai-eval/SKILL.md b/.agents/skills/create-instance-ai-eval/SKILL.md index e388f67c11c..bdc6a8f783e 100644 --- a/.agents/skills/create-instance-ai-eval/SKILL.md +++ b/.agents/skills/create-instance-ai-eval/SKILL.md @@ -847,6 +847,13 @@ pnpm exec dotenvx run -f .env.eval -- pnpm eval:langtracer-push --suite baseline `` (exact file slugs), `--changed` (new/untracked + staged + modified `data/{workflows,agents}/*.json`, ideal right after authoring an uncommitted case), `--filter`/`--tier` (with `--exclude` as a modifier). +- **Validation is selective.** Exact slugs and `--changed` read only the named + files, so an unrelated invalid file in `data/workflows` (a case authored on a + newer branch, a half-written draft) never blocks your push. `--filter` parses + only the files whose slug matches; `--tier` reads the tier from inside each + file, so it parses them all. An invalid file either one parses prints + `⚠ skipped invalid case file …` and the push continues without it. A file you + named still fails the push when it is invalid. - **Multiple positional slugs? Skip pnpm — call the script directly.** `pnpm eval:langtracer-push … slugA slugB` forwards the slugs as one joined argument (`"slugA slugB"`), so no case file matches and nothing is pushed. Either use a diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index d11da09f7c4..0a678f2a78f 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -198,6 +198,10 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai \ In langtracer mode, `--dataset` / `--baseline-prefix` default to a suite-scoped, eval-tagged name (`instance-ai-langtracer-`) so runs don't touch the shared `instance-ai-workflow-evals` cohort and re-runs of a suite upsert one stable dataset. `--filter` / `--exclude` / `--tier` still narrow within the suite. The MCP manifest builder (`eval:build-mcp-manifest`) accepts the same `--source langtracer --suite` flags. +#### Pushing cases to a suite (`eval:langtracer-push`) + +`pnpm eval:langtracer-push --suite [--dry-run]` upserts on-disk cases into a suite: create missing, update changed, leave unchanged. Select with exact slugs, `--changed` (git-new or modified case files), `--filter ` or `--tier `. Validation is selective: exact slugs and `--changed` read only the named files, so an unrelated invalid file in `data/workflows` (a case authored on a newer branch) never blocks a push. `--filter` parses only the files whose slug matches; `--tier` reads the tier from inside each file, so it parses them all. An invalid file either one parses is reported as a warning and skipped. A file you named still fails the push when it is invalid. + ### Outputs Every run produces: diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/load-eval-cases.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/load-eval-cases.test.ts new file mode 100644 index 00000000000..0c8c11df207 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/load-eval-cases.test.ts @@ -0,0 +1,55 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { vi } from 'vitest'; + +import { loadEvalCasesFromDir } from '../utils/load-eval-cases'; + +const validCase = { + conversation: [{ role: 'user', text: 'Build a webhook that posts to Slack' }], + complexity: 'simple', + tags: [], + processExpectations: ['the agent builds a webhook workflow'], +}; + +// A sibling with a key the schema does not know yet (a case authored on a newer +// branch) used to fail every push from master. +describe('loadEvalCasesFromDir', () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'eval-cases-')); + writeFileSync(join(dir, 'good-case.json'), JSON.stringify(validCase)); + writeFileSync( + join(dir, 'newer-schema-case.json'), + JSON.stringify({ ...validCase, keyFromANewerSchema: true }), + ); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('parses only the selected slugs, so an invalid sibling never blocks them', () => { + const loaded = loadEvalCasesFromDir(dir, undefined, undefined, undefined, { + slugs: new Set(['good-case']), + }); + + expect(loaded.map((c) => c.fileSlug)).toEqual(['good-case']); + }); + + it('reports and skips an invalid file when asked to, instead of throwing', () => { + const onInvalid = vi.fn(); + + const loaded = loadEvalCasesFromDir(dir, undefined, undefined, undefined, { onInvalid }); + + expect(loaded.map((c) => c.fileSlug)).toEqual(['good-case']); + expect(onInvalid).toHaveBeenCalledTimes(1); + expect(onInvalid.mock.calls[0][0]).toContain('newer-schema-case.json'); + expect(onInvalid.mock.calls[0][1].message).toContain('keyFromANewerSchema'); + }); + + it('still throws on an invalid file by default, so the runner never grades a partial corpus', () => { + expect(() => loadEvalCasesFromDir(dir)).toThrow(/newer-schema-case\.json/); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts b/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts index a99efad1b19..31f88efecb2 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts @@ -16,6 +16,7 @@ import { LangTracerClient } from '../langtracer/client'; import { resolveLangTracerConfig } from '../langtracer/config'; import { comparableDiff, planPush, toUpdatePatch } from '../langtracer/push'; import { diskCaseToLangTracerCreate } from '../langtracer/to-exported'; +import type { LoadEvalCasesOptions } from '../utils/load-eval-cases'; interface CliArgs { suite: string; @@ -41,6 +42,10 @@ Selectors (at least one required — no accidental push-all): --tier Cases whose datasets include --exclude Substring exclude (modifier, not a selector on its own) +Exact slugs and --changed read only the named files, so an unrelated invalid file +never blocks the push. --filter and --tier report an invalid file they match as a +warning and skip it. + Options: --suite Target suite (required) --set-kind regression (default) | capability_gap @@ -175,12 +180,22 @@ async function main() { throw new Error(`suite "${args.suite}" not found. Available: ${known || '(none)'}.`); } - // Select disk cases: loader applies --filter/--exclude, --tier narrows by the - // case's datasets (mirrors data/source.ts); then narrow to the exact slugs - // from positional args + --changed (if either was given). + // Select disk cases: exact slugs (positional + --changed) are read before + // loading, so unrelated files are never parsed. --filter/--exclude apply in the + // loader by file name; --tier reads each file's datasets, so it parses every + // file. Either way an invalid file is reported and skipped rather than failing + // the push. + const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]); + const loadOptions: LoadEvalCasesOptions = + exactSlugs.size > 0 + ? { slugs: exactSlugs } + : { + onInvalid: (file, error) => + console.warn(`⚠ skipped invalid case file ${basename(file)}: ${error.message}`), + }; const loaded = [ - ...loadWorkflowTestCasesWithFiles(args.filter, args.exclude), - ...loadAgentEvalTestCasesWithFiles(args.filter, args.exclude), + ...loadWorkflowTestCasesWithFiles(args.filter, args.exclude, loadOptions), + ...loadAgentEvalTestCasesWithFiles(args.filter, args.exclude, loadOptions), ]; const dupes = loaded.filter((c, i) => loaded.findIndex((o) => o.fileSlug === c.fileSlug) !== i); if (dupes.length > 0) { @@ -190,7 +205,6 @@ async function main() { } const tier = args.tier; const all = tier ? loaded.filter((c) => c.testCase.datasets.includes(tier)) : loaded; - const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]); const selected = exactSlugs.size > 0 ? all.filter((c) => exactSlugs.has(c.fileSlug)) : all; const missing = [...exactSlugs].filter((s) => !all.some((c) => c.fileSlug === s)); diff --git a/packages/@n8n/instance-ai/evaluations/data/agents/index.ts b/packages/@n8n/instance-ai/evaluations/data/agents/index.ts index 5da6d519257..6705bcc3ae3 100644 --- a/packages/@n8n/instance-ai/evaluations/data/agents/index.ts +++ b/packages/@n8n/instance-ai/evaluations/data/agents/index.ts @@ -6,11 +6,16 @@ // it, push it, and delete the local file. Requires the agents module. // --------------------------------------------------------------------------- -import { loadEvalCasesFromDir, type WorkflowTestCaseWithFile } from '../../utils/load-eval-cases'; +import { + loadEvalCasesFromDir, + type LoadEvalCasesOptions, + type WorkflowTestCaseWithFile, +} from '../../utils/load-eval-cases'; export function loadAgentEvalTestCasesWithFiles( filter?: string, exclude?: string, + options?: LoadEvalCasesOptions, ): WorkflowTestCaseWithFile[] { - return loadEvalCasesFromDir(__dirname, filter, exclude); + return loadEvalCasesFromDir(__dirname, filter, exclude, undefined, options); } diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts b/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts index bd5f4307445..5ee3fe3a469 100644 --- a/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts @@ -1,4 +1,8 @@ -import { loadEvalCasesFromDir, type WorkflowTestCaseWithFile } from '../../utils/load-eval-cases'; +import { + loadEvalCasesFromDir, + type LoadEvalCasesOptions, + type WorkflowTestCaseWithFile, +} from '../../utils/load-eval-cases'; export type { WorkflowTestCaseWithFile } from '../../utils/load-eval-cases'; @@ -6,6 +10,7 @@ export type { WorkflowTestCaseWithFile } from '../../utils/load-eval-cases'; export function loadWorkflowTestCasesWithFiles( filter?: string, exclude?: string, + options?: LoadEvalCasesOptions, ): WorkflowTestCaseWithFile[] { - return loadEvalCasesFromDir(__dirname, filter, exclude); + return loadEvalCasesFromDir(__dirname, filter, exclude, undefined, options); } diff --git a/packages/@n8n/instance-ai/evaluations/utils/load-eval-cases.ts b/packages/@n8n/instance-ai/evaluations/utils/load-eval-cases.ts index 3af860a5c2c..4b87c143cfa 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/load-eval-cases.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/load-eval-cases.ts @@ -11,6 +11,13 @@ export interface WorkflowTestCaseWithFile { fileSlug: string; } +export interface LoadEvalCasesOptions { + /** Parse only these file slugs; other files in the directory are never read. */ + slugs?: ReadonlySet; + /** Report an invalid file and skip it instead of throwing. */ + onInvalid?: (filePath: string, error: Error) => void; +} + function parseTestCaseFile(filePath: string): WorkflowTestCase { const content = readFileSync(filePath, 'utf-8'); @@ -40,12 +47,26 @@ export function loadEvalCasesFromDir( filter?: string, exclude?: string, transform?: (testCase: WorkflowTestCase) => WorkflowTestCase, + options: LoadEvalCasesOptions = {}, ): WorkflowTestCaseWithFile[] { - return getJsonFiles(dataDir, filter, exclude).map((f) => { - const testCase = parseTestCaseFile(f); - return { + const { slugs, onInvalid } = options; + const files = getJsonFiles(dataDir, filter, exclude).filter( + (f) => slugs === undefined || slugs.has(basename(f, '.json')), + ); + const loaded: WorkflowTestCaseWithFile[] = []; + for (const f of files) { + let testCase: WorkflowTestCase; + try { + testCase = parseTestCaseFile(f); + } catch (error) { + if (!onInvalid) throw error; + onInvalid(f, error instanceof Error ? error : new Error(String(error))); + continue; + } + loaded.push({ testCase: transform ? transform(testCase) : testCase, fileSlug: basename(f, '.json'), - }; - }); + }); + } + return loaded; }