diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index f11c3a79d82..f350fe88dba 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -132,6 +132,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai --iterations 3 | `--exclude` | — | Skip test cases whose filename matches any of the substrings. Same comma-separated shape as `--filter`; applied after `--filter` | | `--prebuilt-workflows` | — | Path to a JSON manifest mapping test-case slugs to existing workflow IDs. Skips the orchestrator build for matched test cases — see [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows) | | `--keep-workflows` | `false` | Don't delete built workflows after the run. Pair with the HTML report's "view in n8n" links to inspect each scenario's canvas execution | +| `--delete-prebuilt-workflows` | `false` | With `--prebuilt-workflows`, delete successfully used manifest workflows after the eval run. Mutually exclusive with `--keep-workflows` | | `--base-url` | `http://localhost:5678` | n8n instance URL | | `--email` | E2E test owner | Override login email (or `N8N_EVAL_EMAIL`) | | `--password` | E2E test owner | Override login password (or `N8N_EVAL_PASSWORD`) | @@ -279,7 +280,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai \ --experiment-name mcp-cohort ``` -The harness leaves prebuilt workflows alone after the run (no auto-delete), so the manifest can be re-used across multiple eval runs. +The harness leaves prebuilt workflows alone after the run (no auto-delete), so the manifest can be re-used across multiple eval runs. If the workflows were created only for this eval cohort, pass `--delete-prebuilt-workflows` with `--prebuilt-workflows` to delete every successfully used manifest workflow once after the run. This is destructive: the manifest will still contain the deleted IDs and should not be re-used afterward. ### Producing a manifest diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts index 4de259f7512..0d1b84213a7 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts @@ -41,6 +41,7 @@ describe('parseCliArgs --base-url', () => { describe('parseCliArgs --prebuilt-workflows', () => { it('is undefined by default', () => { expect(parseCliArgs([]).prebuiltWorkflows).toBeUndefined(); + expect(parseCliArgs([]).deletePrebuiltWorkflows).toBe(false); }); it('accepts a path argument', () => { @@ -51,6 +52,32 @@ describe('parseCliArgs --prebuilt-workflows', () => { it('throws when no value is provided', () => { expect(() => parseCliArgs(['--prebuilt-workflows'])).toThrow(/Missing value/); }); + + it('accepts deleting prebuilt workflows when a manifest is provided', () => { + const args = parseCliArgs([ + '--prebuilt-workflows', + './mcp-manifest.json', + '--delete-prebuilt-workflows', + ]); + expect(args.deletePrebuiltWorkflows).toBe(true); + }); + + it('rejects deleting prebuilt workflows without a manifest', () => { + expect(() => parseCliArgs(['--delete-prebuilt-workflows'])).toThrow( + /--delete-prebuilt-workflows requires --prebuilt-workflows/, + ); + }); + + it('rejects deleting and keeping workflows at the same time', () => { + expect(() => + parseCliArgs([ + '--prebuilt-workflows', + './mcp-manifest.json', + '--delete-prebuilt-workflows', + '--keep-workflows', + ]), + ).toThrow(/--delete-prebuilt-workflows cannot be used with --keep-workflows/); + }); }); describe('parseCliArgs --exclude', () => { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/prebuilt-workflows.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/prebuilt-workflows.test.ts index a1c7208f46c..6915b7b5539 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/prebuilt-workflows.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/prebuilt-workflows.test.ts @@ -7,6 +7,7 @@ import type { Mock } from 'vitest'; import type { N8nClient, WorkflowResponse } from '../clients/n8n-client'; import type { EvalLogger } from '../harness/logger'; import { + cleanupPrebuiltWorkflows, fetchPrebuiltBuild, loadPrebuiltManifest, pickPrebuiltWorkflowId, @@ -153,3 +154,45 @@ describe('fetchPrebuiltBuild', () => { expect(result.error).toContain('plain string failure'); }); }); + +describe('cleanupPrebuiltWorkflows', () => { + function makeLogger(): EvalLogger & { info: Mock; warn: Mock } { + return { + info: vi.fn(), + verbose: vi.fn(), + success: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + isVerbose: false, + }; + } + + it('deletes each workflow ID once', async () => { + const deleteWorkflow = vi.fn().mockResolvedValue(undefined); + const client = { deleteWorkflow } as unknown as N8nClient; + const logger = makeLogger(); + + await cleanupPrebuiltWorkflows(client, ['W1', 'W1', 'W2'], logger); + + expect(deleteWorkflow).toHaveBeenCalledTimes(2); + expect(deleteWorkflow).toHaveBeenNthCalledWith(1, 'W1'); + expect(deleteWorkflow).toHaveBeenNthCalledWith(2, 'W2'); + expect(logger.info).toHaveBeenCalledWith('Deleted 2/2 prebuilt workflow(s)'); + expect(logger.warn).not.toHaveBeenCalled(); + }); + + it('continues deleting after a workflow deletion fails', async () => { + const deleteWorkflow = vi + .fn() + .mockRejectedValueOnce(new Error('HTTP 404')) + .mockResolvedValueOnce(undefined); + const client = { deleteWorkflow } as unknown as N8nClient; + const logger = makeLogger(); + + await cleanupPrebuiltWorkflows(client, ['W1', 'W2'], logger); + + expect(deleteWorkflow).toHaveBeenCalledTimes(2); + expect(logger.warn).toHaveBeenCalledWith('Failed to delete prebuilt workflow W1: HTTP 404'); + expect(logger.info).toHaveBeenCalledWith('Deleted 1/2 prebuilt workflow(s)'); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/cli/args.ts b/packages/@n8n/instance-ai/evaluations/cli/args.ts index 30d2fab009f..a8748968f0a 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/args.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/args.ts @@ -35,6 +35,8 @@ export interface CliArgs { prebuiltWorkflows?: string; /** Keep built workflows after evaluation instead of deleting them */ keepWorkflows: boolean; + /** Delete successfully used workflows from --prebuilt-workflows after evaluation */ + deletePrebuiltWorkflows: boolean; /** Directory to write eval-results.json (defaults to cwd) */ outputDir?: string; /** LangSmith dataset name (synced from JSON test cases before each run) */ @@ -72,6 +74,7 @@ const cliArgsSchema = z.object({ exclude: z.string().optional(), prebuiltWorkflows: z.string().optional(), keepWorkflows: z.boolean().default(false), + deletePrebuiltWorkflows: z.boolean().default(false), outputDir: z.string().optional(), dataset: z.string().default('instance-ai-workflow-evals'), concurrency: z.number().int().positive().default(16), @@ -88,6 +91,12 @@ const cliArgsSchema = z.object({ export function parseCliArgs(argv: string[]): CliArgs { const raw = parseRawArgs(argv); const validated = cliArgsSchema.parse(raw); + if (validated.deletePrebuiltWorkflows && !validated.prebuiltWorkflows) { + throw new Error('--delete-prebuilt-workflows requires --prebuilt-workflows'); + } + if (validated.deletePrebuiltWorkflows && validated.keepWorkflows) { + throw new Error('--delete-prebuilt-workflows cannot be used with --keep-workflows'); + } return { timeoutMs: validated.timeoutMs, @@ -99,6 +108,7 @@ export function parseCliArgs(argv: string[]): CliArgs { exclude: validated.exclude, prebuiltWorkflows: validated.prebuiltWorkflows, keepWorkflows: validated.keepWorkflows, + deletePrebuiltWorkflows: validated.deletePrebuiltWorkflows, outputDir: validated.outputDir, dataset: validated.dataset, concurrency: validated.concurrency, @@ -123,6 +133,7 @@ interface RawArgs { exclude?: string; prebuiltWorkflows?: string; keepWorkflows: boolean; + deletePrebuiltWorkflows: boolean; outputDir?: string; dataset: string; concurrency: number; @@ -138,6 +149,7 @@ function parseRawArgs(argv: string[]): RawArgs { baseUrls: ['http://localhost:5678'], verbose: false, keepWorkflows: false, + deletePrebuiltWorkflows: false, outputDir: undefined, dataset: 'instance-ai-workflow-evals', concurrency: 16, @@ -198,6 +210,10 @@ function parseRawArgs(argv: string[]): RawArgs { result.keepWorkflows = true; break; + case '--delete-prebuilt-workflows': + result.deletePrebuiltWorkflows = true; + break; + case '--output-dir': result.outputDir = nextArg(argv, i, '--output-dir'); i++; diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index a2abb46f06d..404bca7915d 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -40,6 +40,7 @@ import type { WorkflowTestCaseWithFile } from '../data/workflows'; import { createLogger } from '../harness/logger'; import type { EvalLogger } from '../harness/logger'; import { + cleanupPrebuiltWorkflows, fetchPrebuiltBuild, loadPrebuiltManifest, pickPrebuiltWorkflowId, @@ -159,6 +160,7 @@ interface RunConfig { lanes: Lane[]; logger: EvalLogger; prebuiltManifest?: PrebuiltManifest; + prebuiltWorkflowIdsToDelete?: Set; } async function main(): Promise { @@ -228,6 +230,7 @@ async function main(): Promise { ); const startTime = Date.now(); + const prebuiltWorkflowIdsToDelete = args.deletePrebuiltWorkflows ? new Set() : undefined; try { const hasLangSmith = Boolean(process.env.LANGSMITH_API_KEY); @@ -239,14 +242,26 @@ async function main(): Promise { if (hasLangSmith) { logger.info('LangSmith API key detected, using evaluate() with experiment tracking'); - const langsmithRun = await runWithLangSmith({ args, lanes, logger, prebuiltManifest }); + const langsmithRun = await runWithLangSmith({ + args, + lanes, + logger, + prebuiltManifest, + prebuiltWorkflowIdsToDelete, + }); evaluation = langsmithRun.evaluation; experimentName = langsmithRun.experimentName; outcome = langsmithRun.outcome; slugByTestCase = langsmithRun.slugByTestCase; } else { logger.info('No LANGSMITH_API_KEY, running direct loop (results in eval-results.json only)'); - evaluation = await runDirectLoop({ args, lanes, logger, prebuiltManifest }); + evaluation = await runDirectLoop({ + args, + lanes, + logger, + prebuiltManifest, + prebuiltWorkflowIdsToDelete, + }); } const totalDuration = Date.now() - startTime; @@ -270,6 +285,9 @@ async function main(): Promise { '\n' + formatComparisonTerminal(evaluation, outcome, { commitSha, slugByTestCase }), ); } finally { + if (prebuiltWorkflowIdsToDelete && lanes[0]) { + await cleanupPrebuiltWorkflows(lanes[0].client, prebuiltWorkflowIdsToDelete, logger); + } await Promise.all( lanes.map(async (lane) => { await cleanupCredentials(lane.client, lane.seedResult.credentialIds).catch(() => {}); @@ -288,7 +306,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{ outcome: ComparisonOutcome; slugByTestCase: Map; }> { - const { args, lanes, logger, prebuiltManifest } = config; + const { args, lanes, logger, prebuiltManifest, prebuiltWorkflowIdsToDelete } = config; const lsClient = new Client(); const datasetName = await syncDataset(lsClient, args.dataset, logger, args.filter, args.exclude); @@ -412,6 +430,9 @@ async function runWithLangSmith(config: RunConfig): Promise<{ const lane = laneStates[0]; const start = Date.now(); const build = await fetchPrebuiltBuild(lane.runner.client, prebuiltId, logger); + if (build.success && build.workflowId) { + prebuiltWorkflowIdsToDelete?.add(build.workflowId); + } const buildDurationMs = Date.now() - start; buildDurations.set(key, buildDurationMs); stashTranscript(build); @@ -934,7 +955,7 @@ function reshapeLangSmithRuns( // --------------------------------------------------------------------------- async function runDirectLoop(config: RunConfig): Promise { - const { args, lanes, logger, prebuiltManifest } = config; + const { args, lanes, logger, prebuiltManifest, prebuiltWorkflowIdsToDelete } = config; const testCasesWithFiles = loadWorkflowTestCasesWithFiles(args.filter, args.exclude, args.tier); if (testCasesWithFiles.length === 0) { @@ -969,8 +990,13 @@ async function runDirectLoop(config: RunConfig): Promise { lanes.length > 1 ? ` [lane ${String(laneIdx + 1)}/${String(lanes.length)}]` : ''; const results = await runWithConcurrency( bucket, - async ({ tc }) => - await runWorkflowTestCase({ + async ({ tc }) => { + const prebuiltWorkflowId = pickPrebuiltWorkflowId( + prebuiltManifest, + tc.fileSlug, + iter, + ); + const result = await runWorkflowTestCase({ client: lane.client, baseUrl: lane.baseUrl, testCase: tc.testCase, @@ -981,9 +1007,18 @@ async function runDirectLoop(config: RunConfig): Promise { logger, keepWorkflows: args.keepWorkflows, laneTag, - prebuiltWorkflowId: pickPrebuiltWorkflowId(prebuiltManifest, tc.fileSlug, iter), + prebuiltWorkflowId, pinAiRoots: args.pinAiRoots, - }), + }); + if ( + prebuiltWorkflowId !== undefined && + result.workflowBuildSuccess && + result.workflowId + ) { + prebuiltWorkflowIdsToDelete?.add(result.workflowId); + } + return result; + }, MAX_CONCURRENT_BUILDS, ); return bucket.map((b, i) => ({ origIdx: b.origIdx, result: results[i] })); diff --git a/packages/@n8n/instance-ai/evaluations/harness/prebuilt-workflows.ts b/packages/@n8n/instance-ai/evaluations/harness/prebuilt-workflows.ts index 6e889a1ca9d..1c16e357889 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/prebuilt-workflows.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/prebuilt-workflows.ts @@ -76,7 +76,7 @@ export function pickPrebuiltWorkflowId( * * `createdWorkflowIds` is intentionally left empty: cleanupBuild() iterates * that array and would delete the workflow otherwise. Prebuilt workflows - * are owned by the caller, not the eval run. + * are owned by the caller unless they opt into cleanupPrebuiltWorkflows(). */ export async function fetchPrebuiltBuild( client: N8nClient, @@ -103,3 +103,28 @@ export async function fetchPrebuiltBuild( }; } } + +/** Explicit opt-in cleanup for workflows supplied via --prebuilt-workflows. */ +export async function cleanupPrebuiltWorkflows( + client: N8nClient, + workflowIds: Iterable, + logger: EvalLogger, +): Promise { + const uniqueWorkflowIds = [...new Set(workflowIds)]; + if (uniqueWorkflowIds.length === 0) return; + + let deleted = 0; + for (const workflowId of uniqueWorkflowIds) { + try { + await client.deleteWorkflow(workflowId); + deleted++; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.warn(`Failed to delete prebuilt workflow ${workflowId}: ${message}`); + } + } + + logger.info( + `Deleted ${String(deleted)}/${String(uniqueWorkflowIds.length)} prebuilt workflow(s)`, + ); +}