feat(core): Allow deleting pre-built workflows after evaluating (no-changelog) (#31960)

packages/@n8n/instance-ai/evaluations/README.md

@@ -132,6 +132,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai --iterations 3
 | `--exclude` | — | Skip test cases whose filename matches any of the substrings. Same comma-separated shape as `--filter`; applied after `--filter` |
 | `--prebuilt-workflows` | — | Path to a JSON manifest mapping test-case slugs to existing workflow IDs. Skips the orchestrator build for matched test cases — see [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows) |
 | `--keep-workflows` | `false` | Don't delete built workflows after the run. Pair with the HTML report's "view in n8n" links to inspect each scenario's canvas execution |
+| `--delete-prebuilt-workflows` | `false` | With `--prebuilt-workflows`, delete successfully used manifest workflows after the eval run. Mutually exclusive with `--keep-workflows` |
 | `--base-url` | `http://localhost:5678` | n8n instance URL |
 | `--email` | E2E test owner | Override login email (or `N8N_EVAL_EMAIL`) |
 | `--password` | E2E test owner | Override login password (or `N8N_EVAL_PASSWORD`) |
@@ -279,7 +280,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai \
   --experiment-name mcp-cohort
 ```
 
-The harness leaves prebuilt workflows alone after the run (no auto-delete), so the manifest can be re-used across multiple eval runs.
+The harness leaves prebuilt workflows alone after the run (no auto-delete), so the manifest can be re-used across multiple eval runs. If the workflows were created only for this eval cohort, pass `--delete-prebuilt-workflows` with `--prebuilt-workflows` to delete every successfully used manifest workflow once after the run. This is destructive: the manifest will still contain the deleted IDs and should not be re-used afterward.
 
 ### Producing a manifest
 

packages/@n8n/instance-ai/evaluations/__tests__/args.test.ts

@@ -41,6 +41,7 @@ describe('parseCliArgs --base-url', () => {
 describe('parseCliArgs --prebuilt-workflows', () => {
 	it('is undefined by default', () => {
 		expect(parseCliArgs([]).prebuiltWorkflows).toBeUndefined();
+		expect(parseCliArgs([]).deletePrebuiltWorkflows).toBe(false);
 	});
 
 	it('accepts a path argument', () => {
@@ -51,6 +52,32 @@ describe('parseCliArgs --prebuilt-workflows', () => {
 	it('throws when no value is provided', () => {
 		expect(() => parseCliArgs(['--prebuilt-workflows'])).toThrow(/Missing value/);
 	});
+
+	it('accepts deleting prebuilt workflows when a manifest is provided', () => {
+		const args = parseCliArgs([
+			'--prebuilt-workflows',
+			'./mcp-manifest.json',
+			'--delete-prebuilt-workflows',
+		]);
+		expect(args.deletePrebuiltWorkflows).toBe(true);
+	});
+
+	it('rejects deleting prebuilt workflows without a manifest', () => {
+		expect(() => parseCliArgs(['--delete-prebuilt-workflows'])).toThrow(
+			/--delete-prebuilt-workflows requires --prebuilt-workflows/,
+		);
+	});
+
+	it('rejects deleting and keeping workflows at the same time', () => {
+		expect(() =>
+			parseCliArgs([
+				'--prebuilt-workflows',
+				'./mcp-manifest.json',
+				'--delete-prebuilt-workflows',
+				'--keep-workflows',
+			]),
+		).toThrow(/--delete-prebuilt-workflows cannot be used with --keep-workflows/);
+	});
 });
 
 describe('parseCliArgs --exclude', () => {

packages/@n8n/instance-ai/evaluations/__tests__/prebuilt-workflows.test.ts

@@ -7,6 +7,7 @@ import type { Mock } from 'vitest';
 import type { N8nClient, WorkflowResponse } from '../clients/n8n-client';
 import type { EvalLogger } from '../harness/logger';
 import {
+	cleanupPrebuiltWorkflows,
 	fetchPrebuiltBuild,
 	loadPrebuiltManifest,
 	pickPrebuiltWorkflowId,
@@ -153,3 +154,45 @@ describe('fetchPrebuiltBuild', () => {
 		expect(result.error).toContain('plain string failure');
 	});
 });
+
+describe('cleanupPrebuiltWorkflows', () => {
+	function makeLogger(): EvalLogger & { info: Mock; warn: Mock } {
+		return {
+			info: vi.fn(),
+			verbose: vi.fn(),
+			success: vi.fn(),
+			warn: vi.fn(),
+			error: vi.fn(),
+			isVerbose: false,
+		};
+	}
+
+	it('deletes each workflow ID once', async () => {
+		const deleteWorkflow = vi.fn().mockResolvedValue(undefined);
+		const client = { deleteWorkflow } as unknown as N8nClient;
+		const logger = makeLogger();
+
+		await cleanupPrebuiltWorkflows(client, ['W1', 'W1', 'W2'], logger);
+
+		expect(deleteWorkflow).toHaveBeenCalledTimes(2);
+		expect(deleteWorkflow).toHaveBeenNthCalledWith(1, 'W1');
+		expect(deleteWorkflow).toHaveBeenNthCalledWith(2, 'W2');
+		expect(logger.info).toHaveBeenCalledWith('Deleted 2/2 prebuilt workflow(s)');
+		expect(logger.warn).not.toHaveBeenCalled();
+	});
+
+	it('continues deleting after a workflow deletion fails', async () => {
+		const deleteWorkflow = vi
+			.fn()
+			.mockRejectedValueOnce(new Error('HTTP 404'))
+			.mockResolvedValueOnce(undefined);
+		const client = { deleteWorkflow } as unknown as N8nClient;
+		const logger = makeLogger();
+
+		await cleanupPrebuiltWorkflows(client, ['W1', 'W2'], logger);
+
+		expect(deleteWorkflow).toHaveBeenCalledTimes(2);
+		expect(logger.warn).toHaveBeenCalledWith('Failed to delete prebuilt workflow W1: HTTP 404');
+		expect(logger.info).toHaveBeenCalledWith('Deleted 1/2 prebuilt workflow(s)');
+	});
+});

packages/@n8n/instance-ai/evaluations/cli/args.ts

@@ -35,6 +35,8 @@ export interface CliArgs {
 	prebuiltWorkflows?: string;
 	/** Keep built workflows after evaluation instead of deleting them */
 	keepWorkflows: boolean;
+	/** Delete successfully used workflows from --prebuilt-workflows after evaluation */
+	deletePrebuiltWorkflows: boolean;
 	/** Directory to write eval-results.json (defaults to cwd) */
 	outputDir?: string;
 	/** LangSmith dataset name (synced from JSON test cases before each run) */
@@ -72,6 +74,7 @@ const cliArgsSchema = z.object({
 	exclude: z.string().optional(),
 	prebuiltWorkflows: z.string().optional(),
 	keepWorkflows: z.boolean().default(false),
+	deletePrebuiltWorkflows: z.boolean().default(false),
 	outputDir: z.string().optional(),
 	dataset: z.string().default('instance-ai-workflow-evals'),
 	concurrency: z.number().int().positive().default(16),
@@ -88,6 +91,12 @@ const cliArgsSchema = z.object({
 export function parseCliArgs(argv: string[]): CliArgs {
 	const raw = parseRawArgs(argv);
 	const validated = cliArgsSchema.parse(raw);
+	if (validated.deletePrebuiltWorkflows && !validated.prebuiltWorkflows) {
+		throw new Error('--delete-prebuilt-workflows requires --prebuilt-workflows');
+	}
+	if (validated.deletePrebuiltWorkflows && validated.keepWorkflows) {
+		throw new Error('--delete-prebuilt-workflows cannot be used with --keep-workflows');
+	}
 
 	return {
 		timeoutMs: validated.timeoutMs,
@@ -99,6 +108,7 @@ export function parseCliArgs(argv: string[]): CliArgs {
 		exclude: validated.exclude,
 		prebuiltWorkflows: validated.prebuiltWorkflows,
 		keepWorkflows: validated.keepWorkflows,
+		deletePrebuiltWorkflows: validated.deletePrebuiltWorkflows,
 		outputDir: validated.outputDir,
 		dataset: validated.dataset,
 		concurrency: validated.concurrency,
@@ -123,6 +133,7 @@ interface RawArgs {
 	exclude?: string;
 	prebuiltWorkflows?: string;
 	keepWorkflows: boolean;
+	deletePrebuiltWorkflows: boolean;
 	outputDir?: string;
 	dataset: string;
 	concurrency: number;
@@ -138,6 +149,7 @@ function parseRawArgs(argv: string[]): RawArgs {
 		baseUrls: ['http://localhost:5678'],
 		verbose: false,
 		keepWorkflows: false,
+		deletePrebuiltWorkflows: false,
 		outputDir: undefined,
 		dataset: 'instance-ai-workflow-evals',
 		concurrency: 16,
@@ -198,6 +210,10 @@ function parseRawArgs(argv: string[]): RawArgs {
 				result.keepWorkflows = true;
 				break;
 
+			case '--delete-prebuilt-workflows':
+				result.deletePrebuiltWorkflows = true;
+				break;
+
 			case '--output-dir':
 				result.outputDir = nextArg(argv, i, '--output-dir');
 				i++;

packages/@n8n/instance-ai/evaluations/cli/index.ts

@@ -40,6 +40,7 @@ import type { WorkflowTestCaseWithFile } from '../data/workflows';
 import { createLogger } from '../harness/logger';
 import type { EvalLogger } from '../harness/logger';
 import {
+	cleanupPrebuiltWorkflows,
 	fetchPrebuiltBuild,
 	loadPrebuiltManifest,
 	pickPrebuiltWorkflowId,
@@ -159,6 +160,7 @@ interface RunConfig {
 	lanes: Lane[];
 	logger: EvalLogger;
 	prebuiltManifest?: PrebuiltManifest;
+	prebuiltWorkflowIdsToDelete?: Set<string>;
 }
 
 async function main(): Promise<void> {
@@ -228,6 +230,7 @@ async function main(): Promise<void> {
 	);
 
 	const startTime = Date.now();
+	const prebuiltWorkflowIdsToDelete = args.deletePrebuiltWorkflows ? new Set<string>() : undefined;
 
 	try {
 		const hasLangSmith = Boolean(process.env.LANGSMITH_API_KEY);
@@ -239,14 +242,26 @@ async function main(): Promise<void> {
 
 		if (hasLangSmith) {
 			logger.info('LangSmith API key detected, using evaluate() with experiment tracking');
-			const langsmithRun = await runWithLangSmith({ args, lanes, logger, prebuiltManifest });
+			const langsmithRun = await runWithLangSmith({
+				args,
+				lanes,
+				logger,
+				prebuiltManifest,
+				prebuiltWorkflowIdsToDelete,
+			});
 			evaluation = langsmithRun.evaluation;
 			experimentName = langsmithRun.experimentName;
 			outcome = langsmithRun.outcome;
 			slugByTestCase = langsmithRun.slugByTestCase;
 		} else {
 			logger.info('No LANGSMITH_API_KEY, running direct loop (results in eval-results.json only)');
-			evaluation = await runDirectLoop({ args, lanes, logger, prebuiltManifest });
+			evaluation = await runDirectLoop({
+				args,
+				lanes,
+				logger,
+				prebuiltManifest,
+				prebuiltWorkflowIdsToDelete,
+			});
 		}
 
 		const totalDuration = Date.now() - startTime;
@@ -270,6 +285,9 @@ async function main(): Promise<void> {
 			'\n' + formatComparisonTerminal(evaluation, outcome, { commitSha, slugByTestCase }),
 		);
 	} finally {
+		if (prebuiltWorkflowIdsToDelete && lanes[0]) {
+			await cleanupPrebuiltWorkflows(lanes[0].client, prebuiltWorkflowIdsToDelete, logger);
+		}
 		await Promise.all(
 			lanes.map(async (lane) => {
 				await cleanupCredentials(lane.client, lane.seedResult.credentialIds).catch(() => {});
@@ -288,7 +306,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{
 	outcome: ComparisonOutcome;
 	slugByTestCase: Map<WorkflowTestCase, string>;
 }> {
-	const { args, lanes, logger, prebuiltManifest } = config;
+	const { args, lanes, logger, prebuiltManifest, prebuiltWorkflowIdsToDelete } = config;
 
 	const lsClient = new Client();
 	const datasetName = await syncDataset(lsClient, args.dataset, logger, args.filter, args.exclude);
@@ -412,6 +430,9 @@ async function runWithLangSmith(config: RunConfig): Promise<{
 				const lane = laneStates[0];
 				const start = Date.now();
 				const build = await fetchPrebuiltBuild(lane.runner.client, prebuiltId, logger);
+				if (build.success && build.workflowId) {
+					prebuiltWorkflowIdsToDelete?.add(build.workflowId);
+				}
 				const buildDurationMs = Date.now() - start;
 				buildDurations.set(key, buildDurationMs);
 				stashTranscript(build);
@@ -934,7 +955,7 @@ function reshapeLangSmithRuns(
 // ---------------------------------------------------------------------------
 
 async function runDirectLoop(config: RunConfig): Promise<MultiRunEvaluation> {
-	const { args, lanes, logger, prebuiltManifest } = config;
+	const { args, lanes, logger, prebuiltManifest, prebuiltWorkflowIdsToDelete } = config;
 
 	const testCasesWithFiles = loadWorkflowTestCasesWithFiles(args.filter, args.exclude, args.tier);
 	if (testCasesWithFiles.length === 0) {
@@ -969,8 +990,13 @@ async function runDirectLoop(config: RunConfig): Promise<MultiRunEvaluation> {
 						lanes.length > 1 ? ` [lane ${String(laneIdx + 1)}/${String(lanes.length)}]` : '';
 					const results = await runWithConcurrency(
 						bucket,
-						async ({ tc }) =>
-							await runWorkflowTestCase({
+						async ({ tc }) => {
+							const prebuiltWorkflowId = pickPrebuiltWorkflowId(
+								prebuiltManifest,
+								tc.fileSlug,
+								iter,
+							);
+							const result = await runWorkflowTestCase({
 								client: lane.client,
 								baseUrl: lane.baseUrl,
 								testCase: tc.testCase,
@@ -981,9 +1007,18 @@ async function runDirectLoop(config: RunConfig): Promise<MultiRunEvaluation> {
 								logger,
 								keepWorkflows: args.keepWorkflows,
 								laneTag,
-								prebuiltWorkflowId: pickPrebuiltWorkflowId(prebuiltManifest, tc.fileSlug, iter),
+								prebuiltWorkflowId,
 								pinAiRoots: args.pinAiRoots,
-							}),
+							});
+							if (
+								prebuiltWorkflowId !== undefined &&
+								result.workflowBuildSuccess &&
+								result.workflowId
+							) {
+								prebuiltWorkflowIdsToDelete?.add(result.workflowId);
+							}
+							return result;
+						},
 						MAX_CONCURRENT_BUILDS,
 					);
 					return bucket.map((b, i) => ({ origIdx: b.origIdx, result: results[i] }));

packages/@n8n/instance-ai/evaluations/harness/prebuilt-workflows.ts

@@ -76,7 +76,7 @@ export function pickPrebuiltWorkflowId(
  *
  * `createdWorkflowIds` is intentionally left empty: cleanupBuild() iterates
  * that array and would delete the workflow otherwise. Prebuilt workflows
- * are owned by the caller, not the eval run.
+ * are owned by the caller unless they opt into cleanupPrebuiltWorkflows().
  */
 export async function fetchPrebuiltBuild(
 	client: N8nClient,
@@ -103,3 +103,28 @@ export async function fetchPrebuiltBuild(
 		};
 	}
 }
+
+/** Explicit opt-in cleanup for workflows supplied via --prebuilt-workflows. */
+export async function cleanupPrebuiltWorkflows(
+	client: N8nClient,
+	workflowIds: Iterable<string>,
+	logger: EvalLogger,
+): Promise<void> {
+	const uniqueWorkflowIds = [...new Set(workflowIds)];
+	if (uniqueWorkflowIds.length === 0) return;
+
+	let deleted = 0;
+	for (const workflowId of uniqueWorkflowIds) {
+		try {
+			await client.deleteWorkflow(workflowId);
+			deleted++;
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			logger.warn(`Failed to delete prebuilt workflow ${workflowId}: ${message}`);
+		}
+	}
+
+	logger.info(
+		`Deleted ${String(deleted)}/${String(uniqueWorkflowIds.length)} prebuilt workflow(s)`,
+	);
+}