n8n/packages/@n8n/instance-ai/evaluations/harness/prebuilt-workflows.ts

// ---------------------------------------------------------------------------
// Prebuilt-workflows manifest + BuildResult adapter
//
// Owns everything specific to `--prebuilt-workflows` mode:
//   • the manifest schema + loader (loadPrebuiltManifest)
//   • the per-iteration workflow-ID picker (pickPrebuiltWorkflowId)
//   • the BuildResult adapter that fetches an existing workflow from n8n
//     and shapes it like a fresh build (fetchPrebuiltBuild)
//
// When `--prebuilt-workflows <path>` is set, the eval CLI uses this module
// in place of the orchestrator build — useful for evaluating workflows
// authored by tools other than Instance AI (e.g. an MCP-driven build,
// a hand-built reference, an older Instance AI snapshot).
//
// Multiple IDs per slug let multi-iteration runs (--iterations N) compare
// across distinct builds; pickPrebuiltWorkflowId rotates with
// iteration % ids.length.
// ---------------------------------------------------------------------------

import { readFileSync } from 'fs';
import { z } from 'zod';

import type { BuildResult } from './build-workflow';
import type { EvalLogger } from './logger';
import type { N8nClient } from '../clients/n8n-client';

export const prebuiltManifestSchema = z
	.record(z.string().min(1), z.array(z.string().min(1)).min(1))
	.refine((v) => Object.keys(v).length > 0, { message: 'manifest must not be empty' });

export type PrebuiltManifest = z.infer<typeof prebuiltManifestSchema>;

export function loadPrebuiltManifest(path: string): PrebuiltManifest {
	let raw: unknown;
	try {
		raw = JSON.parse(readFileSync(path, 'utf-8'));
	} catch (error) {
		const msg = error instanceof Error ? error.message : String(error);
		throw new Error(`Failed to read prebuilt-workflows manifest at ${path}: ${msg}`);
	}
	const result = prebuiltManifestSchema.safeParse(raw);
	if (!result.success) {
		throw new Error(
			`Invalid prebuilt-workflows manifest at ${path}: ${result.error.issues
				.map((i) => `${i.path.join('.')}: ${i.message}`)
				.join('; ')}`,
		);
	}
	return result.data;
}

/**
 * Look up the workflow ID for a given test-case file slug + iteration.
 *
 * Returns `undefined` in two cases — callers cannot distinguish them and
 * shouldn't need to:
 *   • the manifest argument itself is undefined (no `--prebuilt-workflows`)
 *   • the manifest exists but doesn't cover this slug (fall through to the
 *     regular orchestrator build path)
 */
export function pickPrebuiltWorkflowId(
	manifest: PrebuiltManifest | undefined,
	fileSlug: string,
	iteration: number,
): string | undefined {
	if (!manifest) return undefined;
	const ids = manifest[fileSlug];
	if (!ids || ids.length === 0) return undefined;
	return ids[iteration % ids.length];
}

/**
 * Split test cases by whether the manifest provides a build for them. Cases with
 * no workflow in the manifest are skipped (the caller logs them) rather than
 * silently orchestrator-built, which would mix builders in one result set.
 */
export function partitionByPrebuiltCoverage<T extends { fileSlug: string }>(
	cases: T[],
	manifest: PrebuiltManifest,
): { covered: T[]; skipped: T[] } {
	const covered: T[] = [];
	const skipped: T[] = [];
	for (const testCase of cases) {
		if ((manifest[testCase.fileSlug]?.length ?? 0) > 0) covered.push(testCase);
		else skipped.push(testCase);
	}
	return { covered, skipped };
}

/**
 * Build a BuildResult for a workflow that already exists in the n8n instance.
 * Used by --prebuilt-workflows mode to skip the orchestrator and verify a
 * workflow built by some other tool (e.g. an MCP-driven session).
 *
 * `createdWorkflowIds` is intentionally left empty: cleanupBuild() iterates
 * that array and would delete the workflow otherwise. Prebuilt workflows
 * are owned by the caller unless they opt into cleanupPrebuiltWorkflows().
 */
export async function fetchPrebuiltBuild(
	client: N8nClient,
	workflowId: string,
	logger: EvalLogger,
): Promise<BuildResult> {
	logger.info(`  Using prebuilt workflow: ${workflowId}`);
	try {
		const workflow = await client.getWorkflow(workflowId);
		return {
			success: true,
			workflowId,
			workflowJsons: [workflow],
			createdWorkflowIds: [],
			createdDataTableIds: [],
		};
	} catch (error: unknown) {
		return {
			success: false,
			error: `Failed to fetch prebuilt workflow ${workflowId}: ${error instanceof Error ? error.message : String(error)}`,
			workflowJsons: [],
			createdWorkflowIds: [],
			createdDataTableIds: [],
		};
	}
}

/** Explicit opt-in cleanup for workflows supplied via --prebuilt-workflows. */
export async function cleanupPrebuiltWorkflows(
	client: N8nClient,
	workflowIds: Iterable<string>,
	logger: EvalLogger,
): Promise<void> {
	const uniqueWorkflowIds = [...new Set(workflowIds)];
	if (uniqueWorkflowIds.length === 0) return;

	let deleted = 0;
	for (const workflowId of uniqueWorkflowIds) {
		try {
			await client.deleteWorkflow(workflowId);
			deleted++;
		} catch (error) {
			const message = error instanceof Error ? error.message : String(error);
			logger.warn(`Failed to delete prebuilt workflow ${workflowId}: ${message}`);
		}
	}

	logger.info(
		`Deleted ${String(deleted)}/${String(uniqueWorkflowIds.length)} prebuilt workflow(s)`,
	);
}