n8n/packages/@n8n/instance-ai/evaluations/cli/args.ts
José Braulio González Valido b0a10229a0
refactor(ai-builder): Split the eval harness runner into domain modules (no-changelog) (#34834)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-27 09:01:09 +00:00

540 lines
20 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// ---------------------------------------------------------------------------
// CLI argument parser for the instance-ai workflow evaluator
//
// Uses manual parsing (no external CLI lib) to keep dependencies minimal.
// Validates and normalizes arguments into a typed CliArgs interface.
// ---------------------------------------------------------------------------
import { z } from 'zod';
import { DEFAULT_MCP_BUILD_TIMEOUT_MS } from './mcp-builder';
import { BASELINE_EXPERIMENT_PREFIX } from '../comparison/fetch-baseline';
/** Default LangSmith dataset — the shared Instance AI cohort. */
export const DEFAULT_DATASET = 'instance-ai-workflow-evals';
/** Default Anthropic model for `claude -p` MCP builds when ANTHROPIC_MODEL is unset. */
export const DEFAULT_MCP_BUILD_MODEL = 'claude-opus-4-8';
/** Resolve the MCP build model from the environment. `claude` natively reads
* ANTHROPIC_MODEL, so operators/CI set that env var (matching how the AI
* Assistant itself is configured) instead of a CLI flag. We still pin a
* default when it's unset so builds never float with claude-code's bundled
* default, and the resolved value is passed explicitly to `claude --model`
* and recorded as `build_model` experiment metadata. */
function resolveBuildModel(env: NodeJS.ProcessEnv = process.env): string {
// Blank counts as unset: CI passes ANTHROPIC_MODEL through from an optional
// workflow input, so an empty value must still pin the default.
const fromEnv = env.ANTHROPIC_MODEL?.trim() ?? '';
return fromEnv === '' ? DEFAULT_MCP_BUILD_MODEL : fromEnv;
}
// ---------------------------------------------------------------------------
// Public types
// ---------------------------------------------------------------------------
export interface CliArgs {
/** TimeoutMs is defined per iteration, not as the total timeout for all iterations */
timeoutMs: number;
/** One or more n8n base URLs. Multi-lane runs use a work-stealing allocator
* that dispatches each build to a lane that isn't already running its
* prompt, capped per-lane at MAX_CONCURRENT_BUILDS=4. Pass comma-separated
* to `--base-url`. */
baseUrls: string[];
email?: string;
password?: string;
verbose: boolean;
/** Filter workflow test cases by filename substring(s). Accepts a comma-separated
* list with OR semantics, e.g. "contact-form,deduplication". */
filter?: string;
/** Exclude workflow test cases whose filename matches any of the substring(s).
* Same comma-separated shape as --filter; applied after --filter. */
exclude?: string;
/** Path to a JSON manifest mapping test-case file slugs to one or more
* pre-built workflow IDs. When set, the harness skips the orchestrator
* build for matched test cases and verifies the existing workflow instead.
* See evaluations/harness/prebuilt-workflows.ts for the schema. */
prebuiltWorkflows?: string;
/** Keep built workflows after evaluation instead of deleting them */
keepWorkflows: boolean;
/** Delete successfully used workflows from --prebuilt-workflows after evaluation */
deletePrebuiltWorkflows: boolean;
/** Directory to write eval-results.json (defaults to cwd) */
outputDir?: string;
/** LangSmith dataset name (synced from JSON test cases before each run) */
dataset: string;
/** True when `--source langtracer` auto-forked the dataset name off the suite
* (no explicit --dataset) — cohort isolation, surfaced loudly by the driver. */
datasetAutoForked: boolean;
/** Max concurrent target() calls in LangSmith evaluate(). Build concurrency is
* enforced separately by the LaneAllocator (cap=4 per lane). */
concurrency: number;
/** LangSmith experiment name prefix (auto-generated if not set) */
experimentName?: string;
/** Number of iterations to run each test case (default: 1). Each iteration
* gets a fresh build so pass@k / pass^k capture real builder variance. */
iterations: number;
/** AI root nodes (Agent, Chain) to keep pinned — opt-out from the default-on
* wire-server interception path. Useful for A/B comparison or when a
* specific root needs to stay on the pinned baseline. CSV of node names. */
pinAiRoots?: string[];
/** Filter test cases by the `datasets` field (e.g. `pr`, `full`). When set,
* only test cases whose `datasets` array contains this value will run, and
* LangSmith examples are queried via the matching split. Defaults to
* unset → run everything matched by `--filter` / `--exclude`. */
tier?: string;
/** Experiment-name prefix the regression comparison uses to find the
* baseline. Defaults to the Instance AI baseline (`instance-ai-baseline-`).
* Override for an isolated cohort (e.g. `mcp-baseline-`) so the run compares
* against its own baselines instead of the Instance AI one. Pair with a
* dedicated `--dataset` to keep MCP runs fully separate. */
baselinePrefix: string;
/** Test-case source: `disk` (default) reads data/workflows/, `langtracer` pulls a
* suite over MCP (needs LANGTRACER_URL + LANGTRACER_API_KEY). */
source: 'disk' | 'langtracer';
/** lang-tracer suite slug (or numeric id) to export when `--source langtracer`. */
suite?: string;
/** Fused MCP build mode: instead of the Instance AI orchestrator, build each
* workflow by driving the lane's own MCP server with `claude -p`, then verify
* it on that same lane. Works across multiple `--base-url` lanes (each lane
* builds + verifies its own slice). Mutually exclusive with
* `--prebuilt-workflows`. See cli/mcp-builder.ts. */
buildViaMcp: boolean;
/** MCP server name used in the per-lane staged `claude` config + tool allowlist
* (`--build-via-mcp` only). Arbitrary — the eval CLI stages the config itself. */
mcpServerName: string;
/** Anthropic model id passed to `claude -p` for the MCP build (`--build-via-mcp`).
* Sourced from the ANTHROPIC_MODEL env var (the variable `claude` reads
* natively), pinned to DEFAULT_MCP_BUILD_MODEL when unset. Distinct from the
* verifier model (N8N_INSTANCE_AI_MODEL). */
buildModel: string;
/** Working directory for the `claude` build subprocess (`--build-via-mcp`); loads
* that project's Claude config/skills. Defaults to the subprocess default. */
buildCwd?: string;
/** Retries per workflow when `claude` returns no WORKFLOW_ID (`--build-via-mcp`). */
buildMaxAttempts: number;
/** MCP_TIMEOUT (ms) passed to the `claude` build subprocess (`--build-via-mcp`);
* bounds a single MCP tool call. */
buildMcpTimeoutMs: number;
/** Wall-clock cap (ms) for the whole `claude` build subprocess per attempt
* (`--build-via-mcp`). On expiry the process is killed so a hung build can't
* hold its lane. 0 disables. Distinct from `buildMcpTimeoutMs` (per MCP call). */
buildTimeoutMs: number;
}
// ---------------------------------------------------------------------------
// Zod schema for validation
// ---------------------------------------------------------------------------
const cliArgsSchema = z.object({
// Keep in sync with DEFAULT_TIMEOUT_MS in harness/build-workflow.ts (and its
// rationale for why this default must stay conservative).
timeoutMs: z.number().int().positive().default(900_000),
baseUrls: z.array(z.string().url()).min(1).default(['http://localhost:5678']),
email: z.string().optional(),
password: z.string().optional(),
verbose: z.boolean().default(false),
filter: z.string().optional(),
exclude: z.string().optional(),
prebuiltWorkflows: z.string().optional(),
keepWorkflows: z.boolean().default(false),
deletePrebuiltWorkflows: z.boolean().default(false),
outputDir: z.string().optional(),
dataset: z.string().default(DEFAULT_DATASET),
concurrency: z.number().int().positive().default(16),
experimentName: z.string().optional(),
iterations: z.number().int().positive().default(1),
pinAiRoots: z.array(z.string().min(1)).optional(),
tier: z.string().min(1).optional(),
// Normalize to a trailing hyphen. The baseline lookup matches by prefix and
// LangSmith always appends `-<suffix>` to the experiment name, so the hyphen
// anchors the match to that separator — without it `mcp-baseline` would also
// match unrelated names like `mcp-baseline2-...`. Mirrors BASELINE_EXPERIMENT_PREFIX.
baselinePrefix: z
.string()
.min(1)
.transform((s) => (s.endsWith('-') ? s : `${s}-`))
.default(BASELINE_EXPERIMENT_PREFIX),
source: z.enum(['disk', 'langtracer']).default('disk'),
suite: z.string().min(1).optional(),
buildViaMcp: z.boolean().default(false),
mcpServerName: z.string().min(1).default('n8n-local'),
buildModel: z.string().min(1).default('claude-opus-4-8'),
buildCwd: z.string().min(1).optional(),
buildMaxAttempts: z.number().int().positive().default(3),
buildMcpTimeoutMs: z.number().int().positive().default(120_000),
buildTimeoutMs: z.number().int().nonnegative().default(DEFAULT_MCP_BUILD_TIMEOUT_MS),
});
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
export function parseCliArgs(argv: string[]): CliArgs {
const raw = parseRawArgs(argv);
const validated = cliArgsSchema.parse(raw);
// --build-via-mcp checks first: they give clearer guidance than the generic
// --delete-prebuilt-workflows check below when both are combined.
if (validated.buildViaMcp && validated.prebuiltWorkflows) {
throw new Error(
'--build-via-mcp is incompatible with --prebuilt-workflows. --build-via-mcp builds fresh workflows via the MCP server on each lane; --prebuilt-workflows verifies existing ones.',
);
}
if (validated.buildViaMcp && validated.deletePrebuiltWorkflows) {
throw new Error(
'--delete-prebuilt-workflows applies to --prebuilt-workflows. --build-via-mcp already cleans up the workflows it builds unless --keep-workflows is set.',
);
}
// MCP builds are LangSmith-only: the keyless direct loop parallelizes
// iterations without the lane allocator, so its 4-per-lane build cap applies
// PER ITERATION — concurrent `claude` sessions would scale with
// lanes × iterations × 4 and flood the shared Anthropic budget. Fail fast
// instead of teaching the direct loop (tech debt slated for removal) MCP builds.
if (validated.buildViaMcp && !process.env.LANGSMITH_API_KEY) {
throw new Error(
'--build-via-mcp requires LangSmith experiment tracking — set LANGSMITH_API_KEY. The no-LangSmith direct loop does not support MCP builds.',
);
}
// Build knobs without --build-via-mcp would parse fine and then be silently
// ignored — the run would look like it honored them. Fail loudly instead.
if (!validated.buildViaMcp && raw.buildOnlyFlags.length > 0) {
throw new Error(
`${[...new Set(raw.buildOnlyFlags)].join(', ')} only take${raw.buildOnlyFlags.length === 1 ? 's' : ''} effect with --build-via-mcp — pass it, or drop the flag(s).`,
);
}
if (validated.deletePrebuiltWorkflows && !validated.prebuiltWorkflows) {
throw new Error('--delete-prebuilt-workflows requires --prebuilt-workflows');
}
if (validated.deletePrebuiltWorkflows && validated.keepWorkflows) {
throw new Error('--delete-prebuilt-workflows cannot be used with --keep-workflows');
}
if (validated.source === 'langtracer' && !validated.suite) {
throw new Error('--source langtracer requires --suite <slug>');
}
// In langtracer mode, default the dataset + baseline to a suite-scoped, eval-tagged
// name so runs don't pollute the shared cohort and re-runs upsert one stable dataset.
let dataset = validated.dataset;
let datasetAutoForked = false;
let baselinePrefix = validated.baselinePrefix;
if (validated.source === 'langtracer' && validated.suite) {
const suiteSlug = validated.suite
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '');
if (!raw.datasetProvided) {
dataset = `instance-ai-langtracer-${suiteSlug}`;
datasetAutoForked = true;
}
if (!raw.baselineProvided) baselinePrefix = `instance-ai-langtracer-${suiteSlug}-baseline-`;
}
return {
timeoutMs: validated.timeoutMs,
baseUrls: validated.baseUrls,
email: validated.email,
password: validated.password,
verbose: validated.verbose,
filter: validated.filter,
exclude: validated.exclude,
prebuiltWorkflows: validated.prebuiltWorkflows,
keepWorkflows: validated.keepWorkflows,
deletePrebuiltWorkflows: validated.deletePrebuiltWorkflows,
outputDir: validated.outputDir,
dataset,
datasetAutoForked,
concurrency: validated.concurrency,
experimentName: validated.experimentName,
iterations: validated.iterations,
pinAiRoots: validated.pinAiRoots,
tier: validated.tier,
baselinePrefix,
source: validated.source,
suite: validated.suite,
buildViaMcp: validated.buildViaMcp,
mcpServerName: validated.mcpServerName,
buildModel: validated.buildModel,
buildCwd: validated.buildCwd,
buildMaxAttempts: validated.buildMaxAttempts,
buildMcpTimeoutMs: validated.buildMcpTimeoutMs,
buildTimeoutMs: validated.buildTimeoutMs,
};
}
/**
* A dedicated `--dataset` and a dedicated `--baseline-prefix` are the two halves
* of LangSmith cohort isolation (e.g. for MCP runs). Overriding exactly one of
* them still writes to / compares against shared Instance AI data, which is
* almost always a mistake. Returns a warning for that mismatched case, or
* `undefined` when the pairing is consistent (both shared, or both isolated).
*
* Leaving BOTH at their defaults is a normal Instance AI run — including
* `--tier pr`/`mcp` against the shared dataset, whose example upserts are
* idempotent — so it is intentionally not flagged.
*/
export function partialIsolationWarning(
dataset: string,
baselinePrefix: string,
): string | undefined {
const datasetIsolated = dataset !== DEFAULT_DATASET;
const baselineIsolated = baselinePrefix !== BASELINE_EXPERIMENT_PREFIX;
if (datasetIsolated === baselineIsolated) return undefined;
return (
`Partial LangSmith isolation: --dataset="${dataset}" with --baseline-prefix="${baselinePrefix}". ` +
'Override BOTH for an isolated cohort (e.g. MCP), or leave BOTH at their defaults for an ' +
'Instance AI run — overriding only one still touches shared Instance AI data.'
);
}
// ---------------------------------------------------------------------------
// Raw argument parsing
// ---------------------------------------------------------------------------
interface RawArgs {
timeoutMs: number;
baseUrls: string[];
email?: string;
password?: string;
verbose: boolean;
filter?: string;
exclude?: string;
prebuiltWorkflows?: string;
keepWorkflows: boolean;
deletePrebuiltWorkflows: boolean;
outputDir?: string;
dataset: string;
concurrency: number;
experimentName?: string;
iterations: number;
pinAiRoots?: string[];
tier?: string;
baselinePrefix: string;
source: string;
suite?: string;
buildViaMcp: boolean;
mcpServerName: string;
buildModel: string;
buildCwd?: string;
buildMaxAttempts: number;
buildMcpTimeoutMs: number;
buildTimeoutMs: number;
/** Whether --dataset / --baseline-prefix were explicitly passed (langtracer mode
* derives suite-scoped defaults otherwise). */
datasetProvided: boolean;
baselineProvided: boolean;
/** Build-only flags the caller passed. Only meaningful with --build-via-mcp;
* parseCliArgs rejects them otherwise so they can't be silently ignored. */
buildOnlyFlags: string[];
}
function parseRawArgs(argv: string[]): RawArgs {
const result: RawArgs = {
timeoutMs: 900_000,
baseUrls: ['http://localhost:5678'],
verbose: false,
keepWorkflows: false,
deletePrebuiltWorkflows: false,
outputDir: undefined,
dataset: DEFAULT_DATASET,
concurrency: 16,
experimentName: undefined,
iterations: 1,
pinAiRoots: undefined,
baselinePrefix: BASELINE_EXPERIMENT_PREFIX,
source: 'disk',
buildViaMcp: false,
mcpServerName: 'n8n-local',
buildModel: resolveBuildModel(),
buildMaxAttempts: 3,
buildMcpTimeoutMs: 120_000,
buildTimeoutMs: DEFAULT_MCP_BUILD_TIMEOUT_MS,
datasetProvided: false,
baselineProvided: false,
buildOnlyFlags: [],
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
switch (arg) {
case '--timeout-ms':
result.timeoutMs = parseIntArg(argv, i, '--timeout-ms');
i++;
break;
case '--base-url': {
const raw = nextArg(argv, i, '--base-url');
result.baseUrls = raw
.split(',')
.map((s) => s.trim())
.filter((s) => s.length > 0);
i++;
break;
}
case '--email':
result.email = nextArg(argv, i, '--email');
i++;
break;
case '--password':
result.password = nextArg(argv, i, '--password');
i++;
break;
case '--verbose':
result.verbose = true;
break;
case '--filter':
result.filter = nextArg(argv, i, '--filter');
i++;
break;
case '--exclude':
result.exclude = nextArg(argv, i, '--exclude');
i++;
break;
case '--prebuilt-workflows':
result.prebuiltWorkflows = nextArg(argv, i, '--prebuilt-workflows');
i++;
break;
case '--keep-workflows':
result.keepWorkflows = true;
break;
case '--delete-prebuilt-workflows':
result.deletePrebuiltWorkflows = true;
break;
case '--output-dir':
result.outputDir = nextArg(argv, i, '--output-dir');
i++;
break;
case '--iterations':
result.iterations = parseIntArg(argv, i, '--iterations');
i++;
break;
case '--dataset':
result.dataset = nextArg(argv, i, '--dataset');
result.datasetProvided = true;
i++;
break;
case '--concurrency':
result.concurrency = parseIntArg(argv, i, '--concurrency');
i++;
break;
case '--experiment-name':
result.experimentName = nextArg(argv, i, '--experiment-name');
i++;
break;
case '--pin-ai-roots': {
const raw = nextArg(argv, i, '--pin-ai-roots');
result.pinAiRoots = raw
.split(',')
.map((s) => s.trim())
.filter((s) => s.length > 0);
i++;
break;
}
case '--tier':
result.tier = nextArg(argv, i, '--tier');
i++;
break;
case '--baseline-prefix':
result.baselinePrefix = nextArg(argv, i, '--baseline-prefix');
result.baselineProvided = true;
i++;
break;
case '--source':
result.source = nextArg(argv, i, '--source');
i++;
break;
case '--suite':
result.suite = nextArg(argv, i, '--suite');
i++;
break;
case '--build-via-mcp':
result.buildViaMcp = true;
break;
case '--mcp-server':
result.mcpServerName = nextArg(argv, i, '--mcp-server');
result.buildOnlyFlags.push(arg);
i++;
break;
case '--build-cwd':
result.buildCwd = nextArg(argv, i, '--build-cwd');
result.buildOnlyFlags.push(arg);
i++;
break;
case '--build-max-attempts':
result.buildMaxAttempts = parseIntArg(argv, i, '--build-max-attempts');
result.buildOnlyFlags.push(arg);
i++;
break;
case '--build-mcp-timeout-ms':
result.buildMcpTimeoutMs = parseIntArg(argv, i, '--build-mcp-timeout-ms');
result.buildOnlyFlags.push(arg);
i++;
break;
case '--build-timeout-ms':
result.buildTimeoutMs = parseIntArg(argv, i, '--build-timeout-ms');
result.buildOnlyFlags.push(arg);
i++;
break;
default:
// Fail loudly on unknown flags. Strip any =value payload before
// echoing and drop positional values entirely — raw CLI input
// may contain secrets (e.g. --password=... or an accidentally
// pasted token) that would otherwise leak into terminal/CI logs.
if (arg.startsWith('--')) {
const flagName = arg.split('=', 1)[0];
throw new Error(`Unknown flag: ${flagName}`);
}
throw new Error('Unexpected positional argument');
}
}
return result;
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function nextArg(argv: string[], currentIndex: number, flagName: string): string {
const value = argv[currentIndex + 1];
if (value === undefined || value.startsWith('--')) {
throw new Error(`Missing value for ${flagName}`);
}
return value;
}
function parseIntArg(argv: string[], currentIndex: number, flagName: string): number {
const raw = nextArg(argv, currentIndex, flagName);
const parsed = parseInt(raw, 10);
if (Number.isNaN(parsed)) {
// Don't echo raw — a bad shell expansion could leak a secret here.
throw new Error(`Invalid integer for ${flagName}`);
}
return parsed;
}