mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-19 16:55:02 +02:00
391 lines
13 KiB
JavaScript
391 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
||
// ---------------------------------------------------------------------------
|
||
// CLI for browser/computer-use discovery evaluation.
|
||
//
|
||
// Usage:
|
||
// pnpm eval:discovery # run all scenarios, 3 trials each
|
||
// pnpm eval:discovery --filter slack-oauth --verbose
|
||
// pnpm eval:discovery --trials 5
|
||
// pnpm eval:discovery --cases-dir <dir> # routing mode
|
||
//
|
||
// Loads scenarios from evaluations/data/discovery/, runs each scenario × N
|
||
// trials via the in-process runner, reports per-scenario pass-rates, exits
|
||
// non-zero on any scenario below threshold, or on any scenario with zero passes
|
||
// when --fail-on-zero-pass is set.
|
||
//
|
||
// Routing mode (`--cases-dir`) loads `route-*.json` cases from that folder, in
|
||
// the format LangTracer exports. A judge ends each trial as soon as the
|
||
// Assistant picks a route. The CLI prints a pass rate per bucket.
|
||
// It always exits 0: it measures routing and does not gate a merge.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
import { runDiscoveryScenario, runOrchestratorTurn, type DiscoveryRunResult } from './runner';
|
||
import type { DiscoveryTestCase } from './types';
|
||
import { isAgentFeatureEnabled } from '../../src/utils/agent-feature-enabled';
|
||
import { loadDiscoveryTestCasesWithFiles } from '../data/discovery';
|
||
import { loadRoutingCases, ROUTING_BUCKETS, type RoutingBucket } from '../routing/cases';
|
||
import { casePasses, createRouteWatcher, routeLabel, trialPasses } from '../routing/grade';
|
||
import { judgeRoute } from '../routing/judge';
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Args
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface CliArgs {
|
||
filter?: string;
|
||
verbose: boolean;
|
||
trials: number;
|
||
passThreshold: number;
|
||
timeoutMs: number;
|
||
/** Optional iteration cap; unset uses the SDK default. */
|
||
maxSteps?: number;
|
||
modelId: string;
|
||
concurrency: number;
|
||
nodesJsonPath?: string;
|
||
failOnZeroPass: boolean;
|
||
/** Folder of routing cases. Set switches the CLI to routing mode. */
|
||
casesDir?: string;
|
||
}
|
||
|
||
const DEFAULT_MODEL = process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6';
|
||
|
||
function previewArgs(args: Record<string, unknown>): string {
|
||
const entries = Object.entries(args).slice(0, 3);
|
||
if (entries.length === 0) return '';
|
||
return entries
|
||
.map(([k, v]) => {
|
||
let preview: string;
|
||
if (typeof v === 'string') {
|
||
preview = v.length > 40 ? `"${v.slice(0, 40)}…"` : `"${v}"`;
|
||
} else if (typeof v === 'number' || typeof v === 'boolean' || v === null) {
|
||
preview = String(v);
|
||
} else if (Array.isArray(v)) {
|
||
preview = `[${String(v.length)}]`;
|
||
} else {
|
||
preview = '{…}';
|
||
}
|
||
return `${k}=${preview}`;
|
||
})
|
||
.join(', ');
|
||
}
|
||
|
||
function requirePositiveInt(raw: string | undefined, flag: string): number {
|
||
const n = Number(raw);
|
||
if (!Number.isFinite(n) || n < 1 || !Number.isInteger(n)) {
|
||
console.error(`Invalid value for ${flag}: ${String(raw)} (expected a positive integer)`);
|
||
process.exit(1);
|
||
}
|
||
return n;
|
||
}
|
||
|
||
function requireNumberInRange(
|
||
raw: string | undefined,
|
||
flag: string,
|
||
lo: number,
|
||
hi: number,
|
||
): number {
|
||
const n = Number(raw);
|
||
if (!Number.isFinite(n) || n < lo || n > hi) {
|
||
console.error(`Invalid value for ${flag}: ${String(raw)} (expected number in [${lo}, ${hi}])`);
|
||
process.exit(1);
|
||
}
|
||
return n;
|
||
}
|
||
|
||
function parseArgs(argv: string[]): CliArgs {
|
||
const args: CliArgs = {
|
||
verbose: false,
|
||
trials: 3,
|
||
passThreshold: 2 / 3,
|
||
timeoutMs: 60_000,
|
||
modelId: DEFAULT_MODEL,
|
||
concurrency: 3,
|
||
failOnZeroPass: false,
|
||
};
|
||
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const arg = argv[i];
|
||
switch (arg) {
|
||
case '--verbose':
|
||
case '-v':
|
||
args.verbose = true;
|
||
break;
|
||
case '--filter':
|
||
args.filter = argv[++i];
|
||
break;
|
||
case '--trials':
|
||
args.trials = requirePositiveInt(argv[++i], '--trials');
|
||
break;
|
||
case '--pass-threshold':
|
||
args.passThreshold = requireNumberInRange(argv[++i], '--pass-threshold', 0, 1);
|
||
break;
|
||
case '--timeout':
|
||
args.timeoutMs = requirePositiveInt(argv[++i], '--timeout');
|
||
break;
|
||
case '--max-steps':
|
||
args.maxSteps = requirePositiveInt(argv[++i], '--max-steps');
|
||
break;
|
||
case '--model':
|
||
args.modelId = argv[++i];
|
||
break;
|
||
case '--concurrency':
|
||
args.concurrency = requirePositiveInt(argv[++i], '--concurrency');
|
||
break;
|
||
case '--nodes-json':
|
||
args.nodesJsonPath = argv[++i];
|
||
break;
|
||
case '--fail-on-zero-pass':
|
||
args.failOnZeroPass = true;
|
||
break;
|
||
case '--cases-dir':
|
||
args.casesDir = argv[++i];
|
||
// Without a directory the CLI would fall back to the full discovery suite.
|
||
if (!args.casesDir) {
|
||
console.error('--cases-dir needs a directory of routing cases.');
|
||
process.exit(1);
|
||
}
|
||
break;
|
||
default:
|
||
break;
|
||
}
|
||
}
|
||
|
||
return args;
|
||
}
|
||
|
||
/** Runs `args.trials` trials, at most `args.concurrency` at a time. */
|
||
async function runTrials<T>(args: CliArgs, runTrial: () => Promise<T>): Promise<T[]> {
|
||
const results: T[] = [];
|
||
for (let i = 0; i < args.trials; i += args.concurrency) {
|
||
const batchSize = Math.min(args.concurrency, args.trials - i);
|
||
results.push(...(await Promise.all(Array.from({ length: batchSize }, runTrial))));
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Local mode
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface ScenarioAggregate {
|
||
scenario: DiscoveryTestCase;
|
||
results: DiscoveryRunResult[];
|
||
passCount: number;
|
||
passRate: number;
|
||
}
|
||
|
||
async function runLocalMode(args: CliArgs): Promise<void> {
|
||
const cases = loadDiscoveryTestCasesWithFiles(args.filter);
|
||
|
||
if (cases.length === 0) {
|
||
console.log('No discovery scenarios found.');
|
||
return;
|
||
}
|
||
|
||
console.log(
|
||
`Running ${String(cases.length)} discovery scenario(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
|
||
);
|
||
|
||
const aggregates: ScenarioAggregate[] = [];
|
||
|
||
// Per scenario, run trials in parallel up to `concurrency`. Across scenarios, run sequentially
|
||
// — keeps the load profile predictable and per-scenario reports atomic.
|
||
for (const { testCase, fileSlug } of cases) {
|
||
process.stdout.write(`▸ ${fileSlug} ... `);
|
||
const trialResults = await runTrials(
|
||
args,
|
||
async () =>
|
||
await runDiscoveryScenario({
|
||
scenario: testCase,
|
||
modelId: args.modelId,
|
||
maxSteps: args.maxSteps,
|
||
timeoutMs: args.timeoutMs,
|
||
...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
|
||
}),
|
||
);
|
||
|
||
const passCount = trialResults.filter((r) => r.check.pass).length;
|
||
const passRate = passCount / trialResults.length;
|
||
const status = passRate >= args.passThreshold ? '✓' : '✗';
|
||
console.log(
|
||
`${status} ${String(passCount)}/${String(trialResults.length)} passed (${(passRate * 100).toFixed(0)}%)`,
|
||
);
|
||
|
||
if (args.verbose) {
|
||
for (const [idx, r] of trialResults.entries()) {
|
||
const icon = r.check.pass ? ' ✓' : ' ✗';
|
||
const trial = `trial ${String(idx + 1)}`;
|
||
console.log(`${icon} ${trial} (${(r.durationMs / 1000).toFixed(1)}s) — ${r.check.comment}`);
|
||
if (r.runError) console.log(` run error: ${r.runError}`);
|
||
if (r.outcome.toolCalls.length > 0) {
|
||
console.log(' tool calls:');
|
||
for (const tc of r.outcome.toolCalls) {
|
||
const argsPreview = previewArgs(tc.args);
|
||
console.log(` • ${tc.toolName}(${argsPreview})`);
|
||
}
|
||
}
|
||
if (r.outcome.agentActivities.length > 0) {
|
||
console.log(' spawned sub-agents:');
|
||
for (const a of r.outcome.agentActivities) {
|
||
console.log(` • role=${a.role}, tools=[${a.tools.join(', ')}]`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
aggregates.push({ scenario: testCase, results: trialResults, passCount, passRate });
|
||
}
|
||
|
||
printSummary(aggregates, args);
|
||
|
||
const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
|
||
const zeroPassScenarios = aggregates.filter((a) => a.passCount === 0);
|
||
const shouldFail = args.failOnZeroPass
|
||
? zeroPassScenarios.length > 0
|
||
: failingScenarios.length > 0;
|
||
if (shouldFail) {
|
||
process.exitCode = 1;
|
||
}
|
||
}
|
||
|
||
function printSummary(aggregates: ScenarioAggregate[], args: CliArgs): void {
|
||
console.log('\n=== Summary ===');
|
||
const totalTrials = aggregates.reduce((sum, a) => sum + a.results.length, 0);
|
||
const totalPasses = aggregates.reduce((sum, a) => sum + a.passCount, 0);
|
||
const passingScenarios = aggregates.filter((a) => a.passRate >= args.passThreshold).length;
|
||
const totalDurationMs = aggregates
|
||
.flatMap((a) => a.results)
|
||
.reduce((sum, r) => sum + r.durationMs, 0);
|
||
|
||
console.log(
|
||
`Scenarios: ${String(passingScenarios)}/${String(aggregates.length)} above threshold (${(args.passThreshold * 100).toFixed(0)}%)`,
|
||
);
|
||
console.log(
|
||
`Trials: ${String(totalPasses)}/${String(totalTrials)} passed (${((totalPasses / totalTrials) * 100).toFixed(0)}%)`,
|
||
);
|
||
console.log(`Total time: ${(totalDurationMs / 1000).toFixed(1)}s`);
|
||
|
||
const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
|
||
if (failingScenarios.length > 0) {
|
||
console.log('\nFailing scenarios:');
|
||
for (const a of failingScenarios) {
|
||
console.log(` ✗ ${a.scenario.id} (${(a.passRate * 100).toFixed(0)}%)`);
|
||
const firstFailure = a.results.find((r) => !r.check.pass);
|
||
if (firstFailure) {
|
||
console.log(` ${firstFailure.check.comment}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Routing mode
|
||
// ---------------------------------------------------------------------------
|
||
|
||
function percent(passed: number, total: number): string {
|
||
return `${String(passed)}/${String(total)} (${total > 0 ? ((passed / total) * 100).toFixed(0) : '0'}%)`;
|
||
}
|
||
|
||
async function runRoutingMode(args: CliArgs, casesDir: string): Promise<void> {
|
||
const { cases, needsSetup } = loadRoutingCases(casesDir, args.filter);
|
||
if (needsSetup.length > 0) {
|
||
console.log(
|
||
`Skipping ${String(needsSetup.length)} case(s) that need setup (earlier messages, an open workflow or Agent, or accounts): ${needsSetup.join(', ')}`,
|
||
);
|
||
}
|
||
if (cases.length === 0) {
|
||
console.log(`No routing cases found in ${casesDir}.`);
|
||
return;
|
||
}
|
||
if (!isAgentFeatureEnabled()) {
|
||
console.warn(
|
||
'Warning: the Agents module is off, so the Assistant cannot build an Agent and agent cases fail. Set N8N_ENABLED_MODULES=agents,instance-ai.',
|
||
);
|
||
}
|
||
|
||
console.log(
|
||
`Running ${String(cases.length)} routing case(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
|
||
);
|
||
|
||
const results: Array<{ bucket: RoutingBucket; passed: boolean }> = [];
|
||
let runErrors = 0;
|
||
let judgeErrors = 0;
|
||
|
||
for (const routingCase of cases) {
|
||
process.stdout.write(`▸ ${routingCase.id} ... `);
|
||
const trials = await runTrials(args, async () => {
|
||
const watcher = createRouteWatcher(judgeRoute);
|
||
const turn = await runOrchestratorTurn({
|
||
scenario: routingCase,
|
||
modelId: args.modelId,
|
||
maxSteps: args.maxSteps,
|
||
timeoutMs: args.timeoutMs,
|
||
stopBeforeTool: watcher.beforeToolCall,
|
||
...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
|
||
});
|
||
const resolution = await watcher.resolve(turn);
|
||
return { turn, resolution, passed: trialPasses(routingCase, resolution) };
|
||
});
|
||
|
||
const passed = casePasses(trials.filter((t) => t.passed).length, trials.length);
|
||
console.log(
|
||
`${passed ? '✓' : '✗'} ${trials.map((t) => routeLabel(t.resolution)).join(', ')} (accepts ${routingCase.accepts.join(' | ')})`,
|
||
);
|
||
for (const [idx, { turn, resolution, passed: trialPassed }] of trials.entries()) {
|
||
if (turn.runError) runErrors++;
|
||
if (resolution.judgeError) judgeErrors++;
|
||
if (!args.verbose) continue;
|
||
console.log(
|
||
` ${trialPassed ? '✓' : '✗'} trial ${String(idx + 1)} (${(turn.durationMs / 1000).toFixed(1)}s, ${turn.streamStatus}) — ${routeLabel(resolution)} from ${resolution.evidence}`,
|
||
);
|
||
if (resolution.judgeReason) console.log(` judge: ${resolution.judgeReason}`);
|
||
if (resolution.judgeError) console.log(` judge error: ${resolution.judgeError}`);
|
||
if (turn.runError) console.log(` run error: ${turn.runError}`);
|
||
}
|
||
|
||
results.push({ bucket: routingCase.bucket, passed });
|
||
}
|
||
|
||
console.log('\n=== Routing summary (a case passes when 2/3 of its trials pass) ===');
|
||
const summaryLine = (name: string, rows: typeof results) =>
|
||
console.log(` ${name.padEnd(8)} ${percent(rows.filter((r) => r.passed).length, rows.length)}`);
|
||
for (const name of ROUTING_BUCKETS) {
|
||
const rows = results.filter((r) => r.bucket === name);
|
||
if (rows.length > 0) summaryLine(name, rows);
|
||
}
|
||
summaryLine('total', results);
|
||
if (runErrors > 0 || judgeErrors > 0) {
|
||
console.log(`Errored trials: ${String(runErrors)} run, ${String(judgeErrors)} judge.`);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main
|
||
// ---------------------------------------------------------------------------
|
||
|
||
async function main(): Promise<void> {
|
||
const args = parseArgs(process.argv.slice(2));
|
||
|
||
if (!process.env.ANTHROPIC_API_KEY) {
|
||
console.error(
|
||
'Error: ANTHROPIC_API_KEY is required to run discovery evaluations (the runner calls Anthropic in-process).',
|
||
);
|
||
process.exit(1);
|
||
}
|
||
|
||
if (args.casesDir) {
|
||
await runRoutingMode(args, args.casesDir);
|
||
} else {
|
||
await runLocalMode(args);
|
||
}
|
||
}
|
||
|
||
main()
|
||
.then(() => {
|
||
process.stdout.write('', () => process.exit(process.exitCode ?? 0));
|
||
})
|
||
.catch((error) => {
|
||
console.error('Fatal error:', error);
|
||
process.exit(1);
|
||
});
|