mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-18 05:59:42 +02:00
521 lines
18 KiB
JavaScript
521 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
||
// ---------------------------------------------------------------------------
|
||
// CLI for browser/computer-use discovery evaluation.
|
||
//
|
||
// Usage:
|
||
// pnpm eval:discovery # run all scenarios, 3 trials each
|
||
// pnpm eval:discovery --filter slack-oauth --verbose
|
||
// pnpm eval:discovery --trials 5
|
||
// pnpm eval:discovery --cases-dir <dir> # routing mode
|
||
// pnpm eval:discovery --cases-dir <dir> --output-dir <dir>
|
||
//
|
||
// Loads scenarios from evaluations/data/discovery/, runs each scenario × N
|
||
// trials via the in-process runner, reports per-scenario pass-rates, exits
|
||
// non-zero on any scenario below threshold, or on any scenario with zero passes
|
||
// when --fail-on-zero-pass is set.
|
||
//
|
||
// Routing mode (`--cases-dir`) loads `route-*.json` cases from that folder, in
|
||
// the format LangTracer exports. A judge ends each trial as soon as the
|
||
// Assistant picks a route. When a case has stage directions, the user proxy
|
||
// answers up to 2 accepted questions: on the card in the same run, or in a
|
||
// reply turn of the same thread after a question in text. The CLI prints a pass rate per bucket,
|
||
// and per clarity level in each bucket. `--output-dir` also writes `eval-results.json` for
|
||
// LangTracer (see ../routing/eval-results.ts). It always exits 0: it measures routing and does
|
||
// not gate a merge.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
import { runDiscoveryScenario, runOrchestratorTurn, type DiscoveryRunResult } from './runner';
|
||
import { createSeededThread } from './seeded-turn';
|
||
import type { DiscoveryTestCase } from './types';
|
||
import { buildResumeData, toConfirmationData } from '../../src/runtime/confirmation-payload';
|
||
import { loadDiscoveryTestCasesWithFiles } from '../data/discovery';
|
||
import { defaultNodesJsonPath, missingEvalNodeData } from '../harness/stub-services';
|
||
import {
|
||
loadRoutingCases,
|
||
ROUTING_BUCKETS,
|
||
ROUTING_CLARITIES,
|
||
type RoutingCase,
|
||
} from '../routing/cases';
|
||
import { writeRoutingEvalResults, type RoutingCaseResult } from '../routing/eval-results';
|
||
import {
|
||
afterQuestion,
|
||
answeredQuestions,
|
||
canReplyTo,
|
||
casePasses,
|
||
createRouteWatcher,
|
||
MAX_ANSWERS,
|
||
routeLabel,
|
||
trialPasses,
|
||
type RouteResolution,
|
||
} from '../routing/grade';
|
||
import { judgeRoute } from '../routing/judge';
|
||
import type { CapturedEvent } from '../types';
|
||
import { UserProxyLlm } from '../utils/user-proxy';
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Args
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface CliArgs {
|
||
filter?: string;
|
||
verbose: boolean;
|
||
trials: number;
|
||
passThreshold: number;
|
||
timeoutMs: number;
|
||
/** Optional iteration cap; unset uses the SDK default. */
|
||
maxSteps?: number;
|
||
modelId: string;
|
||
concurrency: number;
|
||
nodesJsonPath?: string;
|
||
failOnZeroPass: boolean;
|
||
/** Folder of routing cases. Set switches the CLI to routing mode. */
|
||
casesDir?: string;
|
||
/** Routing mode writes `eval-results.json` here. */
|
||
outputDir?: string;
|
||
}
|
||
|
||
const DEFAULT_MODEL = process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6';
|
||
|
||
function previewArgs(args: Record<string, unknown>): string {
|
||
const entries = Object.entries(args).slice(0, 3);
|
||
if (entries.length === 0) return '';
|
||
return entries
|
||
.map(([k, v]) => {
|
||
let preview: string;
|
||
if (typeof v === 'string') {
|
||
preview = v.length > 40 ? `"${v.slice(0, 40)}…"` : `"${v}"`;
|
||
} else if (typeof v === 'number' || typeof v === 'boolean' || v === null) {
|
||
preview = String(v);
|
||
} else if (Array.isArray(v)) {
|
||
preview = `[${String(v.length)}]`;
|
||
} else {
|
||
preview = '{…}';
|
||
}
|
||
return `${k}=${preview}`;
|
||
})
|
||
.join(', ');
|
||
}
|
||
|
||
function requirePositiveInt(raw: string | undefined, flag: string): number {
|
||
const n = Number(raw);
|
||
if (!Number.isFinite(n) || n < 1 || !Number.isInteger(n)) {
|
||
console.error(`Invalid value for ${flag}: ${String(raw)} (expected a positive integer)`);
|
||
process.exit(1);
|
||
}
|
||
return n;
|
||
}
|
||
|
||
function requireNumberInRange(
|
||
raw: string | undefined,
|
||
flag: string,
|
||
lo: number,
|
||
hi: number,
|
||
): number {
|
||
const n = Number(raw);
|
||
if (!Number.isFinite(n) || n < lo || n > hi) {
|
||
console.error(`Invalid value for ${flag}: ${String(raw)} (expected number in [${lo}, ${hi}])`);
|
||
process.exit(1);
|
||
}
|
||
return n;
|
||
}
|
||
|
||
function parseArgs(argv: string[]): CliArgs {
|
||
const args: CliArgs = {
|
||
verbose: false,
|
||
trials: 3,
|
||
passThreshold: 2 / 3,
|
||
timeoutMs: 60_000,
|
||
modelId: DEFAULT_MODEL,
|
||
concurrency: 3,
|
||
failOnZeroPass: false,
|
||
};
|
||
|
||
for (let i = 0; i < argv.length; i++) {
|
||
const arg = argv[i];
|
||
switch (arg) {
|
||
case '--verbose':
|
||
case '-v':
|
||
args.verbose = true;
|
||
break;
|
||
case '--filter':
|
||
args.filter = argv[++i];
|
||
break;
|
||
case '--trials':
|
||
args.trials = requirePositiveInt(argv[++i], '--trials');
|
||
break;
|
||
case '--pass-threshold':
|
||
args.passThreshold = requireNumberInRange(argv[++i], '--pass-threshold', 0, 1);
|
||
break;
|
||
case '--timeout':
|
||
args.timeoutMs = requirePositiveInt(argv[++i], '--timeout');
|
||
break;
|
||
case '--max-steps':
|
||
args.maxSteps = requirePositiveInt(argv[++i], '--max-steps');
|
||
break;
|
||
case '--model':
|
||
args.modelId = argv[++i];
|
||
break;
|
||
case '--concurrency':
|
||
args.concurrency = requirePositiveInt(argv[++i], '--concurrency');
|
||
break;
|
||
case '--nodes-json':
|
||
args.nodesJsonPath = argv[++i];
|
||
break;
|
||
case '--fail-on-zero-pass':
|
||
args.failOnZeroPass = true;
|
||
break;
|
||
case '--cases-dir':
|
||
args.casesDir = argv[++i];
|
||
// Without a directory the CLI would fall back to the full discovery suite.
|
||
if (!args.casesDir) {
|
||
console.error('--cases-dir needs a directory of routing cases.');
|
||
process.exit(1);
|
||
}
|
||
break;
|
||
case '--output-dir':
|
||
args.outputDir = argv[++i];
|
||
if (!args.outputDir) {
|
||
console.error('--output-dir needs a directory for eval-results.json.');
|
||
process.exit(1);
|
||
}
|
||
break;
|
||
default:
|
||
break;
|
||
}
|
||
}
|
||
|
||
return args;
|
||
}
|
||
|
||
/** Runs `args.trials` trials, at most `args.concurrency` at a time. */
|
||
async function runTrials<T>(args: CliArgs, runTrial: () => Promise<T>): Promise<T[]> {
|
||
const results: T[] = [];
|
||
for (let i = 0; i < args.trials; i += args.concurrency) {
|
||
const batchSize = Math.min(args.concurrency, args.trials - i);
|
||
results.push(...(await Promise.all(Array.from({ length: batchSize }, runTrial))));
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Local mode
|
||
// ---------------------------------------------------------------------------
|
||
|
||
interface ScenarioAggregate {
|
||
scenario: DiscoveryTestCase;
|
||
results: DiscoveryRunResult[];
|
||
passCount: number;
|
||
passRate: number;
|
||
}
|
||
|
||
async function runLocalMode(args: CliArgs): Promise<void> {
|
||
const cases = loadDiscoveryTestCasesWithFiles(args.filter);
|
||
|
||
if (cases.length === 0) {
|
||
console.log('No discovery scenarios found.');
|
||
return;
|
||
}
|
||
|
||
console.log(
|
||
`Running ${String(cases.length)} discovery scenario(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
|
||
);
|
||
|
||
const aggregates: ScenarioAggregate[] = [];
|
||
|
||
// Per scenario, run trials in parallel up to `concurrency`. Across scenarios, run sequentially
|
||
// — keeps the load profile predictable and per-scenario reports atomic.
|
||
for (const { testCase, fileSlug } of cases) {
|
||
process.stdout.write(`▸ ${fileSlug} ... `);
|
||
const trialResults = await runTrials(
|
||
args,
|
||
async () =>
|
||
await runDiscoveryScenario({
|
||
scenario: testCase,
|
||
modelId: args.modelId,
|
||
maxSteps: args.maxSteps,
|
||
timeoutMs: args.timeoutMs,
|
||
...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
|
||
}),
|
||
);
|
||
|
||
const passCount = trialResults.filter((r) => r.check.pass).length;
|
||
const passRate = passCount / trialResults.length;
|
||
const status = passRate >= args.passThreshold ? '✓' : '✗';
|
||
console.log(
|
||
`${status} ${String(passCount)}/${String(trialResults.length)} passed (${(passRate * 100).toFixed(0)}%)`,
|
||
);
|
||
|
||
if (args.verbose) {
|
||
for (const [idx, r] of trialResults.entries()) {
|
||
const icon = r.check.pass ? ' ✓' : ' ✗';
|
||
const trial = `trial ${String(idx + 1)}`;
|
||
console.log(`${icon} ${trial} (${(r.durationMs / 1000).toFixed(1)}s) — ${r.check.comment}`);
|
||
if (r.runError) console.log(` run error: ${r.runError}`);
|
||
if (r.outcome.toolCalls.length > 0) {
|
||
console.log(' tool calls:');
|
||
for (const tc of r.outcome.toolCalls) {
|
||
const argsPreview = previewArgs(tc.args);
|
||
console.log(` • ${tc.toolName}(${argsPreview})`);
|
||
}
|
||
}
|
||
if (r.outcome.agentActivities.length > 0) {
|
||
console.log(' spawned sub-agents:');
|
||
for (const a of r.outcome.agentActivities) {
|
||
console.log(` • role=${a.role}, tools=[${a.tools.join(', ')}]`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
aggregates.push({ scenario: testCase, results: trialResults, passCount, passRate });
|
||
}
|
||
|
||
printSummary(aggregates, args);
|
||
|
||
const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
|
||
const zeroPassScenarios = aggregates.filter((a) => a.passCount === 0);
|
||
const shouldFail = args.failOnZeroPass
|
||
? zeroPassScenarios.length > 0
|
||
: failingScenarios.length > 0;
|
||
if (shouldFail) {
|
||
process.exitCode = 1;
|
||
}
|
||
}
|
||
|
||
function printSummary(aggregates: ScenarioAggregate[], args: CliArgs): void {
|
||
console.log('\n=== Summary ===');
|
||
const totalTrials = aggregates.reduce((sum, a) => sum + a.results.length, 0);
|
||
const totalPasses = aggregates.reduce((sum, a) => sum + a.passCount, 0);
|
||
const passingScenarios = aggregates.filter((a) => a.passRate >= args.passThreshold).length;
|
||
const totalDurationMs = aggregates
|
||
.flatMap((a) => a.results)
|
||
.reduce((sum, r) => sum + r.durationMs, 0);
|
||
|
||
console.log(
|
||
`Scenarios: ${String(passingScenarios)}/${String(aggregates.length)} above threshold (${(args.passThreshold * 100).toFixed(0)}%)`,
|
||
);
|
||
console.log(
|
||
`Trials: ${String(totalPasses)}/${String(totalTrials)} passed (${((totalPasses / totalTrials) * 100).toFixed(0)}%)`,
|
||
);
|
||
console.log(`Total time: ${(totalDurationMs / 1000).toFixed(1)}s`);
|
||
|
||
const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
|
||
if (failingScenarios.length > 0) {
|
||
console.log('\nFailing scenarios:');
|
||
for (const a of failingScenarios) {
|
||
console.log(` ✗ ${a.scenario.id} (${(a.passRate * 100).toFixed(0)}%)`);
|
||
const firstFailure = a.results.find((r) => !r.check.pass);
|
||
if (firstFailure) {
|
||
console.log(` ${firstFailure.check.comment}`);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Routing mode
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/**
|
||
* One routing trial. A question in text ends the turn, so the user proxy
|
||
* replies in a new turn of the same thread.
|
||
*/
|
||
async function runRoutingTrial(args: CliArgs, routingCase: RoutingCase) {
|
||
const proxy = routingCase.direction
|
||
? new UserProxyLlm({
|
||
conversation: [
|
||
{ role: 'user', text: routingCase.userMessage },
|
||
{ role: 'user', text: routingCase.direction },
|
||
],
|
||
})
|
||
: undefined;
|
||
// A reply turn reads the earlier turns from the thread, with their question cards and answers.
|
||
const thread = proxy ? await createSeededThread(routingCase.seed?.messages) : undefined;
|
||
let scenario = routingCase;
|
||
// The text question that the last reply answered, with the questions before it.
|
||
let earlier: RouteResolution | undefined;
|
||
let durationMs = 0;
|
||
// The proxy reads only the events past the ones it has seen, so it gets the events of all turns.
|
||
const events: CapturedEvent[] = [];
|
||
for (;;) {
|
||
const used = earlier ? answeredQuestions(earlier) + 1 : 0;
|
||
const watcher = createRouteWatcher(
|
||
judgeRoute,
|
||
proxy ? (question) => canReplyTo(routingCase, question) : undefined,
|
||
MAX_ANSWERS - used,
|
||
);
|
||
const turn = await runOrchestratorTurn({
|
||
scenario,
|
||
modelId: args.modelId,
|
||
maxSteps: args.maxSteps,
|
||
timeoutMs: args.timeoutMs,
|
||
stopBeforeTool: watcher.beforeToolCall,
|
||
...(proxy
|
||
? {
|
||
thread,
|
||
answerQuestions: async (suspension, turnEvents) => {
|
||
proxy.ingestEvents([...events, ...turnEvents]);
|
||
const answer = await proxy.respondToConfirmation({
|
||
timestamp: Date.now(),
|
||
type: 'confirmation-request',
|
||
data: { payload: suspension.suspendPayload },
|
||
});
|
||
return buildResumeData(toConfirmationData(answer));
|
||
},
|
||
}
|
||
: {}),
|
||
...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
|
||
});
|
||
const turnResolution = await watcher.resolve(turn);
|
||
durationMs += turn.durationMs;
|
||
const resolution = earlier ? afterQuestion(turnResolution, earlier) : turnResolution;
|
||
const result = {
|
||
turn: { ...turn, durationMs },
|
||
resolution,
|
||
passed: trialPasses(routingCase, resolution),
|
||
};
|
||
if (
|
||
!proxy ||
|
||
answeredQuestions(resolution) >= MAX_ANSWERS ||
|
||
turn.streamStatus !== 'completed' ||
|
||
!canReplyTo(routingCase, turnResolution)
|
||
) {
|
||
return result;
|
||
}
|
||
events.push(...turn.events);
|
||
proxy.ingestEvents(events);
|
||
const reply = await proxy.decideFollowUp();
|
||
// ponytail: a proxy that sees nothing to answer leaves the trial graded on the question.
|
||
if (reply.kind !== 'followUp') return result;
|
||
scenario = { ...routingCase, userMessage: reply.message, attach: undefined };
|
||
earlier = resolution;
|
||
}
|
||
}
|
||
|
||
function percent(passed: number, total: number): string {
|
||
return `${String(passed)}/${String(total)} (${total > 0 ? ((passed / total) * 100).toFixed(0) : '0'}%)`;
|
||
}
|
||
|
||
async function runRoutingMode(args: CliArgs, casesDir: string): Promise<void> {
|
||
const { cases, needsSetup } = loadRoutingCases(casesDir, args.filter);
|
||
if (needsSetup.length > 0) {
|
||
console.log(
|
||
`Skipping ${String(needsSetup.length)} case(s) that need setup the stub instance cannot do (thread replay, browser sign-in, folders, or projects): ${needsSetup.join(', ')}`,
|
||
);
|
||
}
|
||
if (cases.length === 0) {
|
||
console.log(`No routing cases found in ${casesDir}.`);
|
||
return;
|
||
}
|
||
|
||
console.log(
|
||
`Running ${String(cases.length)} routing case(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
|
||
);
|
||
|
||
const startedAt = Date.now();
|
||
const results: Array<RoutingCaseResult & { passed: boolean }> = [];
|
||
let runErrors = 0;
|
||
let judgeErrors = 0;
|
||
|
||
for (const routingCase of cases) {
|
||
process.stdout.write(`▸ ${routingCase.id} ... `);
|
||
const trials = await runTrials(args, async () => {
|
||
const trial = await runRoutingTrial(args, routingCase);
|
||
// No route (a timeout, an error, or an empty turn) says nothing about routing, so the trial runs once more.
|
||
return trial.resolution.route === 'none' ? await runRoutingTrial(args, routingCase) : trial;
|
||
});
|
||
|
||
const passed = casePasses(trials);
|
||
const after = routingCase.direction ? `; after a reply ${routingCase.after.join(' | ')}` : '';
|
||
console.log(
|
||
`${passed ? '✓' : '✗'} ${trials.map((t) => routeLabel(t.resolution)).join(', ')} (accepts ${routingCase.accepts.join(' | ')}${after})`,
|
||
);
|
||
for (const [idx, { turn, resolution, passed: trialPassed }] of trials.entries()) {
|
||
if (turn.runError) runErrors++;
|
||
if (resolution.judgeError) judgeErrors++;
|
||
if (!args.verbose) continue;
|
||
console.log(
|
||
` ${trialPassed ? '✓' : '✗'} trial ${String(idx + 1)} (${(turn.durationMs / 1000).toFixed(1)}s, ${turn.streamStatus}) — ${routeLabel(resolution)} from ${resolution.evidence}`,
|
||
);
|
||
if (resolution.judgeReason) console.log(` judge: ${resolution.judgeReason}`);
|
||
if (resolution.judgeError) console.log(` judge error: ${resolution.judgeError}`);
|
||
if (turn.runError) console.log(` run error: ${turn.runError}`);
|
||
}
|
||
|
||
results.push({ routingCase, trials, passed });
|
||
}
|
||
|
||
console.log(
|
||
'\n=== Routing summary (a case passes when more than half of its graded trials pass) ===',
|
||
);
|
||
const summaryLine = (name: string, rows: typeof results) =>
|
||
console.log(` ${name.padEnd(8)} ${percent(rows.filter((r) => r.passed).length, rows.length)}`);
|
||
for (const name of ROUTING_BUCKETS) {
|
||
const rows = results.filter((r) => r.routingCase.bucket === name);
|
||
if (rows.length === 0) continue;
|
||
summaryLine(name, rows);
|
||
for (const clarity of ROUTING_CLARITIES) {
|
||
const cell = rows.filter((r) => r.routingCase.clarity === clarity);
|
||
if (cell.length > 0) summaryLine(` ${clarity}`, cell);
|
||
}
|
||
}
|
||
summaryLine('total', results);
|
||
if (runErrors > 0 || judgeErrors > 0) {
|
||
console.log(`Errored trials: ${String(runErrors)} run, ${String(judgeErrors)} judge.`);
|
||
}
|
||
if (args.outputDir) {
|
||
const jsonPath = writeRoutingEvalResults(
|
||
args.outputDir,
|
||
results,
|
||
args.trials,
|
||
Date.now() - startedAt,
|
||
);
|
||
console.log(`Results: ${jsonPath}`);
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main
|
||
// ---------------------------------------------------------------------------
|
||
|
||
async function main(): Promise<void> {
|
||
const args = parseArgs(process.argv.slice(2));
|
||
|
||
if (!process.env.ANTHROPIC_API_KEY) {
|
||
console.error(
|
||
'Error: ANTHROPIC_API_KEY is required to run discovery evaluations (the runner calls Anthropic in-process).',
|
||
);
|
||
process.exit(1);
|
||
}
|
||
|
||
// The stubs read node data from build outputs. Stop here, because without it
|
||
// every trial runs with no node types and shows no error.
|
||
const missing = missingEvalNodeData(args.nodesJsonPath ?? defaultNodesJsonPath());
|
||
if (missing.length > 0) {
|
||
console.error(
|
||
[
|
||
'Error: node data for the eval stubs is missing:',
|
||
...missing.map((p) => ` - ${p}`),
|
||
'Build the node packages: pnpm --filter n8n-nodes-base --filter @n8n/n8n-nodes-langchain build',
|
||
'Then generate nodes.json: run `pnpm export:nodes` in packages/@n8n/ai-workflow-builder.ee.',
|
||
].join('\n'),
|
||
);
|
||
process.exit(1);
|
||
}
|
||
|
||
if (args.casesDir) {
|
||
await runRoutingMode(args, args.casesDir);
|
||
} else {
|
||
await runLocalMode(args);
|
||
}
|
||
}
|
||
|
||
main()
|
||
.then(() => {
|
||
process.stdout.write('', () => process.exit(process.exitCode ?? 0));
|
||
})
|
||
.catch((error) => {
|
||
console.error('Fatal error:', error);
|
||
process.exit(1);
|
||
});
|