mirror of
https://github.com/n8n-io/n8n.git
synced 2026-07-28 11:35:03 +02:00
feat(core): Enable LangSmith for mcp evaluations (no-changelog) (#32995)
This commit is contained in:
Milorad FIlipović
GitHub
parent
7b495c01db
commit
e12ce0ead0
|
|
@ -139,7 +139,8 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai --iterations 3
|
|||
| `--password` | E2E test owner | Override login password (or `N8N_EVAL_PASSWORD`) |
|
||||
| `--timeout-ms` | `900000` | Per-test-case timeout |
|
||||
| `--output-dir` | cwd | Where to write `eval-results.json` |
|
||||
| `--dataset` | `instance-ai-workflow-evals` | LangSmith dataset name |
|
||||
| `--dataset` | `instance-ai-workflow-evals` | LangSmith dataset name. Synced from the JSON test cases (honoring `--filter`/`--exclude`/`--tier`) before each run — point an isolated cohort (e.g. MCP) at its own dataset to avoid writing to the shared one |
|
||||
| `--baseline-prefix` | `instance-ai-baseline-` | Experiment-name prefix the regression comparison uses to find the baseline. Override (e.g. `mcp-baseline-`) so a cohort compares against its own baselines instead of the Instance AI one |
|
||||
| `--concurrency` | `16` | Max concurrent scenarios (builds are separately capped at 4) |
|
||||
| `--experiment-name` | auto | LangSmith experiment prefix (defaults to `{branch}-{sha}` in CI or `local-{branch}-{sha}-dirty?` locally) |
|
||||
| `--iterations` | `1` | Run each test case N times with fresh builds |
|
||||
|
|
@ -519,6 +520,8 @@ No tools, services, or workflow imports are mocked. The `eval:subagent` command
|
|||
|
||||
When `LANGSMITH_API_KEY` is set, each run is recorded as a LangSmith experiment against the `instance-ai-workflow-evals` dataset (synced from the JSON files before each run). Experiments against the same dataset can be compared side-by-side to spot regressions.
|
||||
|
||||
To record an isolated cohort without touching the shared dataset or baseline — e.g. MCP-built workflows scored via `--prebuilt-workflows` — pass a dedicated `--dataset` and `--baseline-prefix`. The sync then only writes that cohort's cases (filtered by `--tier`) into its own dataset, and regression comparison only looks for baselines under the given prefix. See the [MCP workflow evaluations README](../../../cli/src/modules/mcp/evaluations/README.md#record-runs-in-langsmith).
|
||||
|
||||
## Adding test cases
|
||||
|
||||
Test cases live in `evaluations/data/workflows/*.json`. Drop a file in — the CLI and LangSmith sync pick it up, no registration step. Every case is validated against `data/workflows/schema.ts`.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { parseCliArgs } from '../cli/args';
|
||||
import { parseCliArgs, partialIsolationWarning } from '../cli/args';
|
||||
|
||||
describe('parseCliArgs --base-url', () => {
|
||||
it('defaults to a single localhost URL when --base-url is not provided', () => {
|
||||
|
|
@ -94,3 +94,49 @@ describe('parseCliArgs --exclude', () => {
|
|||
expect(args.exclude).toBe('cross-team,deduplication');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseCliArgs --baseline-prefix', () => {
|
||||
it('defaults to the instance-ai baseline prefix', () => {
|
||||
expect(parseCliArgs([]).baselinePrefix).toBe('instance-ai-baseline-');
|
||||
});
|
||||
|
||||
it('appends the required trailing hyphen when missing', () => {
|
||||
// Anchors the prefix match to LangSmith's `-<suffix>` separator so it can't
|
||||
// catch unrelated experiment names (e.g. `mcp-baseline` vs `mcp-baseline2-`).
|
||||
expect(parseCliArgs(['--baseline-prefix', 'mcp-baseline']).baselinePrefix).toBe(
|
||||
'mcp-baseline-',
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves an existing trailing hyphen intact', () => {
|
||||
expect(parseCliArgs(['--baseline-prefix', 'mcp-baseline-']).baselinePrefix).toBe(
|
||||
'mcp-baseline-',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('partialIsolationWarning', () => {
|
||||
it('returns undefined when both are at their defaults (Instance AI run)', () => {
|
||||
expect(
|
||||
partialIsolationWarning('instance-ai-workflow-evals', 'instance-ai-baseline-'),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined when both are overridden (isolated cohort)', () => {
|
||||
expect(
|
||||
partialIsolationWarning('instance-ai-mcp-workflow-evals', 'mcp-baseline-'),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it('warns when only the dataset is overridden', () => {
|
||||
expect(
|
||||
partialIsolationWarning('instance-ai-mcp-workflow-evals', 'instance-ai-baseline-'),
|
||||
).toMatch(/Partial LangSmith isolation/);
|
||||
});
|
||||
|
||||
it('warns when only the baseline prefix is overridden', () => {
|
||||
expect(partialIsolationWarning('instance-ai-workflow-evals', 'mcp-baseline-')).toMatch(
|
||||
/Partial LangSmith isolation/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -221,4 +221,16 @@ describe('syncDataset', () => {
|
|||
await syncDataset(second.client, 'ds', logger);
|
||||
expect(second.createExamples).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('forwards filter, exclude, and tier to the test-case loader', async () => {
|
||||
// The dataset-isolation guarantee depends on these selection args reaching
|
||||
// the loader unchanged — a dedicated --dataset must sync only its own cohort
|
||||
// (e.g. --tier mcp), never the whole corpus.
|
||||
mockedLoad.mockReturnValue([scenarioFixture('foo', 'happy-path')]);
|
||||
const { client } = buildClient([]);
|
||||
|
||||
await syncDataset(client, 'ds', logger, 'contact-form', 'cross-team', 'mcp');
|
||||
|
||||
expect(mockedLoad).toHaveBeenCalledWith('contact-form', 'cross-team', 'mcp');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
import type { Client } from 'langsmith';
|
||||
import { vi } from 'vitest';
|
||||
import type { Mock } from 'vitest';
|
||||
|
||||
import { BASELINE_EXPERIMENT_PREFIX, findLatestBaseline } from '../comparison/fetch-baseline';
|
||||
|
||||
interface FakeProject {
|
||||
name?: string;
|
||||
start_time?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mock a LangSmith client whose `listProjects` yields the given projects.
|
||||
* `nameContains` is a server-side substring filter, so the mock yields
|
||||
* everything — exercising findLatestBaseline's own `startsWith` guard.
|
||||
*/
|
||||
function clientWith(projects: FakeProject[]): { client: Client; listProjects: Mock } {
|
||||
const listProjects = vi.fn(() =>
|
||||
(async function* () {
|
||||
await Promise.resolve();
|
||||
for (const p of projects) yield p;
|
||||
})(),
|
||||
);
|
||||
return { client: { listProjects } as unknown as Client, listProjects };
|
||||
}
|
||||
|
||||
describe('findLatestBaseline', () => {
|
||||
it('queries with the default instance-ai baseline prefix when none is given', async () => {
|
||||
const { client, listProjects } = clientWith([]);
|
||||
await findLatestBaseline(client);
|
||||
expect(listProjects).toHaveBeenCalledWith({ nameContains: BASELINE_EXPERIMENT_PREFIX });
|
||||
});
|
||||
|
||||
it('queries with a custom prefix (MCP isolation)', async () => {
|
||||
const { client, listProjects } = clientWith([]);
|
||||
await findLatestBaseline(client, 'mcp-baseline-');
|
||||
expect(listProjects).toHaveBeenCalledWith({ nameContains: 'mcp-baseline-' });
|
||||
});
|
||||
|
||||
it('returns the most recently started matching experiment', async () => {
|
||||
const { client } = clientWith([
|
||||
{ name: 'mcp-baseline-old', start_time: '2024-01-01T00:00:00Z' },
|
||||
{ name: 'mcp-baseline-new', start_time: '2024-06-01T00:00:00Z' },
|
||||
{ name: 'mcp-baseline-mid', start_time: '2024-03-01T00:00:00Z' },
|
||||
]);
|
||||
expect(await findLatestBaseline(client, 'mcp-baseline-')).toBe('mcp-baseline-new');
|
||||
});
|
||||
|
||||
it('ignores names that contain but do not start with the prefix', async () => {
|
||||
// The isolation guarantee: a custom MCP prefix must not select an unrelated
|
||||
// cohort whose name merely contains the substring (here, the newer one).
|
||||
const { client } = clientWith([
|
||||
{ name: 'not-mcp-baseline-1', start_time: '2024-09-01T00:00:00Z' },
|
||||
{ name: 'mcp-baseline-1', start_time: '2024-01-01T00:00:00Z' },
|
||||
]);
|
||||
expect(await findLatestBaseline(client, 'mcp-baseline-')).toBe('mcp-baseline-1');
|
||||
});
|
||||
|
||||
it('returns undefined when nothing matches the prefix', async () => {
|
||||
const { client } = clientWith([
|
||||
{ name: 'instance-ai-baseline-1', start_time: '2024-01-01T00:00:00Z' },
|
||||
]);
|
||||
expect(await findLatestBaseline(client, 'mcp-baseline-')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips nameless projects and treats a missing start_time as oldest', async () => {
|
||||
const { client } = clientWith([
|
||||
{ start_time: '2024-09-01T00:00:00Z' }, // no name → skipped despite being newest
|
||||
{ name: 'mcp-baseline-no-ts' }, // no start_time → ts 0, still the only match
|
||||
]);
|
||||
expect(await findLatestBaseline(client, 'mcp-baseline-')).toBe('mcp-baseline-no-ts');
|
||||
});
|
||||
});
|
||||
|
|
@ -7,6 +7,11 @@
|
|||
|
||||
import { z } from 'zod';
|
||||
|
||||
import { BASELINE_EXPERIMENT_PREFIX } from '../comparison/fetch-baseline';
|
||||
|
||||
/** Default LangSmith dataset — the shared Instance AI cohort. */
|
||||
export const DEFAULT_DATASET = 'instance-ai-workflow-evals';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -58,6 +63,12 @@ export interface CliArgs {
|
|||
* LangSmith examples are queried via the matching split. Defaults to
|
||||
* unset → run everything matched by `--filter` / `--exclude`. */
|
||||
tier?: string;
|
||||
/** Experiment-name prefix the regression comparison uses to find the
|
||||
* baseline. Defaults to the Instance AI baseline (`instance-ai-baseline-`).
|
||||
* Override for an isolated cohort (e.g. `mcp-baseline-`) so the run compares
|
||||
* against its own baselines instead of the Instance AI one. Pair with a
|
||||
* dedicated `--dataset` to keep MCP runs fully separate. */
|
||||
baselinePrefix: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -76,12 +87,21 @@ const cliArgsSchema = z.object({
|
|||
keepWorkflows: z.boolean().default(false),
|
||||
deletePrebuiltWorkflows: z.boolean().default(false),
|
||||
outputDir: z.string().optional(),
|
||||
dataset: z.string().default('instance-ai-workflow-evals'),
|
||||
dataset: z.string().default(DEFAULT_DATASET),
|
||||
concurrency: z.number().int().positive().default(16),
|
||||
experimentName: z.string().optional(),
|
||||
iterations: z.number().int().positive().default(1),
|
||||
pinAiRoots: z.array(z.string().min(1)).optional(),
|
||||
tier: z.string().min(1).optional(),
|
||||
// Normalize to a trailing hyphen. The baseline lookup matches by prefix and
|
||||
// LangSmith always appends `-<suffix>` to the experiment name, so the hyphen
|
||||
// anchors the match to that separator — without it `mcp-baseline` would also
|
||||
// match unrelated names like `mcp-baseline2-...`. Mirrors BASELINE_EXPERIMENT_PREFIX.
|
||||
baselinePrefix: z
|
||||
.string()
|
||||
.min(1)
|
||||
.transform((s) => (s.endsWith('-') ? s : `${s}-`))
|
||||
.default(BASELINE_EXPERIMENT_PREFIX),
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -116,9 +136,35 @@ export function parseCliArgs(argv: string[]): CliArgs {
|
|||
iterations: validated.iterations,
|
||||
pinAiRoots: validated.pinAiRoots,
|
||||
tier: validated.tier,
|
||||
baselinePrefix: validated.baselinePrefix,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A dedicated `--dataset` and a dedicated `--baseline-prefix` are the two halves
|
||||
* of LangSmith cohort isolation (e.g. for MCP runs). Overriding exactly one of
|
||||
* them still writes to / compares against shared Instance AI data, which is
|
||||
* almost always a mistake. Returns a warning for that mismatched case, or
|
||||
* `undefined` when the pairing is consistent (both shared, or both isolated).
|
||||
*
|
||||
* Leaving BOTH at their defaults is a normal Instance AI run — including
|
||||
* `--tier pr`/`mcp` against the shared dataset, whose example upserts are
|
||||
* idempotent — so it is intentionally not flagged.
|
||||
*/
|
||||
export function partialIsolationWarning(
|
||||
dataset: string,
|
||||
baselinePrefix: string,
|
||||
): string | undefined {
|
||||
const datasetIsolated = dataset !== DEFAULT_DATASET;
|
||||
const baselineIsolated = baselinePrefix !== BASELINE_EXPERIMENT_PREFIX;
|
||||
if (datasetIsolated === baselineIsolated) return undefined;
|
||||
return (
|
||||
`Partial LangSmith isolation: --dataset="${dataset}" with --baseline-prefix="${baselinePrefix}". ` +
|
||||
'Override BOTH for an isolated cohort (e.g. MCP), or leave BOTH at their defaults for an ' +
|
||||
'Instance AI run — overriding only one still touches shared Instance AI data.'
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Raw argument parsing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -141,6 +187,7 @@ interface RawArgs {
|
|||
iterations: number;
|
||||
pinAiRoots?: string[];
|
||||
tier?: string;
|
||||
baselinePrefix: string;
|
||||
}
|
||||
|
||||
function parseRawArgs(argv: string[]): RawArgs {
|
||||
|
|
@ -151,11 +198,12 @@ function parseRawArgs(argv: string[]): RawArgs {
|
|||
keepWorkflows: false,
|
||||
deletePrebuiltWorkflows: false,
|
||||
outputDir: undefined,
|
||||
dataset: 'instance-ai-workflow-evals',
|
||||
dataset: DEFAULT_DATASET,
|
||||
concurrency: 16,
|
||||
experimentName: undefined,
|
||||
iterations: 1,
|
||||
pinAiRoots: undefined,
|
||||
baselinePrefix: BASELINE_EXPERIMENT_PREFIX,
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
|
|
@ -254,6 +302,11 @@ function parseRawArgs(argv: string[]): RawArgs {
|
|||
i++;
|
||||
break;
|
||||
|
||||
case '--baseline-prefix':
|
||||
result.baselinePrefix = nextArg(argv, i, '--baseline-prefix');
|
||||
i++;
|
||||
break;
|
||||
|
||||
default:
|
||||
// Fail loudly on unknown flags. Strip any =value payload before
|
||||
// echoing and drop positional values entirely — raw CLI input
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import { traceable } from 'langsmith/traceable';
|
|||
import { join } from 'path';
|
||||
|
||||
import { aggregateResults, passAtK, passHatK } from './aggregator';
|
||||
import { parseCliArgs } from './args';
|
||||
import { parseCliArgs, partialIsolationWarning } from './args';
|
||||
import { buildCIMetadata, computeExperimentPrefix } from './ci-metadata';
|
||||
import { LaneAllocator } from './lane-allocator';
|
||||
import { expandWithIterations, partitionRoundRobin } from './lanes';
|
||||
|
|
@ -258,8 +258,20 @@ async function runWithLangSmith(config: RunConfig): Promise<{
|
|||
};
|
||||
}
|
||||
|
||||
// A dedicated dataset and baseline prefix are the two halves of cohort
|
||||
// isolation; overriding only one silently touches shared Instance AI data.
|
||||
const isolationWarning = partialIsolationWarning(args.dataset, args.baselinePrefix);
|
||||
if (isolationWarning) logger.warn(isolationWarning);
|
||||
|
||||
const lsClient = new Client();
|
||||
const datasetName = await syncDataset(lsClient, args.dataset, logger, args.filter, args.exclude);
|
||||
const datasetName = await syncDataset(
|
||||
lsClient,
|
||||
args.dataset,
|
||||
logger,
|
||||
args.filter,
|
||||
args.exclude,
|
||||
args.tier,
|
||||
);
|
||||
|
||||
// Stash transcripts by threadId so reshapeLangSmithRuns can merge them in —
|
||||
// the LangSmith target() output schema doesn't carry the full transcript.
|
||||
|
|
@ -683,7 +695,9 @@ async function runWithLangSmith(config: RunConfig): Promise<{
|
|||
metadata: {
|
||||
filter: args.filter ?? 'all',
|
||||
exclude: args.exclude ?? null,
|
||||
tier: args.tier ?? null,
|
||||
prebuilt: prebuiltManifest !== undefined,
|
||||
baselinePrefix: args.baselinePrefix,
|
||||
concurrency: args.concurrency,
|
||||
maxBuilds: MAX_CONCURRENT_BUILDS,
|
||||
lanes: lanes.length,
|
||||
|
|
@ -736,6 +750,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{
|
|||
prExperimentName: experimentResults.experimentName,
|
||||
evaluation,
|
||||
testCasesWithFiles,
|
||||
baselinePrefix: args.baselinePrefix,
|
||||
logger,
|
||||
});
|
||||
|
||||
|
|
@ -1247,16 +1262,20 @@ async function tryRunComparison(config: {
|
|||
prExperimentName: string;
|
||||
evaluation: MultiRunEvaluation;
|
||||
testCasesWithFiles: WorkflowTestCaseWithFile[];
|
||||
baselinePrefix: string;
|
||||
logger: EvalLogger;
|
||||
}): Promise<ComparisonOutcome> {
|
||||
const { lsClient, prExperimentName, evaluation, testCasesWithFiles, logger } = config;
|
||||
const { lsClient, prExperimentName, evaluation, testCasesWithFiles, baselinePrefix, logger } =
|
||||
config;
|
||||
|
||||
try {
|
||||
const baselineName = await findLatestBaseline(lsClient);
|
||||
const baselineName = await findLatestBaseline(lsClient, baselinePrefix);
|
||||
if (!baselineName) {
|
||||
// Strip the trailing hyphen so the hint names the experiment, not the lookup prefix.
|
||||
const baselineExperimentName = baselinePrefix.replace(/-$/, '');
|
||||
logger.verbose(
|
||||
'No baseline experiment found — skipping comparison. ' +
|
||||
'Run with --experiment-name instance-ai-baseline to create one.',
|
||||
`Run with --experiment-name ${baselineExperimentName} to create one.`,
|
||||
);
|
||||
return { kind: 'no_baseline' };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,12 +51,19 @@ const outputsSchema = z
|
|||
* Return the most recently created baseline experiment, or `undefined` if
|
||||
* none exist. We pick by `start_time` so a re-run of an older snapshot
|
||||
* doesn't displace the latest one.
|
||||
*
|
||||
* `prefix` defaults to the Instance AI baseline. Pass a different prefix (e.g.
|
||||
* `mcp-baseline-`) to scope the lookup to an isolated cohort, so an MCP run
|
||||
* compares MCP-vs-MCP instead of against the Instance AI baseline.
|
||||
*/
|
||||
export async function findLatestBaseline(client: Client): Promise<string | undefined> {
|
||||
export async function findLatestBaseline(
|
||||
client: Client,
|
||||
prefix: string = BASELINE_EXPERIMENT_PREFIX,
|
||||
): Promise<string | undefined> {
|
||||
let latest: { name: string; ts: number } | undefined;
|
||||
for await (const project of client.listProjects({ nameContains: BASELINE_EXPERIMENT_PREFIX })) {
|
||||
for await (const project of client.listProjects({ nameContains: prefix })) {
|
||||
const name = project.name;
|
||||
if (!name?.startsWith(BASELINE_EXPERIMENT_PREFIX)) continue;
|
||||
if (!name?.startsWith(prefix)) continue;
|
||||
const ts = project.start_time ? new Date(project.start_time).getTime() : 0;
|
||||
if (!latest || ts > latest.ts) latest = { name, ts };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,10 @@ export type DatasetExampleMetadata = z.infer<typeof datasetExampleMetadataSchema
|
|||
* - Orders examples round-robin across test cases for optimal parallelism
|
||||
* - Assigns each example to a split (test case file slug) for UI filtering
|
||||
*
|
||||
* `filter`/`exclude`/`tier` mirror the eval CLI selection, so an isolated
|
||||
* cohort (e.g. `--tier mcp` into a dedicated `--dataset`) only syncs its own
|
||||
* cases rather than the whole corpus.
|
||||
*
|
||||
* Never deletes. Orphan cleanup is manual (LangSmith UI or MCP).
|
||||
*
|
||||
* Returns the dataset name for use with evaluate().
|
||||
|
|
@ -59,8 +63,9 @@ export async function syncDataset(
|
|||
logger: EvalLogger,
|
||||
filter?: string,
|
||||
exclude?: string,
|
||||
tier?: string,
|
||||
): Promise<string> {
|
||||
const testCasesWithFiles = loadWorkflowTestCasesWithFiles(filter, exclude);
|
||||
const testCasesWithFiles = loadWorkflowTestCasesWithFiles(filter, exclude, tier);
|
||||
|
||||
// Round-robin ordering ensures evaluate() triggers diverse builds early
|
||||
// rather than burning all concurrency slots on one test case.
|
||||
|
|
|
|||
|
|
@ -40,7 +40,9 @@ N8N_EVAL_PASSWORD=...
|
|||
|
||||
CONTEXT7_API_KEY=ctx7sk-...
|
||||
|
||||
# Not recommended for MCP evals until MCP has a separate LangSmith project.
|
||||
# Optional — record this run to LangSmith. Pair with --dataset and
|
||||
# --baseline-prefix (see "Record runs in LangSmith") so MCP runs never touch
|
||||
# the Instance AI dataset or baseline.
|
||||
# LANGSMITH_API_KEY=ls__...
|
||||
# LANGSMITH_ENDPOINT=https://api.smith.langchain.com
|
||||
```
|
||||
|
|
@ -48,10 +50,11 @@ CONTEXT7_API_KEY=ctx7sk-...
|
|||
Leave `N8N_AI_ANTHROPIC_KEY` unset unless you are intentionally testing that
|
||||
path.
|
||||
|
||||
Do not enable LangSmith reporting for MCP evals yet. It currently writes into
|
||||
the existing Instance AI evaluation project and will overwrite Instance AI
|
||||
baselines. Wait until MCP has a separate LangSmith project before setting
|
||||
`LANGSMITH_API_KEY` for these runs.
|
||||
To record MCP eval runs in LangSmith, always pass a dedicated `--dataset` and
|
||||
`--baseline-prefix` so the run lands in its own dataset and only compares
|
||||
against MCP baselines — never the Instance AI dataset or the
|
||||
`instance-ai-baseline-` experiments. See
|
||||
[Record runs in LangSmith](#record-runs-in-langsmith).
|
||||
|
||||
Start n8n with the same env file in watch mode:
|
||||
|
||||
|
|
@ -74,6 +77,78 @@ On a fresh local DB, create or seed the owner account with the email and
|
|||
password from `.env.mcp-evals`. The full setup is documented in the linked
|
||||
Instance AI evaluation README.
|
||||
|
||||
## Record runs in LangSmith
|
||||
|
||||
LangSmith recording is opt-in via `LANGSMITH_API_KEY` and reuses the Instance AI
|
||||
eval pipeline (the `--prebuilt-workflows` path records exactly like a normal
|
||||
run). To keep MCP runs isolated from the Instance AI dataset and baselines,
|
||||
always pass a dedicated `--dataset` and `--baseline-prefix`:
|
||||
|
||||
```bash
|
||||
LANGSMITH_API_KEY=ls__... dotenvx run -f .env.mcp-evals -- \
|
||||
pnpm --filter @n8n/instance-ai run eval:instance-ai \
|
||||
--base-url http://localhost:5678 \
|
||||
--tier mcp \
|
||||
--prebuilt-workflows /tmp/n8n-mcp-cohort/manifest.json \
|
||||
--dataset mcp-workflow-evals \
|
||||
--baseline-prefix mcp-baseline- \
|
||||
--iterations 3 \
|
||||
--concurrency 3 \
|
||||
--output-dir /tmp/n8n-mcp-cohort-eval
|
||||
```
|
||||
|
||||
- `--dataset` syncs only the `--tier mcp` examples into a dataset of its own;
|
||||
the Instance AI `instance-ai-workflow-evals` dataset is never written to.
|
||||
- `--baseline-prefix` scopes regression comparison to MCP baselines. Until an
|
||||
MCP baseline exists the comparison is simply skipped — an MCP run is never
|
||||
compared against `instance-ai-baseline-`.
|
||||
- `--dataset` and `--baseline-prefix` are the two halves of isolation — pass
|
||||
both together. Overriding only one logs a **partial isolation** warning, since
|
||||
the run would still write to / compare against shared Instance AI data.
|
||||
- The dataset and experiments are created in the workspace your
|
||||
`LANGSMITH_API_KEY` belongs to. Use a personal key/workspace to avoid
|
||||
cluttering the shared team workspace.
|
||||
|
||||
### Create or refresh a baseline
|
||||
|
||||
Refresh the MCP baseline the same way as the Instance AI one, but with the MCP
|
||||
dataset and prefix (high `--iterations` for a low-noise reference point):
|
||||
|
||||
```bash
|
||||
LANGSMITH_API_KEY=ls__... dotenvx run -f .env.mcp-evals -- \
|
||||
pnpm --filter @n8n/instance-ai run eval:instance-ai \
|
||||
--base-url http://localhost:5678 \
|
||||
--tier mcp \
|
||||
--prebuilt-workflows /tmp/n8n-mcp-cohort/manifest.json \
|
||||
--dataset mcp-workflow-evals \
|
||||
--baseline-prefix mcp-baseline- \
|
||||
--experiment-name mcp-baseline \
|
||||
--iterations 10
|
||||
```
|
||||
|
||||
LangSmith appends a random suffix (e.g. `mcp-baseline-7abc1234`); the most
|
||||
recently started `mcp-baseline-` experiment becomes the comparison target on the
|
||||
next MCP run. The comparison is skipped on the baseline-creation run itself.
|
||||
|
||||
### Check baselines in LangSmith
|
||||
|
||||
A baseline is not a special LangSmith object — it's just an experiment whose name
|
||||
starts with `--baseline-prefix` (`mcp-baseline-`). To find them, open your
|
||||
workspace → **Datasets & Experiments** → `mcp-workflow-evals` → the
|
||||
**Experiments** list: baselines are the rows named `mcp-baseline-<suffix>`,
|
||||
while normal runs (e.g. `local-<branch>-<sha>`) are not. With no `mcp-baseline-*`
|
||||
experiment yet, every run's comparison is skipped.
|
||||
|
||||
Two comparison views, kept separate:
|
||||
|
||||
- **Native LangSmith compare** — select two or more experiments in the dataset's
|
||||
**Experiments** list and click **Compare** for a side-by-side metrics view
|
||||
(the `?selectedSessions=…` link printed at the start of a run opens this view).
|
||||
- **This tool's regression report** — computed locally by the eval CLI (reading
|
||||
the latest baseline's runs), written to `eval-pr-comment.md` and
|
||||
`eval-results.json`, and printed to the console. It is not rendered back inside
|
||||
LangSmith.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- `claude` CLI is installed and authenticated.
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user