spike(instance-ai): browser fixture adapter for CU evals [NODE-5547]

FixtureAdapter in @n8n/mcp-browser replays a recorded page-state machine
(aria trees + transitions + captured synthetic secrets) with no real
browser, extension, CDP relay, or daemon. Everything above the adapter
(tool wrappers, redaction, secret capture -> create_credential) runs for
real; only browser I/O is served from the fixture.

- @n8n/mcp-browser: FixtureAdapter (+ resolveRedactedSecret), adapter
  option + factory branch, redacted-key capture fast path
- cli/instance-ai: eval-thread-scoped ephemeral fixture browser server
  (only source:'evals' threads); double-gated on
  N8N_EVAL_BROWSER_ADAPTER=fixture so customer n8n is unaffected;
  per-thread /eval/browser-fixture endpoint (ownership-checked, hard
  errors when not in fixture mode)
- evaluations/harness: browserFixture as a first-class eval case field,
  provisioned per-thread after the credential allowlist

Pairs with the lang-tracer spike branch of the same name. See the RFC.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bernhard Wittmann 2026-07-27 10:06:55 +02:00
parent 6f3d2b9faa
commit 404118dfd1
No known key found for this signature in database
GPG Key ID: C929BD10AA16F8B9
14 changed files with 724 additions and 12 deletions

View File

@ -638,6 +638,20 @@ export class N8nClient {
});
}
/**
* Register a computer-use browser fixture for a build thread (goal:
* computer-use-evals). The server's FIXTURE adapter replays this recorded
* bundle for the thread's browser tools. Provisioned before the first message,
* like the credential allowlist.
* POST /rest/instance-ai/eval/browser-fixture
*/
async setThreadBrowserFixture(threadId: string, fixture: unknown): Promise<void> {
await this.fetch('/rest/instance-ai/eval/browser-fixture', {
method: 'POST',
body: { threadId, fixture },
});
}
/**
* Seed an existing thread with a previously exported conversation: the
* referenced workflows are recreated (node credentials stripped server-side)

View File

@ -516,6 +516,24 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
);
}
// Computer-use (browser) eval (goal: computer-use-evals): register the
// case's recorded browser fixture for this thread before the first message,
// so the FIXTURE adapter replays it deterministically (no real browser).
// Same provisioning shape as the credential allowlist above.
if (config.browserFixture) {
try {
await client.setThreadBrowserFixture(threadId, config.browserFixture);
} catch (error: unknown) {
// An older backend without the endpoint degrades to the
// N8N_EVAL_BROWSER_FIXTURES env-file fallback; any other failure is fatal.
const endpointMissing = error instanceof N8nApiError && error.status === 404;
if (!endpointMissing) throw error;
logger.info(
` Browser-fixture endpoint unavailable, using env fallback${config.laneTag ?? ''}`,
);
}
}
// Restore the seed before the first live message. No degraded mode: a
// seeded case can't run unseeded, so any restore failure fails the build.
if (seed) {

View File

@ -86,6 +86,12 @@ const evalTestCaseObjectSchema = z
}),
)
.optional(),
/** Computer-use (browser) fixture bundle recorded page-states the FIXTURE
* adapter replays deterministically, no real browser (goal:
* computer-use-evals). Provisioned to the run's thread before the build, the
* same way `credentials` are. Opaque here; the adapter validates its shape
* (matches LangTracer's `CuFixtureBundle`). */
browserFixture: z.record(z.unknown()).optional(),
/** Synthetic seed file (relative path), resolved + validated at case load.
* Synthetic fixtures only; real conversations use `seedThread`. */
seedFile: z.string().min(1).optional(),

View File

@ -0,0 +1,17 @@
// Demo: replay a recorded CU fixture bundle through the FixtureAdapter — no
// browser, extension, CDP, or daemon. Usage: node replay-fixture-demo.mjs <bundle.json>
import { readFileSync } from 'node:fs';
import { createBrowserTools } from './dist/index.js';
const bundle = JSON.parse(readFileSync(process.argv[2], 'utf8'));
const { tools, connection } = createBrowserTools({ adapter: 'fixture' }, { fixtures: bundle });
await connection.connect();
console.log(`bundle: states=${bundle.states.length} initial=${bundle.initialStateId} source=${bundle.sourceThreadId ?? '-'}`);
const snapshot = tools.find((t) => t.name === 'browser_snapshot');
const res = await snapshot.execute({}, { dir: '/tmp' });
const tree = res.structuredContent?.snapshot ?? '';
console.log(`\n>>> browser_snapshot replayed (NO real browser). tree length = ${tree.length}`);
console.log(`>>> first lines of the REAL recorded page the agent would see:\n`);
console.log(tree.split('\n').slice(0, 12).join('\n'));
await connection.disconnect();

View File

@ -0,0 +1,85 @@
import { describe, expect, it, beforeEach } from 'vitest';
import { FixtureAdapter, type FixtureBundle } from './fixture';
import { UnsupportedOperationError } from '../errors';
const bundle: FixtureBundle = {
version: 1,
initialStateId: 's0',
states: [
{
id: 's0',
url: 'https://news.ycombinator.com',
ariaTree: '- list:\n - listitem "1. A [ref=e1]"\n - listitem "2. B [ref=e2]"',
html: '<ul><li>1. A</li><li>2. B</li></ul>',
},
{
id: 's1',
url: 'https://news.ycombinator.com/newest',
ariaTree: '- textbox "token" [ref=e9]',
elementValues: { e9: 'sk-ant-secret-value' },
redactedSecrets: { '[REDACTED:anthropic_api_key:1]': 'sk-ant-redacted-value' },
},
],
transitions: [{ from: 's0', action: { tool: 'browser_click', selector: 'text=More' }, to: 's1' }],
};
describe('FixtureAdapter', () => {
let adapter: FixtureAdapter;
beforeEach(async () => {
adapter = new FixtureAdapter(bundle);
await adapter.launch({ browser: 'chrome' });
});
it('serves the initial page-state tree, refCount and url', async () => {
const snap = await adapter.snapshot();
expect(snap.tree).toBe(bundle.states[0].ariaTree);
expect(snap.refCount).toBe(2);
expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com');
});
it('serves recorded HTML from probePageHtml (drives real sensitivity/redaction)', async () => {
const probe = await adapter.probePageHtml();
expect(probe.ok).toBe(true);
expect(probe.root?.html).toBe('<ul><li>1. A</li><li>2. B</li></ul>');
});
it('advances state on a matching click transition and captures the recorded value', async () => {
await adapter.click('p', { selector: 'text=More' });
const snap = await adapter.snapshot();
expect(snap.tree).toBe('- textbox "token" [ref=e9]');
expect(await adapter.getElementValue('p', { ref: 'e9' })).toBe('sk-ant-secret-value');
});
it('navigate falls back to a recorded state with the matching url', async () => {
const res = await adapter.navigate('p', 'https://news.ycombinator.com/newest');
expect(res.status).toBe(200);
expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com/newest');
});
it('resolves a redacted-key marker to its synthetic secret', async () => {
await adapter.click('p', { selector: 'text=More' }); // advance to s1
expect(await adapter.resolveRedactedSecret('p', '[REDACTED:anthropic_api_key:1]')).toBe(
'sk-ant-redacted-value',
);
// Unknown marker ⇒ undefined (caller falls back to the HTML path).
expect(await adapter.resolveRedactedSecret('p', '[REDACTED:nope:9]')).toBeUndefined();
});
it('dead-ends legibly when the agent leaves the recorded trajectory', async () => {
await adapter.click('p', { selector: 'text=does-not-exist' });
const snap = await adapter.snapshot();
expect(snap.tree).toContain('No fixture page for this state');
expect(snap.refCount).toBe(0);
await expect(adapter.getElementValue('p', { ref: 'e9' })).rejects.toBeInstanceOf(
UnsupportedOperationError,
);
});
it('re-launch resets to the initial state', async () => {
await adapter.click('p', { selector: 'text=More' });
expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com/newest');
await adapter.launch({ browser: 'chrome' });
expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com');
});
});

View File

@ -0,0 +1,325 @@
import { UnsupportedOperationError } from '../errors';
import type {
Adapter,
ClickOptions,
ConnectConfig,
Cookie,
ConsoleEntry,
ElementTarget,
HtmlProbeResult,
ModalState,
NavigateResult,
NetworkEntry,
PageInfo,
ScreenshotOptions,
ScrollOptions,
SnapshotResult,
TypeOptions,
WaitOptions,
} from '../types';
// ---------------------------------------------------------------------------
// Fixture bundle — the cross-repo contract with LangTracer's `CuFixtureBundle`
// (packages/shared/src/types/cu-fixture.ts). Keep these in lockstep; bump
// `version` on any shape change. A bundle is a small page-state machine
// recorded from a real browser-use conversation (the LangTracer import IS the
// recorder) and replayed here with no real browser.
// ---------------------------------------------------------------------------
export interface FixturePageState {
id: string;
url: string;
/** ARIA accessibility tree (what `browser_snapshot` returns). */
ariaTree: string;
/** Page HTML for `probePageHtml` optional; import-derived bundles carry
* the tree, not HTML (probe HTML is server-side enrichment only). */
html?: string;
/** ref → value, for `getElementValue` (secret capture by ref). */
elementValues?: Record<string, string>;
/** `[REDACTED:]` marker value, for capture by redacted-key (see
* `resolveRedactedSecret`). Lets the redacted-key capture path replay
* without re-deriving markers from page HTML. */
redactedSecrets?: Record<string, string>;
}
export type FixtureAction =
| { tool: 'browser_navigate'; url: string }
| { tool: 'browser_click'; selector: string }
| { tool: 'browser_type'; selector: string };
export interface FixtureTransition {
from: string;
action: FixtureAction;
to: string;
}
export interface FixtureBundle {
version: number;
initialStateId: string;
states: FixturePageState[];
transitions: FixtureTransition[];
sourceThreadId?: string;
}
/** Env var pointing at a JSON fixture-bundle file, used when the bundle isn't
* passed programmatically (the eval server sets this per scenario). */
export const FIXTURE_BUNDLE_ENV = 'N8N_EVAL_BROWSER_FIXTURES';
/** Load a bundle from the {@link FIXTURE_BUNDLE_ENV} path. Throws a clear error
* when the env var is unset or the file can't be read/parsed the fixture
* adapter is useless without a bundle. */
export async function loadFixtureBundleFromEnv(): Promise<FixtureBundle> {
const path = process.env[FIXTURE_BUNDLE_ENV];
if (!path) {
throw new Error(`Fixture adapter selected but ${FIXTURE_BUNDLE_ENV} is not set`);
}
const { readFile } = await import('node:fs/promises');
const raw = await readFile(path, 'utf8');
return JSON.parse(raw) as FixtureBundle;
}
const PAGE_ID = 'fixture-page-1';
/** Bundle shape this adapter understands. Must match LangTracer's
* `CU_FIXTURE_BUNDLE_VERSION` (cross-repo contract; bump both in lockstep). */
const SUPPORTED_FIXTURE_BUNDLE_VERSION = 1;
// 1x1 transparent PNG — a placeholder for the rare screenshot request.
const BLANK_PNG =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
function selectorOf(target: ElementTarget): string {
return 'selector' in target ? target.selector : target.ref;
}
/**
* Deterministic browser Adapter that replays a recorded fixture bundle no
* real Chrome, extension, CDP relay, or daemon. Everything ABOVE the adapter
* (tool wrappers, enrichment, jsdom sensitivity analysis, redaction, secret
* capture create_credential) runs for real; only browser I/O is served from
* the bundle. See goal `computer-use-evals`.
*/
export class FixtureAdapter implements Adapter {
onDisconnect?: (reason: import('../errors').ConnectionLostReason) => void;
private readonly states: Map<string, FixturePageState>;
private currentId: string;
/** Set when an action has no matching transition the agent walked off the
* recorded trajectory. We serve a legible "no fixture" page rather than
* crash, so the run fails cleanly and the grader can see it. */
private deadEnd = false;
/** Set when the bundle's `version` isn't one this adapter understands a
* cross-repo contract guard. LangTracer's builder stamps `version`; a bundle
* from a drifted/newer shape must fail LOUDLY (a legible error page) rather
* than mis-replay as silent dead-ends indistinguishable from a real failure. */
private readonly unsupportedVersion: boolean;
constructor(private readonly bundle: FixtureBundle) {
this.states = new Map(bundle.states.map((s) => [s.id, s]));
this.currentId = bundle.initialStateId;
if (!this.states.has(this.currentId) && bundle.states.length > 0) {
this.currentId = bundle.states[0].id;
}
this.unsupportedVersion = bundle.version !== SUPPORTED_FIXTURE_BUNDLE_VERSION;
if (this.unsupportedVersion) {
console.warn(
`[fixture] unsupported CuFixtureBundle version ${String(bundle.version)} ` +
`(this adapter supports ${SUPPORTED_FIXTURE_BUNDLE_VERSION}) — serving an error page. ` +
`Rebuild the fixture or align the adapter/LangTracer contract.`,
);
}
}
// --- lifecycle -----------------------------------------------------------
async launch(_config: ConnectConfig): Promise<void> {
this.currentId = this.bundle.initialStateId;
this.deadEnd = false;
}
async close(): Promise<void> {}
// --- current-state helpers ----------------------------------------------
private state(): FixturePageState | undefined {
return this.states.get(this.currentId);
}
private currentUrl(): string {
return this.state()?.url ?? '';
}
private advance(match: (a: FixtureAction) => boolean): void {
const t = this.bundle.transitions.find((tr) => tr.from === this.currentId && match(tr.action));
if (t && this.states.has(t.to)) {
this.currentId = t.to;
this.deadEnd = false;
} else {
this.deadEnd = true;
}
}
// --- tabs (single-tab model) --------------------------------------------
async listTabs(): Promise<PageInfo[]> {
return [{ id: PAGE_ID, title: '', url: this.currentUrl() }];
}
async listTabIds(): Promise<string[]> {
return [PAGE_ID];
}
async listTabSessionIds(): Promise<string[]> {
return [PAGE_ID];
}
async newPage(_url?: string): Promise<PageInfo> {
return { id: PAGE_ID, title: '', url: this.currentUrl() };
}
async closePage(_pageId: string): Promise<void> {}
async focusPage(_pageId: string): Promise<void> {}
// --- navigation ----------------------------------------------------------
private navResult(): NavigateResult {
return { title: '', url: this.currentUrl(), status: this.deadEnd ? 404 : 200 };
}
async navigate(_pageId: string, url: string): Promise<NavigateResult> {
this.advance((a) => a.tool === 'browser_navigate' && a.url === url);
// Fallback: a recorded state with this exact url, reachable directly.
if (this.deadEnd) {
const target = [...this.states.values()].find((s) => s.url === url);
if (target) {
this.currentId = target.id;
this.deadEnd = false;
}
}
return this.navResult();
}
async back(_pageId: string): Promise<NavigateResult> {
return this.navResult();
}
async forward(_pageId: string): Promise<NavigateResult> {
return this.navResult();
}
async reload(_pageId: string): Promise<NavigateResult> {
return this.navResult();
}
// --- interaction ---------------------------------------------------------
async click(_pageId: string, target: ElementTarget, _options?: ClickOptions): Promise<void> {
const selector = selectorOf(target);
this.advance((a) => a.tool === 'browser_click' && a.selector === selector);
}
async type(
_pageId: string,
target: ElementTarget,
_text: string,
_options?: TypeOptions,
): Promise<void> {
const selector = selectorOf(target);
this.advance((a) => a.tool === 'browser_type' && a.selector === selector);
}
async select(_pageId: string, _target: ElementTarget, values: string[]): Promise<string[]> {
return values;
}
async hover(_pageId: string, _target: ElementTarget): Promise<void> {}
async press(_pageId: string, _keys: string): Promise<void> {}
async drag(_pageId: string, _from: ElementTarget, _to: ElementTarget): Promise<void> {}
async scroll(_pageId: string, _target?: ElementTarget, _options?: ScrollOptions): Promise<void> {}
async upload(
_pageId: string,
_target: ElementTarget | undefined,
_files: string[],
): Promise<void> {}
async dialog(_pageId: string, _action: 'accept' | 'dismiss', text?: string): Promise<string> {
return text ?? '';
}
// --- inspection ----------------------------------------------------------
async snapshot(): Promise<SnapshotResult> {
if (this.unsupportedVersion) {
return {
tree: `- text "Fixture bundle version ${String(this.bundle.version)} is not supported by this n8n build (expected ${SUPPORTED_FIXTURE_BUNDLE_VERSION})."`,
refCount: 0,
};
}
if (this.deadEnd) {
const tree = `- text "No fixture page for this state (url: ${this.currentUrl()}); the agent left the recorded trajectory."`;
return { tree, refCount: 0 };
}
const tree = this.state()?.ariaTree ?? '';
const refCount = (tree.match(/\[ref=[^\]]+\]/g) ?? []).length;
return { tree, refCount };
}
async probePageHtml(): Promise<HtmlProbeResult> {
const html = this.deadEnd ? '' : (this.state()?.html ?? '');
return {
ok: true,
root: { kind: 'document', html, url: this.currentUrl(), children: [], errors: [] },
};
}
async screenshot(): Promise<string> {
return BLANK_PNG;
}
async getText(): Promise<string> {
return this.deadEnd ? '' : (this.state()?.ariaTree ?? '');
}
async getContent(): Promise<{ html: string; url: string }> {
return { html: this.state()?.html ?? '', url: this.currentUrl() };
}
async evaluate(): Promise<unknown> {
return null;
}
async getConsole(): Promise<ConsoleEntry[]> {
return [];
}
getConsoleSummary(): { errors: number; warnings: number } {
return { errors: 0, warnings: 0 };
}
getModalStates(): ModalState[] {
return [];
}
async getNetwork(): Promise<NetworkEntry[]> {
return [];
}
async pdf(): Promise<{ data: string; pages: number }> {
throw new UnsupportedOperationError('pdf', 'fixture');
}
// --- wait ----------------------------------------------------------------
async wait(_pageId: string, _options: WaitOptions): Promise<number> {
return 0;
}
async waitForCompletion<T>(_pageId: string, action: () => Promise<T>): Promise<T> {
return await action();
}
// --- state (no persistence) ---------------------------------------------
async getCookies(): Promise<Cookie[]> {
return [];
}
async setCookies(): Promise<void> {}
async clearCookies(): Promise<void> {}
async getStorage(): Promise<Record<string, string>> {
return {};
}
async setStorage(): Promise<void> {}
async clearStorage(): Promise<void> {}
// --- sync helpers --------------------------------------------------------
getPageUrl(): string | undefined {
return this.currentUrl();
}
// --- credential capture --------------------------------------------------
/** Resolve a `[REDACTED:]` marker to its recorded synthetic secret, so the
* redacted-key capture path replays without re-deriving markers from HTML
* (fixtures carry no page HTML). Undefined fall back to the HTML path. */
async resolveRedactedSecret(_pageId: string, marker: string): Promise<string | undefined> {
if (this.deadEnd) return undefined;
return this.state()?.redactedSecrets?.[marker];
}
async getElementValue(_pageId: string, target: ElementTarget): Promise<string> {
if (this.deadEnd) throw new UnsupportedOperationError('getElementValue (dead-end)', 'fixture');
const ref = selectorOf(target);
const value = this.state()?.elementValues?.[ref];
if (value === undefined) {
throw new UnsupportedOperationError(
`getElementValue (no recorded value for ${ref})`,
'fixture',
);
}
return value;
}
}

View File

@ -9,6 +9,7 @@ import {
type ConnectionLostReason,
} from './errors';
import { createLogger } from './logger';
import type { FixtureBundle } from './adapters/fixture';
import type {
Adapter,
BrowserName,
@ -33,6 +34,10 @@ export interface BrowserConnectionOptions {
cdpEndpoint?: string;
/** Headers sent when connecting to {@link cdpEndpoint} (e.g. an auth token). */
cdpConnectHeaders?: Record<string, string>;
/** Fixture bundle for the deterministic `fixture` adapter (eval replay). When
* omitted with `adapter: 'fixture'`, the bundle is loaded from the
* `N8N_EVAL_BROWSER_FIXTURES` file. */
fixtures?: FixtureBundle;
}
export class BrowserConnection {
@ -42,6 +47,7 @@ export class BrowserConnection {
private readonly externalRelay?: CDPRelayServer;
private readonly externalCdpEndpoint?: string;
private readonly cdpConnectHeaders?: Record<string, string>;
private readonly externalFixtures?: FixtureBundle;
/** Adapter kept alive after an extension-connect timeout so its relay URL remains valid. */
private pendingAdapter: Adapter | null = null;
@ -50,6 +56,7 @@ export class BrowserConnection {
this.externalRelay = options?.relay;
this.externalCdpEndpoint = options?.cdpEndpoint;
this.cdpConnectHeaders = options?.cdpConnectHeaders;
this.externalFixtures = options?.fixtures;
// Merge auto-discovery with programmatic overrides
const discovery = getDefaultDiscovery().discover();
@ -96,7 +103,9 @@ export class BrowserConnection {
}
const browser = overrideBrowser ?? this.config.defaultBrowser;
if (this.config.mode !== 'remote') {
// The fixture adapter replays a recorded bundle — it needs no real
// browser binary (that is the point: deterministic eval replay in CI).
if (this.config.mode !== 'remote' && this.config.adapter !== 'fixture') {
this.requireBrowserAvailable(browser);
}
@ -204,6 +213,13 @@ export class BrowserConnection {
}
private async createAdapter(): Promise<Adapter> {
if (this.config.adapter === 'fixture') {
// Deterministic eval replay — no real browser. Wins over mode: a
// fixture run never touches a relay/extension.
const { FixtureAdapter, loadFixtureBundleFromEnv } = await import('./adapters/fixture.js');
const bundle = this.externalFixtures ?? (await loadFixtureBundleFromEnv());
return new FixtureAdapter(bundle);
}
if (this.config.mode === 'remote') {
// Remote mode is only supported by the Playwright adapter
const { PlaywrightAdapter } = await import('./adapters/playwright.js');

View File

@ -65,6 +65,13 @@ function browserCaptureSecret(
let value = '';
if ('redactedKey' in args.element) {
const { redactedKey } = args.element;
// Fast path: an adapter that carries secrets directly (the fixture
// adapter, eval replay) resolves the marker without page HTML.
const direct = await state.adapter.resolveRedactedSecret?.(pageId, redactedKey);
if (direct !== undefined) {
context.secretsBuffer.capture(args.credentialsKey, args.field, direct);
return formatCallToolResult({ ok: true, fieldsCaptured: [args.field] });
}
const sensitivity = analyzeHtmlSensitivity(await state.adapter.probePageHtml(pageId));
if (sensitivity.ok) {
const formatMarker = createRedactionMarkerFormatter(sensitivity.hits);

View File

@ -0,0 +1,58 @@
import { describe, expect, it, beforeEach } from 'vitest';
import { createBrowserTools } from './index';
import { findTool, structuredOf } from './test-helpers';
import type { FixtureBundle } from '../adapters/fixture';
import type { ToolContext } from '../types';
// End-to-end (X-mode) proof: a browser tool call runs through the REAL tool
// pipeline (createConnectedTool → enrichment → jsdom sensitivity → redaction)
// backed by the deterministic FixtureAdapter — no real browser, extension, CDP
// relay, or daemon. Goal: computer-use-evals.
const bundle: FixtureBundle = {
version: 1,
initialStateId: 's0',
states: [
{
id: 's0',
url: 'https://news.ycombinator.com',
ariaTree:
'- list:\n - listitem "1. First post [ref=e1]"\n - listitem "2. Second [ref=e2]"\n - listitem "3. Third [ref=e3]"',
html: '<ol><li>1. First post</li><li>2. Second</li><li>3. Third</li></ol>',
},
{
id: 's1',
url: 'https://news.ycombinator.com/newest',
ariaTree: '- heading "Newest"',
html: '<h1>Newest</h1>',
},
],
transitions: [{ from: 's0', action: { tool: 'browser_click', selector: 'text=More' }, to: 's1' }],
};
const ctx: ToolContext = { dir: '/tmp' };
describe('fixture replay through the real browser tool pipeline', () => {
let toolkit: ReturnType<typeof createBrowserTools>;
beforeEach(async () => {
toolkit = createBrowserTools({ adapter: 'fixture' }, { fixtures: bundle });
await toolkit.connection.connect(); // no real browser required for fixtures
});
it('browser_snapshot returns the recorded aria tree via the tool pipeline', async () => {
const snapshot = findTool(toolkit.tools, 'browser_snapshot');
const result = await snapshot.execute({}, ctx);
expect(structuredOf(result).snapshot).toBe(bundle.states[0].ariaTree);
});
it('browser_click advances the page-state; the next snapshot reflects it', async () => {
const click = findTool(toolkit.tools, 'browser_click');
await click.execute({ element: { selector: 'text=More' } }, ctx);
const snapshot = findTool(toolkit.tools, 'browser_snapshot');
const result = await snapshot.execute({}, ctx);
expect(structuredOf(result).snapshot).toBe(bundle.states[1].ariaTree);
});
});

View File

@ -26,7 +26,7 @@ const browserOverrideSchema = z.object({
export const configSchema = z.object({
defaultBrowser: browserNameSchema.default('chrome'),
browsers: z.record(browserNameSchema, browserOverrideSchema).default({}),
adapter: z.enum(['playwright', 'agent-browser']).default('agent-browser'),
adapter: z.enum(['playwright', 'agent-browser', 'fixture']).default('agent-browser'),
mode: z.enum(['local', 'remote']).default('local'),
});
@ -41,7 +41,7 @@ export interface ResolvedBrowserInfo {
export interface ResolvedConfig {
defaultBrowser: BrowserName;
browsers: Map<BrowserName, ResolvedBrowserInfo>;
adapter: 'playwright' | 'agent-browser';
adapter: 'playwright' | 'agent-browser' | 'fixture';
mode: 'local' | 'remote';
}
@ -125,6 +125,11 @@ export interface Adapter {
getPageUrl(pageId: string): string | undefined;
// Credential capture
getElementValue(pageId: string, target: ElementTarget): Promise<string>;
/** Optional: resolve a `[REDACTED:]` marker directly to its secret value,
* bypassing HTML re-derivation. Used by the deterministic fixture adapter
* (eval replay) which carries synthetic secrets but no page HTML. Real
* adapters omit it the capture tool falls back to the HTML/sensitivity path. */
resolveRedactedSecret?(pageId: string, marker: string): Promise<string | undefined>;
}
export interface ConnectionState {

View File

@ -93,11 +93,14 @@ async function createSession(service: InstanceAiBrowserSessionService) {
const sessionId = relayUrl.pathname.split('/').pop()!;
const extToken = relayUrl.searchParams.get('token')!;
const lastCall = mcpBrowserMock.createBrowserTools.mock.calls.at(-1)!;
const cdpToken = (lastCall[1] as { cdpConnectHeaders: Record<string, string> }).cdpConnectHeaders[
CDP_TOKEN_HEADER
];
const options = lastCall[1] as {
cdpEndpoint: string;
cdpConnectHeaders: Record<string, string>;
};
const cdpToken = options.cdpConnectHeaders[CDP_TOKEN_HEADER];
const cdpEndpoint = options.cdpEndpoint;
const relay = mcpBrowserMock.__createdRelays.at(-1)!;
return { sessionId, extToken, cdpToken, relay, relayEndpoint, connectUrl };
return { sessionId, extToken, cdpToken, cdpEndpoint, relay, relayEndpoint, connectUrl };
}
describe('InstanceAiBrowserSessionService', () => {
@ -106,7 +109,7 @@ describe('InstanceAiBrowserSessionService', () => {
const push = mock<Push>();
const userRepository = mock<UserRepository>();
const credentialsService = mock<CredentialsService>();
const globalConfig = mock<GlobalConfig>({ port: 5678 });
const globalConfig = mock<GlobalConfig>({ port: 5678, listen_address: '::' });
const telemetry = mock<Telemetry>();
let service: InstanceAiBrowserSessionService;
@ -114,6 +117,7 @@ describe('InstanceAiBrowserSessionService', () => {
vi.clearAllMocks();
mcpBrowserMock.__createdRelays.length = 0;
logger.scoped.mockReturnValue(logger);
globalConfig.listen_address = '::';
urlService.getInstanceBaseUrl.mockReturnValue('http://localhost:5678');
service = new InstanceAiBrowserSessionService(
logger,
@ -150,6 +154,17 @@ describe('InstanceAiBrowserSessionService', () => {
expect(second.sessionId).toBe(first.sessionId);
expect(second.extToken).not.toBe(first.extToken);
});
it('builds the CDP endpoint on the IPv6 loopback when listen_address is IPv6', async () => {
const { cdpEndpoint, sessionId } = await createSession(service);
expect(cdpEndpoint).toBe(`ws://[::1]:5678/browser-use/cdp/${sessionId}`);
});
it('builds the CDP endpoint on the IPv4 loopback when listen_address is IPv4', async () => {
globalConfig.listen_address = '0.0.0.0';
const { cdpEndpoint, sessionId } = await createSession(service);
expect(cdpEndpoint).toBe(`ws://127.0.0.1:5678/browser-use/cdp/${sessionId}`);
});
});
describe('handleExtensionUpgrade', () => {
@ -285,3 +300,46 @@ describe('InstanceAiBrowserSessionService', () => {
});
});
});
describe('InstanceAiBrowserSessionService — fixture eval hook (goal: computer-use-evals)', () => {
const logger = mock<Logger>();
const urlService = mock<UrlService>();
const push = mock<Push>();
const userRepository = mock<UserRepository>();
const credentialsService = mock<CredentialsService>();
const globalConfig = mock<GlobalConfig>({ port: 5678, listen_address: '::' });
const telemetry = mock<Telemetry>();
let service: InstanceAiBrowserSessionService;
beforeEach(() => {
vi.clearAllMocks();
logger.scoped.mockReturnValue(logger);
mcpBrowserMock.createBrowserTools.mockReturnValue({
tools: [],
connection: { connect: vi.fn(async () => undefined), shutdown: vi.fn(async () => undefined) },
});
service = new InstanceAiBrowserSessionService(
logger,
urlService,
push,
userRepository,
credentialsService,
globalConfig,
telemetry,
);
});
it('builds an ephemeral fixture MCP server via the fixture adapter', async () => {
const server = await service.buildEphemeralFixtureMcpServer('u1');
expect(server).toBeDefined();
expect(mcpBrowserMock.createBrowserTools).toHaveBeenCalledWith({ adapter: 'fixture' });
});
it('does NOT store the fixture as the user session (normal browser use is untouched)', async () => {
await service.buildEphemeralFixtureMcpServer('u1');
// The ephemeral server must never leak into the user's real session — a
// normal (non-eval) thread on the same account still has no browser session.
expect(service.isConnected('u1')).toBe(false);
expect(service.findMcpServer('u1')).toBeUndefined();
});
});

View File

@ -57,6 +57,13 @@ export class InstanceAiBrowserSessionService {
private readonly sessionsBySessionId = new Map<string, BrowserSession>();
/** Per-thread computer-use fixture bundles (eval replay, goal:
* computer-use-evals). Set by the eval endpoint called by the HARNESS,
* which owns the thread id before the build; consumed (and cleared) when the
* ephemeral fixture browser is composed for that thread. Falls back to the
* N8N_EVAL_BROWSER_FIXTURES env-file when none was registered. */
private readonly fixtureByThread = new Map<string, unknown>();
private readonly logger: Logger;
constructor(
@ -116,6 +123,51 @@ export class InstanceAiBrowserSessionService {
return this.sessions.get(userId)?.connected ?? false;
}
/** Register a per-thread fixture bundle (eval endpoint, harness-called). */
setThreadFixture(threadId: string, fixture: unknown): void {
this.fixtureByThread.set(threadId, fixture);
}
/**
* Eval hook (goal: computer-use-evals). Build an EPHEMERAL, deterministic
* fixture-backed browser MCP server for a single eval run the adapter
* replays a recorded bundle (the harness-registered per-thread fixture, else
* `N8N_EVAL_BROWSER_FIXTURES`), no real browser/extension/CDP relay.
* Deliberately NOT stored in `this.sessions`: fixture replay must apply ONLY
* to the eval run that asked for it (an `evals`-source thread), never to the
* user's normal interactive browser session on the same account. Caller gates
* on the thread being eval-driven.
*/
async buildEphemeralFixtureMcpServer(
userId: string,
threadId?: string,
): Promise<BrowserLocalMcpServer> {
const { createBrowserTools } = await import('@n8n/mcp-browser');
// Prefer the fixture the harness registered for THIS thread (via the eval
// endpoint); fall back to the N8N_EVAL_BROWSER_FIXTURES env file the adapter
// reads when none was registered.
const fixtures = threadId ? this.fixtureByThread.get(threadId) : undefined;
if (threadId) this.fixtureByThread.delete(threadId);
const toolkit = fixtures
? createBrowserTools({ adapter: 'fixture' }, { fixtures: fixtures as never })
: createBrowserTools({ adapter: 'fixture' });
const workDir = join(tmpdir(), 'n8n-instance-ai-browser-fixture', userId);
await mkdir(workDir, { recursive: true });
const toolContext: ToolContext = {
dir: workDir,
secretsBuffer: createInMemorySecretsBuffer(),
createCredential: async (payload: CreateCredentialPayload) =>
await this.createCredential(userId, payload),
};
// The fixture adapter connects with no real browser (see mcp-browser
// connection.connect: the browser-availability gate is skipped for it).
await toolkit.connection.connect();
this.logger.info('Ephemeral fixture browser server ready (eval run)', { userId });
return new BrowserLocalMcpServer(toolkit, toolContext, this.logger);
}
handleExtensionUpgrade(req: BrowserUseUpgradeRequest): void {
const session = this.sessionsBySessionId.get(req.params.sessionId);
const token = typeof req.query.token === 'string' ? req.query.token : null;
@ -303,7 +355,13 @@ export class InstanceAiBrowserSessionService {
}
private buildCdpEndpoint(sessionId: string): string {
return `ws://127.0.0.1:${this.globalConfig.port}${BROWSER_USE_WS_NAMESPACE}/cdp/${sessionId}`;
// The CDP client connects back to this same process over the main HTTP
// server's loopback. Match the loopback family to `listen_address`: with
// the default IPv6 wildcard (`::`) the server is only reachable via
// `[::1]`, and the IPv4 `127.0.0.1` never lands on the socket — the
// WebSocket handshake would hang until Playwright's connect timeout.
const host = this.globalConfig.listen_address.includes(':') ? '[::1]' : '127.0.0.1';
return `ws://${host}:${this.globalConfig.port}${BROWSER_USE_WS_NAMESPACE}/cdp/${sessionId}`;
}
private getPublicWsBaseUrl(): string {

View File

@ -959,6 +959,37 @@ export class InstanceAiController {
return { ok: true };
}
/**
* Register a computer-use browser fixture for a build thread (goal:
* computer-use-evals). The eval HARNESS (which owns the thread id) POSTs the
* case's recorded page-state bundle here before the first message; the FIXTURE
* adapter replays it for that thread's browser tools. Mirrors the credential
* allowlist above.
*/
@Post('/eval/browser-fixture')
@GlobalScope('instanceAi:eval')
async setThreadBrowserFixture(
req: AuthenticatedRequest,
_res: Response,
@Body payload: { threadId: string; fixture: unknown },
) {
this.requireInstanceAiEnabled();
// A fixture is only ever replayed when this n8n runs in fixture mode. If it
// isn't, registering one is a no-op AND the thread would fall through to the
// REAL browser — a CI agent driving a live site. Fail loudly instead: the
// harness treats a non-404 here as fatal, so the build reds at provisioning
// rather than silently going online.
if (process.env.N8N_EVAL_BROWSER_ADAPTER !== 'fixture') {
throw new BadRequestError(
'Case declares a browser fixture but this n8n is not in fixture mode ' +
'(set N8N_EVAL_BROWSER_ADAPTER=fixture on the eval instance).',
);
}
await this.assertThreadAccess(req.user.id, payload.threadId);
this.browserSessionService.setThreadFixture(payload.threadId, payload.fixture);
return { ok: true };
}
/**
* Seed an existing (owned) thread with a previously exported conversation:
* recreate the workflow artifacts the history references (node credentials

View File

@ -2074,6 +2074,11 @@ export class InstanceAiService {
) {
const memory = this.agentMemory;
const boundProjectId = await memory.getThreadProjectId(threadId);
// Computer-use eval fixture is scoped to eval-driven threads ONLY (source
// 'evals', set by the eval harness/CU driver). A normal interactive thread
// on the same account keeps its real browser session even when the fixture
// env is set — so eval replay and normal browser use coexist on one n8n.
const threadSource = (await memory.getThread(threadId))?.metadata?.source;
if (!boundProjectId) {
throw new UnexpectedError(
`Instance AI thread "${threadId}" has no bound project; it must be created via POST /instance-ai/threads before a run can start`,
@ -2118,9 +2123,18 @@ export class InstanceAiService {
this.gatewayService.applyToolPolicy(user.id);
const gatewayMcpServer =
!localGatewayDisabledForUser && userGateway?.isConnected ? userGateway : undefined;
const browserMcpServer = browserUseEnabledGlobally
? this.browserSessionService.findMcpServer(user.id)
: undefined;
// Eval hook (goal: computer-use-evals): an eval-driven thread (source
// 'evals') with N8N_EVAL_BROWSER_ADAPTER=fixture gets an EPHEMERAL
// fixture-backed browser server (deterministic replay, no real browser).
// Every other thread — normal interactive use — gets the user's real
// browser session, so the two coexist on the same instance/account.
const useFixtureBrowser =
process.env.N8N_EVAL_BROWSER_ADAPTER === 'fixture' && threadSource === 'evals';
const browserMcpServer = !browserUseEnabledGlobally
? undefined
: useFixtureBrowser
? await this.browserSessionService.buildEphemeralFixtureMcpServer(user.id, threadId)
: this.browserSessionService.findMcpServer(user.id);
const localMcpServer = composeLocalMcpServers(gatewayMcpServer, browserMcpServer);
if (localMcpServer) {
context.localMcpServer = localMcpServer;