diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index d5efb18eb32..8ea7c3f9bab 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -638,6 +638,20 @@ export class N8nClient { }); } + /** + * Register a computer-use browser fixture for a build thread (goal: + * computer-use-evals). The server's FIXTURE adapter replays this recorded + * bundle for the thread's browser tools. Provisioned before the first message, + * like the credential allowlist. + * POST /rest/instance-ai/eval/browser-fixture + */ + async setThreadBrowserFixture(threadId: string, fixture: unknown): Promise { + await this.fetch('/rest/instance-ai/eval/browser-fixture', { + method: 'POST', + body: { threadId, fixture }, + }); + } + /** * Seed an existing thread with a previously exported conversation: the * referenced workflows are recreated (node credentials stripped server-side) diff --git a/packages/@n8n/instance-ai/evaluations/harness/runner.ts b/packages/@n8n/instance-ai/evaluations/harness/runner.ts index 12280d268bd..c36eac2d77e 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/runner.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/runner.ts @@ -516,6 +516,24 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise +import { readFileSync } from 'node:fs'; +import { createBrowserTools } from './dist/index.js'; + +const bundle = JSON.parse(readFileSync(process.argv[2], 'utf8')); +const { tools, connection } = createBrowserTools({ adapter: 'fixture' }, { fixtures: bundle }); +await connection.connect(); + +console.log(`bundle: states=${bundle.states.length} initial=${bundle.initialStateId} source=${bundle.sourceThreadId ?? '-'}`); +const snapshot = tools.find((t) => t.name === 'browser_snapshot'); +const res = await snapshot.execute({}, { dir: '/tmp' }); +const tree = res.structuredContent?.snapshot ?? ''; +console.log(`\n>>> browser_snapshot replayed (NO real browser). tree length = ${tree.length}`); +console.log(`>>> first lines of the REAL recorded page the agent would see:\n`); +console.log(tree.split('\n').slice(0, 12).join('\n')); +await connection.disconnect(); diff --git a/packages/@n8n/mcp-browser/src/adapters/fixture.test.ts b/packages/@n8n/mcp-browser/src/adapters/fixture.test.ts new file mode 100644 index 00000000000..74151c36904 --- /dev/null +++ b/packages/@n8n/mcp-browser/src/adapters/fixture.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it, beforeEach } from 'vitest'; + +import { FixtureAdapter, type FixtureBundle } from './fixture'; +import { UnsupportedOperationError } from '../errors'; + +const bundle: FixtureBundle = { + version: 1, + initialStateId: 's0', + states: [ + { + id: 's0', + url: 'https://news.ycombinator.com', + ariaTree: '- list:\n - listitem "1. A [ref=e1]"\n - listitem "2. B [ref=e2]"', + html: '', + }, + { + id: 's1', + url: 'https://news.ycombinator.com/newest', + ariaTree: '- textbox "token" [ref=e9]', + elementValues: { e9: 'sk-ant-secret-value' }, + redactedSecrets: { '[REDACTED:anthropic_api_key:1]': 'sk-ant-redacted-value' }, + }, + ], + transitions: [{ from: 's0', action: { tool: 'browser_click', selector: 'text=More' }, to: 's1' }], +}; + +describe('FixtureAdapter', () => { + let adapter: FixtureAdapter; + beforeEach(async () => { + adapter = new FixtureAdapter(bundle); + await adapter.launch({ browser: 'chrome' }); + }); + + it('serves the initial page-state tree, refCount and url', async () => { + const snap = await adapter.snapshot(); + expect(snap.tree).toBe(bundle.states[0].ariaTree); + expect(snap.refCount).toBe(2); + expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com'); + }); + + it('serves recorded HTML from probePageHtml (drives real sensitivity/redaction)', async () => { + const probe = await adapter.probePageHtml(); + expect(probe.ok).toBe(true); + expect(probe.root?.html).toBe(''); + }); + + it('advances state on a matching click transition and captures the recorded value', async () => { + await adapter.click('p', { selector: 'text=More' }); + const snap = await adapter.snapshot(); + expect(snap.tree).toBe('- textbox "token" [ref=e9]'); + expect(await adapter.getElementValue('p', { ref: 'e9' })).toBe('sk-ant-secret-value'); + }); + + it('navigate falls back to a recorded state with the matching url', async () => { + const res = await adapter.navigate('p', 'https://news.ycombinator.com/newest'); + expect(res.status).toBe(200); + expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com/newest'); + }); + + it('resolves a redacted-key marker to its synthetic secret', async () => { + await adapter.click('p', { selector: 'text=More' }); // advance to s1 + expect(await adapter.resolveRedactedSecret('p', '[REDACTED:anthropic_api_key:1]')).toBe( + 'sk-ant-redacted-value', + ); + // Unknown marker ⇒ undefined (caller falls back to the HTML path). + expect(await adapter.resolveRedactedSecret('p', '[REDACTED:nope:9]')).toBeUndefined(); + }); + + it('dead-ends legibly when the agent leaves the recorded trajectory', async () => { + await adapter.click('p', { selector: 'text=does-not-exist' }); + const snap = await adapter.snapshot(); + expect(snap.tree).toContain('No fixture page for this state'); + expect(snap.refCount).toBe(0); + await expect(adapter.getElementValue('p', { ref: 'e9' })).rejects.toBeInstanceOf( + UnsupportedOperationError, + ); + }); + + it('re-launch resets to the initial state', async () => { + await adapter.click('p', { selector: 'text=More' }); + expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com/newest'); + await adapter.launch({ browser: 'chrome' }); + expect(adapter.getPageUrl()).toBe('https://news.ycombinator.com'); + }); +}); diff --git a/packages/@n8n/mcp-browser/src/adapters/fixture.ts b/packages/@n8n/mcp-browser/src/adapters/fixture.ts new file mode 100644 index 00000000000..78c86abde3a --- /dev/null +++ b/packages/@n8n/mcp-browser/src/adapters/fixture.ts @@ -0,0 +1,325 @@ +import { UnsupportedOperationError } from '../errors'; +import type { + Adapter, + ClickOptions, + ConnectConfig, + Cookie, + ConsoleEntry, + ElementTarget, + HtmlProbeResult, + ModalState, + NavigateResult, + NetworkEntry, + PageInfo, + ScreenshotOptions, + ScrollOptions, + SnapshotResult, + TypeOptions, + WaitOptions, +} from '../types'; + +// --------------------------------------------------------------------------- +// Fixture bundle — the cross-repo contract with LangTracer's `CuFixtureBundle` +// (packages/shared/src/types/cu-fixture.ts). Keep these in lockstep; bump +// `version` on any shape change. A bundle is a small page-state machine +// recorded from a real browser-use conversation (the LangTracer import IS the +// recorder) and replayed here with no real browser. +// --------------------------------------------------------------------------- + +export interface FixturePageState { + id: string; + url: string; + /** ARIA accessibility tree (what `browser_snapshot` returns). */ + ariaTree: string; + /** Page HTML for `probePageHtml` — optional; import-derived bundles carry + * the tree, not HTML (probe HTML is server-side enrichment only). */ + html?: string; + /** ref → value, for `getElementValue` (secret capture by ref). */ + elementValues?: Record; + /** `[REDACTED:…]` marker → value, for capture by redacted-key (see + * `resolveRedactedSecret`). Lets the redacted-key capture path replay + * without re-deriving markers from page HTML. */ + redactedSecrets?: Record; +} + +export type FixtureAction = + | { tool: 'browser_navigate'; url: string } + | { tool: 'browser_click'; selector: string } + | { tool: 'browser_type'; selector: string }; + +export interface FixtureTransition { + from: string; + action: FixtureAction; + to: string; +} + +export interface FixtureBundle { + version: number; + initialStateId: string; + states: FixturePageState[]; + transitions: FixtureTransition[]; + sourceThreadId?: string; +} + +/** Env var pointing at a JSON fixture-bundle file, used when the bundle isn't + * passed programmatically (the eval server sets this per scenario). */ +export const FIXTURE_BUNDLE_ENV = 'N8N_EVAL_BROWSER_FIXTURES'; + +/** Load a bundle from the {@link FIXTURE_BUNDLE_ENV} path. Throws a clear error + * when the env var is unset or the file can't be read/parsed — the fixture + * adapter is useless without a bundle. */ +export async function loadFixtureBundleFromEnv(): Promise { + const path = process.env[FIXTURE_BUNDLE_ENV]; + if (!path) { + throw new Error(`Fixture adapter selected but ${FIXTURE_BUNDLE_ENV} is not set`); + } + const { readFile } = await import('node:fs/promises'); + const raw = await readFile(path, 'utf8'); + return JSON.parse(raw) as FixtureBundle; +} + +const PAGE_ID = 'fixture-page-1'; +/** Bundle shape this adapter understands. Must match LangTracer's + * `CU_FIXTURE_BUNDLE_VERSION` (cross-repo contract; bump both in lockstep). */ +const SUPPORTED_FIXTURE_BUNDLE_VERSION = 1; +// 1x1 transparent PNG — a placeholder for the rare screenshot request. +const BLANK_PNG = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + +function selectorOf(target: ElementTarget): string { + return 'selector' in target ? target.selector : target.ref; +} + +/** + * Deterministic browser Adapter that replays a recorded fixture bundle — no + * real Chrome, extension, CDP relay, or daemon. Everything ABOVE the adapter + * (tool wrappers, enrichment, jsdom sensitivity analysis, redaction, secret + * capture → create_credential) runs for real; only browser I/O is served from + * the bundle. See goal `computer-use-evals`. + */ +export class FixtureAdapter implements Adapter { + onDisconnect?: (reason: import('../errors').ConnectionLostReason) => void; + + private readonly states: Map; + private currentId: string; + /** Set when an action has no matching transition — the agent walked off the + * recorded trajectory. We serve a legible "no fixture" page rather than + * crash, so the run fails cleanly and the grader can see it. */ + private deadEnd = false; + + /** Set when the bundle's `version` isn't one this adapter understands — a + * cross-repo contract guard. LangTracer's builder stamps `version`; a bundle + * from a drifted/newer shape must fail LOUDLY (a legible error page) rather + * than mis-replay as silent dead-ends indistinguishable from a real failure. */ + private readonly unsupportedVersion: boolean; + + constructor(private readonly bundle: FixtureBundle) { + this.states = new Map(bundle.states.map((s) => [s.id, s])); + this.currentId = bundle.initialStateId; + if (!this.states.has(this.currentId) && bundle.states.length > 0) { + this.currentId = bundle.states[0].id; + } + this.unsupportedVersion = bundle.version !== SUPPORTED_FIXTURE_BUNDLE_VERSION; + if (this.unsupportedVersion) { + console.warn( + `[fixture] unsupported CuFixtureBundle version ${String(bundle.version)} ` + + `(this adapter supports ${SUPPORTED_FIXTURE_BUNDLE_VERSION}) — serving an error page. ` + + `Rebuild the fixture or align the adapter/LangTracer contract.`, + ); + } + } + + // --- lifecycle ----------------------------------------------------------- + async launch(_config: ConnectConfig): Promise { + this.currentId = this.bundle.initialStateId; + this.deadEnd = false; + } + async close(): Promise {} + + // --- current-state helpers ---------------------------------------------- + private state(): FixturePageState | undefined { + return this.states.get(this.currentId); + } + private currentUrl(): string { + return this.state()?.url ?? ''; + } + private advance(match: (a: FixtureAction) => boolean): void { + const t = this.bundle.transitions.find((tr) => tr.from === this.currentId && match(tr.action)); + if (t && this.states.has(t.to)) { + this.currentId = t.to; + this.deadEnd = false; + } else { + this.deadEnd = true; + } + } + + // --- tabs (single-tab model) -------------------------------------------- + async listTabs(): Promise { + return [{ id: PAGE_ID, title: '', url: this.currentUrl() }]; + } + async listTabIds(): Promise { + return [PAGE_ID]; + } + async listTabSessionIds(): Promise { + return [PAGE_ID]; + } + async newPage(_url?: string): Promise { + return { id: PAGE_ID, title: '', url: this.currentUrl() }; + } + async closePage(_pageId: string): Promise {} + async focusPage(_pageId: string): Promise {} + + // --- navigation ---------------------------------------------------------- + private navResult(): NavigateResult { + return { title: '', url: this.currentUrl(), status: this.deadEnd ? 404 : 200 }; + } + async navigate(_pageId: string, url: string): Promise { + this.advance((a) => a.tool === 'browser_navigate' && a.url === url); + // Fallback: a recorded state with this exact url, reachable directly. + if (this.deadEnd) { + const target = [...this.states.values()].find((s) => s.url === url); + if (target) { + this.currentId = target.id; + this.deadEnd = false; + } + } + return this.navResult(); + } + async back(_pageId: string): Promise { + return this.navResult(); + } + async forward(_pageId: string): Promise { + return this.navResult(); + } + async reload(_pageId: string): Promise { + return this.navResult(); + } + + // --- interaction --------------------------------------------------------- + async click(_pageId: string, target: ElementTarget, _options?: ClickOptions): Promise { + const selector = selectorOf(target); + this.advance((a) => a.tool === 'browser_click' && a.selector === selector); + } + async type( + _pageId: string, + target: ElementTarget, + _text: string, + _options?: TypeOptions, + ): Promise { + const selector = selectorOf(target); + this.advance((a) => a.tool === 'browser_type' && a.selector === selector); + } + async select(_pageId: string, _target: ElementTarget, values: string[]): Promise { + return values; + } + async hover(_pageId: string, _target: ElementTarget): Promise {} + async press(_pageId: string, _keys: string): Promise {} + async drag(_pageId: string, _from: ElementTarget, _to: ElementTarget): Promise {} + async scroll(_pageId: string, _target?: ElementTarget, _options?: ScrollOptions): Promise {} + async upload( + _pageId: string, + _target: ElementTarget | undefined, + _files: string[], + ): Promise {} + async dialog(_pageId: string, _action: 'accept' | 'dismiss', text?: string): Promise { + return text ?? ''; + } + + // --- inspection ---------------------------------------------------------- + async snapshot(): Promise { + if (this.unsupportedVersion) { + return { + tree: `- text "Fixture bundle version ${String(this.bundle.version)} is not supported by this n8n build (expected ${SUPPORTED_FIXTURE_BUNDLE_VERSION})."`, + refCount: 0, + }; + } + if (this.deadEnd) { + const tree = `- text "No fixture page for this state (url: ${this.currentUrl()}); the agent left the recorded trajectory."`; + return { tree, refCount: 0 }; + } + const tree = this.state()?.ariaTree ?? ''; + const refCount = (tree.match(/\[ref=[^\]]+\]/g) ?? []).length; + return { tree, refCount }; + } + async probePageHtml(): Promise { + const html = this.deadEnd ? '' : (this.state()?.html ?? ''); + return { + ok: true, + root: { kind: 'document', html, url: this.currentUrl(), children: [], errors: [] }, + }; + } + async screenshot(): Promise { + return BLANK_PNG; + } + async getText(): Promise { + return this.deadEnd ? '' : (this.state()?.ariaTree ?? ''); + } + async getContent(): Promise<{ html: string; url: string }> { + return { html: this.state()?.html ?? '', url: this.currentUrl() }; + } + async evaluate(): Promise { + return null; + } + async getConsole(): Promise { + return []; + } + getConsoleSummary(): { errors: number; warnings: number } { + return { errors: 0, warnings: 0 }; + } + getModalStates(): ModalState[] { + return []; + } + async getNetwork(): Promise { + return []; + } + async pdf(): Promise<{ data: string; pages: number }> { + throw new UnsupportedOperationError('pdf', 'fixture'); + } + + // --- wait ---------------------------------------------------------------- + async wait(_pageId: string, _options: WaitOptions): Promise { + return 0; + } + async waitForCompletion(_pageId: string, action: () => Promise): Promise { + return await action(); + } + + // --- state (no persistence) --------------------------------------------- + async getCookies(): Promise { + return []; + } + async setCookies(): Promise {} + async clearCookies(): Promise {} + async getStorage(): Promise> { + return {}; + } + async setStorage(): Promise {} + async clearStorage(): Promise {} + + // --- sync helpers -------------------------------------------------------- + getPageUrl(): string | undefined { + return this.currentUrl(); + } + + // --- credential capture -------------------------------------------------- + /** Resolve a `[REDACTED:…]` marker to its recorded synthetic secret, so the + * redacted-key capture path replays without re-deriving markers from HTML + * (fixtures carry no page HTML). Undefined ⇒ fall back to the HTML path. */ + async resolveRedactedSecret(_pageId: string, marker: string): Promise { + if (this.deadEnd) return undefined; + return this.state()?.redactedSecrets?.[marker]; + } + + async getElementValue(_pageId: string, target: ElementTarget): Promise { + if (this.deadEnd) throw new UnsupportedOperationError('getElementValue (dead-end)', 'fixture'); + const ref = selectorOf(target); + const value = this.state()?.elementValues?.[ref]; + if (value === undefined) { + throw new UnsupportedOperationError( + `getElementValue (no recorded value for ${ref})`, + 'fixture', + ); + } + return value; + } +} diff --git a/packages/@n8n/mcp-browser/src/connection.ts b/packages/@n8n/mcp-browser/src/connection.ts index 917817e1a28..d2639389dae 100644 --- a/packages/@n8n/mcp-browser/src/connection.ts +++ b/packages/@n8n/mcp-browser/src/connection.ts @@ -9,6 +9,7 @@ import { type ConnectionLostReason, } from './errors'; import { createLogger } from './logger'; +import type { FixtureBundle } from './adapters/fixture'; import type { Adapter, BrowserName, @@ -33,6 +34,10 @@ export interface BrowserConnectionOptions { cdpEndpoint?: string; /** Headers sent when connecting to {@link cdpEndpoint} (e.g. an auth token). */ cdpConnectHeaders?: Record; + /** Fixture bundle for the deterministic `fixture` adapter (eval replay). When + * omitted with `adapter: 'fixture'`, the bundle is loaded from the + * `N8N_EVAL_BROWSER_FIXTURES` file. */ + fixtures?: FixtureBundle; } export class BrowserConnection { @@ -42,6 +47,7 @@ export class BrowserConnection { private readonly externalRelay?: CDPRelayServer; private readonly externalCdpEndpoint?: string; private readonly cdpConnectHeaders?: Record; + private readonly externalFixtures?: FixtureBundle; /** Adapter kept alive after an extension-connect timeout so its relay URL remains valid. */ private pendingAdapter: Adapter | null = null; @@ -50,6 +56,7 @@ export class BrowserConnection { this.externalRelay = options?.relay; this.externalCdpEndpoint = options?.cdpEndpoint; this.cdpConnectHeaders = options?.cdpConnectHeaders; + this.externalFixtures = options?.fixtures; // Merge auto-discovery with programmatic overrides const discovery = getDefaultDiscovery().discover(); @@ -96,7 +103,9 @@ export class BrowserConnection { } const browser = overrideBrowser ?? this.config.defaultBrowser; - if (this.config.mode !== 'remote') { + // The fixture adapter replays a recorded bundle — it needs no real + // browser binary (that is the point: deterministic eval replay in CI). + if (this.config.mode !== 'remote' && this.config.adapter !== 'fixture') { this.requireBrowserAvailable(browser); } @@ -204,6 +213,13 @@ export class BrowserConnection { } private async createAdapter(): Promise { + if (this.config.adapter === 'fixture') { + // Deterministic eval replay — no real browser. Wins over mode: a + // fixture run never touches a relay/extension. + const { FixtureAdapter, loadFixtureBundleFromEnv } = await import('./adapters/fixture.js'); + const bundle = this.externalFixtures ?? (await loadFixtureBundleFromEnv()); + return new FixtureAdapter(bundle); + } if (this.config.mode === 'remote') { // Remote mode is only supported by the Playwright adapter const { PlaywrightAdapter } = await import('./adapters/playwright.js'); diff --git a/packages/@n8n/mcp-browser/src/tools/credential.ts b/packages/@n8n/mcp-browser/src/tools/credential.ts index 2a6022562ff..892b9fe670e 100644 --- a/packages/@n8n/mcp-browser/src/tools/credential.ts +++ b/packages/@n8n/mcp-browser/src/tools/credential.ts @@ -65,6 +65,13 @@ function browserCaptureSecret( let value = ''; if ('redactedKey' in args.element) { const { redactedKey } = args.element; + // Fast path: an adapter that carries secrets directly (the fixture + // adapter, eval replay) resolves the marker without page HTML. + const direct = await state.adapter.resolveRedactedSecret?.(pageId, redactedKey); + if (direct !== undefined) { + context.secretsBuffer.capture(args.credentialsKey, args.field, direct); + return formatCallToolResult({ ok: true, fieldsCaptured: [args.field] }); + } const sensitivity = analyzeHtmlSensitivity(await state.adapter.probePageHtml(pageId)); if (sensitivity.ok) { const formatMarker = createRedactionMarkerFormatter(sensitivity.hits); diff --git a/packages/@n8n/mcp-browser/src/tools/fixture-replay.test.ts b/packages/@n8n/mcp-browser/src/tools/fixture-replay.test.ts new file mode 100644 index 00000000000..798cccbc20d --- /dev/null +++ b/packages/@n8n/mcp-browser/src/tools/fixture-replay.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, beforeEach } from 'vitest'; + +import { createBrowserTools } from './index'; +import { findTool, structuredOf } from './test-helpers'; +import type { FixtureBundle } from '../adapters/fixture'; +import type { ToolContext } from '../types'; + +// End-to-end (X-mode) proof: a browser tool call runs through the REAL tool +// pipeline (createConnectedTool → enrichment → jsdom sensitivity → redaction) +// backed by the deterministic FixtureAdapter — no real browser, extension, CDP +// relay, or daemon. Goal: computer-use-evals. + +const bundle: FixtureBundle = { + version: 1, + initialStateId: 's0', + states: [ + { + id: 's0', + url: 'https://news.ycombinator.com', + ariaTree: + '- list:\n - listitem "1. First post [ref=e1]"\n - listitem "2. Second [ref=e2]"\n - listitem "3. Third [ref=e3]"', + html: '
  1. 1. First post
  2. 2. Second
  3. 3. Third
', + }, + { + id: 's1', + url: 'https://news.ycombinator.com/newest', + ariaTree: '- heading "Newest"', + html: '

Newest

', + }, + ], + transitions: [{ from: 's0', action: { tool: 'browser_click', selector: 'text=More' }, to: 's1' }], +}; + +const ctx: ToolContext = { dir: '/tmp' }; + +describe('fixture replay through the real browser tool pipeline', () => { + let toolkit: ReturnType; + + beforeEach(async () => { + toolkit = createBrowserTools({ adapter: 'fixture' }, { fixtures: bundle }); + await toolkit.connection.connect(); // no real browser required for fixtures + }); + + it('browser_snapshot returns the recorded aria tree via the tool pipeline', async () => { + const snapshot = findTool(toolkit.tools, 'browser_snapshot'); + const result = await snapshot.execute({}, ctx); + expect(structuredOf(result).snapshot).toBe(bundle.states[0].ariaTree); + }); + + it('browser_click advances the page-state; the next snapshot reflects it', async () => { + const click = findTool(toolkit.tools, 'browser_click'); + await click.execute({ element: { selector: 'text=More' } }, ctx); + + const snapshot = findTool(toolkit.tools, 'browser_snapshot'); + const result = await snapshot.execute({}, ctx); + expect(structuredOf(result).snapshot).toBe(bundle.states[1].ariaTree); + }); +}); diff --git a/packages/@n8n/mcp-browser/src/types.ts b/packages/@n8n/mcp-browser/src/types.ts index 8af5744a81e..6e2d5a0d96f 100644 --- a/packages/@n8n/mcp-browser/src/types.ts +++ b/packages/@n8n/mcp-browser/src/types.ts @@ -26,7 +26,7 @@ const browserOverrideSchema = z.object({ export const configSchema = z.object({ defaultBrowser: browserNameSchema.default('chrome'), browsers: z.record(browserNameSchema, browserOverrideSchema).default({}), - adapter: z.enum(['playwright', 'agent-browser']).default('agent-browser'), + adapter: z.enum(['playwright', 'agent-browser', 'fixture']).default('agent-browser'), mode: z.enum(['local', 'remote']).default('local'), }); @@ -41,7 +41,7 @@ export interface ResolvedBrowserInfo { export interface ResolvedConfig { defaultBrowser: BrowserName; browsers: Map; - adapter: 'playwright' | 'agent-browser'; + adapter: 'playwright' | 'agent-browser' | 'fixture'; mode: 'local' | 'remote'; } @@ -125,6 +125,11 @@ export interface Adapter { getPageUrl(pageId: string): string | undefined; // Credential capture getElementValue(pageId: string, target: ElementTarget): Promise; + /** Optional: resolve a `[REDACTED:…]` marker directly to its secret value, + * bypassing HTML re-derivation. Used by the deterministic fixture adapter + * (eval replay) which carries synthetic secrets but no page HTML. Real + * adapters omit it → the capture tool falls back to the HTML/sensitivity path. */ + resolveRedactedSecret?(pageId: string, marker: string): Promise; } export interface ConnectionState { diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-browser-session.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-browser-session.service.test.ts index 35d448208d6..8d28cc09fee 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-browser-session.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-browser-session.service.test.ts @@ -93,11 +93,14 @@ async function createSession(service: InstanceAiBrowserSessionService) { const sessionId = relayUrl.pathname.split('/').pop()!; const extToken = relayUrl.searchParams.get('token')!; const lastCall = mcpBrowserMock.createBrowserTools.mock.calls.at(-1)!; - const cdpToken = (lastCall[1] as { cdpConnectHeaders: Record }).cdpConnectHeaders[ - CDP_TOKEN_HEADER - ]; + const options = lastCall[1] as { + cdpEndpoint: string; + cdpConnectHeaders: Record; + }; + const cdpToken = options.cdpConnectHeaders[CDP_TOKEN_HEADER]; + const cdpEndpoint = options.cdpEndpoint; const relay = mcpBrowserMock.__createdRelays.at(-1)!; - return { sessionId, extToken, cdpToken, relay, relayEndpoint, connectUrl }; + return { sessionId, extToken, cdpToken, cdpEndpoint, relay, relayEndpoint, connectUrl }; } describe('InstanceAiBrowserSessionService', () => { @@ -106,7 +109,7 @@ describe('InstanceAiBrowserSessionService', () => { const push = mock(); const userRepository = mock(); const credentialsService = mock(); - const globalConfig = mock({ port: 5678 }); + const globalConfig = mock({ port: 5678, listen_address: '::' }); const telemetry = mock(); let service: InstanceAiBrowserSessionService; @@ -114,6 +117,7 @@ describe('InstanceAiBrowserSessionService', () => { vi.clearAllMocks(); mcpBrowserMock.__createdRelays.length = 0; logger.scoped.mockReturnValue(logger); + globalConfig.listen_address = '::'; urlService.getInstanceBaseUrl.mockReturnValue('http://localhost:5678'); service = new InstanceAiBrowserSessionService( logger, @@ -150,6 +154,17 @@ describe('InstanceAiBrowserSessionService', () => { expect(second.sessionId).toBe(first.sessionId); expect(second.extToken).not.toBe(first.extToken); }); + + it('builds the CDP endpoint on the IPv6 loopback when listen_address is IPv6', async () => { + const { cdpEndpoint, sessionId } = await createSession(service); + expect(cdpEndpoint).toBe(`ws://[::1]:5678/browser-use/cdp/${sessionId}`); + }); + + it('builds the CDP endpoint on the IPv4 loopback when listen_address is IPv4', async () => { + globalConfig.listen_address = '0.0.0.0'; + const { cdpEndpoint, sessionId } = await createSession(service); + expect(cdpEndpoint).toBe(`ws://127.0.0.1:5678/browser-use/cdp/${sessionId}`); + }); }); describe('handleExtensionUpgrade', () => { @@ -285,3 +300,46 @@ describe('InstanceAiBrowserSessionService', () => { }); }); }); + +describe('InstanceAiBrowserSessionService — fixture eval hook (goal: computer-use-evals)', () => { + const logger = mock(); + const urlService = mock(); + const push = mock(); + const userRepository = mock(); + const credentialsService = mock(); + const globalConfig = mock({ port: 5678, listen_address: '::' }); + const telemetry = mock(); + let service: InstanceAiBrowserSessionService; + + beforeEach(() => { + vi.clearAllMocks(); + logger.scoped.mockReturnValue(logger); + mcpBrowserMock.createBrowserTools.mockReturnValue({ + tools: [], + connection: { connect: vi.fn(async () => undefined), shutdown: vi.fn(async () => undefined) }, + }); + service = new InstanceAiBrowserSessionService( + logger, + urlService, + push, + userRepository, + credentialsService, + globalConfig, + telemetry, + ); + }); + + it('builds an ephemeral fixture MCP server via the fixture adapter', async () => { + const server = await service.buildEphemeralFixtureMcpServer('u1'); + expect(server).toBeDefined(); + expect(mcpBrowserMock.createBrowserTools).toHaveBeenCalledWith({ adapter: 'fixture' }); + }); + + it('does NOT store the fixture as the user session (normal browser use is untouched)', async () => { + await service.buildEphemeralFixtureMcpServer('u1'); + // The ephemeral server must never leak into the user's real session — a + // normal (non-eval) thread on the same account still has no browser session. + expect(service.isConnected('u1')).toBe(false); + expect(service.findMcpServer('u1')).toBeUndefined(); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/browser/instance-ai-browser-session.service.ts b/packages/cli/src/modules/instance-ai/browser/instance-ai-browser-session.service.ts index 83f3d3bfbae..67aac915e4d 100644 --- a/packages/cli/src/modules/instance-ai/browser/instance-ai-browser-session.service.ts +++ b/packages/cli/src/modules/instance-ai/browser/instance-ai-browser-session.service.ts @@ -57,6 +57,13 @@ export class InstanceAiBrowserSessionService { private readonly sessionsBySessionId = new Map(); + /** Per-thread computer-use fixture bundles (eval replay, goal: + * computer-use-evals). Set by the eval endpoint — called by the HARNESS, + * which owns the thread id — before the build; consumed (and cleared) when the + * ephemeral fixture browser is composed for that thread. Falls back to the + * N8N_EVAL_BROWSER_FIXTURES env-file when none was registered. */ + private readonly fixtureByThread = new Map(); + private readonly logger: Logger; constructor( @@ -116,6 +123,51 @@ export class InstanceAiBrowserSessionService { return this.sessions.get(userId)?.connected ?? false; } + /** Register a per-thread fixture bundle (eval endpoint, harness-called). */ + setThreadFixture(threadId: string, fixture: unknown): void { + this.fixtureByThread.set(threadId, fixture); + } + + /** + * Eval hook (goal: computer-use-evals). Build an EPHEMERAL, deterministic + * fixture-backed browser MCP server for a single eval run — the adapter + * replays a recorded bundle (the harness-registered per-thread fixture, else + * `N8N_EVAL_BROWSER_FIXTURES`), no real browser/extension/CDP relay. + * Deliberately NOT stored in `this.sessions`: fixture replay must apply ONLY + * to the eval run that asked for it (an `evals`-source thread), never to the + * user's normal interactive browser session on the same account. Caller gates + * on the thread being eval-driven. + */ + async buildEphemeralFixtureMcpServer( + userId: string, + threadId?: string, + ): Promise { + const { createBrowserTools } = await import('@n8n/mcp-browser'); + // Prefer the fixture the harness registered for THIS thread (via the eval + // endpoint); fall back to the N8N_EVAL_BROWSER_FIXTURES env file the adapter + // reads when none was registered. + const fixtures = threadId ? this.fixtureByThread.get(threadId) : undefined; + if (threadId) this.fixtureByThread.delete(threadId); + const toolkit = fixtures + ? createBrowserTools({ adapter: 'fixture' }, { fixtures: fixtures as never }) + : createBrowserTools({ adapter: 'fixture' }); + + const workDir = join(tmpdir(), 'n8n-instance-ai-browser-fixture', userId); + await mkdir(workDir, { recursive: true }); + const toolContext: ToolContext = { + dir: workDir, + secretsBuffer: createInMemorySecretsBuffer(), + createCredential: async (payload: CreateCredentialPayload) => + await this.createCredential(userId, payload), + }; + + // The fixture adapter connects with no real browser (see mcp-browser + // connection.connect: the browser-availability gate is skipped for it). + await toolkit.connection.connect(); + this.logger.info('Ephemeral fixture browser server ready (eval run)', { userId }); + return new BrowserLocalMcpServer(toolkit, toolContext, this.logger); + } + handleExtensionUpgrade(req: BrowserUseUpgradeRequest): void { const session = this.sessionsBySessionId.get(req.params.sessionId); const token = typeof req.query.token === 'string' ? req.query.token : null; @@ -303,7 +355,13 @@ export class InstanceAiBrowserSessionService { } private buildCdpEndpoint(sessionId: string): string { - return `ws://127.0.0.1:${this.globalConfig.port}${BROWSER_USE_WS_NAMESPACE}/cdp/${sessionId}`; + // The CDP client connects back to this same process over the main HTTP + // server's loopback. Match the loopback family to `listen_address`: with + // the default IPv6 wildcard (`::`) the server is only reachable via + // `[::1]`, and the IPv4 `127.0.0.1` never lands on the socket — the + // WebSocket handshake would hang until Playwright's connect timeout. + const host = this.globalConfig.listen_address.includes(':') ? '[::1]' : '127.0.0.1'; + return `ws://${host}:${this.globalConfig.port}${BROWSER_USE_WS_NAMESPACE}/cdp/${sessionId}`; } private getPublicWsBaseUrl(): string { diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index cc8c845e79f..f8ceb5fa8af 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -959,6 +959,37 @@ export class InstanceAiController { return { ok: true }; } + /** + * Register a computer-use browser fixture for a build thread (goal: + * computer-use-evals). The eval HARNESS (which owns the thread id) POSTs the + * case's recorded page-state bundle here before the first message; the FIXTURE + * adapter replays it for that thread's browser tools. Mirrors the credential + * allowlist above. + */ + @Post('/eval/browser-fixture') + @GlobalScope('instanceAi:eval') + async setThreadBrowserFixture( + req: AuthenticatedRequest, + _res: Response, + @Body payload: { threadId: string; fixture: unknown }, + ) { + this.requireInstanceAiEnabled(); + // A fixture is only ever replayed when this n8n runs in fixture mode. If it + // isn't, registering one is a no-op AND the thread would fall through to the + // REAL browser — a CI agent driving a live site. Fail loudly instead: the + // harness treats a non-404 here as fatal, so the build reds at provisioning + // rather than silently going online. + if (process.env.N8N_EVAL_BROWSER_ADAPTER !== 'fixture') { + throw new BadRequestError( + 'Case declares a browser fixture but this n8n is not in fixture mode ' + + '(set N8N_EVAL_BROWSER_ADAPTER=fixture on the eval instance).', + ); + } + await this.assertThreadAccess(req.user.id, payload.threadId); + this.browserSessionService.setThreadFixture(payload.threadId, payload.fixture); + return { ok: true }; + } + /** * Seed an existing (owned) thread with a previously exported conversation: * recreate the workflow artifacts the history references (node credentials diff --git a/packages/cli/src/modules/instance-ai/instance-ai.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.service.ts index eb715f9d4a4..e8deda67f8b 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.service.ts @@ -2074,6 +2074,11 @@ export class InstanceAiService { ) { const memory = this.agentMemory; const boundProjectId = await memory.getThreadProjectId(threadId); + // Computer-use eval fixture is scoped to eval-driven threads ONLY (source + // 'evals', set by the eval harness/CU driver). A normal interactive thread + // on the same account keeps its real browser session even when the fixture + // env is set — so eval replay and normal browser use coexist on one n8n. + const threadSource = (await memory.getThread(threadId))?.metadata?.source; if (!boundProjectId) { throw new UnexpectedError( `Instance AI thread "${threadId}" has no bound project; it must be created via POST /instance-ai/threads before a run can start`, @@ -2118,9 +2123,18 @@ export class InstanceAiService { this.gatewayService.applyToolPolicy(user.id); const gatewayMcpServer = !localGatewayDisabledForUser && userGateway?.isConnected ? userGateway : undefined; - const browserMcpServer = browserUseEnabledGlobally - ? this.browserSessionService.findMcpServer(user.id) - : undefined; + // Eval hook (goal: computer-use-evals): an eval-driven thread (source + // 'evals') with N8N_EVAL_BROWSER_ADAPTER=fixture gets an EPHEMERAL + // fixture-backed browser server (deterministic replay, no real browser). + // Every other thread — normal interactive use — gets the user's real + // browser session, so the two coexist on the same instance/account. + const useFixtureBrowser = + process.env.N8N_EVAL_BROWSER_ADAPTER === 'fixture' && threadSource === 'evals'; + const browserMcpServer = !browserUseEnabledGlobally + ? undefined + : useFixtureBrowser + ? await this.browserSessionService.buildEphemeralFixtureMcpServer(user.id, threadId) + : this.browserSessionService.findMcpServer(user.id); const localMcpServer = composeLocalMcpServers(gatewayMcpServer, browserMcpServer); if (localMcpServer) { context.localMcpServer = localMcpServer;