From 7f3713702aaa73a5ab474cb3ba08a3e8bf673203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dimitri=20Lavren=C3=BCk?= <20122620+dlavrenuek@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:24:33 +0200 Subject: [PATCH] feat: Add snapshot option to every browser use interaction tool (#36111) Co-authored-by: Claude Fable 5 --- .../@n8n/mcp-browser/src/tools/helpers.ts | 14 +- .../mcp-browser/src/tools/interaction.test.ts | 64 ++++ .../@n8n/mcp-browser/src/tools/interaction.ts | 11 + .../src/tools/response-envelope.test.ts | 301 ++++++++++++++++++ .../src/tools/response-envelope.ts | 46 ++- .../@n8n/mcp-browser/src/tools/schemas.ts | 7 + .../@n8n/mcp-browser/src/tools/tabs.test.ts | 2 +- 7 files changed, 426 insertions(+), 19 deletions(-) create mode 100644 packages/@n8n/mcp-browser/src/tools/response-envelope.test.ts diff --git a/packages/@n8n/mcp-browser/src/tools/helpers.ts b/packages/@n8n/mcp-browser/src/tools/helpers.ts index fb16e2d72bb..9c2b1b63d8c 100644 --- a/packages/@n8n/mcp-browser/src/tools/helpers.ts +++ b/packages/@n8n/mcp-browser/src/tools/helpers.ts @@ -24,6 +24,7 @@ export { elementTargetSchema, modalStateSchema, pageIdField, + snapshotField, withSnapshotEnvelope, } from './schemas'; export type { ElementTargetInput } from './schemas'; @@ -32,7 +33,7 @@ export type { ElementTargetInput } from './schemas'; // Connected tool input constraint — every tool must have at least pageId // --------------------------------------------------------------------------- -type ConnectedToolInput = { pageId?: string }; +type ConnectedToolInput = { pageId?: string; snapshot?: 'interactive' | 'non-interactive' }; // --------------------------------------------------------------------------- // Connected tool options @@ -41,6 +42,8 @@ type ConnectedToolInput = { pageId?: string }; export interface ConnectedToolOptions { /** Append an accessibility snapshot to the response after the action. */ autoSnapshot?: boolean; + /** Annotate the auto-snapshot with interactive refs. Defaults to the adapter default (true). */ + snapshotInteractive?: boolean; /** Wrap the action in waitForCompletion (network/navigation settle). */ waitForCompletion?: boolean; /** Skip post-action enrichment (snapshot, tab diff, etc.). Use for destructive actions like tab close. */ @@ -91,6 +94,9 @@ export function createConnectedTool< inputSchema, outputSchema, async execute(args: z.infer, context: ToolContext) { + const effectiveOptions: ConnectedToolOptions = args.snapshot + ? { ...options, autoSnapshot: true, snapshotInteractive: args.snapshot === 'interactive' } + : (options ?? {}); try { const { state, pageId } = resolvePageContext(connection, args); @@ -111,7 +117,7 @@ export function createConnectedTool< if (!options?.skipEnrichment) { // Re-resolve: tab-creating actions (tab_open) update activePageId const enrichPageId = state.activePageId || pageId; - await enrichResponse(result, state, enrichPageId, options ?? {}, tabsBefore); + await enrichResponse(result, state, enrichPageId, effectiveOptions, tabsBefore); } // Sync live URL back to state.pages so the cache stays fresh const currentUrl = state.adapter.getPageUrl(pageId); @@ -130,12 +136,12 @@ export function createConnectedTool< new ConnectionLostError('browser_closed'), connection, args, - options ?? {}, + effectiveOptions, ), ); } return redactCallToolResult( - await buildErrorResponse(error, connection, args, options ?? {}), + await buildErrorResponse(error, connection, args, effectiveOptions), ); } }, diff --git a/packages/@n8n/mcp-browser/src/tools/interaction.test.ts b/packages/@n8n/mcp-browser/src/tools/interaction.test.ts index d525643366b..7d713ad70eb 100644 --- a/packages/@n8n/mcp-browser/src/tools/interaction.test.ts +++ b/packages/@n8n/mcp-browser/src/tools/interaction.test.ts @@ -536,4 +536,68 @@ describe('createInteractionTools', () => { }); }); }); + + // ----------------------------------------------------------------------- + // snapshot input param (shared via createConnectedTool) + // ----------------------------------------------------------------------- + + describe('snapshot input param', () => { + const validInputs: Record> = { + browser_click: { element: { ref: 'e1' } }, + browser_type: { element: { ref: 'e1' }, text: 'hi' }, + browser_select: { element: { ref: 'e1' }, values: ['a'] }, + browser_drag: { from: { ref: 'e1' }, to: { ref: 'e2' } }, + browser_hover: { element: { ref: 'e1' } }, + browser_press: { keys: 'Enter' }, + browser_scroll: { mode: 'direction', direction: 'down' }, + browser_upload: { files: ['/tmp/a.txt'] }, + browser_dialog: { action: 'accept' }, + }; + + it.each(Object.keys(validInputs))('%s accepts snapshot values and rejects invalid', (name) => { + const schema = findTool(tools, name).inputSchema; + expect(() => schema.parse({ ...validInputs[name], snapshot: 'interactive' })).not.toThrow(); + expect(() => + schema.parse({ ...validInputs[name], snapshot: 'non-interactive' }), + ).not.toThrow(); + expect(() => schema.parse({ ...validInputs[name], snapshot: 'full' })).toThrow(); + }); + + it('returns an interactive snapshot when snapshot is "interactive"', async () => { + mockConnection.adapter.snapshot.mockResolvedValue({ tree: '- button "OK"', refCount: 1 }); + + const result = await findTool(tools, 'browser_click').execute( + { element: { ref: 'e1' }, snapshot: 'interactive' }, + TOOL_CONTEXT, + ); + const data = structuredOf(result); + + expect(mockConnection.adapter.snapshot).toHaveBeenCalledWith('page1', undefined, true); + expect(data.snapshot).toBe('- button "OK"'); + }); + + it('returns a plain snapshot when snapshot is "non-interactive"', async () => { + mockConnection.adapter.snapshot.mockResolvedValue({ tree: '- text "Done"', refCount: 0 }); + + const result = await findTool(tools, 'browser_click').execute( + { element: { ref: 'e1' }, snapshot: 'non-interactive' }, + TOOL_CONTEXT, + ); + const data = structuredOf(result); + + expect(mockConnection.adapter.snapshot).toHaveBeenCalledWith('page1', undefined, false); + expect(data.snapshot).toBe('- text "Done"'); + }); + + it('does not snapshot when the param is omitted', async () => { + const result = await findTool(tools, 'browser_click').execute( + { element: { ref: 'e1' } }, + TOOL_CONTEXT, + ); + const data = structuredOf(result); + + expect(mockConnection.adapter.snapshot).not.toHaveBeenCalled(); + expect(data.snapshot).toBeUndefined(); + }); + }); }); diff --git a/packages/@n8n/mcp-browser/src/tools/interaction.ts b/packages/@n8n/mcp-browser/src/tools/interaction.ts index baceffd2b50..635f26af12d 100644 --- a/packages/@n8n/mcp-browser/src/tools/interaction.ts +++ b/packages/@n8n/mcp-browser/src/tools/interaction.ts @@ -7,6 +7,7 @@ import { createConnectedTool, elementTargetSchema, pageIdField, + snapshotField, withSnapshotEnvelope, } from './helpers'; @@ -41,6 +42,7 @@ const browserClickSchema = z .optional() .describe('Modifier keys to hold'), pageId: pageIdField, + snapshot: snapshotField, }) .describe('Click an element'); @@ -83,6 +85,7 @@ const browserTypeSchema = z submit: z.boolean().optional().describe('Press Enter after typing'), delay: z.number().optional().describe('Delay between keystrokes in ms'), pageId: pageIdField, + snapshot: snapshotField, }) .describe('Type text into an element'); @@ -124,6 +127,7 @@ const browserSelectSchema = z element: elementTargetSchema.describe('Select element to interact with'), values: z.array(z.string()).describe('Option values or labels to select'), pageId: pageIdField, + snapshot: snapshotField, }) .describe('Select option(s) in a