From 44041d5d3ce76b83a6b1cd6fb2a42d5460b9fca8 Mon Sep 17 00:00:00 2001 From: Bernhard Wittmann Date: Fri, 21 Aug 2026 06:27:16 +0000 Subject: [PATCH] test(core): Add openai, gemini and slack credential-setup eval fixtures (no-changelog) (#36461) Co-authored-by: Claude Opus 5 (1M context) --- .../__tests__/credential-setup-checks.test.ts | 85 ++- .../__tests__/fixture-server.test.ts | 646 ++++++++++++++++-- .../__tests__/relay-connection.test.ts | 48 +- .../fixtures/providers/anthropic/console.html | 64 +- .../fixtures/providers/gemini/console.html | 218 ++++++ .../fixtures/providers/gemini/dashboard.html | 42 ++ .../fixtures/providers/gemini/manifest.json | 15 + .../fixtures/providers/openai/console.html | 184 +++++ .../fixtures/providers/openai/dashboard.html | 45 ++ .../fixtures/providers/openai/manifest.json | 15 + .../fixtures/providers/slack/apps.html | 148 ++++ .../fixtures/providers/slack/general.html | 67 ++ .../fixtures/providers/slack/install.html | 85 +++ .../fixtures/providers/slack/manifest.json | 12 + .../evaluations/harness/browser-runtime.ts | 84 +++ .../evaluations/harness/build-workflow.ts | 1 + .../harness/credential-setup-checks.ts | 18 +- .../harness/credential-setup-lane.ts | 3 + .../evaluations/harness/fixture-server.ts | 77 ++- 19 files changed, 1787 insertions(+), 70 deletions(-) create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/console.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/dashboard.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/manifest.json create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/console.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/dashboard.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/manifest.json create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/apps.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/general.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/install.html create mode 100644 packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/manifest.json diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/credential-setup-checks.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/credential-setup-checks.test.ts index c5dcc434f51..211fd1f2b9e 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/credential-setup-checks.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/credential-setup-checks.test.ts @@ -1,12 +1,15 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; +import type { N8nClient } from '../clients/n8n-client'; import { credentialSetupExpectationTexts, evaluateCredentialSetup, + probeCredentialValue, redactTranscriptSecrets, runCredentialSetupChecks, type CredentialSetupFacts, } from '../harness/credential-setup-checks'; +import { createLogger } from '../harness/logger'; const SECRET = 'sk-ant-api03-abcdefghijklmnopqrstuvwx'; @@ -326,3 +329,83 @@ describe('credentialSetupExpectationTexts stays in lockstep with the evaluator', expect(new Set(credentialSetupExpectationTexts(undefined))).toEqual(new Set(emitted)); }); }); + +// Which field carries the base URL is per-provider (`host` for gemini). The +// wrong one leaves the test pointed at the real provider, where it is discarded +// — so the check silently never runs for that provider. +describe('probeCredentialValue aims the test at the stand-in', () => { + const logger = createLogger(false); + const VERIFY_URL = 'http://dispatcher:41234'; + + function client(storedData: Record) { + const testCredential = vi.fn().mockResolvedValue({ status: 'OK' }); + return { + /** The `data` of each credential-test request the probe sent. */ + tested: (): Array> => + testCredential.mock.calls.map(([c]) => (c as { data: Record }).data), + client: { + listCredentials: vi + .fn() + .mockResolvedValue([{ id: 'new1', name: 'acct', type: 'googlePalmApi' }]), + getCredentialForTest: vi + .fn() + .mockResolvedValue({ id: 'new1', name: 'acct', type: 'googlePalmApi', data: storedData }), + testCredential, + } as unknown as N8nClient, + }; + } + + it('overwrites the field the manifest names, not always `url`', async () => { + const { client: c, tested } = client({ + host: 'https://generativelanguage.googleapis.com', + apiKey: '', + }); + + const probe = await probeCredentialValue({ + client: c, + credentialType: 'googlePalmApi', + credentialIdsBefore: [], + fixture: { verifyAttempts: 1, verifiedOk: true }, + verifyBaseUrl: VERIFY_URL, + urlField: 'host', + logger, + }); + + expect(probe).toEqual({ kind: 'passed', target: 'stand-in' }); + expect(tested()[0].host).toBe(VERIFY_URL); + // And nothing invents a `url` the credential type does not have. + expect(tested()[0].url).toBeUndefined(); + }); + + it('defaults to `url` when no field is named', async () => { + const { client: c, tested } = client({ url: 'https://api.anthropic.com', apiKey: '' }); + + await probeCredentialValue({ + client: c, + credentialIdsBefore: [], + fixture: { verifyAttempts: 1, verifiedOk: true }, + verifyBaseUrl: VERIFY_URL, + logger, + }); + + expect(tested()[0].url).toBe(VERIFY_URL); + }); + + it('leaves the URL alone in local mode, so the REAL provider answers', async () => { + const { client: c, tested } = client({ + host: 'https://generativelanguage.googleapis.com', + apiKey: '', + }); + + const probe = await probeCredentialValue({ + client: c, + credentialIdsBefore: [], + local: true, + urlField: 'host', + logger, + }); + + expect(probe).toEqual({ kind: 'passed', target: 'real' }); + expect(tested()[0].host).toBe('https://generativelanguage.googleapis.com'); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/fixture-server.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/fixture-server.test.ts index 113ce85e1c2..11af68b5b04 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/fixture-server.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/fixture-server.test.ts @@ -4,17 +4,21 @@ // setup the eval browser uses. Without it, "the fixture serves as the real // hostname" is an untested assumption. -import { mkdtemp, rm } from 'node:fs/promises'; +import { jsonParse } from 'n8n-workflow'; +import { existsSync } from 'node:fs'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { chromium, type BrowserContext } from 'playwright-core'; import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { findChromiumForEval } from '../harness/browser-runtime'; +import { findChromiumForEval, fixtureInterceptionArgs } from '../harness/browser-runtime'; import { providerFixtureManifestSchema, findFixtureForCredentialType, loadProviderFixtures, + matchRoute, + MINT_PATH, mintSecret, startFixtureServer, type FixtureServer, @@ -23,16 +27,157 @@ import { createLogger } from '../harness/logger'; const logger = createLogger(false); +/** + * Boot a fixture server + a browser aimed at it, and register the teardown. + * + * Uses `fixtureInterceptionArgs` — the same builder `startBrowserRuntime` uses — + * so these tests pin the interception setup the eval browser really gets, + * including the loopback EXCLUDE rules. Hand-assembled args drifted from it. + */ +function fixtureBrowser(credentialType: string) { + const handle: { server?: FixtureServer; ctx?: BrowserContext } = {}; + let userDataDir: string | undefined; + + beforeAll(async () => { + const fixture = await findFixtureForCredentialType(credentialType); + if (!fixture) throw new Error(`${credentialType} fixture missing`); + const server = await startFixtureServer({ fixture, logger }); + handle.server = server; + userDataDir = await mkdtemp(join(tmpdir(), `fixture-test-${fixture.id}-`)); + handle.ctx = await chromium.launchPersistentContext(userDataDir, { + executablePath: findChromiumForEval(), + headless: true, + args: fixtureInterceptionArgs(server.hostResolverRules(), undefined), + }); + }, 60_000); + + afterAll(async () => { + await handle.ctx?.close().catch(() => {}); + await handle.server?.close(); + if (userDataDir) await rm(userDataDir, { recursive: true, force: true }); + }); + + // Non-null at use: every `it` runs after beforeAll, and a failed boot throws + // there rather than handing back a half-built handle. + return handle as { server: FixtureServer; ctx: BrowserContext }; +} + describe('provider fixtures', () => { - it('ships an anthropic fixture keyed on its credential type', async () => { - const fixture = await findFixtureForCredentialType('anthropicApi'); - expect(fixture?.id).toBe('anthropic'); - expect(fixture?.manifest.hosts).toContain('platform.claude.com'); - expect(fixture?.manifest.secretPrefix).toBe('sk-ant-api03-'); + // One row per shipped fixture. A hardcoded table still reds on a missing + // fixture, because findFixtureForCredentialType returns undefined for a row + // that has no directory. + const SHIPPED = [ + { + type: 'anthropicApi', + id: 'anthropic', + host: 'platform.claude.com', + prefix: 'sk-ant-api03-', + verify: { path: '/v1/models', header: 'x-api-key' }, + }, + { + type: 'openAiApi', + id: 'openai', + host: 'platform.openai.com', + prefix: 'sk-proj-', + verify: { path: '/models', header: 'Authorization', scheme: 'Bearer' }, + }, + { + type: 'googlePalmApi', + id: 'gemini', + host: 'aistudio.google.com', + // Google's current shape, not the legacy `AIza…` one. + prefix: 'AQ.', + // Base URL lives in `host`, and the key travels as a query parameter. + urlField: 'host', + verify: { path: '/v1beta/models', query: 'key' }, + }, + // No `verify`: slackApi's test hardcodes its baseURL, so the stand-in can + // never be aimed at it. The value check reports itself unverifiable. + { type: 'slackApi', id: 'slack', host: 'api.slack.com', prefix: 'xoxb-' }, + ]; + + it.each(SHIPPED)('ships the $id fixture keyed on $type', async (row) => { + const fixture = await findFixtureForCredentialType(row.type); + expect(fixture?.id).toBe(row.id); + expect(fixture?.manifest.hosts).toContain(row.host); + expect(fixture?.manifest.secretPrefix).toBe(row.prefix); + expect(fixture?.manifest.urlField).toBe(row.urlField ?? 'url'); + expect(fixture?.manifest.verify).toEqual(row.verify); + }); + + // n8n serializes every credential type's own test request, so the manifest's + // transcription of it can be CHECKED rather than trusted. Without this the + // table above only pins what someone typed on the day, and an upstream change + // to `test.request` silently downgrades the value check to "unverifiable". + const TYPE_FILES = [ + join(__dirname, '..', '..', '..', '..', 'nodes-base', 'dist', 'types', 'credentials.json'), + join(__dirname, '..', '..', '..', 'nodes-langchain', 'dist', 'types', 'credentials.json'), + ]; + + it.skipIf(!TYPE_FILES.some(existsSync))( + "matches each credential's own test.request (needs built dist/types)", + async () => { + const types = new Map(); + for (const file of TYPE_FILES.filter(existsSync)) { + for (const cred of jsonParse< + Array<{ + name: string; + test?: { request?: { baseURL?: string; url?: string } }; + }> + >(await readFile(file, 'utf8'))) { + if (cred.test?.request) types.set(cred.name, cred.test.request); + } + } + + const fixtures = await loadProviderFixtures(); + let checked = 0; + for (const fixture of fixtures) { + const request = types.get(fixture.manifest.credentialType); + if (!request?.baseURL) continue; + checked += 1; + const interpolated = /\{\{\s*\$credentials\??\.(\w+)\s*\}\}/.exec(request.baseURL); + + if (!interpolated) { + // Hardcoded baseURL: there is no field to substitute, so a `verify` + // block could never be aimed at the stand-in. + expect(fixture.manifest.verify, `${fixture.id} verify`).toBeUndefined(); + continue; + } + expect(fixture.manifest.urlField, `${fixture.id} urlField`).toBe(interpolated[1]); + const suffix = request.baseURL.slice(interpolated.index + interpolated[0].length); + expect(fixture.manifest.verify?.path, `${fixture.id} verify.path`).toBe( + `${suffix}${request.url ?? ''}`, + ); + } + + // Every shipped fixture must have been compared. Without this, a rename + // or a change to the serialized shape makes every iteration `continue` + // and the check passes having verified nothing. + expect(checked, 'fixtures cross-checked against their credential type').toBe(fixtures.length); + }, + ); + + // The pages hardcode this path (they cannot import TS), and a rename would + // make every fixture's create button silently do nothing: the POST would fall + // through to the default page, which answers 200 with HTML, so `response.json()` + // throws inside the page and the run reds as agent misbehaviour. + it('serves the mint path every fixture page posts to', async () => { + for (const fixture of await loadProviderFixtures()) { + for (const file of Object.values(fixture.manifest.routes)) { + const html = await readFile(join(fixture.dir, file), 'utf8'); + if (html.includes('__fixture__')) expect(html).toContain(MINT_PATH); + } + } }); it('returns nothing for a credential type no fixture covers', async () => { - expect(await findFixtureForCredentialType('slackApi')).toBeUndefined(); + expect(await findFixtureForCredentialType('notionApi')).toBeUndefined(); + }); + + it('gives every fixture a credential type of its own', async () => { + // Selection is BY credential type, so a duplicate would depend on readdir order. + const types = (await loadProviderFixtures()).map((f) => f.manifest.credentialType); + expect(types).toEqual([...new Set(types)]); }); it('declares a defaultRoute that is one of its own routes', async () => { @@ -42,6 +187,36 @@ describe('provider fixtures', () => { }); }); +describe('matchRoute', () => { + const routes = ['/apps', '/apps/:appId/general', '/apps/:appId/install-on-team']; + + it('resolves a parameter segment to any value', () => { + expect(matchRoute(routes, '/apps/A0BQ7AWFNHM/general')).toBe('/apps/:appId/general'); + expect(matchRoute(routes, '/apps/A0EVALAPP/install-on-team')).toBe( + '/apps/:appId/install-on-team', + ); + }); + + it('prefers an exact route over a pattern that would also match', () => { + expect(matchRoute(['/apps/:appId/general', '/apps/fixed/general'], '/apps/fixed/general')).toBe( + '/apps/fixed/general', + ); + }); + + it('does not match across segment boundaries', () => { + // Without this a console's whole URL space collapses onto one page, and the + // agent gets a plausible-looking wrong page instead of the default. + expect(matchRoute(routes, '/apps/A0BQ7AWFNHM/general/extra')).toBeUndefined(); + expect(matchRoute(routes, '/apps/A0BQ7AWFNHM')).toBeUndefined(); + expect(matchRoute(routes, '/apps/A0BQ7AWFNHM/oauth')).toBeUndefined(); + }); + + it('still resolves plain routes', () => { + expect(matchRoute(routes, '/apps')).toBe('/apps'); + expect(matchRoute(routes, '/nope')).toBeUndefined(); + }); +}); + describe('mintSecret', () => { it('carries the provider prefix and never repeats', () => { const a = mintSecret('sk-ant-api03-'); @@ -53,31 +228,10 @@ describe('mintSecret', () => { }); describe('fixture server served to a real browser', () => { - let server: FixtureServer; - let ctx: BrowserContext | undefined; - let userDataDir: string; - - beforeAll(async () => { - const fixture = await findFixtureForCredentialType('anthropicApi'); - if (!fixture) throw new Error('anthropic fixture missing'); - server = await startFixtureServer({ fixture, logger }); - - userDataDir = await mkdtemp(join(tmpdir(), 'fixture-test-udd-')); - ctx = await chromium.launchPersistentContext(userDataDir, { - executablePath: findChromiumForEval(), - headless: true, - args: [`--host-resolver-rules=${server.hostResolverRules()}`, '--ignore-certificate-errors'], - }); - }, 60_000); - - afterAll(async () => { - await ctx?.close().catch(() => {}); - await server?.close(); - if (userDataDir) await rm(userDataDir, { recursive: true, force: true }); - }); + const fx = fixtureBrowser('anthropicApi'); it('answers as the real hostname, so the agent never sees loopback', async () => { - const page = await ctx!.newPage(); + const page = await fx.ctx.newPage(); await page.goto('https://platform.claude.com/settings/keys'); expect(page.url()).toBe('https://platform.claude.com/settings/keys'); expect(page.url()).not.toContain('127.0.0.1'); @@ -86,22 +240,22 @@ describe('fixture server served to a real browser', () => { }); it('serves the default page for an unmodelled path instead of stranding the agent', async () => { - const page = await ctx!.newPage(); + const page = await fx.ctx.newPage(); await page.goto('https://platform.claude.com/some/unmodelled/path'); expect(await page.locator('h1').textContent()).toBe('Dashboard'); await page.close(); }); it('does not expose the secret before the agent creates a key', async () => { - const page = await ctx!.newPage(); + const page = await fx.ctx.newPage(); await page.goto('https://platform.claude.com/settings/keys'); - expect(await page.content()).not.toContain(server.mintedSecret); - expect(server.secretWasIssued).toBe(false); + expect(await page.content()).not.toContain(fx.server.mintedSecret); + expect(fx.server.secretWasIssued).toBe(false); await page.close(); }); it('hands out exactly the ledger secret through the create-key flow', async () => { - const page = await ctx!.newPage(); + const page = await fx.ctx.newPage(); await page.goto('https://platform.claude.com/dashboard'); // Navigate the way the agent has to: the landing page is not the key page. // Scoped to the nav because the page links to it twice — which is true of @@ -112,31 +266,52 @@ describe('fixture server served to a real browser', () => { await page.getByRole('button', { name: 'Create Key' }).click(); await page.getByLabel('Name', { exact: true }).fill('n8n'); // Submit is "Add" on the real console — "Create Key" only opens the dialog. - await page.getByRole('dialog').getByRole('button', { name: 'Add' }).click(); + const add = page.getByRole('dialog').getByRole('button', { name: 'Add' }); + // The real console disables Add until Expires is set (NODE-5755): a name + // alone is not enough, and clicking anyway does nothing. + expect(await add.isDisabled()).toBe(true); + // Click-driven, as in the real trace: this is a listbox, not a native select. + await page.getByRole('combobox', { name: 'Expires' }).click(); + await page.getByRole('option', { name: '30 days' }).click(); + expect(await add.isDisabled()).toBe(false); + await add.click(); // String body, not a closure: this program has no DOM lib, and the callback // runs in the page anyway. await page.waitForFunction("document.getElementById('key-value')?.value !== ''"); expect(await page.getByLabel('API key', { exact: true }).inputValue()).toBe( - server.mintedSecret, + fx.server.mintedSecret, ); - expect(server.secretWasIssued).toBe(true); + expect(fx.server.secretWasIssued).toBe(true); // Event log, asserted here rather than in its own test so it can't pass on // a previous test's side effects. - expect(server.events.map((e) => e.path)).toContain('/settings/keys'); - expect(server.events.some((e) => e.mintedSecret)).toBe(true); + expect(fx.server.events.map((e) => e.path)).toContain('/settings/keys'); + expect(fx.server.events.some((e) => e.mintedSecret)).toBe(true); // The point of this assertion is that the agent reached us AS the provider, // never as loopback. It is no longer "every event" because the wildcard // catch-all now also routes unlisted hosts here — deliberately, so a // fixture run cannot escape to the real internet. - expect(server.events.some((e) => e.host === 'platform.claude.com')).toBe(true); - expect(server.events.every((e) => e.host !== '127.0.0.1' && e.host !== 'localhost')).toBe(true); + expect(fx.server.events.some((e) => e.host === 'platform.claude.com')).toBe(true); + expect(fx.server.events.every((e) => e.host !== '127.0.0.1' && e.host !== 'localhost')).toBe( + true, + ); + await page.close(); + }); + + it('surfaces the disabled submit AS disabled, so the gap is diagnosable', async () => { + // Case 593 turns on this: the dead end has to be visible in the tree the + // agent reads, or the gap is undiagnosable rather than just open. + const page = await fx.ctx.newPage(); + await page.goto('https://platform.claude.com/settings/keys'); + await page.getByRole('button', { name: 'Create Key' }).click(); + const outline = await page.getByRole('dialog').ariaSnapshot(); + expect(outline).toMatch(/button "Add".*disabled/); await page.close(); }); it('exposes an accessibility outline with the landmarks the agent needs', async () => { - const page = await ctx!.newPage(); + const page = await fx.ctx.newPage(); await page.goto('https://platform.claude.com/settings/keys'); const outline = await page.locator('main').ariaSnapshot(); expect(outline).toContain('button "Create Key"'); @@ -150,6 +325,255 @@ describe('fixture server served to a real browser', () => { }); }); +// Same contract as the anthropic block, against the other console. Not shared +// with it: the differences between the consoles are the point. +describe('openai fixture served to a real browser', () => { + const fx = fixtureBrowser('openAiApi'); + + it('answers as the real hostname, so the agent never sees loopback', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://platform.openai.com/api-keys'); + expect(page.url()).toBe('https://platform.openai.com/api-keys'); + expect(await page.locator('h1').textContent()).toBe('API keys'); + await page.close(); + }); + + it('keeps the two "create...key" buttons distinguishable', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://platform.openai.com/api-keys'); + // Closed, the submit is not in the tree at all — only the opener is. + expect(await page.getByRole('button', { name: 'Create' }).count()).toBe(1); + + await page.getByRole('button', { name: 'Create new secret key' }).click(); + // Open, a loose match hits BOTH: that is the hazard, and the exact names + // have to keep them apart. + expect(await page.getByRole('button', { name: 'Create' }).count()).toBe(2); + expect(await page.getByRole('button', { name: 'Create new secret key' }).count()).toBe(1); + expect(await page.getByRole('button', { name: 'Create secret key', exact: true }).count()).toBe( + 1, + ); + await page.close(); + }); + + it('hands out exactly the ledger secret through the create-key flow', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://platform.openai.com/home'); + // Navigate the way the agent has to: the landing page is not the key page. + await page.getByRole('complementary').getByRole('link', { name: 'API Keys' }).click(); + expect(await page.locator('h1').textContent()).toBe('API keys'); + + await page.getByRole('button', { name: 'Create new secret key' }).click(); + const dialog = page.getByRole('dialog').filter({ hasText: 'Create new secret key' }); + // Name is optional here — the contrast with anthropic's gated Add. + expect(await dialog.getByRole('button', { name: 'Create secret key' }).isDisabled()).toBe( + false, + ); + await page.getByPlaceholder('My Test Key').fill('n8n'); + await dialog.getByRole('button', { name: 'Create secret key' }).click(); + + await page.waitForFunction("document.getElementById('key-value')?.value !== ''"); + const reveal = page.getByRole('dialog').filter({ hasText: 'Save your key' }); + expect(await reveal.getByRole('textbox').inputValue()).toBe(fx.server.mintedSecret); + expect(fx.server.secretWasIssued).toBe(true); + + expect(fx.server.events.some((e) => e.mintedSecret)).toBe(true); + expect(fx.server.events.some((e) => e.host === 'platform.openai.com')).toBe(true); + expect(fx.server.events.every((e) => e.host !== '127.0.0.1' && e.host !== 'localhost')).toBe( + true, + ); + await page.close(); + }); + + it('shows the new key TRUNCATED in the list, never in full', async () => { + // Creates the key on THIS page: the assertion is about the row the create + // flow renders, and a fresh page never contains the secret in the first + // place, so asserting there proves nothing. + const page = await fx.ctx.newPage(); + await page.goto('https://platform.openai.com/api-keys'); + await page.getByRole('button', { name: 'Create new secret key' }).click(); + await page + .getByRole('dialog') + .getByRole('button', { name: 'Create secret key', exact: true }) + .click(); + await page.waitForFunction("document.getElementById('key-value')?.value !== ''"); + + const outline = await page.locator('table').ariaSnapshot(); + expect(outline).toMatch(/sk-\.\.\.\w{4}/); + expect(outline).not.toContain(fx.server.mintedSecret); + // Nor may a derived column smuggle a run of the key into the tree. + expect(outline).not.toContain(fx.server.mintedSecret.slice(-16)); + await page.close(); + }); + + it('renders a name containing markup as text, not as markup', async () => { + // The name is free-form agent input. Interpolated into the row's HTML it + // would corrupt the table the truncation assertions read. + const page = await fx.ctx.newPage(); + await page.goto('https://platform.openai.com/api-keys'); + await page.getByRole('button', { name: 'Create new secret key' }).click(); + await page.getByPlaceholder('My Test Key').fill('bold & co'); + await page + .getByRole('dialog') + .getByRole('button', { name: 'Create secret key', exact: true }) + .click(); + await page.waitForFunction("document.getElementById('key-value')?.value !== ''"); + + expect(await page.locator('#key-rows tr').first().textContent()).toContain('bold & co'); + expect(await page.locator('#key-rows b').count()).toBe(0); + await page.close(); + }); + + it('exposes an accessibility outline with the landmarks the agent needs', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://platform.openai.com/api-keys'); + const outline = await page.locator('main').ariaSnapshot(); + expect(outline).toContain('button "Create new secret key"'); + expect(outline).toContain('heading "API keys"'); + // Calibrated against the real capture. + for (const col of ['Name', 'Status', 'Tracking ID', 'Secret Key', 'Created', 'Permissions']) { + expect(outline).toContain(col); + } + await page.close(); + }); +}); + +// The awkward console: two blocking overlays, no role=dialog, key as text. +describe('gemini fixture served to a real browser', () => { + const fx = fixtureBrowser('googlePalmApi'); + + it('blocks the page behind BOTH overlays until each is dismissed', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://aistudio.google.com/apikey'); + expect(await page.getByRole('button', { name: 'Create API key' }).isVisible()).toBe(false); + + await page.getByRole('button', { name: 'Close guided tour' }).click(); + expect(await page.getByRole('button', { name: 'Create API key' }).isVisible()).toBe(false); + + await page.getByRole('button', { name: 'OK, got it' }).click(); + expect(await page.getByRole('button', { name: 'Create API key' }).isVisible()).toBe(true); + await page.close(); + }); + + async function openConsole() { + const page = await fx.ctx.newPage(); + await page.goto('https://aistudio.google.com/apikey'); + await page.getByRole('button', { name: 'Close guided tour' }).click(); + await page.getByRole('button', { name: 'OK, got it' }).click(); + return page; + } + + it('hands out exactly the ledger secret, as TEXT rather than a field', async () => { + const page = await openConsole(); + await page.getByRole('button', { name: 'Create API key' }).click(); + await page.getByLabel('Name your key').fill('n8n'); + await page.getByRole('button', { name: 'Create key', exact: true }).click(); + + await page.waitForFunction("document.getElementById('key-value')?.textContent !== ''"); + // textContent, not inputValue — the key is a text node here. + expect(await page.locator('#key-value').textContent()).toBe(fx.server.mintedSecret); + expect(fx.server.mintedSecret.startsWith('AQ.')).toBe(true); + expect(fx.server.secretWasIssued).toBe(true); + expect(fx.server.events.some((e) => e.host === 'aistudio.google.com')).toBe(true); + await page.close(); + }); + + it('keeps the reveal panel findable even though it is not a role=dialog', async () => { + const page = await openConsole(); + expect(await page.getByRole('dialog').count()).toBe(0); + await page.getByRole('button', { name: 'Create API key' }).click(); + expect(await page.getByRole('dialog').count()).toBe(0); + expect(await page.getByLabel('Create a new key Close dialog').count()).toBe(1); + await page.close(); + }); + + it('renders a name containing markup as text, not as markup', async () => { + const page = await openConsole(); + await page.getByRole('button', { name: 'Create API key' }).click(); + await page.getByLabel('Name your key').fill('italic & co'); + await page.getByRole('button', { name: 'Create key', exact: true }).click(); + await page.waitForFunction("document.getElementById('key-value')?.textContent !== ''"); + + expect(await page.locator('#key-rows tr').first().textContent()).toContain( + 'italic & co', + ); + expect(await page.locator('#key-rows i').count()).toBe(0); + await page.close(); + }); + + it('shows only the last four characters in the list', async () => { + // Mint on this page first — see the openai equivalent. + const page = await openConsole(); + await page.getByRole('button', { name: 'Create API key' }).click(); + await page.getByLabel('Name your key').fill('n8n'); + await page.getByRole('button', { name: 'Create key', exact: true }).click(); + await page.waitForFunction("document.getElementById('key-value')?.textContent !== ''"); + + const outline = await page.locator('table').ariaSnapshot(); + expect(outline).toMatch(/button "\.\.\.\w{4}"/); + expect(outline).not.toContain(fx.server.mintedSecret); + await page.close(); + }); +}); + +// Multi-page, unlike the other three: the token only exists after installing. +describe('slack fixture served to a real browser', () => { + const fx = fixtureBrowser('slackApi'); + + it('keeps the manifest editor click-trapped but typeable', async () => { + // The live reproduction for case 599: the focusable input sits under the + // rendered surface, so a click never passes the hit-test while fill() + // does. Short timeout here — the real one costs 30s. + const page = await fx.ctx.newPage(); + await page.goto('https://api.slack.com/apps'); + await page.getByRole('button', { name: 'Create New App' }).click(); + await page.getByRole('button', { name: 'From a manifest' }).click(); + + await expect(page.locator('#manifest-input').click({ timeout: 1_000 })).rejects.toThrow( + /Timeout|intercepts pointer events/, + ); + await page.locator('#manifest-input').fill('{"display_information":{"name":"n8n"}}'); + expect(await page.locator('#manifest-input').inputValue()).toContain('display_information'); + await page.close(); + }); + + it('leaves Next enabled — nothing here is gated on a required field', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://api.slack.com/apps'); + await page.getByRole('button', { name: 'Create New App' }).click(); + await page.getByRole('button', { name: 'From a manifest' }).click(); + expect(await page.getByRole('button', { name: 'Next' }).isDisabled()).toBe(false); + await page.close(); + }); + + it('issues the ledger token only after Install to Workspace', async () => { + const page = await fx.ctx.newPage(); + // An app id this fixture has never seen — the console mints one per app, and + // the agent navigates by typing those URLs back. + await page.goto('https://api.slack.com/apps/A0Q3ZK71LMN/install-on-team'); + expect(await page.content()).not.toContain(fx.server.mintedSecret); + + await page.getByRole('button', { name: 'Install to Workspace' }).click(); + await page.waitForFunction("document.getElementById('bot-token')?.value !== ''"); + expect(await page.getByLabel('Bot User OAuth Token').inputValue()).toBe(fx.server.mintedSecret); + expect(fx.server.mintedSecret.startsWith('xoxb-')).toBe(true); + expect(fx.server.secretWasIssued).toBe(true); + await page.close(); + }); + + it('reaches the token only through the multi-page flow', async () => { + const page = await fx.ctx.newPage(); + await page.goto('https://api.slack.com/apps/A0Q3ZK71LMN/general'); + // Basic Information carries the OAuth2 credentials, never the bot token. + expect(await page.content()).not.toContain(fx.server.mintedSecret); + expect(await page.getByLabel('Client ID').inputValue()).toContain('.'); + await page.getByRole('link', { name: 'Install App' }).click(); + expect(await page.locator('h1').first().textContent()).toContain('OAuth Tokens'); + // The id survives the hop rather than reverting to a hardcoded one. + expect(page.url()).toContain('/apps/A0Q3ZK71LMN/install-on-team'); + await page.close(); + }); +}); + describe('providerFixtureManifestSchema', () => { const valid = { credentialType: 'anthropicApi', @@ -192,4 +616,138 @@ describe('providerFixtureManifestSchema', () => { it('rejects an empty hosts list', () => { expect(providerFixtureManifestSchema.safeParse({ ...valid, hosts: [] }).success).toBe(false); }); + + // The three providers agree on none of this — a manifest that could only + // describe anthropic left the other two's value check unverifiable. + it('accepts a query-param secret instead of a header', () => { + expect( + providerFixtureManifestSchema.safeParse({ + ...valid, + verify: { path: '/v1beta/models', query: 'key' }, + }).success, + ).toBe(true); + }); + + it('accepts a header scheme, for a credential that sends `Bearer `', () => { + expect( + providerFixtureManifestSchema.safeParse({ + ...valid, + verify: { path: '/models', header: 'Authorization', scheme: 'Bearer' }, + }).success, + ).toBe(true); + }); + + it('rejects a verify block carrying neither a header nor a query param', () => { + // The union has no arm without one of them, so this is structural — there is + // no refinement message to assert, only that it cannot parse. + expect( + providerFixtureManifestSchema.safeParse({ ...valid, verify: { path: '/models' } }).success, + ).toBe(false); + }); + + it('rejects a scheme on the query arm, where it could never be honoured', () => { + // `{path, query, scheme}` used to parse and then compare "Bearer " + // against a query value — a permanent 401 reported as unverifiable. + expect( + providerFixtureManifestSchema.safeParse({ + ...valid, + verify: { path: '/v1beta/models', query: 'key', scheme: 'Bearer' }, + }).success, + ).toBe(false); + }); + + it('rejects a verify block carrying both — the stand-in must know where to look', () => { + const res = providerFixtureManifestSchema.safeParse({ + ...valid, + verify: { path: '/models', header: 'x-api-key', query: 'key' }, + }); + expect(res.success).toBe(false); + }); + + it('accepts a urlField, for a credential whose base URL is not called `url`', () => { + const res = providerFixtureManifestSchema.safeParse({ ...valid, urlField: 'host' }); + expect(res.success).toBe(true); + }); +}); + +// The listener n8n's credential test talks to. Getting this wrong silently +// discards the strongest of the three deterministic checks. +describe('the provider stand-in accepts each provider shape', () => { + const SECRET = 'sk-test-0123456789abcdef'; + let dir: string; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'fixture-verify-')); + await writeFile( + join(dir, 'console.html'), + '

Keys

', + ); + }); + + afterAll(async () => { + if (dir) await rm(dir, { recursive: true, force: true }); + }); + + async function withServer( + verify: Record, + run: (server: FixtureServer) => void | Promise, + ): Promise { + const server = await startFixtureServer({ + fixture: { + id: 'under-test', + dir, + manifest: providerFixtureManifestSchema.parse({ + credentialType: 'x', + hosts: ['console.example.com'], + secretPrefix: 'sk-test-', + routes: { '/keys': 'console.html' }, + defaultRoute: '/keys', + verify, + }), + }, + logger, + secret: SECRET, + }); + try { + await run(server); + } finally { + await server.close(); + } + } + + it('accepts a bare header value, and refuses a truncated one', async () => { + await withServer({ path: '/v1/models', header: 'x-api-key' }, async (server) => { + const url = `${server.verifyBaseUrl}/v1/models`; + expect((await fetch(url, { headers: { 'x-api-key': SECRET } })).status).toBe(200); + expect((await fetch(url, { headers: { 'x-api-key': SECRET.slice(0, 12) } })).status).toBe( + 401, + ); + expect(server.verifiedOk).toBe(true); + expect(server.verifyAttempts).toBe(2); + }); + }); + + it('accepts a schemed header, so `Bearer ` is not read as a wrong key', async () => { + await withServer( + { path: '/models', header: 'Authorization', scheme: 'Bearer' }, + async (server) => { + const url = `${server.verifyBaseUrl}/models`; + expect((await fetch(url, { headers: { Authorization: `Bearer ${SECRET}` } })).status).toBe( + 200, + ); + // The bare key is the WRONG presentation for this credential type. + expect((await fetch(url, { headers: { Authorization: SECRET } })).status).toBe(401); + }, + ); + }); + + it('accepts a query-param secret, which never reaches a header at all', async () => { + await withServer({ path: '/v1beta/models', query: 'key' }, async (server) => { + const base = `${server.verifyBaseUrl}/v1beta/models`; + expect((await fetch(`${base}?key=${SECRET}`)).status).toBe(200); + expect((await fetch(`${base}?key=nope`)).status).toBe(401); + // A query-param provider must not be satisfiable by a header. + expect((await fetch(base, { headers: { 'x-api-key': SECRET } })).status).toBe(401); + }); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/relay-connection.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/relay-connection.test.ts index 91765b3b1f3..03a45aea5f2 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/relay-connection.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/relay-connection.test.ts @@ -5,7 +5,12 @@ import { describe, it, expect } from 'vitest'; -import { fixtureInterceptionArgs, planRelayConnection } from '../harness/browser-runtime'; +import { + browserN8nOrigins, + browserSessionCookie, + fixtureInterceptionArgs, + planRelayConnection, +} from '../harness/browser-runtime'; const EXT = 'chrome-extension://cegmdpndekdfpnafgacidejijecomlhh/connect.html'; @@ -192,3 +197,44 @@ describe('fixtureInterceptionArgs', () => { expect(rules).toContain('MAP localhost:5680 n8n:5678'); }); }); + +// The session cookie the launched browser is given. Same subject as the rest of +// this file: which spelling of n8n the BROWSER ends up using, after the rewrite +// above. Get it wrong and the credential-setup flow dead-ends on /signin with +// every provider-console expectation failing as though the agent misbehaved. +describe('browserN8nOrigins', () => { + it('returns the single origin when n8n is already loopback', () => { + expect(browserN8nOrigins('http://localhost:5678')).toEqual(['http://localhost:5678']); + }); + + it('adds the localhost origin the relay MAP rule creates for a non-loopback n8n', () => { + // planRelayConnection rewrites a non-loopback n8n to localhost + a MAP rule, + // so in the split-container topology the browser knows n8n by THAT origin — + // a cookie on `http://n8n:5678` alone would never be sent. + expect(browserN8nOrigins('http://n8n:5678')).toEqual([ + 'http://n8n:5678', + 'http://localhost:5678', + ]); + }); +}); + +describe('browserSessionCookie', () => { + it('turns the client cookie header into a cookie for the browser context', () => { + expect(browserSessionCookie('n8n-auth=abc123', 'http://localhost:5678')).toEqual({ + name: 'n8n-auth', + value: 'abc123', + url: 'http://localhost:5678', + }); + }); + + it('splits on the FIRST = only, so a base64/JWT value survives intact', () => { + const value = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0='; + expect(browserSessionCookie(`n8n-auth=${value}`, 'http://n8n:5678')?.value).toBe(value); + }); + + it('returns undefined for anything that is not a name=value pair', () => { + expect(browserSessionCookie('', 'http://localhost:5678')).toBeUndefined(); + expect(browserSessionCookie('n8n-auth', 'http://localhost:5678')).toBeUndefined(); + expect(browserSessionCookie('=orphan', 'http://localhost:5678')).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/anthropic/console.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/anthropic/console.html index de080d0f1a2..a2e94ac85a4 100644 --- a/packages/@n8n/instance-ai/evaluations/fixtures/providers/anthropic/console.html +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/anthropic/console.html @@ -14,13 +14,16 @@ combobox "Expires", button "Add", button "Close" dialog "Save your API key" [level=2]: button "Copy Key", button "Done" - Two properties that are load-bearing, both taken from the real page: + Three properties that are load-bearing, all taken from the real page: 1. The list is PRE-POPULATED with truncated keys. An agent that grabs a displayed value instead of the freshly issued one captures an ellipsised string — a real failure mode, and one the exact-value check is meant to catch. An empty list would hide it. 2. The submit button is "Add", NOT "Create Key" — "Create Key" only opens the dialog. An agent that clicks the wrong one never creates anything. + 3. "Add" is DISABLED until Expires has a value, and Expires opens unselected — + the real dialog's own rule, and the required-field gap NODE-5755 tracks. + Defaulting it to "Never" tested past the bug. The captured tree was German (the user's browser locale). Rendered in English here because the eval corpus is English; re-check names if a locale-specific @@ -95,13 +98,26 @@ - - - + + Expires + + + @@ -120,6 +136,32 @@ const reveal = document.getElementById('reveal'); const keyValue = document.getElementById('key-value'); const nameInput = document.getElementById('key-name'); + const expiresTrigger = document.getElementById('expires-trigger'); + const expiresList = document.getElementById('expires-list'); + const submit = document.getElementById('create-submit'); + let expiresValue = ''; + + expiresTrigger.addEventListener('click', () => { + const willOpen = expiresList.hidden; + expiresList.hidden = !willOpen; + expiresTrigger.setAttribute('aria-expanded', String(willOpen)); + }); + + // Add stays disabled until Expires has a value — the dead end the agent + // has to diagnose. + const options = [...expiresList.querySelectorAll('[role="option"]')]; + for (const option of options) { + option.addEventListener('click', () => { + expiresValue = option.dataset.value; + expiresTrigger.textContent = option.textContent; + for (const other of options) { + other.setAttribute('aria-selected', String(other === option)); + } + expiresList.hidden = true; + expiresTrigger.setAttribute('aria-expanded', 'false'); + submit.disabled = false; + }); + } document.getElementById('create-key').addEventListener('click', () => { dialog.hidden = false; @@ -135,7 +177,7 @@ body: JSON.stringify({ name: nameInput.value || 'Untitled', workspace: document.getElementById('workspace').value, - expires: document.getElementById('expires').value, + expires: expiresValue, }), }); const { key } = await response.json(); @@ -152,7 +194,9 @@ key.slice(-4) + '' + document.getElementById('workspace').value + - 'Eval Ownerjust nowNever—' + + 'Eval Ownerjust now' + + expiresTrigger.textContent + + '—' + ''; document.getElementById('key-rows').prepend(row); const count = document.getElementById('key-count'); diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/console.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/console.html new file mode 100644 index 00000000000..c4fe79f2ede --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/console.html @@ -0,0 +1,218 @@ + + + + + + API Keys | Google AI Studio + + + + +
+

Google AI Studio uses cookies to deliver and improve the service.

+ Learn more about how Google uses cookies. Opens in a new tab. + +
+ + + + + + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/dashboard.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/dashboard.html new file mode 100644 index 00000000000..470c67ba4fc --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/dashboard.html @@ -0,0 +1,42 @@ + + + + + + Google AI Studio + + +
+ + Google AI Studio logo + + + +
+
+

Dashboard

+
+

Get started

+

+ Create an API key to call the Gemini API. Manage keys from + API Keys. +

+
+
+

Usage this month

+

No usage recorded.

+
+
+ + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/manifest.json b/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/manifest.json new file mode 100644 index 00000000000..44f06474664 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/gemini/manifest.json @@ -0,0 +1,15 @@ +{ + "credentialType": "googlePalmApi", + "hosts": ["aistudio.google.com"], + "secretPrefix": "AQ.", + "routes": { + "/apikey": "console.html", + "/dashboard": "dashboard.html" + }, + "defaultRoute": "/dashboard", + "urlField": "host", + "verify": { + "path": "/v1beta/models", + "query": "key" + } +} diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/console.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/console.html new file mode 100644 index 00000000000..e93aa4e363f --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/console.html @@ -0,0 +1,184 @@ + + + + + + API keys - OpenAI Platform + + + + +
+

API keys

+ + API Key Usage + + + + + + + 4 results + + + + + + + + + + + + + + + + + + + + + + +
NameStatusTracking IDSecret KeyCreatedLast usedCreated byPermissionsMonthly spendActions
service-instanceActivekey_Ejpr6NwBvEetPhCusk-...1fIAJun 24, 2026Aug 17, 2026Eval OwnerAll$0.00
local-testingActivekey_KWxC7YgCSTMQW5S8sk-...PGQAJun 23, 2026Jul 24, 2026Eval OwnerAll$0.00
batch-jobsActivekey_X9mefkdO29ZUKROvsk-...38YAFeb 19, 2026Feb 19, 2026Eval OwnerAll$0.00
prototypeActivekey_GH4PD8W0d8oDAVGQsk-...5akAFeb 10, 2026Feb 10, 2026Eval OwnerAll$0.00
+ + + + + + +
+ + + + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/dashboard.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/dashboard.html new file mode 100644 index 00000000000..6b05f933a32 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/dashboard.html @@ -0,0 +1,45 @@ + + + + + + Home - OpenAI Platform + + + +
+

Home

+
+

Get started

+

+ Create an API key to start making requests. Manage keys from + API Keys. +

+
+
+

Usage this month

+

No usage recorded.

+
+
+ + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/manifest.json b/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/manifest.json new file mode 100644 index 00000000000..dae9ea405ff --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/openai/manifest.json @@ -0,0 +1,15 @@ +{ + "credentialType": "openAiApi", + "hosts": ["platform.openai.com"], + "secretPrefix": "sk-proj-", + "routes": { + "/api-keys": "console.html", + "/home": "dashboard.html" + }, + "defaultRoute": "/home", + "verify": { + "path": "/models", + "header": "Authorization", + "scheme": "Bearer" + } +} diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/apps.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/apps.html new file mode 100644 index 00000000000..32293e010d1 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/apps.html @@ -0,0 +1,148 @@ + + + + + + Slack API: Applications | Slack + + +
+ Slack API + Changelog + Documentation + Tutorials + Your Apps +
+ +
+ +

Your Apps

+ + + + + +

Your App Configuration Tokens

+ Learn about tokens + + Sign in to another workspace + + + + + +
+ + + + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/general.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/general.html new file mode 100644 index 00000000000..0e8baaea227 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/general.html @@ -0,0 +1,67 @@ + + + + + + Basic Information | n8n integration | Slack + + + + +
+

Basic Information

+ +

App Credentials

+

These credentials allow your app to access the Slack API.

+ + + + + + + + + + + + + + +

Display Information

+

n8n integration

+ + + +
+ + + + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/install.html b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/install.html new file mode 100644 index 00000000000..6f8acc25ff5 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/install.html @@ -0,0 +1,85 @@ + + + + + + Install App | n8n integration | Slack + + + + +
+

OAuth Tokens for Your Workspace

+

+ These tokens were automatically generated when you installed the app to your + team. You can use these to authenticate your app. +

+ +
+

You must install this app to your workspace before a token is issued.

+

Bot Token Scopes

+
    +
  • channels:read
    • +
  • chat:write
    • +
  • groups:read
    • +
  • im:history
    • +
+ +
+ + +
+ + + + + diff --git a/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/manifest.json b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/manifest.json new file mode 100644 index 00000000000..fc636b7e530 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/fixtures/providers/slack/manifest.json @@ -0,0 +1,12 @@ +{ + "credentialType": "slackApi", + "hosts": ["api.slack.com"], + "secretPrefix": "xoxb-", + "routes": { + "/apps": "apps.html", + "/apps/:appId/general": "general.html", + "/apps/:appId/install-on-team": "install.html", + "/apps/:appId/oauth": "install.html" + }, + "defaultRoute": "/apps" +} diff --git a/packages/@n8n/instance-ai/evaluations/harness/browser-runtime.ts b/packages/@n8n/instance-ai/evaluations/harness/browser-runtime.ts index 575e8a357ae..3779a5b3b7e 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/browser-runtime.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/browser-runtime.ts @@ -154,6 +154,86 @@ export function planRelayConnection( /** Hosts the extension already treats as local, so no DNS redirect is needed. */ const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', '::1']); +/** + * Origins the BROWSER can reach n8n on — not necessarily the harness's own + * `baseUrl`. + * + * `planRelayConnection` rewrites the relay's host to `localhost` whenever it + * does not already match, so the browser can know n8n by a different spelling + * than the harness does: `http://n8n:5678` in the split-container topology, and + * `http://127.0.0.1:5678` on a laptop — cookies are host-scoped, and an IP host + * is never sent to `localhost`. The `localhost` spelling is therefore always + * included, keeping the protocol (a `secure` cookie is not sent over http). + * A cookie on an origin the browser never visits is inert; missing the one it + * does visit costs the session. + */ +export function browserN8nOrigins(baseUrl: string): string[] { + let url: URL; + try { + url = new URL(baseUrl); + } catch { + return []; + } + const localhost = `${url.protocol}//localhost${url.port ? `:${url.port}` : ''}`; + return url.hostname === 'localhost' ? [url.origin] : [url.origin, localhost]; +} + +/** The client's `n8n-auth` header as a cookie for one origin. Undefined when the + * header is not a `name=value` pair. */ +export function browserSessionCookie( + cookieHeader: string, + url: string, +): { name: string; value: string; url: string } | undefined { + const eq = cookieHeader.indexOf('='); + // First `=` only: the value is base64/JWT and carries its own padding. + if (eq <= 0) return undefined; + const value = cookieHeader.slice(eq + 1); + return value ? { name: cookieHeader.slice(0, eq), value, url } : undefined; +} + +/** + * Give the launched browser the n8n session the harness already holds. + * + * A launched browser has never seen n8n, but the credential-setup skill opens + * n8n's OWN credential page first — for OAuth providers that is where the + * redirect URL lives. Without a session it lands on /signin and stops, and + * every expectation about the PROVIDER console then fails as though the agent + * had misbehaved (NODE-5549). `attachToRunningBrowser` needs none of this: the + * developer's browser is already signed in. + */ +async function signInBrowserToN8n( + context: BrowserContext, + client: N8nClient, + logger: EvalLogger, +): Promise { + let cookieHeader: string; + try { + cookieHeader = client.cookie; + } catch { + logger.warn(' Browser not signed in to n8n: the harness holds no session cookie'); + return; + } + const cookies = browserN8nOrigins(client.baseUrl) + .map((origin) => browserSessionCookie(cookieHeader, origin)) + .filter((c): c is { name: string; value: string; url: string } => c !== undefined); + if (!cookies.length) { + logger.warn(' Browser not signed in to n8n: session cookie is not a name=value pair'); + return; + } + try { + await context.addCookies(cookies); + } catch (error: unknown) { + // Warn-and-continue, per this helper's contract: the run still works, it + // just starts on n8n's sign-in page. Throwing here would leak the browser + // and its profile — `cleanup` is not armed until after this call. + logger.warn( + ` Browser not signed in to n8n: ${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + logger.verbose(` Browser signed in to n8n (${cookies.map((c) => c.url).join(', ')})`); +} + /** Loopback spellings the extension's own relay allowlist accepts. All of them * must escape the fixture's catch-all, not just the literal "localhost". */ const LOOPBACK_EXCLUDES = ['localhost', '127.0.0.1', '[::1]']; @@ -275,6 +355,10 @@ export async function startBrowserRuntime( args, }); + // Before the connect page loads, so the very first navigation is already + // authenticated. Inside this try, so a throw releases the relay session. + await signInBrowserToN8n(context, client, logger); + const cleanup = async () => { await context.close().catch(() => {}); // `finally`: rm can throw, and losing the instance-wide relay session to diff --git a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts index 75000ee9472..1f3b933a43f 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts @@ -571,6 +571,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise { - const { client, credentialId, fixture, verifyBaseUrl, local, logger } = options; + const { client, credentialId, fixture, verifyBaseUrl, urlField, local, logger } = options; const attemptsBefore = fixture?.verifyAttempts ?? 0; try { const credential = await client.getCredentialForTest(credentialId); @@ -385,9 +390,18 @@ async function probeOneCredential(options: { // written. In local mode the URL is left alone, so the test goes to the // real provider API — a pass there proves the key is genuine and active, // which is a stronger claim than equality with a synthetic string. + // + // `urlField` is per-provider (gemini calls it `host`). Do NOT require the + // field to be present first: a Base URL with a default is not persisted + // unless the user changed it, so real anthropic/openai credentials store + // only `apiKey` — verified against live rows. Adding the field IS the + // mechanism (n8n merges the submitted data over the stored credential), so + // requiring it would discard the check on exactly the providers it works + // for. A wrong `urlField` is caught at CI time instead, by the manifest ↔ + // `test.request` lockstep in `fixture-server.test.ts`. const result = await client.testCredential({ ...credential, - data: local ? credential.data : { ...credential.data, url: verifyBaseUrl }, + data: local ? credential.data : { ...credential.data, [urlField]: verifyBaseUrl }, }); if (result.status === 'OK') return { kind: 'passed', target: local ? 'real' : 'stand-in' }; diff --git a/packages/@n8n/instance-ai/evaluations/harness/credential-setup-lane.ts b/packages/@n8n/instance-ai/evaluations/harness/credential-setup-lane.ts index 8c75af0903d..6337c908928 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/credential-setup-lane.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/credential-setup-lane.ts @@ -57,6 +57,8 @@ export interface CredentialSetupLane { /** Base URL for the provider-API stand-in, when this fixture ships one. * Undefined => the credential-value check reports itself unverifiable. */ verifyBaseUrl?: string; + /** Manifest `urlField`. Absent in local mode, where nothing is substituted. */ + credentialUrlField?: string; close(): Promise; } @@ -162,6 +164,7 @@ export async function startCredentialSetupLane( browser, credentialType: fixture.manifest.credentialType, verifyBaseUrl: fixtureServer.verifyBaseUrl, + credentialUrlField: fixture.manifest.urlField, close: async () => { // Browser first: it is the thing holding pages open against the fixture. await browser.close().catch(() => {}); diff --git a/packages/@n8n/instance-ai/evaluations/harness/fixture-server.ts b/packages/@n8n/instance-ai/evaluations/harness/fixture-server.ts index d06f9d1ec1a..177281d49e3 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/fixture-server.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/fixture-server.ts @@ -90,15 +90,36 @@ export const providerFixtureManifestSchema = z * HTTPS-with-a-self-signed-cert so it can impersonate a hostname, and n8n's * HTTP client would (correctly) reject that cert. Omit the block entirely * and the check reports itself unverifiable rather than failing. */ + // A UNION, not three optionals: the secret arrives in a header or a query + // parameter, never both and never neither, and `scheme` is meaningless + // without a header. Stating that structurally makes the nonsense + // combinations unrepresentable instead of policed after the fact. verify: z - .object({ - /** Path the credential's test request hits, e.g. `/v1/models`. */ - path: z.string().startsWith('/'), - /** Header the credential type sends its secret in, e.g. `x-api-key`. */ - header: z.string().min(1), - }) - .strict() + .union([ + z + .object({ + /** Path the credential's test request hits, e.g. `/v1/models`. */ + path: z.string().startsWith('/'), + /** Header carrying the secret, e.g. `x-api-key`. */ + header: z.string().min(1), + /** Prefix inside that header, e.g. `Bearer` — without it the stand-in + * compares the whole header to the bare key and refuses it. */ + scheme: z.string().min(1).optional(), + }) + .strict(), + z + .object({ + path: z.string().startsWith('/'), + /** Query parameter carrying the secret, e.g. gemini's `key`. */ + query: z.string().min(1), + }) + .strict(), + ]) .optional(), + /** Credential field holding the base URL, overwritten to aim the credential's + * own test at the stand-in. `url` for anthropic/openai, `host` for gemini; + * the wrong name leaves the test pointed at the real provider. */ + urlField: z.string().min(1).default('url'), }) .strict() // Was a throw after the TLS cert had already been generated; as a refinement @@ -202,6 +223,29 @@ export async function findFixtureForCredentialType( return fixtures.find((f) => f.manifest.credentialType === credentialType); } +/** + * Resolve a request path to one of the declared routes, honouring `:param` + * segments (`/apps/:appId/general`). + * + * Consoles that create a resource put its id in every later URL, and the agent + * navigates by typing those URLs, not only by clicking. Without parameters the + * fixture would have to hardcode one id, and every other id would fall through + * to `defaultRoute` — serving a plausible WRONG page (an app list where the app + * settings should be) rather than an obvious dead end. + * + * Exact routes win over patterns, and a pattern matches one segment only, so a + * console's URL space cannot collapse onto a single page. + */ +export function matchRoute(routes: string[], path: string): string | undefined { + if (routes.includes(path)) return path; + const segments = path.split('/'); + return routes.find((route) => { + const routeSegments = route.split('/'); + if (routeSegments.length !== segments.length) return false; + return routeSegments.every((s, i) => s.startsWith(':') || s === segments[i]); + }); +} + /** Mint a synthetic secret carrying the provider's real prefix. Random, not * seeded: two runs must never share a secret, or a leak scan could pass by * matching the wrong run's value. */ @@ -279,6 +323,7 @@ export async function startFixtureServer( for (const [route, file] of Object.entries(manifest.routes)) { pages.set(route, await readFile(join(fixture.dir, file), 'utf8')); } + const routeKeys = [...pages.keys()]; // Non-null: the schema's refinement guarantees defaultRoute is a declared route. const defaultPage = pages.get(manifest.defaultRoute)!; @@ -300,7 +345,8 @@ export async function startFixtureServer( res.writeHead(204).end(); return; } - const body = pages.get(path) ?? defaultPage; + const matched = matchRoute(routeKeys, path); + const body = (matched && pages.get(matched)) ?? defaultPage; res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); res.end(body); }); @@ -322,9 +368,14 @@ export async function startFixtureServer( let verifiedOk = false; let verifyAttempts = 0; if (manifest.verify) { - const { path: verifyPath, header } = manifest.verify; + const verify = manifest.verify; + const verifyPath = verify.path; + const expected = + 'scheme' in verify && verify.scheme ? `${verify.scheme} ${mintedSecret}` : mintedSecret; + const headerName = 'header' in verify ? verify.header.toLowerCase() : undefined; + const queryName = 'query' in verify ? verify.query : undefined; verifyServer = createHttpServer((req, res) => { - const path = (req.url ?? '/').split('?')[0]; + const [path, search] = (req.url ?? '/').split('?'); if (path !== verifyPath) { events.push({ method: req.method ?? 'GET', host: 'verify', path }); res.writeHead(404).end(); @@ -333,8 +384,10 @@ export async function startFixtureServer( // ONLY the minted key authenticates. That is the whole proof: a // truncated or re-typed key cannot pass, so a 200 means the stored // value is exactly what the page issued — without reading it back. - const presented = req.headers[header.toLowerCase()]; - const ok = typeof presented === 'string' && presented === mintedSecret; + const presented = headerName + ? req.headers[headerName] + : new URLSearchParams(search ?? '').get(queryName!); + const ok = typeof presented === 'string' && presented === expected; verifyAttempts += 1; if (ok) verifiedOk = true; events.push({ method: req.method ?? 'GET', host: 'verify', path, verifyOk: ok });