mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 05:29:44 +02:00
test(core): Add openai, gemini and slack credential-setup eval fixtures (no-changelog) (#36461)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bernhard Wittmann
GitHub
parent
2c5db29175
commit
44041d5d3c
|
|
@ -1,12 +1,15 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
|
||||
import type { N8nClient } from '../clients/n8n-client';
|
||||
import {
|
||||
credentialSetupExpectationTexts,
|
||||
evaluateCredentialSetup,
|
||||
probeCredentialValue,
|
||||
redactTranscriptSecrets,
|
||||
runCredentialSetupChecks,
|
||||
type CredentialSetupFacts,
|
||||
} from '../harness/credential-setup-checks';
|
||||
import { createLogger } from '../harness/logger';
|
||||
|
||||
const SECRET = 'sk-ant-api03-abcdefghijklmnopqrstuvwx';
|
||||
|
||||
|
|
@ -326,3 +329,83 @@ describe('credentialSetupExpectationTexts stays in lockstep with the evaluator',
|
|||
expect(new Set(credentialSetupExpectationTexts(undefined))).toEqual(new Set(emitted));
|
||||
});
|
||||
});
|
||||
|
||||
// Which field carries the base URL is per-provider (`host` for gemini). The
|
||||
// wrong one leaves the test pointed at the real provider, where it is discarded
|
||||
// — so the check silently never runs for that provider.
|
||||
describe('probeCredentialValue aims the test at the stand-in', () => {
|
||||
const logger = createLogger(false);
|
||||
const VERIFY_URL = 'http://dispatcher:41234';
|
||||
|
||||
function client(storedData: Record<string, unknown>) {
|
||||
const testCredential = vi.fn().mockResolvedValue({ status: 'OK' });
|
||||
return {
|
||||
/** The `data` of each credential-test request the probe sent. */
|
||||
tested: (): Array<Record<string, unknown>> =>
|
||||
testCredential.mock.calls.map(([c]) => (c as { data: Record<string, unknown> }).data),
|
||||
client: {
|
||||
listCredentials: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: 'new1', name: 'acct', type: 'googlePalmApi' }]),
|
||||
getCredentialForTest: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ id: 'new1', name: 'acct', type: 'googlePalmApi', data: storedData }),
|
||||
testCredential,
|
||||
} as unknown as N8nClient,
|
||||
};
|
||||
}
|
||||
|
||||
it('overwrites the field the manifest names, not always `url`', async () => {
|
||||
const { client: c, tested } = client({
|
||||
host: 'https://generativelanguage.googleapis.com',
|
||||
apiKey: '',
|
||||
});
|
||||
|
||||
const probe = await probeCredentialValue({
|
||||
client: c,
|
||||
credentialType: 'googlePalmApi',
|
||||
credentialIdsBefore: [],
|
||||
fixture: { verifyAttempts: 1, verifiedOk: true },
|
||||
verifyBaseUrl: VERIFY_URL,
|
||||
urlField: 'host',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(probe).toEqual({ kind: 'passed', target: 'stand-in' });
|
||||
expect(tested()[0].host).toBe(VERIFY_URL);
|
||||
// And nothing invents a `url` the credential type does not have.
|
||||
expect(tested()[0].url).toBeUndefined();
|
||||
});
|
||||
|
||||
it('defaults to `url` when no field is named', async () => {
|
||||
const { client: c, tested } = client({ url: 'https://api.anthropic.com', apiKey: '' });
|
||||
|
||||
await probeCredentialValue({
|
||||
client: c,
|
||||
credentialIdsBefore: [],
|
||||
fixture: { verifyAttempts: 1, verifiedOk: true },
|
||||
verifyBaseUrl: VERIFY_URL,
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(tested()[0].url).toBe(VERIFY_URL);
|
||||
});
|
||||
|
||||
it('leaves the URL alone in local mode, so the REAL provider answers', async () => {
|
||||
const { client: c, tested } = client({
|
||||
host: 'https://generativelanguage.googleapis.com',
|
||||
apiKey: '',
|
||||
});
|
||||
|
||||
const probe = await probeCredentialValue({
|
||||
client: c,
|
||||
credentialIdsBefore: [],
|
||||
local: true,
|
||||
urlField: 'host',
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(probe).toEqual({ kind: 'passed', target: 'real' });
|
||||
expect(tested()[0].host).toBe('https://generativelanguage.googleapis.com');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,17 +4,21 @@
|
|||
// setup the eval browser uses. Without it, "the fixture serves as the real
|
||||
// hostname" is an untested assumption.
|
||||
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { jsonParse } from 'n8n-workflow';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { chromium, type BrowserContext } from 'playwright-core';
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
|
||||
import { findChromiumForEval } from '../harness/browser-runtime';
|
||||
import { findChromiumForEval, fixtureInterceptionArgs } from '../harness/browser-runtime';
|
||||
import {
|
||||
providerFixtureManifestSchema,
|
||||
findFixtureForCredentialType,
|
||||
loadProviderFixtures,
|
||||
matchRoute,
|
||||
MINT_PATH,
|
||||
mintSecret,
|
||||
startFixtureServer,
|
||||
type FixtureServer,
|
||||
|
|
@ -23,16 +27,157 @@ import { createLogger } from '../harness/logger';
|
|||
|
||||
const logger = createLogger(false);
|
||||
|
||||
/**
|
||||
* Boot a fixture server + a browser aimed at it, and register the teardown.
|
||||
*
|
||||
* Uses `fixtureInterceptionArgs` — the same builder `startBrowserRuntime` uses —
|
||||
* so these tests pin the interception setup the eval browser really gets,
|
||||
* including the loopback EXCLUDE rules. Hand-assembled args drifted from it.
|
||||
*/
|
||||
function fixtureBrowser(credentialType: string) {
|
||||
const handle: { server?: FixtureServer; ctx?: BrowserContext } = {};
|
||||
let userDataDir: string | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
const fixture = await findFixtureForCredentialType(credentialType);
|
||||
if (!fixture) throw new Error(`${credentialType} fixture missing`);
|
||||
const server = await startFixtureServer({ fixture, logger });
|
||||
handle.server = server;
|
||||
userDataDir = await mkdtemp(join(tmpdir(), `fixture-test-${fixture.id}-`));
|
||||
handle.ctx = await chromium.launchPersistentContext(userDataDir, {
|
||||
executablePath: findChromiumForEval(),
|
||||
headless: true,
|
||||
args: fixtureInterceptionArgs(server.hostResolverRules(), undefined),
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await handle.ctx?.close().catch(() => {});
|
||||
await handle.server?.close();
|
||||
if (userDataDir) await rm(userDataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Non-null at use: every `it` runs after beforeAll, and a failed boot throws
|
||||
// there rather than handing back a half-built handle.
|
||||
return handle as { server: FixtureServer; ctx: BrowserContext };
|
||||
}
|
||||
|
||||
describe('provider fixtures', () => {
|
||||
it('ships an anthropic fixture keyed on its credential type', async () => {
|
||||
const fixture = await findFixtureForCredentialType('anthropicApi');
|
||||
expect(fixture?.id).toBe('anthropic');
|
||||
expect(fixture?.manifest.hosts).toContain('platform.claude.com');
|
||||
expect(fixture?.manifest.secretPrefix).toBe('sk-ant-api03-');
|
||||
// One row per shipped fixture. A hardcoded table still reds on a missing
|
||||
// fixture, because findFixtureForCredentialType returns undefined for a row
|
||||
// that has no directory.
|
||||
const SHIPPED = [
|
||||
{
|
||||
type: 'anthropicApi',
|
||||
id: 'anthropic',
|
||||
host: 'platform.claude.com',
|
||||
prefix: 'sk-ant-api03-',
|
||||
verify: { path: '/v1/models', header: 'x-api-key' },
|
||||
},
|
||||
{
|
||||
type: 'openAiApi',
|
||||
id: 'openai',
|
||||
host: 'platform.openai.com',
|
||||
prefix: 'sk-proj-',
|
||||
verify: { path: '/models', header: 'Authorization', scheme: 'Bearer' },
|
||||
},
|
||||
{
|
||||
type: 'googlePalmApi',
|
||||
id: 'gemini',
|
||||
host: 'aistudio.google.com',
|
||||
// Google's current shape, not the legacy `AIza…` one.
|
||||
prefix: 'AQ.',
|
||||
// Base URL lives in `host`, and the key travels as a query parameter.
|
||||
urlField: 'host',
|
||||
verify: { path: '/v1beta/models', query: 'key' },
|
||||
},
|
||||
// No `verify`: slackApi's test hardcodes its baseURL, so the stand-in can
|
||||
// never be aimed at it. The value check reports itself unverifiable.
|
||||
{ type: 'slackApi', id: 'slack', host: 'api.slack.com', prefix: 'xoxb-' },
|
||||
];
|
||||
|
||||
it.each(SHIPPED)('ships the $id fixture keyed on $type', async (row) => {
|
||||
const fixture = await findFixtureForCredentialType(row.type);
|
||||
expect(fixture?.id).toBe(row.id);
|
||||
expect(fixture?.manifest.hosts).toContain(row.host);
|
||||
expect(fixture?.manifest.secretPrefix).toBe(row.prefix);
|
||||
expect(fixture?.manifest.urlField).toBe(row.urlField ?? 'url');
|
||||
expect(fixture?.manifest.verify).toEqual(row.verify);
|
||||
});
|
||||
|
||||
// n8n serializes every credential type's own test request, so the manifest's
|
||||
// transcription of it can be CHECKED rather than trusted. Without this the
|
||||
// table above only pins what someone typed on the day, and an upstream change
|
||||
// to `test.request` silently downgrades the value check to "unverifiable".
|
||||
const TYPE_FILES = [
|
||||
join(__dirname, '..', '..', '..', '..', 'nodes-base', 'dist', 'types', 'credentials.json'),
|
||||
join(__dirname, '..', '..', '..', 'nodes-langchain', 'dist', 'types', 'credentials.json'),
|
||||
];
|
||||
|
||||
it.skipIf(!TYPE_FILES.some(existsSync))(
|
||||
"matches each credential's own test.request (needs built dist/types)",
|
||||
async () => {
|
||||
const types = new Map<string, { baseURL?: string; url?: string }>();
|
||||
for (const file of TYPE_FILES.filter(existsSync)) {
|
||||
for (const cred of jsonParse<
|
||||
Array<{
|
||||
name: string;
|
||||
test?: { request?: { baseURL?: string; url?: string } };
|
||||
}>
|
||||
>(await readFile(file, 'utf8'))) {
|
||||
if (cred.test?.request) types.set(cred.name, cred.test.request);
|
||||
}
|
||||
}
|
||||
|
||||
const fixtures = await loadProviderFixtures();
|
||||
let checked = 0;
|
||||
for (const fixture of fixtures) {
|
||||
const request = types.get(fixture.manifest.credentialType);
|
||||
if (!request?.baseURL) continue;
|
||||
checked += 1;
|
||||
const interpolated = /\{\{\s*\$credentials\??\.(\w+)\s*\}\}/.exec(request.baseURL);
|
||||
|
||||
if (!interpolated) {
|
||||
// Hardcoded baseURL: there is no field to substitute, so a `verify`
|
||||
// block could never be aimed at the stand-in.
|
||||
expect(fixture.manifest.verify, `${fixture.id} verify`).toBeUndefined();
|
||||
continue;
|
||||
}
|
||||
expect(fixture.manifest.urlField, `${fixture.id} urlField`).toBe(interpolated[1]);
|
||||
const suffix = request.baseURL.slice(interpolated.index + interpolated[0].length);
|
||||
expect(fixture.manifest.verify?.path, `${fixture.id} verify.path`).toBe(
|
||||
`${suffix}${request.url ?? ''}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Every shipped fixture must have been compared. Without this, a rename
|
||||
// or a change to the serialized shape makes every iteration `continue`
|
||||
// and the check passes having verified nothing.
|
||||
expect(checked, 'fixtures cross-checked against their credential type').toBe(fixtures.length);
|
||||
},
|
||||
);
|
||||
|
||||
// The pages hardcode this path (they cannot import TS), and a rename would
|
||||
// make every fixture's create button silently do nothing: the POST would fall
|
||||
// through to the default page, which answers 200 with HTML, so `response.json()`
|
||||
// throws inside the page and the run reds as agent misbehaviour.
|
||||
it('serves the mint path every fixture page posts to', async () => {
|
||||
for (const fixture of await loadProviderFixtures()) {
|
||||
for (const file of Object.values(fixture.manifest.routes)) {
|
||||
const html = await readFile(join(fixture.dir, file), 'utf8');
|
||||
if (html.includes('__fixture__')) expect(html).toContain(MINT_PATH);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('returns nothing for a credential type no fixture covers', async () => {
|
||||
expect(await findFixtureForCredentialType('slackApi')).toBeUndefined();
|
||||
expect(await findFixtureForCredentialType('notionApi')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('gives every fixture a credential type of its own', async () => {
|
||||
// Selection is BY credential type, so a duplicate would depend on readdir order.
|
||||
const types = (await loadProviderFixtures()).map((f) => f.manifest.credentialType);
|
||||
expect(types).toEqual([...new Set(types)]);
|
||||
});
|
||||
|
||||
it('declares a defaultRoute that is one of its own routes', async () => {
|
||||
|
|
@ -42,6 +187,36 @@ describe('provider fixtures', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('matchRoute', () => {
|
||||
const routes = ['/apps', '/apps/:appId/general', '/apps/:appId/install-on-team'];
|
||||
|
||||
it('resolves a parameter segment to any value', () => {
|
||||
expect(matchRoute(routes, '/apps/A0BQ7AWFNHM/general')).toBe('/apps/:appId/general');
|
||||
expect(matchRoute(routes, '/apps/A0EVALAPP/install-on-team')).toBe(
|
||||
'/apps/:appId/install-on-team',
|
||||
);
|
||||
});
|
||||
|
||||
it('prefers an exact route over a pattern that would also match', () => {
|
||||
expect(matchRoute(['/apps/:appId/general', '/apps/fixed/general'], '/apps/fixed/general')).toBe(
|
||||
'/apps/fixed/general',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not match across segment boundaries', () => {
|
||||
// Without this a console's whole URL space collapses onto one page, and the
|
||||
// agent gets a plausible-looking wrong page instead of the default.
|
||||
expect(matchRoute(routes, '/apps/A0BQ7AWFNHM/general/extra')).toBeUndefined();
|
||||
expect(matchRoute(routes, '/apps/A0BQ7AWFNHM')).toBeUndefined();
|
||||
expect(matchRoute(routes, '/apps/A0BQ7AWFNHM/oauth')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('still resolves plain routes', () => {
|
||||
expect(matchRoute(routes, '/apps')).toBe('/apps');
|
||||
expect(matchRoute(routes, '/nope')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('mintSecret', () => {
|
||||
it('carries the provider prefix and never repeats', () => {
|
||||
const a = mintSecret('sk-ant-api03-');
|
||||
|
|
@ -53,31 +228,10 @@ describe('mintSecret', () => {
|
|||
});
|
||||
|
||||
describe('fixture server served to a real browser', () => {
|
||||
let server: FixtureServer;
|
||||
let ctx: BrowserContext | undefined;
|
||||
let userDataDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
const fixture = await findFixtureForCredentialType('anthropicApi');
|
||||
if (!fixture) throw new Error('anthropic fixture missing');
|
||||
server = await startFixtureServer({ fixture, logger });
|
||||
|
||||
userDataDir = await mkdtemp(join(tmpdir(), 'fixture-test-udd-'));
|
||||
ctx = await chromium.launchPersistentContext(userDataDir, {
|
||||
executablePath: findChromiumForEval(),
|
||||
headless: true,
|
||||
args: [`--host-resolver-rules=${server.hostResolverRules()}`, '--ignore-certificate-errors'],
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await ctx?.close().catch(() => {});
|
||||
await server?.close();
|
||||
if (userDataDir) await rm(userDataDir, { recursive: true, force: true });
|
||||
});
|
||||
const fx = fixtureBrowser('anthropicApi');
|
||||
|
||||
it('answers as the real hostname, so the agent never sees loopback', async () => {
|
||||
const page = await ctx!.newPage();
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.claude.com/settings/keys');
|
||||
expect(page.url()).toBe('https://platform.claude.com/settings/keys');
|
||||
expect(page.url()).not.toContain('127.0.0.1');
|
||||
|
|
@ -86,22 +240,22 @@ describe('fixture server served to a real browser', () => {
|
|||
});
|
||||
|
||||
it('serves the default page for an unmodelled path instead of stranding the agent', async () => {
|
||||
const page = await ctx!.newPage();
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.claude.com/some/unmodelled/path');
|
||||
expect(await page.locator('h1').textContent()).toBe('Dashboard');
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('does not expose the secret before the agent creates a key', async () => {
|
||||
const page = await ctx!.newPage();
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.claude.com/settings/keys');
|
||||
expect(await page.content()).not.toContain(server.mintedSecret);
|
||||
expect(server.secretWasIssued).toBe(false);
|
||||
expect(await page.content()).not.toContain(fx.server.mintedSecret);
|
||||
expect(fx.server.secretWasIssued).toBe(false);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('hands out exactly the ledger secret through the create-key flow', async () => {
|
||||
const page = await ctx!.newPage();
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.claude.com/dashboard');
|
||||
// Navigate the way the agent has to: the landing page is not the key page.
|
||||
// Scoped to the nav because the page links to it twice — which is true of
|
||||
|
|
@ -112,31 +266,52 @@ describe('fixture server served to a real browser', () => {
|
|||
await page.getByRole('button', { name: 'Create Key' }).click();
|
||||
await page.getByLabel('Name', { exact: true }).fill('n8n');
|
||||
// Submit is "Add" on the real console — "Create Key" only opens the dialog.
|
||||
await page.getByRole('dialog').getByRole('button', { name: 'Add' }).click();
|
||||
const add = page.getByRole('dialog').getByRole('button', { name: 'Add' });
|
||||
// The real console disables Add until Expires is set (NODE-5755): a name
|
||||
// alone is not enough, and clicking anyway does nothing.
|
||||
expect(await add.isDisabled()).toBe(true);
|
||||
// Click-driven, as in the real trace: this is a listbox, not a native select.
|
||||
await page.getByRole('combobox', { name: 'Expires' }).click();
|
||||
await page.getByRole('option', { name: '30 days' }).click();
|
||||
expect(await add.isDisabled()).toBe(false);
|
||||
await add.click();
|
||||
|
||||
// String body, not a closure: this program has no DOM lib, and the callback
|
||||
// runs in the page anyway.
|
||||
await page.waitForFunction("document.getElementById('key-value')?.value !== ''");
|
||||
expect(await page.getByLabel('API key', { exact: true }).inputValue()).toBe(
|
||||
server.mintedSecret,
|
||||
fx.server.mintedSecret,
|
||||
);
|
||||
expect(server.secretWasIssued).toBe(true);
|
||||
expect(fx.server.secretWasIssued).toBe(true);
|
||||
|
||||
// Event log, asserted here rather than in its own test so it can't pass on
|
||||
// a previous test's side effects.
|
||||
expect(server.events.map((e) => e.path)).toContain('/settings/keys');
|
||||
expect(server.events.some((e) => e.mintedSecret)).toBe(true);
|
||||
expect(fx.server.events.map((e) => e.path)).toContain('/settings/keys');
|
||||
expect(fx.server.events.some((e) => e.mintedSecret)).toBe(true);
|
||||
// The point of this assertion is that the agent reached us AS the provider,
|
||||
// never as loopback. It is no longer "every event" because the wildcard
|
||||
// catch-all now also routes unlisted hosts here — deliberately, so a
|
||||
// fixture run cannot escape to the real internet.
|
||||
expect(server.events.some((e) => e.host === 'platform.claude.com')).toBe(true);
|
||||
expect(server.events.every((e) => e.host !== '127.0.0.1' && e.host !== 'localhost')).toBe(true);
|
||||
expect(fx.server.events.some((e) => e.host === 'platform.claude.com')).toBe(true);
|
||||
expect(fx.server.events.every((e) => e.host !== '127.0.0.1' && e.host !== 'localhost')).toBe(
|
||||
true,
|
||||
);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('surfaces the disabled submit AS disabled, so the gap is diagnosable', async () => {
|
||||
// Case 593 turns on this: the dead end has to be visible in the tree the
|
||||
// agent reads, or the gap is undiagnosable rather than just open.
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.claude.com/settings/keys');
|
||||
await page.getByRole('button', { name: 'Create Key' }).click();
|
||||
const outline = await page.getByRole('dialog').ariaSnapshot();
|
||||
expect(outline).toMatch(/button "Add".*disabled/);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('exposes an accessibility outline with the landmarks the agent needs', async () => {
|
||||
const page = await ctx!.newPage();
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.claude.com/settings/keys');
|
||||
const outline = await page.locator('main').ariaSnapshot();
|
||||
expect(outline).toContain('button "Create Key"');
|
||||
|
|
@ -150,6 +325,255 @@ describe('fixture server served to a real browser', () => {
|
|||
});
|
||||
});
|
||||
|
||||
// Same contract as the anthropic block, against the other console. Not shared
|
||||
// with it: the differences between the consoles are the point.
|
||||
describe('openai fixture served to a real browser', () => {
|
||||
const fx = fixtureBrowser('openAiApi');
|
||||
|
||||
it('answers as the real hostname, so the agent never sees loopback', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.openai.com/api-keys');
|
||||
expect(page.url()).toBe('https://platform.openai.com/api-keys');
|
||||
expect(await page.locator('h1').textContent()).toBe('API keys');
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('keeps the two "create...key" buttons distinguishable', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.openai.com/api-keys');
|
||||
// Closed, the submit is not in the tree at all — only the opener is.
|
||||
expect(await page.getByRole('button', { name: 'Create' }).count()).toBe(1);
|
||||
|
||||
await page.getByRole('button', { name: 'Create new secret key' }).click();
|
||||
// Open, a loose match hits BOTH: that is the hazard, and the exact names
|
||||
// have to keep them apart.
|
||||
expect(await page.getByRole('button', { name: 'Create' }).count()).toBe(2);
|
||||
expect(await page.getByRole('button', { name: 'Create new secret key' }).count()).toBe(1);
|
||||
expect(await page.getByRole('button', { name: 'Create secret key', exact: true }).count()).toBe(
|
||||
1,
|
||||
);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('hands out exactly the ledger secret through the create-key flow', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.openai.com/home');
|
||||
// Navigate the way the agent has to: the landing page is not the key page.
|
||||
await page.getByRole('complementary').getByRole('link', { name: 'API Keys' }).click();
|
||||
expect(await page.locator('h1').textContent()).toBe('API keys');
|
||||
|
||||
await page.getByRole('button', { name: 'Create new secret key' }).click();
|
||||
const dialog = page.getByRole('dialog').filter({ hasText: 'Create new secret key' });
|
||||
// Name is optional here — the contrast with anthropic's gated Add.
|
||||
expect(await dialog.getByRole('button', { name: 'Create secret key' }).isDisabled()).toBe(
|
||||
false,
|
||||
);
|
||||
await page.getByPlaceholder('My Test Key').fill('n8n');
|
||||
await dialog.getByRole('button', { name: 'Create secret key' }).click();
|
||||
|
||||
await page.waitForFunction("document.getElementById('key-value')?.value !== ''");
|
||||
const reveal = page.getByRole('dialog').filter({ hasText: 'Save your key' });
|
||||
expect(await reveal.getByRole('textbox').inputValue()).toBe(fx.server.mintedSecret);
|
||||
expect(fx.server.secretWasIssued).toBe(true);
|
||||
|
||||
expect(fx.server.events.some((e) => e.mintedSecret)).toBe(true);
|
||||
expect(fx.server.events.some((e) => e.host === 'platform.openai.com')).toBe(true);
|
||||
expect(fx.server.events.every((e) => e.host !== '127.0.0.1' && e.host !== 'localhost')).toBe(
|
||||
true,
|
||||
);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('shows the new key TRUNCATED in the list, never in full', async () => {
|
||||
// Creates the key on THIS page: the assertion is about the row the create
|
||||
// flow renders, and a fresh page never contains the secret in the first
|
||||
// place, so asserting there proves nothing.
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.openai.com/api-keys');
|
||||
await page.getByRole('button', { name: 'Create new secret key' }).click();
|
||||
await page
|
||||
.getByRole('dialog')
|
||||
.getByRole('button', { name: 'Create secret key', exact: true })
|
||||
.click();
|
||||
await page.waitForFunction("document.getElementById('key-value')?.value !== ''");
|
||||
|
||||
const outline = await page.locator('table').ariaSnapshot();
|
||||
expect(outline).toMatch(/sk-\.\.\.\w{4}/);
|
||||
expect(outline).not.toContain(fx.server.mintedSecret);
|
||||
// Nor may a derived column smuggle a run of the key into the tree.
|
||||
expect(outline).not.toContain(fx.server.mintedSecret.slice(-16));
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('renders a name containing markup as text, not as markup', async () => {
|
||||
// The name is free-form agent input. Interpolated into the row's HTML it
|
||||
// would corrupt the table the truncation assertions read.
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.openai.com/api-keys');
|
||||
await page.getByRole('button', { name: 'Create new secret key' }).click();
|
||||
await page.getByPlaceholder('My Test Key').fill('<b>bold</b> & co');
|
||||
await page
|
||||
.getByRole('dialog')
|
||||
.getByRole('button', { name: 'Create secret key', exact: true })
|
||||
.click();
|
||||
await page.waitForFunction("document.getElementById('key-value')?.value !== ''");
|
||||
|
||||
expect(await page.locator('#key-rows tr').first().textContent()).toContain('<b>bold</b> & co');
|
||||
expect(await page.locator('#key-rows b').count()).toBe(0);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('exposes an accessibility outline with the landmarks the agent needs', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://platform.openai.com/api-keys');
|
||||
const outline = await page.locator('main').ariaSnapshot();
|
||||
expect(outline).toContain('button "Create new secret key"');
|
||||
expect(outline).toContain('heading "API keys"');
|
||||
// Calibrated against the real capture.
|
||||
for (const col of ['Name', 'Status', 'Tracking ID', 'Secret Key', 'Created', 'Permissions']) {
|
||||
expect(outline).toContain(col);
|
||||
}
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
// The awkward console: two blocking overlays, no role=dialog, key as text.
|
||||
describe('gemini fixture served to a real browser', () => {
|
||||
const fx = fixtureBrowser('googlePalmApi');
|
||||
|
||||
it('blocks the page behind BOTH overlays until each is dismissed', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://aistudio.google.com/apikey');
|
||||
expect(await page.getByRole('button', { name: 'Create API key' }).isVisible()).toBe(false);
|
||||
|
||||
await page.getByRole('button', { name: 'Close guided tour' }).click();
|
||||
expect(await page.getByRole('button', { name: 'Create API key' }).isVisible()).toBe(false);
|
||||
|
||||
await page.getByRole('button', { name: 'OK, got it' }).click();
|
||||
expect(await page.getByRole('button', { name: 'Create API key' }).isVisible()).toBe(true);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
async function openConsole() {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://aistudio.google.com/apikey');
|
||||
await page.getByRole('button', { name: 'Close guided tour' }).click();
|
||||
await page.getByRole('button', { name: 'OK, got it' }).click();
|
||||
return page;
|
||||
}
|
||||
|
||||
it('hands out exactly the ledger secret, as TEXT rather than a field', async () => {
|
||||
const page = await openConsole();
|
||||
await page.getByRole('button', { name: 'Create API key' }).click();
|
||||
await page.getByLabel('Name your key').fill('n8n');
|
||||
await page.getByRole('button', { name: 'Create key', exact: true }).click();
|
||||
|
||||
await page.waitForFunction("document.getElementById('key-value')?.textContent !== ''");
|
||||
// textContent, not inputValue — the key is a text node here.
|
||||
expect(await page.locator('#key-value').textContent()).toBe(fx.server.mintedSecret);
|
||||
expect(fx.server.mintedSecret.startsWith('AQ.')).toBe(true);
|
||||
expect(fx.server.secretWasIssued).toBe(true);
|
||||
expect(fx.server.events.some((e) => e.host === 'aistudio.google.com')).toBe(true);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('keeps the reveal panel findable even though it is not a role=dialog', async () => {
|
||||
const page = await openConsole();
|
||||
expect(await page.getByRole('dialog').count()).toBe(0);
|
||||
await page.getByRole('button', { name: 'Create API key' }).click();
|
||||
expect(await page.getByRole('dialog').count()).toBe(0);
|
||||
expect(await page.getByLabel('Create a new key Close dialog').count()).toBe(1);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('renders a name containing markup as text, not as markup', async () => {
|
||||
const page = await openConsole();
|
||||
await page.getByRole('button', { name: 'Create API key' }).click();
|
||||
await page.getByLabel('Name your key').fill('<i>italic</i> & co');
|
||||
await page.getByRole('button', { name: 'Create key', exact: true }).click();
|
||||
await page.waitForFunction("document.getElementById('key-value')?.textContent !== ''");
|
||||
|
||||
expect(await page.locator('#key-rows tr').first().textContent()).toContain(
|
||||
'<i>italic</i> & co',
|
||||
);
|
||||
expect(await page.locator('#key-rows i').count()).toBe(0);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('shows only the last four characters in the list', async () => {
|
||||
// Mint on this page first — see the openai equivalent.
|
||||
const page = await openConsole();
|
||||
await page.getByRole('button', { name: 'Create API key' }).click();
|
||||
await page.getByLabel('Name your key').fill('n8n');
|
||||
await page.getByRole('button', { name: 'Create key', exact: true }).click();
|
||||
await page.waitForFunction("document.getElementById('key-value')?.textContent !== ''");
|
||||
|
||||
const outline = await page.locator('table').ariaSnapshot();
|
||||
expect(outline).toMatch(/button "\.\.\.\w{4}"/);
|
||||
expect(outline).not.toContain(fx.server.mintedSecret);
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
// Multi-page, unlike the other three: the token only exists after installing.
|
||||
describe('slack fixture served to a real browser', () => {
|
||||
const fx = fixtureBrowser('slackApi');
|
||||
|
||||
it('keeps the manifest editor click-trapped but typeable', async () => {
|
||||
// The live reproduction for case 599: the focusable input sits under the
|
||||
// rendered surface, so a click never passes the hit-test while fill()
|
||||
// does. Short timeout here — the real one costs 30s.
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://api.slack.com/apps');
|
||||
await page.getByRole('button', { name: 'Create New App' }).click();
|
||||
await page.getByRole('button', { name: 'From a manifest' }).click();
|
||||
|
||||
await expect(page.locator('#manifest-input').click({ timeout: 1_000 })).rejects.toThrow(
|
||||
/Timeout|intercepts pointer events/,
|
||||
);
|
||||
await page.locator('#manifest-input').fill('{"display_information":{"name":"n8n"}}');
|
||||
expect(await page.locator('#manifest-input').inputValue()).toContain('display_information');
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('leaves Next enabled — nothing here is gated on a required field', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://api.slack.com/apps');
|
||||
await page.getByRole('button', { name: 'Create New App' }).click();
|
||||
await page.getByRole('button', { name: 'From a manifest' }).click();
|
||||
expect(await page.getByRole('button', { name: 'Next' }).isDisabled()).toBe(false);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('issues the ledger token only after Install to Workspace', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
// An app id this fixture has never seen — the console mints one per app, and
|
||||
// the agent navigates by typing those URLs back.
|
||||
await page.goto('https://api.slack.com/apps/A0Q3ZK71LMN/install-on-team');
|
||||
expect(await page.content()).not.toContain(fx.server.mintedSecret);
|
||||
|
||||
await page.getByRole('button', { name: 'Install to Workspace' }).click();
|
||||
await page.waitForFunction("document.getElementById('bot-token')?.value !== ''");
|
||||
expect(await page.getByLabel('Bot User OAuth Token').inputValue()).toBe(fx.server.mintedSecret);
|
||||
expect(fx.server.mintedSecret.startsWith('xoxb-')).toBe(true);
|
||||
expect(fx.server.secretWasIssued).toBe(true);
|
||||
await page.close();
|
||||
});
|
||||
|
||||
it('reaches the token only through the multi-page flow', async () => {
|
||||
const page = await fx.ctx.newPage();
|
||||
await page.goto('https://api.slack.com/apps/A0Q3ZK71LMN/general');
|
||||
// Basic Information carries the OAuth2 credentials, never the bot token.
|
||||
expect(await page.content()).not.toContain(fx.server.mintedSecret);
|
||||
expect(await page.getByLabel('Client ID').inputValue()).toContain('.');
|
||||
await page.getByRole('link', { name: 'Install App' }).click();
|
||||
expect(await page.locator('h1').first().textContent()).toContain('OAuth Tokens');
|
||||
// The id survives the hop rather than reverting to a hardcoded one.
|
||||
expect(page.url()).toContain('/apps/A0Q3ZK71LMN/install-on-team');
|
||||
await page.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe('providerFixtureManifestSchema', () => {
|
||||
const valid = {
|
||||
credentialType: 'anthropicApi',
|
||||
|
|
@ -192,4 +616,138 @@ describe('providerFixtureManifestSchema', () => {
|
|||
it('rejects an empty hosts list', () => {
|
||||
expect(providerFixtureManifestSchema.safeParse({ ...valid, hosts: [] }).success).toBe(false);
|
||||
});
|
||||
|
||||
// The three providers agree on none of this — a manifest that could only
|
||||
// describe anthropic left the other two's value check unverifiable.
|
||||
it('accepts a query-param secret instead of a header', () => {
|
||||
expect(
|
||||
providerFixtureManifestSchema.safeParse({
|
||||
...valid,
|
||||
verify: { path: '/v1beta/models', query: 'key' },
|
||||
}).success,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('accepts a header scheme, for a credential that sends `Bearer <key>`', () => {
|
||||
expect(
|
||||
providerFixtureManifestSchema.safeParse({
|
||||
...valid,
|
||||
verify: { path: '/models', header: 'Authorization', scheme: 'Bearer' },
|
||||
}).success,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects a verify block carrying neither a header nor a query param', () => {
|
||||
// The union has no arm without one of them, so this is structural — there is
|
||||
// no refinement message to assert, only that it cannot parse.
|
||||
expect(
|
||||
providerFixtureManifestSchema.safeParse({ ...valid, verify: { path: '/models' } }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects a scheme on the query arm, where it could never be honoured', () => {
|
||||
// `{path, query, scheme}` used to parse and then compare "Bearer <secret>"
|
||||
// against a query value — a permanent 401 reported as unverifiable.
|
||||
expect(
|
||||
providerFixtureManifestSchema.safeParse({
|
||||
...valid,
|
||||
verify: { path: '/v1beta/models', query: 'key', scheme: 'Bearer' },
|
||||
}).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects a verify block carrying both — the stand-in must know where to look', () => {
|
||||
const res = providerFixtureManifestSchema.safeParse({
|
||||
...valid,
|
||||
verify: { path: '/models', header: 'x-api-key', query: 'key' },
|
||||
});
|
||||
expect(res.success).toBe(false);
|
||||
});
|
||||
|
||||
it('accepts a urlField, for a credential whose base URL is not called `url`', () => {
|
||||
const res = providerFixtureManifestSchema.safeParse({ ...valid, urlField: 'host' });
|
||||
expect(res.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// The listener n8n's credential test talks to. Getting this wrong silently
|
||||
// discards the strongest of the three deterministic checks.
|
||||
describe('the provider stand-in accepts each provider shape', () => {
|
||||
const SECRET = 'sk-test-0123456789abcdef';
|
||||
let dir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), 'fixture-verify-'));
|
||||
await writeFile(
|
||||
join(dir, 'console.html'),
|
||||
'<!doctype html><html><body><h1>Keys</h1></body></html>',
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (dir) await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function withServer(
|
||||
verify: Record<string, string>,
|
||||
run: (server: FixtureServer) => void | Promise<void>,
|
||||
): Promise<void> {
|
||||
const server = await startFixtureServer({
|
||||
fixture: {
|
||||
id: 'under-test',
|
||||
dir,
|
||||
manifest: providerFixtureManifestSchema.parse({
|
||||
credentialType: 'x',
|
||||
hosts: ['console.example.com'],
|
||||
secretPrefix: 'sk-test-',
|
||||
routes: { '/keys': 'console.html' },
|
||||
defaultRoute: '/keys',
|
||||
verify,
|
||||
}),
|
||||
},
|
||||
logger,
|
||||
secret: SECRET,
|
||||
});
|
||||
try {
|
||||
await run(server);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
}
|
||||
|
||||
it('accepts a bare header value, and refuses a truncated one', async () => {
|
||||
await withServer({ path: '/v1/models', header: 'x-api-key' }, async (server) => {
|
||||
const url = `${server.verifyBaseUrl}/v1/models`;
|
||||
expect((await fetch(url, { headers: { 'x-api-key': SECRET } })).status).toBe(200);
|
||||
expect((await fetch(url, { headers: { 'x-api-key': SECRET.slice(0, 12) } })).status).toBe(
|
||||
401,
|
||||
);
|
||||
expect(server.verifiedOk).toBe(true);
|
||||
expect(server.verifyAttempts).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts a schemed header, so `Bearer <key>` is not read as a wrong key', async () => {
|
||||
await withServer(
|
||||
{ path: '/models', header: 'Authorization', scheme: 'Bearer' },
|
||||
async (server) => {
|
||||
const url = `${server.verifyBaseUrl}/models`;
|
||||
expect((await fetch(url, { headers: { Authorization: `Bearer ${SECRET}` } })).status).toBe(
|
||||
200,
|
||||
);
|
||||
// The bare key is the WRONG presentation for this credential type.
|
||||
expect((await fetch(url, { headers: { Authorization: SECRET } })).status).toBe(401);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts a query-param secret, which never reaches a header at all', async () => {
|
||||
await withServer({ path: '/v1beta/models', query: 'key' }, async (server) => {
|
||||
const base = `${server.verifyBaseUrl}/v1beta/models`;
|
||||
expect((await fetch(`${base}?key=${SECRET}`)).status).toBe(200);
|
||||
expect((await fetch(`${base}?key=nope`)).status).toBe(401);
|
||||
// A query-param provider must not be satisfiable by a header.
|
||||
expect((await fetch(base, { headers: { 'x-api-key': SECRET } })).status).toBe(401);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,7 +5,12 @@
|
|||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
|
||||
import { fixtureInterceptionArgs, planRelayConnection } from '../harness/browser-runtime';
|
||||
import {
|
||||
browserN8nOrigins,
|
||||
browserSessionCookie,
|
||||
fixtureInterceptionArgs,
|
||||
planRelayConnection,
|
||||
} from '../harness/browser-runtime';
|
||||
|
||||
const EXT = 'chrome-extension://cegmdpndekdfpnafgacidejijecomlhh/connect.html';
|
||||
|
||||
|
|
@ -192,3 +197,44 @@ describe('fixtureInterceptionArgs', () => {
|
|||
expect(rules).toContain('MAP localhost:5680 n8n:5678');
|
||||
});
|
||||
});
|
||||
|
||||
// The session cookie the launched browser is given. Same subject as the rest of
|
||||
// this file: which spelling of n8n the BROWSER ends up using, after the rewrite
|
||||
// above. Get it wrong and the credential-setup flow dead-ends on /signin with
|
||||
// every provider-console expectation failing as though the agent misbehaved.
|
||||
describe('browserN8nOrigins', () => {
|
||||
it('returns the single origin when n8n is already loopback', () => {
|
||||
expect(browserN8nOrigins('http://localhost:5678')).toEqual(['http://localhost:5678']);
|
||||
});
|
||||
|
||||
it('adds the localhost origin the relay MAP rule creates for a non-loopback n8n', () => {
|
||||
// planRelayConnection rewrites a non-loopback n8n to localhost + a MAP rule,
|
||||
// so in the split-container topology the browser knows n8n by THAT origin —
|
||||
// a cookie on `http://n8n:5678` alone would never be sent.
|
||||
expect(browserN8nOrigins('http://n8n:5678')).toEqual([
|
||||
'http://n8n:5678',
|
||||
'http://localhost:5678',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('browserSessionCookie', () => {
|
||||
it('turns the client cookie header into a cookie for the browser context', () => {
|
||||
expect(browserSessionCookie('n8n-auth=abc123', 'http://localhost:5678')).toEqual({
|
||||
name: 'n8n-auth',
|
||||
value: 'abc123',
|
||||
url: 'http://localhost:5678',
|
||||
});
|
||||
});
|
||||
|
||||
it('splits on the FIRST = only, so a base64/JWT value survives intact', () => {
|
||||
const value = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0=';
|
||||
expect(browserSessionCookie(`n8n-auth=${value}`, 'http://n8n:5678')?.value).toBe(value);
|
||||
});
|
||||
|
||||
it('returns undefined for anything that is not a name=value pair', () => {
|
||||
expect(browserSessionCookie('', 'http://localhost:5678')).toBeUndefined();
|
||||
expect(browserSessionCookie('n8n-auth', 'http://localhost:5678')).toBeUndefined();
|
||||
expect(browserSessionCookie('=orphan', 'http://localhost:5678')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,13 +14,16 @@
|
|||
combobox "Expires", button "Add", button "Close"
|
||||
dialog "Save your API key" [level=2]: button "Copy Key", button "Done"
|
||||
|
||||
Two properties that are load-bearing, both taken from the real page:
|
||||
Three properties that are load-bearing, all taken from the real page:
|
||||
1. The list is PRE-POPULATED with truncated keys. An agent that grabs a
|
||||
displayed value instead of the freshly issued one captures an ellipsised
|
||||
string — a real failure mode, and one the exact-value check is meant to
|
||||
catch. An empty list would hide it.
|
||||
2. The submit button is "Add", NOT "Create Key" — "Create Key" only opens the
|
||||
dialog. An agent that clicks the wrong one never creates anything.
|
||||
3. "Add" is DISABLED until Expires has a value, and Expires opens unselected —
|
||||
the real dialog's own rule, and the required-field gap NODE-5755 tracks.
|
||||
Defaulting it to "Never" tested past the bug.
|
||||
|
||||
The captured tree was German (the user's browser locale). Rendered in English
|
||||
here because the eval corpus is English; re-check names if a locale-specific
|
||||
|
|
@ -95,13 +98,26 @@
|
|||
</select>
|
||||
<label for="key-name">Name</label>
|
||||
<input id="key-name" name="key-name" type="text" value="" />
|
||||
<label for="expires">Expires</label>
|
||||
<select id="expires" name="expires">
|
||||
<option value="never" selected>Never</option>
|
||||
<option value="30d">30 days</option>
|
||||
<option value="90d">90 days</option>
|
||||
</select>
|
||||
<button id="create-submit" type="button">Add</button>
|
||||
<!-- A custom dropdown, not a native <select>: in the real trace the
|
||||
agent set this by CLICKING items. A native <option> cannot be
|
||||
clicked, which would invent a dead end the real console lacks. -->
|
||||
<span id="expires-label">Expires</span>
|
||||
<button
|
||||
id="expires-trigger"
|
||||
type="button"
|
||||
role="combobox"
|
||||
aria-labelledby="expires-label"
|
||||
aria-expanded="false"
|
||||
aria-controls="expires-list"
|
||||
>
|
||||
Select an option
|
||||
</button>
|
||||
<ul id="expires-list" role="listbox" aria-labelledby="expires-label" hidden>
|
||||
<li role="option" data-value="never">Never</li>
|
||||
<li role="option" data-value="30d">30 days</li>
|
||||
<li role="option" data-value="90d">90 days</li>
|
||||
</ul>
|
||||
<button id="create-submit" type="button" disabled>Add</button>
|
||||
</div>
|
||||
|
||||
<!-- Reveal dialog — the ONLY place the full key ever appears. -->
|
||||
|
|
@ -120,6 +136,32 @@
|
|||
const reveal = document.getElementById('reveal');
|
||||
const keyValue = document.getElementById('key-value');
|
||||
const nameInput = document.getElementById('key-name');
|
||||
const expiresTrigger = document.getElementById('expires-trigger');
|
||||
const expiresList = document.getElementById('expires-list');
|
||||
const submit = document.getElementById('create-submit');
|
||||
let expiresValue = '';
|
||||
|
||||
expiresTrigger.addEventListener('click', () => {
|
||||
const willOpen = expiresList.hidden;
|
||||
expiresList.hidden = !willOpen;
|
||||
expiresTrigger.setAttribute('aria-expanded', String(willOpen));
|
||||
});
|
||||
|
||||
// Add stays disabled until Expires has a value — the dead end the agent
|
||||
// has to diagnose.
|
||||
const options = [...expiresList.querySelectorAll('[role="option"]')];
|
||||
for (const option of options) {
|
||||
option.addEventListener('click', () => {
|
||||
expiresValue = option.dataset.value;
|
||||
expiresTrigger.textContent = option.textContent;
|
||||
for (const other of options) {
|
||||
other.setAttribute('aria-selected', String(other === option));
|
||||
}
|
||||
expiresList.hidden = true;
|
||||
expiresTrigger.setAttribute('aria-expanded', 'false');
|
||||
submit.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('create-key').addEventListener('click', () => {
|
||||
dialog.hidden = false;
|
||||
|
|
@ -135,7 +177,7 @@
|
|||
body: JSON.stringify({
|
||||
name: nameInput.value || 'Untitled',
|
||||
workspace: document.getElementById('workspace').value,
|
||||
expires: document.getElementById('expires').value,
|
||||
expires: expiresValue,
|
||||
}),
|
||||
});
|
||||
const { key } = await response.json();
|
||||
|
|
@ -152,7 +194,9 @@
|
|||
key.slice(-4) +
|
||||
'</code></td><td>' +
|
||||
document.getElementById('workspace').value +
|
||||
'</td><td>Eval Owner</td><td>just now</td><td>Never</td><td>—</td>' +
|
||||
'</td><td>Eval Owner</td><td>just now</td><td>' +
|
||||
expiresTrigger.textContent +
|
||||
'</td><td>—</td>' +
|
||||
'<td><button type="button">Delete</button></td>';
|
||||
document.getElementById('key-rows').prepend(row);
|
||||
const count = document.getElementById('key-count');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,218 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike Google AI Studio API-keys console. Our own generic content — no
|
||||
provider source, no recorded responses, no real tokens. Calibrated against a
|
||||
real captured accessibility tree (LangTracer thread 708172f1, 2026-08-17).
|
||||
|
||||
Load-bearing, all taken from the real page:
|
||||
- TWO overlays (guided tour, cookie notice) block the page until dismissed
|
||||
- the key is a TEXT NODE with a copy button, not a field, so reading
|
||||
`input.value` finds nothing here
|
||||
- neither panel is role=dialog — they are containers with accessible names
|
||||
- the list truncates to the last four characters, rendered as a button
|
||||
- keys carry Google's current `AQ.` prefix, and minting the same shape is what
|
||||
lets the hermetic case notice a partially captured value (NODE-5768) instead
|
||||
of passing on a friendlier format.
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>API Keys | Google AI Studio</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="tour" role="dialog" aria-label="Guided Tour" aria-modal="true">
|
||||
<h2>credit_card New: Control your API cost</h2>
|
||||
<p>Set a monthly spend limit per project and get alerts before you hit it.</p>
|
||||
<a href="/docs/billing">Further resources</a>
|
||||
<button id="tour-close" type="button">Close guided tour</button>
|
||||
</div>
|
||||
|
||||
<div id="cookies">
|
||||
<p>Google AI Studio uses cookies to deliver and improve the service.</p>
|
||||
<a href="/docs/cookies">Learn more about how Google uses cookies. Opens in a new tab.</a>
|
||||
<button id="cookies-ok" type="button">OK, got it</button>
|
||||
</div>
|
||||
|
||||
<div id="page" hidden>
|
||||
<button type="button">Skip to main content</button>
|
||||
<header>
|
||||
<button type="button">Toggle navigation menu</button>
|
||||
<a href="/">Google AI Studio logo</a>
|
||||
<button type="button">View related products</button>
|
||||
<nav aria-label="Main">
|
||||
<ul>
|
||||
<li><a href="/dashboard">Dashboard</a></li>
|
||||
<li><a href="/apikey" aria-current="page">API Keys</a></li>
|
||||
<li><a href="/projects">Projects</a></li>
|
||||
<li><a href="/usage">Usage</a></li>
|
||||
<li><a href="/rate-limit">Rate Limit</a></li>
|
||||
<li><a href="/spend">Spend</a></li>
|
||||
<li><a href="/billing">Billing</a></li>
|
||||
<li><a href="/logs">Logs and Datasets</a></li>
|
||||
<li><a href="/changelog">Changelog</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
<button type="button">What's new</button>
|
||||
<button type="button">Settings</button>
|
||||
<button type="button">Search</button>
|
||||
<button type="button">Get API key</button>
|
||||
<button type="button">Google Account: Eval Owner</button>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<h2>API Keys</h2>
|
||||
<a href="/docs/quickstart">API quickstart</a>
|
||||
<button id="open-create" type="button">Create API key</button>
|
||||
|
||||
<span>Group by</span>
|
||||
<button type="button">API Key</button>
|
||||
<button type="button">Project</button>
|
||||
<span>Filter by</span>
|
||||
<label for="project-filter">Filter by</label>
|
||||
<select id="project-filter" name="project-filter">
|
||||
<option value="all" selected>All projects</option>
|
||||
<option value="default">Default Gemini Project</option>
|
||||
</select>
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Key</th>
|
||||
<th scope="col">Project</th>
|
||||
<th scope="col"><button type="button">Created</button></th>
|
||||
<th scope="col">Billing Tier</th>
|
||||
<th scope="col"></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="key-rows">
|
||||
<tr>
|
||||
<td><button type="button">...iIJg</button><span>Test local model selection</span></td>
|
||||
<td><button type="button">Default Gemini Project</button><span>gen-lang-client-0228122468</span></td>
|
||||
<td>Jul 8, 2026</td>
|
||||
<td><button type="button">Set up billing</button><span>Free tier</span></td>
|
||||
<td>
|
||||
<button type="button">Copy API key</button>
|
||||
<button type="button">View spend</button>
|
||||
<button type="button">View usage</button>
|
||||
<button type="button">View more actions</button>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<p>Can't find your API keys here?</p>
|
||||
<p>
|
||||
This list only shows API keys for projects imported into Google AI
|
||||
Studio. Import other projects to manage their associated API Keys. You
|
||||
can also create a new API Key above.
|
||||
<a href="/docs/api-key#import-projects">Learn more</a>
|
||||
</p>
|
||||
<button type="button">Import projects</button>
|
||||
|
||||
<!-- Not role=dialog — the real one isn't either. -->
|
||||
<div id="create-panel" aria-label="Create a new key Close dialog" hidden>
|
||||
<h2>Create a new key <button id="create-close" type="button">Close dialog</button></h2>
|
||||
<label for="key-name">Name your key</label>
|
||||
<input id="key-name" name="key-name" type="text" placeholder="Name your key" value="" />
|
||||
<label for="key-project">Project</label>
|
||||
<select id="key-project" name="key-project">
|
||||
<option value="default" selected>Default Gemini Project</option>
|
||||
<option value="new">Create a new project</option>
|
||||
</select>
|
||||
<button id="create-cancel" type="button">Cancel</button>
|
||||
<button id="create-submit" type="button">Create key</button>
|
||||
</div>
|
||||
|
||||
<div id="reveal" aria-label="API key details close" hidden>
|
||||
<h2>API key details <button id="reveal-close" type="button">close</button></h2>
|
||||
<p>
|
||||
info For details on using auth keys with Gemini API, see our
|
||||
<a href="/docs/api-key">API key documentation</a>.
|
||||
</p>
|
||||
<span>API Key</span>
|
||||
<span id="key-value"></span>
|
||||
<button id="copy-key" type="button">Copy key to clipboard</button>
|
||||
<span>Name</span>
|
||||
<span id="reveal-name">Gemini API Key</span>
|
||||
<button type="button">Copy display name to clipboard</button>
|
||||
<span>Project name</span>
|
||||
<span>projects/58324130107</span>
|
||||
<button type="button">Copy project name to clipboard</button>
|
||||
<span>Project number</span>
|
||||
<span>58324130107</span>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const tour = document.getElementById('tour');
|
||||
const cookies = document.getElementById('cookies');
|
||||
const page = document.getElementById('page');
|
||||
const panel = document.getElementById('create-panel');
|
||||
const reveal = document.getElementById('reveal');
|
||||
const keyValue = document.getElementById('key-value');
|
||||
const nameInput = document.getElementById('key-name');
|
||||
|
||||
// Hidden until BOTH overlays are dismissed.
|
||||
const revealPage = () => {
|
||||
if (tour.hidden && cookies.hidden) page.hidden = false;
|
||||
};
|
||||
document.getElementById('tour-close').addEventListener('click', () => {
|
||||
tour.hidden = true;
|
||||
revealPage();
|
||||
});
|
||||
document.getElementById('cookies-ok').addEventListener('click', () => {
|
||||
cookies.hidden = true;
|
||||
revealPage();
|
||||
});
|
||||
|
||||
document.getElementById('open-create').addEventListener('click', () => {
|
||||
panel.hidden = false;
|
||||
nameInput.focus();
|
||||
});
|
||||
document.getElementById('create-cancel').addEventListener('click', () => {
|
||||
panel.hidden = true;
|
||||
});
|
||||
document.getElementById('create-close').addEventListener('click', () => {
|
||||
panel.hidden = true;
|
||||
});
|
||||
|
||||
document.getElementById('create-submit').addEventListener('click', async () => {
|
||||
const response = await fetch('/__fixture__/create-key', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: nameInput.value || 'Gemini API Key',
|
||||
project: document.getElementById('key-project').value,
|
||||
}),
|
||||
});
|
||||
const { key } = await response.json();
|
||||
keyValue.textContent = key;
|
||||
document.getElementById('reveal-name').textContent = nameInput.value || 'Gemini API Key';
|
||||
panel.hidden = true;
|
||||
reveal.hidden = false;
|
||||
|
||||
const row = document.createElement('tr');
|
||||
row.innerHTML =
|
||||
'<td><button type="button">...' +
|
||||
key.slice(-4) +
|
||||
'</button><span class="key-name"></span></td>' +
|
||||
'<td><button type="button">Default Gemini Project</button><span>gen-lang-client-0228122468</span></td>' +
|
||||
'<td>just now</td>' +
|
||||
'<td><button type="button">Set up billing</button><span>Free tier</span></td>' +
|
||||
'<td><button type="button">Copy API key</button><button type="button">View spend</button>' +
|
||||
'<button type="button">View usage</button><button type="button">View more actions</button></td>';
|
||||
// textContent, not markup — see the openai fixture.
|
||||
row.querySelector('.key-name').textContent = nameInput.value || 'Gemini API Key';
|
||||
document.getElementById('key-rows').prepend(row);
|
||||
});
|
||||
|
||||
document.getElementById('copy-key').addEventListener('click', async () => {
|
||||
await navigator.clipboard.writeText(keyValue.textContent).catch(() => {});
|
||||
});
|
||||
document.getElementById('reveal-close').addEventListener('click', () => {
|
||||
reveal.hidden = true;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike AI Studio landing page, and this fixture's defaultRoute. Exists so
|
||||
the agent has to NAVIGATE to the key page rather than starting on it.
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Google AI Studio</title>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<button type="button">Toggle navigation menu</button>
|
||||
<a href="/">Google AI Studio logo</a>
|
||||
<nav aria-label="Main">
|
||||
<ul>
|
||||
<li><a href="/dashboard" aria-current="page">Dashboard</a></li>
|
||||
<li><a href="/apikey">API Keys</a></li>
|
||||
<li><a href="/projects">Projects</a></li>
|
||||
<li><a href="/usage">Usage</a></li>
|
||||
<li><a href="/billing">Billing</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
<button type="button">Get API key</button>
|
||||
<button type="button">Google Account: Eval Owner</button>
|
||||
</header>
|
||||
<main>
|
||||
<h1>Dashboard</h1>
|
||||
<section aria-labelledby="get-started">
|
||||
<h2 id="get-started">Get started</h2>
|
||||
<p>
|
||||
Create an API key to call the Gemini API. Manage keys from
|
||||
<a href="/apikey">API Keys</a>.
|
||||
</p>
|
||||
</section>
|
||||
<section aria-labelledby="usage">
|
||||
<h2 id="usage">Usage this month</h2>
|
||||
<p>No usage recorded.</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"credentialType": "googlePalmApi",
|
||||
"hosts": ["aistudio.google.com"],
|
||||
"secretPrefix": "AQ.",
|
||||
"routes": {
|
||||
"/apikey": "console.html",
|
||||
"/dashboard": "dashboard.html"
|
||||
},
|
||||
"defaultRoute": "/dashboard",
|
||||
"urlField": "host",
|
||||
"verify": {
|
||||
"path": "/v1beta/models",
|
||||
"query": "key"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,184 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike OpenAI API-keys console. Our own generic content — no provider
|
||||
source, no recorded responses, no real tokens. Calibrated against a real
|
||||
captured accessibility tree (LangTracer thread 5dc90eeb, 2026-08-17).
|
||||
|
||||
Load-bearing, all taken from the real page:
|
||||
- keys are listed TRUNCATED (`sk-...1fIA`), so a lifted display value is far
|
||||
too short to be the key — the exact-value check catches it
|
||||
- the submit is "Create secret key"; "Create new secret key" only OPENS the
|
||||
dialog. One word apart, so a loose locator hits the wrong control
|
||||
- Name is optional and nothing gates the submit — the contrast with the
|
||||
anthropic fixture, whose Add is disabled until Expires is set
|
||||
- the reveal textbox has no accessible name
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>API keys - OpenAI Platform</title>
|
||||
</head>
|
||||
<body>
|
||||
<aside aria-label="Main navigation">
|
||||
<button type="button">Default project</button>
|
||||
<button type="button">Collapse sidebar</button>
|
||||
<ul>
|
||||
<li><a href="/home">Home</a></li>
|
||||
<li><a href="/chat/edit">Chat</a></li>
|
||||
<li><a href="/audio">Audio</a></li>
|
||||
<li><a href="/playground/images">Images</a></li>
|
||||
<li><a href="/codex">Codex</a></li>
|
||||
<li><a href="/api-keys" aria-current="page">API Keys</a></li>
|
||||
<li><a href="/usage">Usage</a></li>
|
||||
<li><a href="/logs">Logs</a></li>
|
||||
<li><a href="/batches">Batches</a></li>
|
||||
<li><a href="/storage">Storage</a></li>
|
||||
<li><a href="/plugins">Plugins</a></li>
|
||||
<li><a href="/settings/proj_evalowner">Settings</a></li>
|
||||
</ul>
|
||||
<button type="button">More navigation items</button>
|
||||
<button type="button">P Personal Organization</button>
|
||||
</aside>
|
||||
|
||||
<main>
|
||||
<h1>API keys</h1>
|
||||
<button type="button">API key security guidance</button>
|
||||
<a href="/usage">API Key Usage</a>
|
||||
<button id="open-create" type="button">Create new secret key</button>
|
||||
|
||||
<label for="search-keys">Search API keys</label>
|
||||
<input id="search-keys" name="search-keys" type="search" />
|
||||
<button type="button">Active Clear current value</button>
|
||||
<button type="button">Add filter</button>
|
||||
<span id="result-count">4 results</span>
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Name</th>
|
||||
<th scope="col">Status</th>
|
||||
<th scope="col">Tracking ID</th>
|
||||
<th scope="col">Secret Key</th>
|
||||
<th scope="col">Created</th>
|
||||
<th scope="col">Last used</th>
|
||||
<th scope="col">Created by</th>
|
||||
<th scope="col">Permissions</th>
|
||||
<th scope="col">Monthly spend</th>
|
||||
<th scope="col">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="key-rows">
|
||||
<tr><td>service-instance</td><td>Active</td><td>key_Ejpr6NwBvEetPhCu</td><td>sk-...1fIA</td><td>Jun 24, 2026</td><td>Aug 17, 2026</td><td>Eval Owner</td><td>All</td><td>$0.00</td><td><button type="button">Edit</button><button type="button">Delete</button></td></tr>
|
||||
<tr><td>local-testing</td><td>Active</td><td>key_KWxC7YgCSTMQW5S8</td><td>sk-...PGQA</td><td>Jun 23, 2026</td><td>Jul 24, 2026</td><td>Eval Owner</td><td>All</td><td>$0.00</td><td><button type="button">Edit</button><button type="button">Delete</button></td></tr>
|
||||
<tr><td>batch-jobs</td><td>Active</td><td>key_X9mefkdO29ZUKROv</td><td>sk-...38YA</td><td>Feb 19, 2026</td><td>Feb 19, 2026</td><td>Eval Owner</td><td>All</td><td>$0.00</td><td><button type="button">Edit</button><button type="button">Delete</button></td></tr>
|
||||
<tr><td>prototype</td><td>Active</td><td>key_GH4PD8W0d8oDAVGQ</td><td>sk-...5akA</td><td>Feb 10, 2026</td><td>Feb 10, 2026</td><td>Eval Owner</td><td>All</td><td>$0.00</td><td><button type="button">Edit</button><button type="button">Delete</button></td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<!-- Submit is "Create secret key", not the opener. -->
|
||||
<div id="create-dialog" role="dialog" aria-labelledby="create-title" aria-modal="true" hidden>
|
||||
<h2 id="create-title">Create new secret key</h2>
|
||||
<span>Owned by</span>
|
||||
<div role="radiogroup" aria-label="API key owner">
|
||||
<label><input type="radio" name="owner" value="you" checked /> You</label>
|
||||
<label><input type="radio" name="owner" value="service-account" /> Service account</label>
|
||||
</div>
|
||||
<p>
|
||||
This API key is tied to your user and can make requests against the
|
||||
selected project. If you are removed from the organization or project,
|
||||
this key will be disabled.
|
||||
</p>
|
||||
<span>Name</span>
|
||||
<span>Optional</span>
|
||||
<input id="key-name" name="key-name" type="text" placeholder="My Test Key" value="" />
|
||||
<span>Project</span>
|
||||
<button type="button">Default project</button>
|
||||
<span>Permissions</span>
|
||||
<div role="radiogroup" aria-label="Permissions">
|
||||
<label><input type="radio" name="permissions" value="all" checked /> All</label>
|
||||
<label><input type="radio" name="permissions" value="restricted" /> Restricted</label>
|
||||
<label><input type="radio" name="permissions" value="read-only" /> Read only</label>
|
||||
</div>
|
||||
<button id="create-cancel" type="button">Cancel</button>
|
||||
<button id="create-submit" type="button">Create secret key</button>
|
||||
</div>
|
||||
|
||||
<!-- The only place the full key appears; the textbox has no name. -->
|
||||
<div id="reveal" role="dialog" aria-labelledby="reveal-title" aria-modal="true" hidden>
|
||||
<h2 id="reveal-title">Save your key</h2>
|
||||
<p>
|
||||
Please save your secret key in a safe place since
|
||||
<strong>you won't be able to view it again</strong>. Keep it secure, as
|
||||
anyone with your API key can make requests on your behalf. If you do
|
||||
lose it, you'll need to generate a new one.
|
||||
</p>
|
||||
<a href="/docs/api-key-best-practices">Learn more about API key best practices</a>
|
||||
<input id="key-value" name="key-value" type="text" readonly value="" />
|
||||
<button id="copy-key" type="button">Copy</button>
|
||||
<span>Permissions</span>
|
||||
<span>Read and write API resources</span>
|
||||
<button id="reveal-done" type="button">Done</button>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const dialog = document.getElementById('create-dialog');
|
||||
const reveal = document.getElementById('reveal');
|
||||
const keyValue = document.getElementById('key-value');
|
||||
const nameInput = document.getElementById('key-name');
|
||||
|
||||
document.getElementById('open-create').addEventListener('click', () => {
|
||||
dialog.hidden = false;
|
||||
nameInput.focus();
|
||||
});
|
||||
document.getElementById('create-cancel').addEventListener('click', () => {
|
||||
dialog.hidden = true;
|
||||
});
|
||||
|
||||
document.getElementById('create-submit').addEventListener('click', async () => {
|
||||
const response = await fetch('/__fixture__/create-key', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: nameInput.value || 'Secret key',
|
||||
owner: document.querySelector('input[name="owner"]:checked').value,
|
||||
permissions: document.querySelector('input[name="permissions"]:checked').value,
|
||||
}),
|
||||
});
|
||||
const { key } = await response.json();
|
||||
keyValue.value = key;
|
||||
dialog.hidden = true;
|
||||
reveal.hidden = false;
|
||||
|
||||
// 3 leading chars + last 4, as the real console truncates.
|
||||
const row = document.createElement('tr');
|
||||
row.innerHTML =
|
||||
// Tracking id independent of the key, as the real console's is — a
|
||||
// value derived from it would put a run of the key in the page for
|
||||
// no reason.
|
||||
'<td class="key-name"></td><td>Active</td><td>key_' +
|
||||
Math.random().toString(36).slice(2, 18) +
|
||||
'</td><td>' +
|
||||
key.slice(0, 3) +
|
||||
'...' +
|
||||
key.slice(-4) +
|
||||
'</td><td>just now</td><td>—</td><td>Eval Owner</td><td>All</td><td>$0.00</td>' +
|
||||
'<td><button type="button">Edit</button><button type="button">Delete</button></td>';
|
||||
// textContent, not markup: the name is free-form agent input, and a
|
||||
// value containing `<` or `&` would otherwise corrupt the row the
|
||||
// truncation assertions read.
|
||||
row.querySelector('.key-name').textContent = nameInput.value || 'Secret key';
|
||||
document.getElementById('key-rows').prepend(row);
|
||||
const count = document.getElementById('result-count');
|
||||
count.textContent = `${Number(count.textContent.split(' ')[0]) + 1} results`;
|
||||
});
|
||||
|
||||
document.getElementById('copy-key').addEventListener('click', async () => {
|
||||
await navigator.clipboard.writeText(keyValue.value).catch(() => {});
|
||||
});
|
||||
document.getElementById('reveal-done').addEventListener('click', () => {
|
||||
reveal.hidden = true;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike OpenAI landing page. Exists so the agent has to NAVIGATE to the keys
|
||||
page rather than starting on it — reaching the right page is the step that
|
||||
broke in NODE-5384.
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Home - OpenAI Platform</title>
|
||||
</head>
|
||||
<body>
|
||||
<aside aria-label="Main navigation">
|
||||
<button type="button">Default project</button>
|
||||
<ul>
|
||||
<li><a href="/home" aria-current="page">Home</a></li>
|
||||
<li><a href="/chat/edit">Chat</a></li>
|
||||
<li><a href="/audio">Audio</a></li>
|
||||
<li><a href="/playground/images">Images</a></li>
|
||||
<li><a href="/codex">Codex</a></li>
|
||||
<li><a href="/api-keys">API Keys</a></li>
|
||||
<li><a href="/usage">Usage</a></li>
|
||||
<li><a href="/logs">Logs</a></li>
|
||||
<li><a href="/batches">Batches</a></li>
|
||||
<li><a href="/storage">Storage</a></li>
|
||||
<li><a href="/plugins">Plugins</a></li>
|
||||
</ul>
|
||||
<button type="button">P Personal Organization</button>
|
||||
</aside>
|
||||
<main>
|
||||
<h1>Home</h1>
|
||||
<section aria-labelledby="get-started">
|
||||
<h2 id="get-started">Get started</h2>
|
||||
<p>
|
||||
Create an API key to start making requests. Manage keys from
|
||||
<a href="/api-keys">API Keys</a>.
|
||||
</p>
|
||||
</section>
|
||||
<section aria-labelledby="usage">
|
||||
<h2 id="usage">Usage this month</h2>
|
||||
<p>No usage recorded.</p>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"credentialType": "openAiApi",
|
||||
"hosts": ["platform.openai.com"],
|
||||
"secretPrefix": "sk-proj-",
|
||||
"routes": {
|
||||
"/api-keys": "console.html",
|
||||
"/home": "dashboard.html"
|
||||
},
|
||||
"defaultRoute": "/home",
|
||||
"verify": {
|
||||
"path": "/models",
|
||||
"header": "Authorization",
|
||||
"scheme": "Bearer"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike Slack app console (Your Apps + create-from-manifest). Our own generic
|
||||
content — no provider source, no recorded responses, no real tokens.
|
||||
Calibrated against a real captured accessibility tree (LangTracer thread
|
||||
eca9cd8c, 2026-08-17).
|
||||
|
||||
Load-bearing, all taken from the real page:
|
||||
- getting a credential is MULTI-PAGE: create an app here, then Install App,
|
||||
and only the install page reveals a token
|
||||
- the app id is MINTED on create and carried in every later URL, so the fixture
|
||||
routes on `/apps/:appId/...` rather than a hardcoded id
|
||||
- the manifest editor's focusable input sits UNDER the rendered surface, so a
|
||||
click on it times out while typing into it works. That is the real
|
||||
CodeMirror behaviour and the live reproduction for LangTracer case 599 —
|
||||
replacing it with a plain visible textarea deletes the trap
|
||||
- "Next" is ENABLED throughout. The real run assumed it was disabled and spent
|
||||
three browser_evaluate scripts on that theory; it never was (not NODE-5755)
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Slack API: Applications | Slack</title>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<a href="/">Slack API</a>
|
||||
<a href="/changelog">Changelog</a>
|
||||
<a href="/docs">Documentation</a>
|
||||
<a href="/tutorials">Tutorials</a>
|
||||
<a href="/apps">Your Apps</a>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<button id="create-new-app" type="button">Create New App</button>
|
||||
<h1>Your Apps</h1>
|
||||
<label for="filter-apps">Filter apps by name or workspace</label>
|
||||
<input id="filter-apps" name="filter-apps" type="text" />
|
||||
|
||||
<ul id="app-list">
|
||||
<li><a href="/apps/A0EVALOLD/general">eval workspace bot</a><span>Eval Workspace</span></li>
|
||||
<li><a href="/apps/A0EVALOL2/general">notes relay</a><span>Eval Workspace</span></li>
|
||||
</ul>
|
||||
|
||||
<h4>Your App Configuration Tokens</h4>
|
||||
<a href="/docs/tokens">Learn about tokens</a>
|
||||
<button type="button">Generate Token</button>
|
||||
<a href="/signin">Sign in to another workspace</a>
|
||||
|
||||
<!-- Chooser shown by "Create New App". -->
|
||||
<div id="choose-dialog" role="alertdialog" aria-labelledby="choose-title" hidden>
|
||||
<h1 id="choose-title">Create an app</h1>
|
||||
<button id="from-manifest" type="button">From a manifest</button>
|
||||
<button id="from-scratch" type="button">From scratch</button>
|
||||
<button id="choose-close" type="button">Close</button>
|
||||
</div>
|
||||
|
||||
<div id="manifest-dialog" role="alertdialog" aria-labelledby="manifest-title" hidden>
|
||||
<h1 id="manifest-title">Create from a manifest</h1>
|
||||
<p><strong>Manifest</strong></p>
|
||||
<p>
|
||||
This is your app’s manifest containing basic info, scopes, settings, and
|
||||
features.
|
||||
<a href="https://docs.slack.dev/app-manifests/configuring-apps-with-app-manifests/">Learn more about manifests</a>
|
||||
or <a href="https://docs.slack.dev/samples/">check out examples</a>.
|
||||
</p>
|
||||
<p id="manifest-error" hidden>We can’t translate a manifest with errors.</p>
|
||||
<p>
|
||||
Add scopes so your app can be installed when you create it.
|
||||
<a href="https://docs.slack.dev/reference/scopes/">Learn more about scopes</a>
|
||||
</p>
|
||||
|
||||
<div role="tablist" aria-label="Manifest format">
|
||||
<button role="tab" aria-selected="true" type="button">JSON</button>
|
||||
<button role="tab" aria-selected="false" type="button">YAML</button>
|
||||
</div>
|
||||
<div role="tabpanel" aria-label="JSON">
|
||||
<!-- The trap: the textarea is covered by the rendered surface, so a
|
||||
click never passes the hit-test while fill() still works. -->
|
||||
<div style="position: relative; height: 140px">
|
||||
<textarea
|
||||
id="manifest-input"
|
||||
style="position: absolute; top: 0; left: 0; width: 100%; height: 140px"
|
||||
></textarea>
|
||||
<div
|
||||
id="manifest-view"
|
||||
style="position: absolute; top: 0; left: 0; width: 100%; height: 140px; background: #fff"
|
||||
>
|
||||
<span>1</span> <span>{}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button id="manifest-cancel" type="button">Cancel</button>
|
||||
<button id="manifest-next" data-qa="new_app_modal_go" type="button">Next</button>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const chooser = document.getElementById('choose-dialog');
|
||||
const dialog = document.getElementById('manifest-dialog');
|
||||
const input = document.getElementById('manifest-input');
|
||||
const view = document.getElementById('manifest-view');
|
||||
const error = document.getElementById('manifest-error');
|
||||
|
||||
document.getElementById('create-new-app').addEventListener('click', () => {
|
||||
chooser.hidden = false;
|
||||
});
|
||||
document.getElementById('choose-close').addEventListener('click', () => {
|
||||
chooser.hidden = true;
|
||||
});
|
||||
document.getElementById('from-scratch').addEventListener('click', () => {
|
||||
chooser.hidden = true;
|
||||
dialog.hidden = false;
|
||||
});
|
||||
document.getElementById('from-manifest').addEventListener('click', () => {
|
||||
chooser.hidden = true;
|
||||
dialog.hidden = false;
|
||||
});
|
||||
|
||||
// Mirror typed text into the rendered surface, as the editor does.
|
||||
input.addEventListener('input', () => {
|
||||
const lines = input.value.split('\n');
|
||||
view.innerHTML = lines
|
||||
.map((line, i) => '<span>' + (i + 1) + '</span> <span>' + line.replace(/</g, '<') + '</span>')
|
||||
.join('<br />');
|
||||
});
|
||||
|
||||
document.getElementById('manifest-cancel').addEventListener('click', () => {
|
||||
dialog.hidden = true;
|
||||
});
|
||||
|
||||
// Enabled throughout, like the real one; it just refuses empty/invalid JSON.
|
||||
document.getElementById('manifest-next').addEventListener('click', () => {
|
||||
try {
|
||||
JSON.parse(input.value);
|
||||
} catch {
|
||||
error.hidden = false;
|
||||
return;
|
||||
}
|
||||
// The real console mints the id here and puts it in every later URL;
|
||||
// the agent then navigates by typing those URLs back.
|
||||
const appId = 'A0' + Math.random().toString(36).slice(2, 11).toUpperCase();
|
||||
window.location.href = '/apps/' + appId + '/general';
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike Slack app "Basic Information" page. Calibrated against a real
|
||||
captured tree (LangTracer thread eca9cd8c, 2026-08-17).
|
||||
|
||||
The credentials shown HERE (client id/secret) drive the OAuth2 path, which
|
||||
cannot complete hermetically: the authorize redirect and the token exchange
|
||||
both leave the browser. The reachable hermetic path is the bot token on
|
||||
Install App, so the sidebar link to it is the one that matters.
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Basic Information | n8n integration | Slack</title>
|
||||
</head>
|
||||
<body>
|
||||
<nav aria-label="App settings">
|
||||
<span>n8n integration</span>
|
||||
<!-- Only routed pages are linked. An unrouted link falls through to
|
||||
defaultRoute and serves the app list, which looks like a working page
|
||||
and is the trap `matchRoute` exists to avoid. -->
|
||||
<h4>Settings</h4>
|
||||
<ul>
|
||||
<li><a href="/apps/A0EVALAPP/general" aria-current="page">Basic Information</a></li>
|
||||
<li><a href="/apps/A0EVALAPP/install-on-team">Install App</a></li>
|
||||
</ul>
|
||||
<h4>Features</h4>
|
||||
<ul>
|
||||
<li><a href="/apps/A0EVALAPP/oauth">OAuth & Permissions</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main>
|
||||
<h1>Basic Information</h1>
|
||||
|
||||
<h2>App Credentials</h2>
|
||||
<p>These credentials allow your app to access the Slack API.</p>
|
||||
|
||||
<label for="client-id">Client ID</label>
|
||||
<input id="client-id" name="client-id" type="text" readonly value="553213193971.11823370532599" />
|
||||
|
||||
<label for="client-secret">Client Secret</label>
|
||||
<input id="client-secret" name="client-secret" type="password" readonly value="8f2c41a7d9b64e05ac13ff721b0d6e94" />
|
||||
<button type="button">Show</button>
|
||||
<button type="button">Regenerate</button>
|
||||
|
||||
<label for="signing-secret">Signing Secret</label>
|
||||
<input id="signing-secret" name="signing-secret" type="password" readonly value="4d61b0e7c2a94f83bb75e10c9f3a2d68" />
|
||||
<button type="button">Show</button>
|
||||
<button type="button">Regenerate</button>
|
||||
|
||||
<h2>Display Information</h2>
|
||||
<p>n8n integration</p>
|
||||
|
||||
<button type="button" disabled>Discard Changes</button>
|
||||
<button type="button" disabled>Save Changes</button>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
// Keep every in-app link on the id this page was reached with.
|
||||
const appId = window.location.pathname.split('/')[2];
|
||||
for (const a of document.querySelectorAll('a[href^="/apps/"]')) {
|
||||
a.setAttribute('href', a.getAttribute('href').replace(/^\/apps\/[^/]+/, '/apps/' + appId));
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
<!doctype html>
|
||||
<!--
|
||||
Lookalike Slack "Install App" page — the hermetic path to a credential.
|
||||
|
||||
The token only exists after Install to Workspace, and it is rendered in a
|
||||
field LABELLED as a token, which is what gets it redacted in the snapshot the
|
||||
agent reads (the same treatment the real console's Client Secret got). So the
|
||||
agent captures it through the marker, never by reading a raw value.
|
||||
-->
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Install App | n8n integration | Slack</title>
|
||||
</head>
|
||||
<body>
|
||||
<nav aria-label="App settings">
|
||||
<span>n8n integration</span>
|
||||
<h4>Settings</h4>
|
||||
<ul>
|
||||
<li><a href="/apps/A0EVALAPP/general">Basic Information</a></li>
|
||||
<li><a href="/apps/A0EVALAPP/install-on-team" aria-current="page">Install App</a></li>
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<main>
|
||||
<h1>OAuth Tokens for Your Workspace</h1>
|
||||
<p>
|
||||
These tokens were automatically generated when you installed the app to your
|
||||
team. You can use these to authenticate your app.
|
||||
</p>
|
||||
|
||||
<section id="before-install">
|
||||
<p>You must install this app to your workspace before a token is issued.</p>
|
||||
<h2>Bot Token Scopes</h2>
|
||||
<ul>
|
||||
<li>channels:read</li>
|
||||
<li>chat:write</li>
|
||||
<li>groups:read</li>
|
||||
<li>im:history</li>
|
||||
</ul>
|
||||
<button id="install" type="button">Install to Workspace</button>
|
||||
</section>
|
||||
|
||||
<section id="after-install" hidden>
|
||||
<h2>OAuth Tokens for Your Workspace</h2>
|
||||
<label for="bot-token">Bot User OAuth Token</label>
|
||||
<input id="bot-token" name="bot-token" type="text" readonly value="" />
|
||||
<button id="copy-token" type="button">Copy</button>
|
||||
<p>Installed to <strong>Eval Workspace</strong> by Eval Owner.</p>
|
||||
<button type="button">Reinstall to Workspace</button>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
|
||||
<script>
|
||||
// Keep every in-app link on the id this page was reached with.
|
||||
const appId = window.location.pathname.split('/')[2];
|
||||
for (const a of document.querySelectorAll('a[href^="/apps/"]')) {
|
||||
a.setAttribute('href', a.getAttribute('href').replace(/^\/apps\/[^/]+/, '/apps/' + appId));
|
||||
}
|
||||
|
||||
const before = document.getElementById('before-install');
|
||||
const after = document.getElementById('after-install');
|
||||
const token = document.getElementById('bot-token');
|
||||
|
||||
document.getElementById('install').addEventListener('click', async () => {
|
||||
// The real console routes through an authorize screen; hermetically the
|
||||
// grant is implicit, since the redirect would leave the fixture.
|
||||
const response = await fetch('/__fixture__/create-key', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ app: 'n8n integration', workspace: 'Eval Workspace' }),
|
||||
});
|
||||
const { key } = await response.json();
|
||||
token.value = key;
|
||||
before.hidden = true;
|
||||
after.hidden = false;
|
||||
});
|
||||
|
||||
document.getElementById('copy-token').addEventListener('click', async () => {
|
||||
await navigator.clipboard.writeText(token.value).catch(() => {});
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"credentialType": "slackApi",
|
||||
"hosts": ["api.slack.com"],
|
||||
"secretPrefix": "xoxb-",
|
||||
"routes": {
|
||||
"/apps": "apps.html",
|
||||
"/apps/:appId/general": "general.html",
|
||||
"/apps/:appId/install-on-team": "install.html",
|
||||
"/apps/:appId/oauth": "install.html"
|
||||
},
|
||||
"defaultRoute": "/apps"
|
||||
}
|
||||
|
|
@ -154,6 +154,86 @@ export function planRelayConnection(
|
|||
/** Hosts the extension already treats as local, so no DNS redirect is needed. */
|
||||
const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '[::1]', '::1']);
|
||||
|
||||
/**
|
||||
* Origins the BROWSER can reach n8n on — not necessarily the harness's own
|
||||
* `baseUrl`.
|
||||
*
|
||||
* `planRelayConnection` rewrites the relay's host to `localhost` whenever it
|
||||
* does not already match, so the browser can know n8n by a different spelling
|
||||
* than the harness does: `http://n8n:5678` in the split-container topology, and
|
||||
* `http://127.0.0.1:5678` on a laptop — cookies are host-scoped, and an IP host
|
||||
* is never sent to `localhost`. The `localhost` spelling is therefore always
|
||||
* included, keeping the protocol (a `secure` cookie is not sent over http).
|
||||
* A cookie on an origin the browser never visits is inert; missing the one it
|
||||
* does visit costs the session.
|
||||
*/
|
||||
export function browserN8nOrigins(baseUrl: string): string[] {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(baseUrl);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const localhost = `${url.protocol}//localhost${url.port ? `:${url.port}` : ''}`;
|
||||
return url.hostname === 'localhost' ? [url.origin] : [url.origin, localhost];
|
||||
}
|
||||
|
||||
/** The client's `n8n-auth` header as a cookie for one origin. Undefined when the
|
||||
* header is not a `name=value` pair. */
|
||||
export function browserSessionCookie(
|
||||
cookieHeader: string,
|
||||
url: string,
|
||||
): { name: string; value: string; url: string } | undefined {
|
||||
const eq = cookieHeader.indexOf('=');
|
||||
// First `=` only: the value is base64/JWT and carries its own padding.
|
||||
if (eq <= 0) return undefined;
|
||||
const value = cookieHeader.slice(eq + 1);
|
||||
return value ? { name: cookieHeader.slice(0, eq), value, url } : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Give the launched browser the n8n session the harness already holds.
|
||||
*
|
||||
* A launched browser has never seen n8n, but the credential-setup skill opens
|
||||
* n8n's OWN credential page first — for OAuth providers that is where the
|
||||
* redirect URL lives. Without a session it lands on /signin and stops, and
|
||||
* every expectation about the PROVIDER console then fails as though the agent
|
||||
* had misbehaved (NODE-5549). `attachToRunningBrowser` needs none of this: the
|
||||
* developer's browser is already signed in.
|
||||
*/
|
||||
async function signInBrowserToN8n(
|
||||
context: BrowserContext,
|
||||
client: N8nClient,
|
||||
logger: EvalLogger,
|
||||
): Promise<void> {
|
||||
let cookieHeader: string;
|
||||
try {
|
||||
cookieHeader = client.cookie;
|
||||
} catch {
|
||||
logger.warn(' Browser not signed in to n8n: the harness holds no session cookie');
|
||||
return;
|
||||
}
|
||||
const cookies = browserN8nOrigins(client.baseUrl)
|
||||
.map((origin) => browserSessionCookie(cookieHeader, origin))
|
||||
.filter((c): c is { name: string; value: string; url: string } => c !== undefined);
|
||||
if (!cookies.length) {
|
||||
logger.warn(' Browser not signed in to n8n: session cookie is not a name=value pair');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await context.addCookies(cookies);
|
||||
} catch (error: unknown) {
|
||||
// Warn-and-continue, per this helper's contract: the run still works, it
|
||||
// just starts on n8n's sign-in page. Throwing here would leak the browser
|
||||
// and its profile — `cleanup` is not armed until after this call.
|
||||
logger.warn(
|
||||
` Browser not signed in to n8n: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
logger.verbose(` Browser signed in to n8n (${cookies.map((c) => c.url).join(', ')})`);
|
||||
}
|
||||
|
||||
/** Loopback spellings the extension's own relay allowlist accepts. All of them
|
||||
* must escape the fixture's catch-all, not just the literal "localhost". */
|
||||
const LOOPBACK_EXCLUDES = ['localhost', '127.0.0.1', '[::1]'];
|
||||
|
|
@ -275,6 +355,10 @@ export async function startBrowserRuntime(
|
|||
args,
|
||||
});
|
||||
|
||||
// Before the connect page loads, so the very first navigation is already
|
||||
// authenticated. Inside this try, so a throw releases the relay session.
|
||||
await signInBrowserToN8n(context, client, logger);
|
||||
|
||||
const cleanup = async () => {
|
||||
await context.close().catch(() => {});
|
||||
// `finally`: rm can throw, and losing the instance-wide relay session to
|
||||
|
|
|
|||
|
|
@ -571,6 +571,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
|
|||
foreignCredentialIds,
|
||||
fixture: lane.fixture,
|
||||
verifyBaseUrl: lane.verifyBaseUrl,
|
||||
urlField: lane.credentialUrlField,
|
||||
local: lane.local,
|
||||
logger,
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -305,6 +305,8 @@ export async function probeCredentialValue(options: {
|
|||
/** Absent in local mode — there is no stand-in to have received anything. */
|
||||
fixture?: { verifyAttempts: number; verifiedOk: boolean };
|
||||
verifyBaseUrl?: string;
|
||||
/** Base-URL field from the fixture manifest's `urlField`. */
|
||||
urlField?: string;
|
||||
/** Local mode: test against the REAL provider API instead of a stand-in. */
|
||||
local?: boolean;
|
||||
logger: EvalLogger;
|
||||
|
|
@ -316,6 +318,7 @@ export async function probeCredentialValue(options: {
|
|||
foreignCredentialIds,
|
||||
fixture,
|
||||
verifyBaseUrl,
|
||||
urlField = 'url',
|
||||
local,
|
||||
logger,
|
||||
} = options;
|
||||
|
|
@ -349,6 +352,7 @@ export async function probeCredentialValue(options: {
|
|||
credentialId,
|
||||
fixture,
|
||||
verifyBaseUrl,
|
||||
urlField,
|
||||
local,
|
||||
logger,
|
||||
});
|
||||
|
|
@ -373,10 +377,11 @@ async function probeOneCredential(options: {
|
|||
credentialId: string;
|
||||
fixture?: { verifyAttempts: number; verifiedOk: boolean };
|
||||
verifyBaseUrl?: string;
|
||||
urlField: string;
|
||||
local?: boolean;
|
||||
logger: EvalLogger;
|
||||
}): Promise<CredentialValueProbe> {
|
||||
const { client, credentialId, fixture, verifyBaseUrl, local, logger } = options;
|
||||
const { client, credentialId, fixture, verifyBaseUrl, urlField, local, logger } = options;
|
||||
const attemptsBefore = fixture?.verifyAttempts ?? 0;
|
||||
try {
|
||||
const credential = await client.getCredentialForTest(credentialId);
|
||||
|
|
@ -385,9 +390,18 @@ async function probeOneCredential(options: {
|
|||
// written. In local mode the URL is left alone, so the test goes to the
|
||||
// real provider API — a pass there proves the key is genuine and active,
|
||||
// which is a stronger claim than equality with a synthetic string.
|
||||
//
|
||||
// `urlField` is per-provider (gemini calls it `host`). Do NOT require the
|
||||
// field to be present first: a Base URL with a default is not persisted
|
||||
// unless the user changed it, so real anthropic/openai credentials store
|
||||
// only `apiKey` — verified against live rows. Adding the field IS the
|
||||
// mechanism (n8n merges the submitted data over the stored credential), so
|
||||
// requiring it would discard the check on exactly the providers it works
|
||||
// for. A wrong `urlField` is caught at CI time instead, by the manifest ↔
|
||||
// `test.request` lockstep in `fixture-server.test.ts`.
|
||||
const result = await client.testCredential({
|
||||
...credential,
|
||||
data: local ? credential.data : { ...credential.data, url: verifyBaseUrl },
|
||||
data: local ? credential.data : { ...credential.data, [urlField]: verifyBaseUrl },
|
||||
});
|
||||
if (result.status === 'OK') return { kind: 'passed', target: local ? 'real' : 'stand-in' };
|
||||
|
||||
|
|
|
|||
|
|
@ -57,6 +57,8 @@ export interface CredentialSetupLane {
|
|||
/** Base URL for the provider-API stand-in, when this fixture ships one.
|
||||
* Undefined => the credential-value check reports itself unverifiable. */
|
||||
verifyBaseUrl?: string;
|
||||
/** Manifest `urlField`. Absent in local mode, where nothing is substituted. */
|
||||
credentialUrlField?: string;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
|
|
@ -162,6 +164,7 @@ export async function startCredentialSetupLane(
|
|||
browser,
|
||||
credentialType: fixture.manifest.credentialType,
|
||||
verifyBaseUrl: fixtureServer.verifyBaseUrl,
|
||||
credentialUrlField: fixture.manifest.urlField,
|
||||
close: async () => {
|
||||
// Browser first: it is the thing holding pages open against the fixture.
|
||||
await browser.close().catch(() => {});
|
||||
|
|
|
|||
|
|
@ -90,15 +90,36 @@ export const providerFixtureManifestSchema = z
|
|||
* HTTPS-with-a-self-signed-cert so it can impersonate a hostname, and n8n's
|
||||
* HTTP client would (correctly) reject that cert. Omit the block entirely
|
||||
* and the check reports itself unverifiable rather than failing. */
|
||||
// A UNION, not three optionals: the secret arrives in a header or a query
|
||||
// parameter, never both and never neither, and `scheme` is meaningless
|
||||
// without a header. Stating that structurally makes the nonsense
|
||||
// combinations unrepresentable instead of policed after the fact.
|
||||
verify: z
|
||||
.object({
|
||||
/** Path the credential's test request hits, e.g. `/v1/models`. */
|
||||
path: z.string().startsWith('/'),
|
||||
/** Header the credential type sends its secret in, e.g. `x-api-key`. */
|
||||
header: z.string().min(1),
|
||||
})
|
||||
.strict()
|
||||
.union([
|
||||
z
|
||||
.object({
|
||||
/** Path the credential's test request hits, e.g. `/v1/models`. */
|
||||
path: z.string().startsWith('/'),
|
||||
/** Header carrying the secret, e.g. `x-api-key`. */
|
||||
header: z.string().min(1),
|
||||
/** Prefix inside that header, e.g. `Bearer` — without it the stand-in
|
||||
* compares the whole header to the bare key and refuses it. */
|
||||
scheme: z.string().min(1).optional(),
|
||||
})
|
||||
.strict(),
|
||||
z
|
||||
.object({
|
||||
path: z.string().startsWith('/'),
|
||||
/** Query parameter carrying the secret, e.g. gemini's `key`. */
|
||||
query: z.string().min(1),
|
||||
})
|
||||
.strict(),
|
||||
])
|
||||
.optional(),
|
||||
/** Credential field holding the base URL, overwritten to aim the credential's
|
||||
* own test at the stand-in. `url` for anthropic/openai, `host` for gemini;
|
||||
* the wrong name leaves the test pointed at the real provider. */
|
||||
urlField: z.string().min(1).default('url'),
|
||||
})
|
||||
.strict()
|
||||
// Was a throw after the TLS cert had already been generated; as a refinement
|
||||
|
|
@ -202,6 +223,29 @@ export async function findFixtureForCredentialType(
|
|||
return fixtures.find((f) => f.manifest.credentialType === credentialType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a request path to one of the declared routes, honouring `:param`
|
||||
* segments (`/apps/:appId/general`).
|
||||
*
|
||||
* Consoles that create a resource put its id in every later URL, and the agent
|
||||
* navigates by typing those URLs, not only by clicking. Without parameters the
|
||||
* fixture would have to hardcode one id, and every other id would fall through
|
||||
* to `defaultRoute` — serving a plausible WRONG page (an app list where the app
|
||||
* settings should be) rather than an obvious dead end.
|
||||
*
|
||||
* Exact routes win over patterns, and a pattern matches one segment only, so a
|
||||
* console's URL space cannot collapse onto a single page.
|
||||
*/
|
||||
export function matchRoute(routes: string[], path: string): string | undefined {
|
||||
if (routes.includes(path)) return path;
|
||||
const segments = path.split('/');
|
||||
return routes.find((route) => {
|
||||
const routeSegments = route.split('/');
|
||||
if (routeSegments.length !== segments.length) return false;
|
||||
return routeSegments.every((s, i) => s.startsWith(':') || s === segments[i]);
|
||||
});
|
||||
}
|
||||
|
||||
/** Mint a synthetic secret carrying the provider's real prefix. Random, not
|
||||
* seeded: two runs must never share a secret, or a leak scan could pass by
|
||||
* matching the wrong run's value. */
|
||||
|
|
@ -279,6 +323,7 @@ export async function startFixtureServer(
|
|||
for (const [route, file] of Object.entries(manifest.routes)) {
|
||||
pages.set(route, await readFile(join(fixture.dir, file), 'utf8'));
|
||||
}
|
||||
const routeKeys = [...pages.keys()];
|
||||
// Non-null: the schema's refinement guarantees defaultRoute is a declared route.
|
||||
const defaultPage = pages.get(manifest.defaultRoute)!;
|
||||
|
||||
|
|
@ -300,7 +345,8 @@ export async function startFixtureServer(
|
|||
res.writeHead(204).end();
|
||||
return;
|
||||
}
|
||||
const body = pages.get(path) ?? defaultPage;
|
||||
const matched = matchRoute(routeKeys, path);
|
||||
const body = (matched && pages.get(matched)) ?? defaultPage;
|
||||
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
|
||||
res.end(body);
|
||||
});
|
||||
|
|
@ -322,9 +368,14 @@ export async function startFixtureServer(
|
|||
let verifiedOk = false;
|
||||
let verifyAttempts = 0;
|
||||
if (manifest.verify) {
|
||||
const { path: verifyPath, header } = manifest.verify;
|
||||
const verify = manifest.verify;
|
||||
const verifyPath = verify.path;
|
||||
const expected =
|
||||
'scheme' in verify && verify.scheme ? `${verify.scheme} ${mintedSecret}` : mintedSecret;
|
||||
const headerName = 'header' in verify ? verify.header.toLowerCase() : undefined;
|
||||
const queryName = 'query' in verify ? verify.query : undefined;
|
||||
verifyServer = createHttpServer((req, res) => {
|
||||
const path = (req.url ?? '/').split('?')[0];
|
||||
const [path, search] = (req.url ?? '/').split('?');
|
||||
if (path !== verifyPath) {
|
||||
events.push({ method: req.method ?? 'GET', host: 'verify', path });
|
||||
res.writeHead(404).end();
|
||||
|
|
@ -333,8 +384,10 @@ export async function startFixtureServer(
|
|||
// ONLY the minted key authenticates. That is the whole proof: a
|
||||
// truncated or re-typed key cannot pass, so a 200 means the stored
|
||||
// value is exactly what the page issued — without reading it back.
|
||||
const presented = req.headers[header.toLowerCase()];
|
||||
const ok = typeof presented === 'string' && presented === mintedSecret;
|
||||
const presented = headerName
|
||||
? req.headers[headerName]
|
||||
: new URLSearchParams(search ?? '').get(queryName!);
|
||||
const ok = typeof presented === 'string' && presented === expected;
|
||||
verifyAttempts += 1;
|
||||
if (ok) verifiedOk = true;
|
||||
events.push({ method: req.method ?? 'GET', host: 'verify', path, verifyOk: ok });
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user