diff --git a/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md b/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md index b6ac43523d2..bd4eb74e712 100644 --- a/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md +++ b/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md @@ -130,8 +130,10 @@ the cursor instead. n8n — it targets a long-lived container per slot, `restart: unless-stopped`, booted once for the whole sweep and never reset between cases. The nightly runs ~12 of them (runners x slots), so one n8n process serves dozens of cases - back to back. Isolation is per-case *user* (`run/lane-users.ts` - `provisionCaseBuildUser`), not per-case instance. Assume anything the backend + back to back. Isolation is per build, not per instance (`run/lane-users.ts`): + an MCP build acts as a fresh member (`provisionCaseBuildUser`), an + orchestrator build runs in a fresh member's personal project + (`provisionBuildProject`). Assume anything the backend holds in memory outlives the case that created it, and size it for a multi-hour process rather than one run. - **`eval-pr-comment.md`** is posted verbatim by CI. The comment uses an diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 16795a60f48..f3c651bbb90 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -856,6 +856,10 @@ A direction governs only what it covers; otherwise the proxy answers every quest - Edge cases — empty data, missing fields, single vs multiple items - Error scenarios only if the workflow is expected to handle them gracefully. Most agent-built workflows don't include error handling, so "the workflow crashes on invalid input" is a legitimate finding, not a test-case failure. +### Build isolation + +Each orchestrator build runs in its own empty project: the personal project of a freshly invited member, named like the lane owner. The owner still drives the build; the member only owns the project. An unlicensed instance cannot create team projects, so a new user is the only way to get a new project. Cleanup deletes the member, which also deletes its project and anything a timed-out build left there; under `--keep-workflows` the members stay, with their projects. Like the MCP lane, this needs a lane without SMTP, so invites return their accept token. + ### Credentials By default a build sees **no credentials**: the harness pins every build thread's credential view to the case's declared set (empty unless declared), so concurrent cases — and whatever happens to live on the instance — can never leak into a build. Every node mocks during verification. diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/agent-execution-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/agent-execution-seed.test.ts index 57a1a78e5d2..36d1539b7d6 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/agent-execution-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/agent-execution-seed.test.ts @@ -119,4 +119,30 @@ describe('executeAgentScenario', () => { expect(reseedScenarioTables).not.toHaveBeenCalled(); expect(calls).toEqual(['execute']); }); + + it('runs the agent in the project it was built in', async () => { + const client = clientRecording([]); + + await executeAgentScenario( + client, + 'agent-1', + scenario, + 'AGENT CONTEXT', + silentLogger, + 10_000, + undefined, + undefined, + undefined, + undefined, + 'build-project', + ); + + expect(client.executeAgentWithLlmMock).toHaveBeenCalledWith( + 'agent-1', + 'build-project', + scenario.dataSetup, + 10_000, + ); + expect(client.getPersonalProjectId).not.toHaveBeenCalled(); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-agent-handler.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-agent-handler.test.ts index bc543e92435..c6e85688b18 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-agent-handler.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-agent-handler.test.ts @@ -103,6 +103,23 @@ describe('agentHandler', () => { expect(rendered).not.toContain('abcdef1234567890'); }); + it('reads the agent from the project the ref names', async () => { + const getPersonalProjectId: Mock = vi.fn(); + const getAgentConfig: Mock = vi.fn().mockResolvedValue(validConfig); + const getAgentSkills: Mock = vi.fn().mockResolvedValue({}); + const client = { + getPersonalProjectId, + getAgentConfig, + getAgentSkills, + } as unknown as N8nClient; + + await agentHandler.fetch({ type: 'agent', id: 'agent-1', projectId: 'build-project' }, client); + + expect(getAgentConfig).toHaveBeenCalledWith('build-project', 'agent-1'); + expect(getAgentSkills).toHaveBeenCalledWith('build-project', 'agent-1'); + expect(getPersonalProjectId).not.toHaveBeenCalled(); + }); + it('keeps node-tool credential configs available to the judge and redacts them for export', async () => { const config = { ...validConfig, diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts index 799a5843ceb..c5199c396ac 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts @@ -132,6 +132,31 @@ describe('configEvalHandler', () => { expect(getDataTableRows).not.toHaveBeenCalled(); }); + it('fetch() reads the dataset table from the project the ref names', async () => { + const workflowId = 'wf-3'; + const dataTableId = 'dt-3'; + const getPersonalProjectId: Mock = vi.fn(); + const getDataTableColumns: Mock = vi.fn().mockResolvedValue([]); + const getDataTableRows: Mock = vi.fn().mockResolvedValue({ count: 0, data: [] }); + const client = { + getWorkflowEvaluationConfigs: vi + .fn() + .mockResolvedValue([dataTableConfig(workflowId, dataTableId)]), + getPersonalProjectId, + getDataTableColumns, + getDataTableRows, + } as unknown as N8nClient; + + await configEvalHandler.fetch( + { type: 'config-eval', id: workflowId, projectId: 'build-project' }, + client, + ); + + expect(getDataTableColumns).toHaveBeenCalledWith('build-project', dataTableId); + expect(getDataTableRows).toHaveBeenCalledWith('build-project', dataTableId); + expect(getPersonalProjectId).not.toHaveBeenCalled(); + }); + it('renderArtifact() surfaces config + dataset detail and never mentions the owning workflow', () => { const workflowId = 'wf-should-not-appear'; const configs = [dataTableConfig(workflowId, 'dt-1')]; diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-build-project.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-build-project.test.ts new file mode 100644 index 00000000000..bc6915a6009 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-build-project.test.ts @@ -0,0 +1,129 @@ +import { vi } from 'vitest'; + +import type { N8nClient } from '../clients/n8n-client'; +import { buildWorkflow } from '../harness/build-workflow'; +import type { EvalLogger } from '../harness/logger'; +import { extractOutcomeFromEvents } from '../outcome/event-parser'; + +// Only thread setup and result bookkeeping matter here; the loop and the proxy +// are stubbed as in build-workflow-observer-threshold. +vi.mock('../harness/chat-loop', () => ({ + SSE_SETTLE_DELAY_MS: 0, + startSseConnection: vi.fn().mockResolvedValue(undefined), + waitForAllActivity: vi.fn().mockResolvedValue(undefined), + runMultiTurnConversation: vi.fn().mockResolvedValue(undefined), + recordUserTurn: vi.fn(), +})); + +vi.mock('../utils/user-proxy', () => ({ + UserProxyLlm: class { + respondToConfirmation = vi.fn().mockResolvedValue({ approve: true }); + ingestEvents = vi.fn(); + decideFollowUp = vi.fn().mockResolvedValue({ kind: 'done' }); + getDecisionStats = vi.fn().mockReturnValue({}); + }, +})); + +vi.mock('../outcome/workflow-discovery', () => ({ + buildAgentOutcome: vi.fn().mockResolvedValue({ + workflowsCreated: [], + executionsRun: [], + dataTablesCreated: [], + finalText: 'done', + workflowJsons: [], + }), + extractWorkflowIdsFromMessages: vi.fn().mockReturnValue([]), +})); + +vi.mock('../outcome/event-parser', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, extractOutcomeFromEvents: vi.fn(actual.extractOutcomeFromEvents) }; +}); + +const silentLogger: EvalLogger = { + info: () => {}, + verbose: () => {}, + success: () => {}, + warn: () => {}, + error: () => {}, + isVerbose: false, +}; + +function makeClient(sendMessage = vi.fn().mockResolvedValue({ runId: 'run-1' })) { + const mocks = { + getPersonalProjectId: vi.fn().mockResolvedValue('owner-project'), + ensureThread: vi.fn().mockResolvedValue(undefined), + createCredential: vi.fn().mockResolvedValue({ id: 'cred-1' }), + setThreadCredentialAllowlist: vi.fn().mockResolvedValue(undefined), + sendMessage, + getThreadMessages: vi.fn().mockResolvedValue({ messages: [] }), + listWorkflows: vi.fn().mockResolvedValue([]), + }; + return { client: mocks as unknown as N8nClient, mocks }; +} + +const baseConfig = { + skipWorkflowChecks: true, + preRunWorkflowIds: new Set(), + claimedWorkflowIds: new Set(), + logger: silentLogger, + conversation: [{ role: 'user' as const, text: 'build a thing' }], + buildProject: vi.fn().mockResolvedValue({ userId: 'build-user', projectId: 'build-project' }), +}; + +describe('buildWorkflow build project', () => { + it("binds the thread to the build's own project and creates the case's credentials there", async () => { + const { client, mocks } = makeClient(); + + const result = await buildWorkflow({ + client, + ...baseConfig, + credentials: [{ type: 'slackApi' }], + }); + + expect(mocks.ensureThread.mock.calls[0]?.[1]).toBe('build-project'); + expect(mocks.createCredential.mock.calls[0]?.[4]).toBe('build-project'); + expect(mocks.getPersonalProjectId).not.toHaveBeenCalled(); + expect(result).toMatchObject({ buildProjectId: 'build-project', buildUserId: 'build-user' }); + }); + + it('keeps the build user on a failed build, so cleanup still removes it', async () => { + const { client } = makeClient(vi.fn().mockRejectedValue(new Error('chat send failed'))); + + const result = await buildWorkflow({ client, ...baseConfig }); + + expect(result.success).toBe(false); + expect(result).toMatchObject({ buildProjectId: 'build-project', buildUserId: 'build-user' }); + }); + + it('fails as a harness problem, before any thread exists, when no project can be provisioned', async () => { + const { client, mocks } = makeClient(); + + const result = await buildWorkflow({ + client, + ...baseConfig, + buildProject: vi.fn().mockRejectedValue(new Error('no accept token')), + }); + + expect(result.success).toBe(false); + expect(result.seedingFailed).toBe(true); + expect(mocks.ensureThread).not.toHaveBeenCalled(); + expect(mocks.sendMessage).not.toHaveBeenCalled(); + }); + + it("tags the artifact refs with the build's project, so later reads look there", async () => { + const actual = + await vi.importActual('../outcome/event-parser'); + vi.mocked(extractOutcomeFromEvents).mockImplementationOnce((events) => ({ + ...actual.extractOutcomeFromEvents(events), + artifactRefs: [{ type: 'agent', id: 'agent-1' }], + })); + const { client } = makeClient(); + + const result = await buildWorkflow({ client, ...baseConfig, workflowExpected: false }); + + expect(result.artifactRefs).toEqual([ + { type: 'agent', id: 'agent-1', projectId: 'build-project' }, + ]); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts index e86171e8f09..6e4f08cf80a 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-seed-cleanup.test.ts @@ -253,6 +253,7 @@ describe('buildWorkflow declared credentials', () => { 'httpHeaderAuth', {}, undefined, + undefined, ); expect(setThreadCredentialAllowlist).toHaveBeenCalledWith( expect.any(String), diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts index aad4396b675..ed88427d45f 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts @@ -92,6 +92,51 @@ describe('cleanupBuild', () => { expect(mocks.deleteThread).toHaveBeenCalledWith('T1'); }); + it("cleans the build's own project, then deletes the user that owns it", async () => { + const { client, mocks } = makeClient({ + deleteAgent: vi.fn().mockResolvedValue(undefined), + deleteUser: vi.fn().mockResolvedValue(undefined), + }); + const build = { + ...makeBuild(), + artifactRefs: [{ type: 'agent' as const, id: 'agent-1' }], + buildProjectId: 'build-project', + buildUserId: 'build-user', + }; + + await expect(cleanupBuild(client, build, silentLogger)).resolves.toBe(true); + + expect(mocks.deleteDataTable).toHaveBeenCalledWith('build-project', 'DT1'); + expect(mocks.deleteAgent).toHaveBeenCalledWith('build-project', 'agent-1'); + expect(mocks.getPersonalProjectId).not.toHaveBeenCalled(); + expect(mocks.deleteUser).toHaveBeenCalledWith('build-user'); + // The thread goes through its own route first, which also stops its sandbox. + expect(mocks.deleteUser.mock.invocationCallOrder[0]).toBeGreaterThan( + mocks.deleteThread.mock.invocationCallOrder[0], + ); + }); + + it('keeps the build user when another delete fails, so the retry still finds its project', async () => { + const { client, mocks } = makeClient({ + deleteDataTable: vi.fn().mockRejectedValue(new Error('HTTP 502')), + deleteUser: vi.fn().mockResolvedValue(undefined), + }); + const build = { ...makeBuild(), buildProjectId: 'build-project', buildUserId: 'build-user' }; + + await expect(cleanupBuild(client, build, silentLogger)).resolves.toBe(false); + + expect(mocks.deleteUser).not.toHaveBeenCalled(); + }); + + it('reports not clean when the build user cannot be deleted, so the caller can retry', async () => { + const { client } = makeClient({ + deleteUser: vi.fn().mockRejectedValue(new Error('HTTP 502')), + }); + const build = { ...makeBuild(), buildProjectId: 'build-project', buildUserId: 'build-user' }; + + await expect(cleanupBuild(client, build, silentLogger)).resolves.toBe(false); + }); + it('never calls deleteAgent for a build without an agent ref', async () => { const { client, mocks } = makeClient({ deleteAgent: vi.fn() }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-lanes.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-lanes.test.ts new file mode 100644 index 00000000000..ee1940950d2 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-lanes.test.ts @@ -0,0 +1,49 @@ +import { vi } from 'vitest'; + +import { silentLogger } from './fixtures'; +import type { N8nClient } from '../clients/n8n-client'; +import type { Lane } from '../run/build-orchestrator'; +import { cleanupLanes } from '../run/lane-setup'; +import { LaneUserPool } from '../run/lane-users'; + +function laneWithPool() { + const deleteUser = vi.fn().mockResolvedValue(undefined); + const inviteMembers = vi.fn( + async (emails: string[]) => + await Promise.resolve( + emails.map((email, i) => ({ id: `u${String(i + 1)}`, email, acceptToken: 'token' })), + ), + ); + const client = { deleteUser, inviteMembers } as unknown as N8nClient; + const buildUserPool = new LaneUserPool(client, 3); + const lane = { + client, + baseUrl: 'http://n8n.test', + preRunWorkflowIds: new Set(), + claimedWorkflowIds: new Set(), + createdCredentialIds: new Set(), + workflowIdsToDelete: new Set(), + buildUserPool, + } as Lane; + return { lane, buildUserPool, deleteUser }; +} + +describe('cleanupLanes', () => { + it('deletes every invited build user, claimed or not, when workflows are throwaway', async () => { + const { lane, buildUserPool, deleteUser } = laneWithPool(); + await buildUserPool.claim(); + + await cleanupLanes([lane], { workflows: false, buildUsers: true }, silentLogger); + + expect(deleteUser.mock.calls.map(([id]) => id as string)).toEqual(['u1', 'u2', 'u3']); + }); + + it('keeps the build users, and the workflows in their projects, under --keep-workflows', async () => { + const { lane, buildUserPool, deleteUser } = laneWithPool(); + await buildUserPool.claim(); + + await cleanupLanes([lane], { workflows: false, buildUsers: false }, silentLogger); + + expect(deleteUser).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/lane-users.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/lane-users.test.ts index 98dc4b448da..a84ea3a657e 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/lane-users.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/lane-users.test.ts @@ -1,6 +1,11 @@ import { silentLogger } from './fixtures'; import type { N8nClient } from '../clients/n8n-client'; -import { cleanupLaneUsers, LaneUserPool, provisionCaseBuildUser } from '../run/lane-users'; +import { + cleanupLaneUsers, + LaneUserPool, + provisionBuildProject, + provisionCaseBuildUser, +} from '../run/lane-users'; import type { TestCaseCredential } from '../types'; type InvitedUser = { id: string; email: string; acceptToken: string }; @@ -172,6 +177,31 @@ describe('provisionCaseBuildUser', () => { }); }); +describe('provisionBuildProject', () => { + it("accepts a pooled invite under the given name and returns that user's personal project", async () => { + const { client: ownerClient, inviteBatches } = fakeOwnerClient(); + const pool = new LaneUserPool(ownerClient, 1); + const member = { + acceptInvitation: vi.fn().mockResolvedValue(undefined), + getPersonalProjectId: vi.fn().mockResolvedValue('member-project'), + }; + + const project = await provisionBuildProject({ + pool, + memberClient: member as unknown as N8nClient, + name: { firstName: 'Eval', lastName: 'Bot' }, + }); + + expect(member.acceptInvitation).toHaveBeenCalledWith({ + token: `token-${inviteBatches[0][0]}`, + firstName: 'Eval', + lastName: 'Bot', + password: pool.password, + }); + expect(project).toEqual({ userId: 'u1', projectId: 'member-project' }); + }); +}); + describe('cleanupLaneUsers', () => { it('deletes every invited user best-effort, surviving individual failures', async () => { const { client: ownerClient } = fakeOwnerClient(); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts index 22cbca4ec20..e0fb43240f2 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts @@ -907,6 +907,7 @@ describe('UserProxyLlm.respondToConfirmation', () => { 'httpTemplatedCustomAuth', expect.objectContaining({ template: expect.stringContaining('Bearer') }), undefined, + undefined, ); // Filling the slot means the credential works, so its test is bypassed. expect(setThreadCredentialAllowlist).toHaveBeenCalledWith( @@ -956,6 +957,7 @@ describe('UserProxyLlm.respondToConfirmation', () => { 'slackApi', expect.any(Object), undefined, + undefined, ); // The allowlist call must include the pre-existing id, not just the new // one — setThreadCredentialAllowlist replaces the whole list. @@ -1137,6 +1139,42 @@ describe('UserProxyLlm.respondToConfirmation', () => { expect(proxy.getDecisionStats()['credential-test-bypassed']).toBeUndefined(); }); + it("workflows(action='setup'): creates the credential in the build's project", async () => { + const agent = new FakeAgent(); + agent.enqueue({ + action: 'apply_setup_wizard', + nodeParametersJson: '{}', + nodeCredentialsJson: JSON.stringify({ 'Post To Slack': { slackApi: 'new' } }), + }); + const { client, createCredential } = fakeCredentialClient('cred-fresh'); + const proxy = new UserProxyLlm({ + conversation: [ + { role: 'user', text: 'Post to Slack every morning.' }, + { role: 'user', text: '[Set up the Slack credential now.]' }, + ], + agent, + credentialCreation: { + client, + threadId: 'thread-1', + allowlistedCredentialIds: [], + projectId: 'build-project', + }, + }); + + await proxy.respondToConfirmation( + setupWizardEvent('req-sw-build-project', [ + { + nodeId: 'n1', + nodeName: 'Post To Slack', + credentialType: 'slackApi', + existingCredentials: [], + }, + ]), + ); + + expect(createCredential.mock.calls[0]?.[4]).toBe('build-project'); + }); + it("workflows(action='setup'): keeps the seeded credentials bypassed when it creates another", async () => { const agent = new FakeAgent(); agent.enqueue({ @@ -1527,6 +1565,7 @@ describe('UserProxyLlm.respondToConfirmation', () => { 'slackApi', expect.any(Object), undefined, + undefined, ); expect(setThreadCredentialAllowlist).toHaveBeenCalledWith('thread-1', ['cred-fresh'], []); expect(response.kind).toBe('credentialSelection'); diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index 05bf3ce235d..9dba8487e9c 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -179,7 +179,11 @@ async function main(): Promise { experimentName: args.experimentName, }); } finally { - await cleanupLanes(lanes, cleanupBuiltWorkflows, logger); + await cleanupLanes( + lanes, + { workflows: cleanupBuiltWorkflows, buildUsers: !args.keepWorkflows }, + logger, + ); } } diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 176b49733c1..44310eea582 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -275,6 +275,9 @@ export class N8nClient { * never logged in takes the strict path. */ private hasGlobalWorkflowDelete = false; + /** The logged-in user's name, when the login payload carries both parts. */ + userName?: { firstName: string; lastName: string }; + /** Public: the browser runtime needs to know where n8n ACTUALLY is, which is * not always what n8n reports as its own base URL (see `planRelayConnection`). */ constructor(readonly baseUrl: string) {} @@ -293,13 +296,22 @@ export class N8nClient { const result = (await this.fetch('/rest/login', { method: 'POST', body: { emailOrLdapLoginId: loginEmail, password: loginPassword }, - })) as { data?: { globalScopes?: string[]; isOwner?: boolean } }; + })) as { + data?: { + globalScopes?: string[]; + isOwner?: boolean; + firstName?: string | null; + lastName?: string | null; + }; + }; // `/rest/login` returns the public user with `withScopes: true`. `isOwner` // is the fallback for a payload that carries no scope list: an owner always // holds the scope, so the two agree wherever both are present. this.hasGlobalWorkflowDelete = result.data?.globalScopes?.includes('workflow:delete') ?? result.data?.isOwner ?? false; + const { firstName, lastName } = result.data ?? {}; + this.userName = firstName && lastName ? { firstName, lastName } : undefined; if (!this.sessionCookie) { throw new Error('Failed to authenticate with n8n — no session cookie received'); @@ -813,18 +825,25 @@ export class N8nClient { } /** - * Create a credential. - * POST /rest/credentials body: { name, type, data } + * Create a credential, in `projectId` when given (else the caller's personal project). + * POST /rest/credentials body: { name, type, data, projectId? } */ async createCredential( name: string, type: string, data: Record, description?: string | null, + projectId?: string, ): Promise<{ id: string }> { const result = (await this.fetch('/rest/credentials', { method: 'POST', - body: { name, type, data, ...(description !== undefined ? { description } : {}) }, + body: { + name, + type, + data, + ...(description !== undefined ? { description } : {}), + ...(projectId !== undefined ? { projectId } : {}), + }, })) as { data: { id: string } }; return { id: result.data.id }; } diff --git a/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts b/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts index bf10e55f21e..97ab3ec9b07 100644 --- a/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts +++ b/packages/@n8n/instance-ai/evaluations/credentials/__tests__/seeder.test.ts @@ -180,6 +180,7 @@ describe('credential descriptions', () => { 'httpTemplatedCustomAuth', expect.any(Object), 'Production reports', + undefined, ); }); }); diff --git a/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts b/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts index 3521d95ef83..15b141254b2 100644 --- a/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts +++ b/packages/@n8n/instance-ai/evaluations/credentials/seeder.ts @@ -165,6 +165,8 @@ export async function createOneCredential( description?: string | null; /** Seed with no field values, modelling a credential the user saved empty. */ blank?: boolean; + /** The build's project. Absent: the caller's personal project. */ + projectId?: string; }, ): Promise { if (credentialType === 'httpTemplatedCustomAuth') { @@ -194,6 +196,7 @@ export async function createOneCredential( credentialType, options?.blank ? {} : template.buildData(token), options?.description, + options?.projectId, ); return { id, name: resolvedName, type: credentialType }; } @@ -213,6 +216,7 @@ async function createTemplatedCustomAuthCredential( logger?: EvalLogger; setupHint?: InstanceAiCredentialSetupHint; description?: string | null; + projectId?: string; }, ): Promise { const hint = options?.setupHint; @@ -259,6 +263,7 @@ async function createTemplatedCustomAuthCredential( acceptedStatusCodes: hint.acceptedStatusCodes ? JSON.stringify(hint.acceptedStatusCodes) : '', }, options?.description, + options?.projectId, ); return { id, name: resolvedName, type: 'httpTemplatedCustomAuth' }; } @@ -286,6 +291,8 @@ export async function createDeclaredCredentials( onCreated?: (id: string) => void; logger?: EvalLogger; nameCounts?: Map; + /** The build's project. Absent: the caller's personal project. */ + projectId?: string; }, ): Promise { const logger = options?.logger; @@ -297,6 +304,7 @@ export async function createDeclaredCredentials( logger, ...(decl.description !== undefined ? { description: decl.description } : {}), ...(decl.blank ? { blank: true } : {}), + ...(options?.projectId !== undefined ? { projectId: options.projectId } : {}), }); options?.onCreated?.(cred.id); created.push(cred); diff --git a/packages/@n8n/instance-ai/evaluations/harness/agent-execution.ts b/packages/@n8n/instance-ai/evaluations/harness/agent-execution.ts index 3206c5cc15a..e2de35f4eed 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/agent-execution.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/agent-execution.ts @@ -170,6 +170,8 @@ export async function executeAgentScenario( buildTrace?: BuildTrace, outputDir?: string, seedContext?: ScenarioSeedContext, + /** The build's project. Absent: the client's personal project. */ + projectId?: string, ): Promise { if (seedContext) { await reseedScenarioTables( @@ -182,10 +184,10 @@ export async function executeAgentScenario( } const execStart = Date.now(); - const projectId = await client.getPersonalProjectId(); + const agentProjectId = projectId ?? (await client.getPersonalProjectId()); let evalResult = await client.executeAgentWithLlmMock( agentId, - projectId, + agentProjectId, scenario.dataSetup, timeoutMs, ); @@ -203,7 +205,7 @@ export async function executeAgentScenario( await delay(500 * attempt); evalResult = await client.executeAgentWithLlmMock( agentId, - projectId, + agentProjectId, scenario.dataSetup, timeoutMs, ); diff --git a/packages/@n8n/instance-ai/evaluations/harness/artifacts/agent-handler.ts b/packages/@n8n/instance-ai/evaluations/harness/artifacts/agent-handler.ts index 2d27a584a42..44049630079 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/artifacts/agent-handler.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/artifacts/agent-handler.ts @@ -17,8 +17,8 @@ export const agentHandler: ArtifactHandler = { return ctx.artifactRefs.filter((ref) => ref.type === 'agent'); }, async fetch(ref, client) { - // Agent routes are project-scoped; the harness builds in the user's personal project. - const projectId = await client.getPersonalProjectId(); + // Agent routes are project-scoped. + const projectId = ref.projectId ?? (await client.getPersonalProjectId()); const [config, skills] = await Promise.all([ client.getAgentConfig(projectId, ref.id), client.getAgentSkills(projectId, ref.id), diff --git a/packages/@n8n/instance-ai/evaluations/harness/artifacts/config-eval-handler.ts b/packages/@n8n/instance-ai/evaluations/harness/artifacts/config-eval-handler.ts index d7e51a8ac14..0a021b8928e 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/artifacts/config-eval-handler.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/artifacts/config-eval-handler.ts @@ -23,7 +23,7 @@ export const configEvalHandler: ArtifactHandler = { let dataTable: ConfigEvalArtifact['dataTable']; if (dtConfig?.datasetSource === 'data_table') { // re-check narrows the discriminated union -- .find() alone doesn't - const projectId = await client.getPersonalProjectId(); + const projectId = ref.projectId ?? (await client.getPersonalProjectId()); const dataTableId = dtConfig.datasetRef.dataTableId; const [columns, rows] = await Promise.all([ client.getDataTableColumns(projectId, dataTableId), diff --git a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts index 363125b5b78..3aef627aa29 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts @@ -163,6 +163,8 @@ interface MultiTurnDriverConfig { /** Shared with `createDeclaredCredentials`'s pre-run seeding — see * `CredentialCreationConfig.nameCounts`. */ credentialNameCounts?: Map; + /** The build's project, where a mid-run credential is created. */ + credentialProjectId?: string; /** Resource references sent with the FIRST message only — an attachment is a * hand-off, not something a user re-sends every turn. */ openingAttachments?: InstanceAiResourceAttachment[]; @@ -203,6 +205,7 @@ async function driveMultiTurnConversation( bypassCredentialTestIds: config.bypassCredentialTestIds, createdCredentialIds: config.createdCredentialIds, nameCounts: config.credentialNameCounts, + projectId: config.credentialProjectId, }, } : {}), @@ -286,6 +289,9 @@ export interface BuildResult { * a regression ever did let the agent write into one, an early delete would * destroy the workflow under grading and read as a build failure. */ createdProjectIds?: string[]; + /** The build's own project; `cleanupBuild` deletes the user that owns it. */ + buildProjectId?: string; + buildUserId?: string; /** The ROOT folders a seed created in the thread's project (a folder delete * cascades to its subfolders). Deleted in `cleanupBuild` after the workflows, * because a folder delete archives what it holds. */ @@ -499,6 +505,8 @@ export interface BuildWorkflowConfig { credentials?: TestCaseCredential[]; /** Run-level registry the created credential IDs are added to for cleanup. */ createdCredentialIds?: Set; + /** Gives the build a fresh project. Absent: the owner's personal project. */ + buildProject?: () => Promise<{ userId: string; projectId: string }>; /** History restored before the live message — carried in the case * (`mode: 'inline'`) or reconstructed from a trace (`mode: 'replay'`, which * also supplies the live turn). */ @@ -585,6 +593,12 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise + buildProject + ? { buildProjectId: buildProject.projectId, buildUserId: buildProject.userId } + : {}; /** The agent the seeded history last targeted — graded and executed first. */ let seedActiveAgentId: string | undefined; // Scenario seed tables are created empty before the build @@ -736,7 +750,17 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise config.createdCredentialIds?.add(id), logger, nameCounts: credentialNameCounts, + projectId: buildProject?.projectId, }); const seededCredentialIds = createdCredentials.map((c) => c.id); // `createDeclaredCredentials` returns one entry per `declaredCredentials`, in @@ -1163,6 +1188,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise id !== seedActiveAgentId)] : restoredAgentIds; - const artifactRefs: ArtifactRef[] = [ + const builtRefs: ArtifactRef[] = [ ...eventOutcome.artifactRefs, ...restoredAgentOrder .filter((id) => !seenAgentIds.has(id)) .map((id) => ({ type: 'agent' as const, id })), ]; + const buildProjectId = buildProject?.projectId; + const artifactRefs = buildProjectId + ? builtRefs.map((ref) => ({ ...ref, projectId: buildProjectId })) + : builtRefs; const buildTrace: BuildTrace = { finalText: eventOutcome.finalText.length > 0 ? eventOutcome.finalText : lastAgentText(transcript), @@ -1304,6 +1334,7 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise => { let ok = true; try { - const projectId = await client.getPersonalProjectId(); + const projectId = build.buildProjectId ?? (await client.getPersonalProjectId()); for (const id of ids) { try { await remove(projectId, id); @@ -239,6 +239,16 @@ export async function cleanupBuild( } } + // Last, and only once the rest is gone: the user's delete takes its project, so a + // retry of a failed delete needs the project to still exist. + if (build.buildUserId && clean) { + try { + await client.deleteUser(build.buildUserId); + } catch (error: unknown) { + if (!(error instanceof N8nApiError && error.status === 404)) clean = false; + } + } + return clean; } diff --git a/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts b/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts index 9aba21140c3..f626d57522b 100644 --- a/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts +++ b/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts @@ -96,9 +96,8 @@ export interface Lane { * per-lane because prebuilt/MCP-built workflows only exist on their own lane — * deleting them via another lane's client would 404. */ workflowIdsToDelete: Set; - /** Pool of invited member users for `--build-via-mcp` — each build claims one - * so its MCP credential/workflow view is isolated to that user. */ - mcpUserPool?: LaneUserPool; + /** Invited members, one per build: MCP builds act as it, orchestrator builds use its project. */ + buildUserPool?: LaneUserPool; } /** One `claude` build's Anthropic spend (`--build-via-mcp` only). Mirrors @@ -164,6 +163,7 @@ export interface LaneState { timeoutMs: number; testCaseName?: string; seedContext?: ScenarioSeedContext; + projectId?: string; }) => Promise>>; } @@ -211,14 +211,14 @@ async function buildWorkflowViaMcpOnLane(config: { createdWorkflowIds: [], createdDataTableIds: [], }); - if (!lane.mcpUserPool) { - return failure(`Lane ${lane.baseUrl} has no MCP build user pool — cannot build via MCP`); + if (!lane.buildUserPool) { + return failure(`Lane ${lane.baseUrl} has no build user pool — cannot build via MCP`); } let mcpApiKey: string; try { mcpApiKey = await provisionCaseBuildUser({ - pool: lane.mcpUserPool, + pool: lane.buildUserPool, memberClient: new N8nClient(lane.baseUrl), credentials, onCredentialCreated: (id) => lane.createdCredentialIds.add(id), diff --git a/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts b/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts index ae1c9402b60..f663419bacd 100644 --- a/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts +++ b/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts @@ -426,6 +426,7 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline { ), testCaseName: inputs.testCaseFile, seedContext, + projectId: agentRef.projectId, }); break; } catch (error: unknown) { diff --git a/packages/@n8n/instance-ai/evaluations/run/eval-session.ts b/packages/@n8n/instance-ai/evaluations/run/eval-session.ts index b11fbbbdb13..cbd79bdacca 100644 --- a/packages/@n8n/instance-ai/evaluations/run/eval-session.ts +++ b/packages/@n8n/instance-ai/evaluations/run/eval-session.ts @@ -24,7 +24,9 @@ import { } from './build-orchestrator'; import { createCasePipeline, type CasePipeline } from './case-pipeline'; import { LaneAllocator } from './lane-allocator'; +import { provisionBuildProject } from './lane-users'; import type { CliArgs } from '../cli/args'; +import { N8nClient } from '../clients/n8n-client'; import type { WorkflowTestCaseWithFile } from '../data/workflows'; import { executeAgentScenario, type AgentScenarioContext } from '../harness/agent-execution'; import { @@ -135,6 +137,16 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { const laneStates: LaneState[] = lanes.map((lane, idx) => { const laneNum = idx + 1; const laneTag = lanes.length > 1 ? ` [lane ${String(laneNum)}/${String(lanes.length)}]` : ''; + const { buildUserPool } = lane; + // Named like the owner, so the agent's project line reads as it does for a real user. + const buildProject = buildUserPool + ? async () => + await provisionBuildProject({ + pool: buildUserPool, + memberClient: new N8nClient(lane.baseUrl), + name: lane.client.userName ?? { firstName: 'Eval', lastName: 'Builder' }, + }) + : undefined; return { runner: lane, laneNum, @@ -160,6 +172,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { seed: buildArgs.seed, executionScenarios: buildArgs.executionScenarios, createdCredentialIds: lane.createdCredentialIds, + buildProject, timeoutMs: buildArgs.timeoutMs, preRunWorkflowIds: lane.preRunWorkflowIds, preRunDataTableIds: lane.preRunDataTableIds, @@ -212,6 +225,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { timeoutMs: number; testCaseName?: string; seedContext?: ScenarioSeedContext; + projectId?: string; }) => await executeAgentScenario( lane.client, @@ -224,6 +238,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { execArgs.buildTrace, args.outputDir, execArgs.seedContext, + execArgs.projectId, ), ), }; diff --git a/packages/@n8n/instance-ai/evaluations/run/lane-setup.ts b/packages/@n8n/instance-ai/evaluations/run/lane-setup.ts index 07506cd3569..abd407d085d 100644 --- a/packages/@n8n/instance-ai/evaluations/run/lane-setup.ts +++ b/packages/@n8n/instance-ai/evaluations/run/lane-setup.ts @@ -41,14 +41,12 @@ export async function setupLanes(args: CliArgs, logger: EvalLogger): Promise { await Promise.all( lanes.map(async (lane) => { - if (cleanupBuiltWorkflows && lane.workflowIdsToDelete.size > 0) { + if (cleanup.workflows && lane.workflowIdsToDelete.size > 0) { await cleanupPrebuiltWorkflows(lane.client, lane.workflowIdsToDelete, logger); } await cleanupCredentials(lane.client, [...lane.createdCredentialIds]).catch(() => {}); // Deleting a user deletes their remaining data, so keep the build // users when workflows are kept. - if (cleanupBuiltWorkflows && lane.mcpUserPool) { - await cleanupLaneUsers(lane.client, lane.mcpUserPool, logger); + if (cleanup.buildUsers && lane.buildUserPool) { + await cleanupLaneUsers(lane.client, lane.buildUserPool, logger); } }), ); diff --git a/packages/@n8n/instance-ai/evaluations/run/lane-users.ts b/packages/@n8n/instance-ai/evaluations/run/lane-users.ts index d6fc274d8f1..1ea88dcd1f6 100644 --- a/packages/@n8n/instance-ai/evaluations/run/lane-users.ts +++ b/packages/@n8n/instance-ai/evaluations/run/lane-users.ts @@ -1,10 +1,7 @@ // --------------------------------------------------------------------------- -// Per-case build users for `--build-via-mcp`. -// -// MCP credential/workflow visibility is user-scoped, so running each build as -// its own freshly-invited member gives it an isolated credential view holding -// exactly the case's declared credentials — the MCP analog of the -// orchestrator's per-thread credential pinning. +// Per-build users. `--build-via-mcp` builds act as a fresh member; orchestrator +// builds keep the owner but run in a fresh member's personal project, the only +// new project an unlicensed instance can create. // --------------------------------------------------------------------------- import { randomBytes } from 'crypto'; @@ -58,7 +55,7 @@ export class LaneUserPool { // One batched POST per chunk — /rest/invitations is IP-rate-limited in production. const emails = Array.from({ length: this.chunkSize }, () => { this.seq += 1; - return `eval-mcp-${this.nonce}-${String(this.seq)}@n8n-evals.invalid`; + return `eval-build-${this.nonce}-${String(this.seq)}@n8n-evals.invalid`; }); const invited = await this.ownerClient.inviteMembers(emails); // Record every shell before judging usability: n8n creates them all up @@ -108,6 +105,30 @@ export async function provisionCaseBuildUser(opts: { return mcpApiKey; } +export interface BuildProject { + userId: string; + projectId: string; +} + +/** Accept a pooled invite and return that user's personal project, so a build + * starts in a project no other build has touched. */ +export async function provisionBuildProject(opts: { + pool: LaneUserPool; + /** A fresh, logged-out client for the lane — accepting the invite logs it in. */ + memberClient: N8nClient; + /** Becomes the project name the agent sees. */ + name: { firstName: string; lastName: string }; +}): Promise { + const { pool, memberClient } = opts; + const user = await pool.claim(); + await memberClient.acceptInvitation({ + token: user.acceptToken, + ...opts.name, + password: pool.password, + }); + return { userId: user.id, projectId: await memberClient.getPersonalProjectId() }; +} + /** Best-effort deletion of the pool's users. Deleting a user also deletes * what's left in their personal project, so only call this when built * workflows are throwaway. */ @@ -129,5 +150,5 @@ export async function cleanupLaneUsers( // best-effort } } - logger.verbose(`Deleted ${String(deleted)}/${String(ids.length)} MCP build user(s)`); + logger.verbose(`Deleted ${String(deleted)}/${String(ids.length)} build user(s)`); } diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 73e7aea6e45..440f6effdbe 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -202,6 +202,8 @@ export type ArtifactType = (typeof ARTIFACT_TYPES)[number]; export interface ArtifactRef { type: ArtifactType; id: string; + /** The project the build ran in. Absent means the owner's personal project. */ + projectId?: string; } /** Structured agent preview. Capture redacts it; persistence validates and caps it. */ diff --git a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts index 502fb8063c3..b473ebf35db 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts @@ -51,6 +51,8 @@ export interface CredentialCreationConfig { * credentials with no way to tell which one an agent picked). Defaults to * a fresh, unshared `Map` if omitted. */ nameCounts?: Map; + /** The build's project. Absent: the client's personal project. */ + projectId?: string; } /** @@ -272,13 +274,13 @@ export class UserProxyLlm { // throw here means that invariant broke, not a normal runtime failure. throw new Error('createCredential invoked without a credentialCreation config'); } - const { client, threadId, createdCredentialIds } = this.credentialCreation; + const { client, threadId, createdCredentialIds, projectId } = this.credentialCreation; const created = await createOneCredential( client, credentialType, undefined, this.createdCredentialNameCounts, - { logger: this.logger, setupHint: options?.setupHint }, + { logger: this.logger, setupHint: options?.setupHint, projectId }, ); createdCredentialIds?.add(created.id); this.allowlistedCredentialIds = [...this.allowlistedCredentialIds, created.id];