diff --git a/.agents/skills/create-instance-ai-eval/SKILL.md b/.agents/skills/create-instance-ai-eval/SKILL.md index b76a1ef21c1..73d30f4b571 100644 --- a/.agents/skills/create-instance-ai-eval/SKILL.md +++ b/.agents/skills/create-instance-ai-eval/SKILL.md @@ -600,9 +600,15 @@ nodes get LLM-generated pin data). So: static data** (`removeItemsSeenInPreviousExecutions`, `$getWorkflowStaticData`) is **not** seedable, because static data starts empty every run, so such a scenario reds vacuously (it sees everything as "new"). To get a seedable - change-detection scenario, steer the build toward a Data Table; otherwise - accept the static-data red as a harness limit and carry the logic in + change-detection scenario, say "keep the state in a data table" in the prompt; + otherwise accept the static-data red as a harness limit and carry the logic in `outcomeExpectations`. Note the agent may *choose* static-data dedup on its own. +- **The agent's own runs are mocked too.** When the agent runs a seeded or built + workflow itself inside the eval thread, its HTTP goes through the same mock + layer; a seeded workflow reuses the case's last `seed.priorRuns` hint for it. + Internal faults are not steered by hints, so a seeded fault in a Code, Set, IF + or Merge node must really fail in n8n (see + [`case-shapes.md`](case-shapes.md#seeded-faults-must-really-fail)). - Don't assert exact counts that depend on mock generation ("exactly 7 posts"). Say "fewer than the original 10". diff --git a/.agents/skills/create-instance-ai-eval/case-shapes.md b/.agents/skills/create-instance-ai-eval/case-shapes.md index b7ddbd4bd5f..cd3596679ff 100644 --- a/.agents/skills/create-instance-ai-eval/case-shapes.md +++ b/.agents/skills/create-instance-ai-eval/case-shapes.md @@ -662,6 +662,29 @@ really had one — and the agent has to guess from prose that deliberately names nothing. It will list workflows and pick, or ask which one, and you will score a clarification failure the real user never hit. +#### Seeded faults must really fail + +A debug case stages a failure in the seed. Two layers are involved, and only one +of them listens to `seed.priorRuns[].hints`: + +- **Hints steer outbound mocks only.** The staged prior run and the agent's own + rerun of the workflow both go through the HTTP mock, and the last hint for the + workflow steers both, so "the API returns 500" reproduces on the rerun. Code, + Set, IF, Filter and Merge run for real; a hint cannot make them fail. +- **An internal fault must really fail in n8n.** A Code node that returns a plain + object does not fail: n8n wraps it as one item and the run continues. Real Code + failures: return an array of plain values, reference a node that does not exist + (`$('Missing')`), or throw. Check the node's own validation before you rely on + it. +- **An empty branch needs a Filter that keeps nothing.** A Split Out over an empty + array still emits a placeholder item, so "Merge waits for an empty branch" never + arms that way. +- **Walk the item flow before you assert counts.** A Slack node fed three items + posts three times; "exactly one message" then fails the faithful fix. Add an + Aggregate step to the seed or assert per-item behaviour. +- **Put the seeded fault early.** External calls before it are mocked on the + agent's run, but each one is generated data the case does not control. + #### Before you ship a seeded case — three checks 1. **The defect still bites.** A seed whose workflow isn't broken any more makes the diff --git a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts index b03956d42bc..3a164e6a839 100644 --- a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts +++ b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts @@ -2738,6 +2738,8 @@ const instanceAiEvalSeedWorkflowSchema = z.object({ /** The seed folder (`folders[].id`) the workflow is created in. Omit for the * project root. Must name a declared folder; see `findSeedFolderIssues`. */ parentFolderId: instanceAiEvalSeedArtifactIdSchema.optional(), + /** Steers the mock layer when the agent itself runs this workflow in the eval thread. */ + executionMockHints: z.string().min(1).max(2000).optional(), }); export type InstanceAiEvalSeedWorkflow = z.infer; diff --git a/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md b/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md index 86feb127343..e34e8878a1a 100644 --- a/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md +++ b/packages/@n8n/instance-ai/evaluations/ARCHITECTURE.md @@ -59,6 +59,23 @@ split them out of the old `runner.ts` monolith): - `harness/cleanup.ts` — `cleanupBuild`, per-case timeout policy, bounded concurrency, binary workflow checks and shared failure summaries. +## Execution and mocking paths + +Three paths execute a workflow during a case. All three end in the same +LLM mock (`packages/cli/src/modules/instance-ai/eval/mock-handler.ts`): + +``` +scenario execution harness → POST eval/execute-with-llm-mock (dataSetup as hints, + Phase-1 hint pass, pinned triggers/AI roots) +prior run (seed) harness → POST eval/execute-with-llm-mock (priorRuns[].hints) +agent-triggered run agent → executions[run|step] / execute-node + → adapter sees the thread's eval allowlist entry + → configureAdditionalData attaches the mock + (seed workflow: its last priorRuns hint, carried on + restore-thread as workflows[].executionMockHints; + otherwise generic realistic responses) +``` + ## Where to add things | You want to… | Touch exactly | diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index df101b51a32..b8b5644dc9d 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -39,6 +39,7 @@ Each run: - **Mocked nodes** — anything that makes HTTP requests (Gmail, Slack, Google Sheets, HTTP Request, Notion…). The request is intercepted before it leaves the process; an LLM generates the response. - **Pinned nodes** — nodes that don't go through the HTTP layer (triggers/webhooks, LangChain/AI nodes, database nodes). They receive LLM-generated pin data. - **Real nodes** — logic nodes (Code, Set, Merge, Filter, IF, Switch) execute on the mocked data. +- **Agent-triggered runs** — when the agent runs, step-runs or node-tests a workflow inside an eval thread, its HTTP goes through the same mock layer. A seeded workflow reuses the case's last `seed.priorRuns` hint for it; anything else gets the mock's generic realistic responses. The thread's credential allowlist entry is what marks it as an eval thread. ~95% of node types are covered. See [Known limitations](#known-limitations) for the gaps. @@ -859,6 +860,10 @@ requires the agent to read it. - **A run that never happens is not a staged failure.** If no execution record lands, the case is reported as a framework issue rather than scored — it would otherwise be graded against history the instance does not have. +- **The agent's own rerun sees the same failure.** The last `hints` for a workflow also + steer the mock when the agent runs that workflow itself during the live turn, so "run it + again" reproduces the staged state instead of a fresh success. Internal faults (Code, + Set, IF, Merge) are not steered by hints — they must really fail in n8n. - Runs execute sequentially in declared order, before the live turn, on a 120s budget — tighter than a scenario execution, which gets the case's build budget (900s by default). @@ -1216,6 +1221,7 @@ packages/cli/src/modules/instance-ai/eval/ ## Known limitations - **LangChain/AI nodes** — use their own SDKs, not the HTTP mock layer. They fail with credential errors; use pin data instead. +- **Agent-triggered runs** — mocked at the HTTP layer only. AI sub-nodes on those runs still call their vendor SDK with the placeholder credential, and no Phase-1 consistency pass runs, so data may not line up across nodes the way it does in a scenario execution. - **Binary / file nodes** — minimal-valid synthetic fixtures (PDF, PNG, JPEG, OGG/Opus, WAV, MP3, MP4, ZIP, plaintext) are generated per content type and round-trip correctly through `prepareBinaryData`. Image-content correctness and OOXML formats (docx/xlsx — currently mime-sniffed as `application/zip`) remain out of scope. See [Binary / file scenarios](#binary--file-scenarios) for the synthesis path. - **Streaming nodes** — mocks return complete responses, not streams. - **GraphQL APIs** — response shape depends on the query, not just the endpoint. Quality depends on the LLM knowing the API schema. diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/prior-runs.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/prior-runs.test.ts index c3e9bc76f5e..53c9feadbcf 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/prior-runs.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/prior-runs.test.ts @@ -4,7 +4,11 @@ import { mock } from 'vitest-mock-extended'; import type { N8nClient } from '../clients/n8n-client'; import { remapSeedArtifactIds, type ConversationSeed } from '../harness/conversation-seed'; import type { EvalLogger } from '../harness/logger'; -import { executePriorRuns, STAGING_TIMEOUT_MS } from '../harness/prior-runs'; +import { + executePriorRuns, + executionMockHintsByWorkflow, + STAGING_TIMEOUT_MS, +} from '../harness/prior-runs'; import { EvalTestCaseSchema } from '../harness/schema'; import { MAX_EXEC_ATTEMPTS } from '../harness/transient-error'; @@ -369,6 +373,30 @@ describe('executePriorRuns', () => { }); }); +describe('executionMockHintsByWorkflow', () => { + it('keys the last declared hint by the restored workflow id', () => { + const hints = executionMockHintsByWorkflow( + [ + { workflow: 'dS8xQ2mV6bTn4Kp1', hints: 'the run succeeds' }, + { workflow: 'dS8xQ2mV6bTn4Kp1', hints: 'the HTTP node returns 500' }, + ], + seeded, + ); + + expect(hints).toEqual(new Map([['wf-1', 'the HTTP node returns 500']])); + }); + + it('skips runs without hints and unknown workflows, and tolerates no prior runs', () => { + expect( + executionMockHintsByWorkflow( + [{ workflow: 'dS8xQ2mV6bTn4Kp1' }, { workflow: 'nOtDeClArEd12345', hints: 'x' }], + seeded, + ), + ).toEqual(new Map()); + expect(executionMockHintsByWorkflow(undefined, seeded)).toEqual(new Map()); + }); +}); + describe('priorRuns schema validation', () => { function caseWith(priorRuns: Array<{ workflow: string; hints?: string }>) { return { diff --git a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts index 597c684164a..72a0491aa34 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts @@ -51,7 +51,7 @@ import { import { loadProviderFixtures } from './fixture-server'; import { reconstructSeedFromThread } from './langsmith-seed'; import type { EvalLogger } from './logger'; -import { executePriorRuns } from './prior-runs'; +import { executePriorRuns, executionMockHintsByWorkflow } from './prior-runs'; import { redactSecretsInTextDeep } from './redact'; import type { CaseSeed } from './schema'; import { @@ -836,11 +836,21 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise 0 || remapped.agents.length > 0 || remapped.folders.length > 0; + // The agent's own run of a seeded workflow is mocked with the case's last + // prior-run hint for it, so a rerun reproduces the staged failure. + const mockHints = executionMockHintsByWorkflow( + config.seed?.mode === 'inline' ? config.seed.priorRuns : undefined, + seedWorkflowsBySeedId, + ); + const workflowsToRestore = remapped.workflows.map((workflow) => { + const executionMockHints = mockHints.get(workflow.id); + return executionMockHints ? { ...workflow, executionMockHints } : workflow; + }); const restoreResult = hasThreadScopedSeed ? await client.restoreThread( threadId, remapped.messages, - remapped.workflows, + workflowsToRestore, remapped.dataTables, remapped.agents, { folders: remapped.folders }, diff --git a/packages/@n8n/instance-ai/evaluations/harness/prior-runs.ts b/packages/@n8n/instance-ai/evaluations/harness/prior-runs.ts index a0253c3a4ca..5c7ebadaede 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/prior-runs.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/prior-runs.ts @@ -49,6 +49,22 @@ export interface PriorRunsOptions { sleep?: (ms: number) => Promise; } +/** + * The last declared hint per seeded workflow, keyed by the id the restore creates, so + * the agent's own run of the workflow is mocked the way the staged run was. + */ +export function executionMockHintsByWorkflow( + priorRuns: SeedPriorRun[] | undefined, + seedWorkflows: Map, +): Map { + const hints = new Map(); + for (const priorRun of priorRuns ?? []) { + const restored = seedWorkflows.get(priorRun.workflow); + if (restored && priorRun.hints) hints.set(restored.id, priorRun.hints); + } + return hints; +} + /** * Run seeded workflows BEFORE the graded turn, so the instance carries real execution * history the agent can look up. diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts index 027cefb167b..8a30de6aa9f 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts @@ -48,6 +48,10 @@ vi.mock('@n8n/ai-utilities', () => ({ searxngSearch: vi.fn(), })); +vi.mock('../eval/thread-run-mock', () => ({ + createEvalThreadRunConfigurer: vi.fn(() => vi.fn()), +})); + import type { PolicyCleared } from '@n8n/decorators'; import { Container } from '@n8n/di'; import { generateWorkflowCode, parseWorkflowCode } from '@n8n/workflow-sdk'; @@ -86,6 +90,8 @@ import type { NodeCatalogService } from '@/node-catalog'; import type { NodeTypes } from '@/node-types'; import { McpRegistryService } from '@/modules/mcp-registry/registry/mcp-registry.service'; +import { EvalThreadCredentialAllowlistService } from '../eval/thread-credential-allowlist.service'; +import { createEvalThreadRunConfigurer } from '../eval/thread-run-mock'; import { InstanceAiMcpRegistryService } from '../mcp'; import type { InstanceContextService } from '../instance-context.service'; @@ -4552,6 +4558,7 @@ function createRunAdapterForTests( threadId?: string; queueMode?: boolean; allowSendingParameterValues?: boolean; + evalThreadAllowlists?: EvalThreadCredentialAllowlistService; }, ) { const mockWorkflowFinderService = { @@ -4640,6 +4647,15 @@ function createRunAdapterForTests( createMockPolicyEnforcementService() as unknown as ConstructorParameters< typeof InstanceAiAdapterService >[35], + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + options?.evalThreadAllowlists, ); const adapter = service.createContext(mockUser, { threadId: options?.threadId }).executionService; @@ -4688,6 +4704,69 @@ describe('createExecutionAdapter run()', () => { expect(live).not.toHaveProperty('injectedTriggerNodeName'); }); + it('routes an eval thread run through the mock layer with the seeded workflow hints', async () => { + const evalThreadAllowlists = new EvalThreadCredentialAllowlistService(); + evalThreadAllowlists.set('thread-eval', []); + evalThreadAllowlists.setExecutionMockHints('thread-eval', { + 'wf-1': 'the HTTP node returns 500', + }); + const { adapter, mockWorkflowRunner } = createRunAdapterForTests( + { id: 'wf-1', nodes: [], connections: {} }, + { + threadId: 'thread-eval', + evalThreadAllowlists, + execution: makeExecution({ status: 'success' }), + }, + ); + + await adapter.run('wf-1'); + + expect(createEvalThreadRunConfigurer).toHaveBeenCalledWith( + 'the HTTP node returns 500', + expect.anything(), + ); + expect(mockWorkflowRunner.run.mock.calls[0][0].configureAdditionalData).toBe( + vi.mocked(createEvalThreadRunConfigurer).mock.results[0].value, + ); + }); + + it('leaves a run outside an eval thread unmocked', async () => { + const evalThreadAllowlists = new EvalThreadCredentialAllowlistService(); + evalThreadAllowlists.set('thread-eval', []); + const { adapter, mockWorkflowRunner } = createRunAdapterForTests( + { id: 'wf-1', nodes: [], connections: {} }, + { + threadId: 'thread-prod', + evalThreadAllowlists, + execution: makeExecution({ status: 'success' }), + }, + ); + + await adapter.run('wf-1'); + + expect(createEvalThreadRunConfigurer).not.toHaveBeenCalled(); + expect(mockWorkflowRunner.run.mock.calls[0][0]).not.toHaveProperty('configureAdditionalData'); + }); + + it('skips the mock in queue mode, where the hook cannot reach the worker', async () => { + const evalThreadAllowlists = new EvalThreadCredentialAllowlistService(); + evalThreadAllowlists.set('thread-eval', []); + const { adapter, mockWorkflowRunner } = createRunAdapterForTests( + { id: 'wf-1', nodes: [], connections: {} }, + { + threadId: 'thread-eval', + evalThreadAllowlists, + queueMode: true, + execution: makeExecution({ status: 'success' }), + }, + ); + + await adapter.run('wf-1'); + + expect(createEvalThreadRunConfigurer).not.toHaveBeenCalled(); + expect(mockWorkflowRunner.run.mock.calls[0][0]).not.toHaveProperty('configureAdditionalData'); + }); + it('reports workflow-pinned nodes on the run result', async () => { const { adapter } = createRunAdapterForTests( { @@ -5427,6 +5506,22 @@ describe('createExecutionAdapter runStep()', () => { return { ...harness, result, runData: harness.mockWorkflowRunner.run.mock.calls[0]?.[0] }; } + it('routes an eval thread step run through the mock layer', async () => { + const evalThreadAllowlists = new EvalThreadCredentialAllowlistService(); + evalThreadAllowlists.set('thread-eval', []); + + const { runData } = await runStepOn(chainWorkflow, 'Send', undefined, { + threadId: 'thread-eval', + evalThreadAllowlists, + }); + + // No hints for a workflow the case never staged: the mock runs in its generic mode. + expect(createEvalThreadRunConfigurer).toHaveBeenCalledWith(undefined, expect.anything()); + expect(runData.configureAdditionalData).toBe( + vi.mocked(createEvalThreadRunConfigurer).mock.results[0].value, + ); + }); + it('runs the chain up to the target when given no input', async () => { const { runData, result } = await runStepOn(chainWorkflow, 'Send'); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts index 8512b7e1db5..85b217430e4 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts @@ -1067,6 +1067,26 @@ describe('InstanceAiController', () => { expect(result).toMatchObject({ dataTableIds: ['dt-new'] }); }); + it('should keep the seed workflows execution mock hints on the thread', async () => { + memoryService.checkThreadOwnership.mockResolvedValue('owned'); + memoryService.getThreadProjectId.mockResolvedValue('project-1'); + memoryService.restoreThreadMessages.mockResolvedValue({ restored: 1 }); + evalThreadRestore.restoreDataTables.mockResolvedValue(new Map()); + + await controller.restoreEvalThread(req, res, { + ...payload, + workflows: [ + { ...seedWorkflow, executionMockHints: 'the HTTP node returns 500' }, + { ...seedWorkflow, id: 'wf-2' }, + ], + } as InstanceAiEvalRestoreThreadRequest); + + expect(evalCredentialAllowlists.getExecutionMockHints(THREAD_ID, 'wf-1')).toBe( + 'the HTTP node returns 500', + ); + expect(evalCredentialAllowlists.getExecutionMockHints(THREAD_ID, 'wf-2')).toBeUndefined(); + }); + it('should hand the pinned credential allowlist to the workflow restore', async () => { memoryService.checkThreadOwnership.mockResolvedValue('owned'); memoryService.getThreadProjectId.mockResolvedValue('project-1'); diff --git a/packages/cli/src/modules/instance-ai/eval/README.md b/packages/cli/src/modules/instance-ai/eval/README.md index 03b0cdb4981..42c9f8df0e2 100644 --- a/packages/cli/src/modules/instance-ai/eval/README.md +++ b/packages/cli/src/modules/instance-ai/eval/README.md @@ -54,6 +54,17 @@ The bypass is queried per test rather than snapshotted into the adapter: the harness registers ids mid-run, when the simulated user creates a credential on a card, long after the run's context was built. +## Agent-triggered runs in eval threads + +The allowlist entry also marks the thread as an eval thread. When the agent +runs, step-runs or node-tests a workflow from such a thread, the execution +adapter attaches `evalLlmMockHandler` and the `EvalMockedCredentialsHelper` +through `configureAdditionalData` (`thread-run-mock.ts`), so HTTP nodes are +mocked the way a scenario execution mocks them. A seeded workflow's hints +arrive on `restore-thread` as `workflows[].executionMockHints` and are kept per +thread and workflow id. Two deliberate gaps: no Phase-1 hint pass runs, and AI +roots are not routed to the wire server. + ## Agent model catalogs The eval credential allowlist also marks Agent Builder sessions as evaluations. diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/thread-credential-allowlist.service.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/thread-credential-allowlist.service.test.ts index c71404e821a..68bd1f36c48 100644 --- a/packages/cli/src/modules/instance-ai/eval/__tests__/thread-credential-allowlist.service.test.ts +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/thread-credential-allowlist.service.test.ts @@ -54,4 +54,31 @@ describe('EvalThreadCredentialAllowlistService', () => { expect(service.shouldBypassTest('thread-1', 'cred-a')).toBe(false); expect(service.get('thread-1')).toBeUndefined(); }); + + // An entry is what marks a thread as an eval thread; an empty allowlist still counts. + it('marks a thread as eval once an allowlist is set, even an empty one', () => { + service.set('thread-1', []); + + expect(service.isEvalThread('thread-1')).toBe(true); + expect(service.isEvalThread('thread-2')).toBe(false); + }); + + it('keeps execution mock hints per workflow and merges across restores', () => { + service.setExecutionMockHints('thread-1', { 'wf-1': 'the HTTP node returns 500' }); + service.setExecutionMockHints('thread-1', { 'wf-2': 'the API returns no rows' }); + + expect(service.getExecutionMockHints('thread-1', 'wf-1')).toBe('the HTTP node returns 500'); + expect(service.getExecutionMockHints('thread-1', 'wf-2')).toBe('the API returns no rows'); + expect(service.getExecutionMockHints('thread-1', 'wf-3')).toBeUndefined(); + expect(service.getExecutionMockHints('thread-2', 'wf-1')).toBeUndefined(); + }); + + it('forgets the execution mock hints when the thread is cleared', () => { + service.set('thread-1', []); + service.setExecutionMockHints('thread-1', { 'wf-1': 'the HTTP node returns 500' }); + service.clearThread('thread-1'); + + expect(service.isEvalThread('thread-1')).toBe(false); + expect(service.getExecutionMockHints('thread-1', 'wf-1')).toBeUndefined(); + }); }); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/thread-run-mock.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/thread-run-mock.test.ts new file mode 100644 index 00000000000..2a6bcf713a2 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/thread-run-mock.test.ts @@ -0,0 +1,46 @@ +import type { Logger } from '@n8n/backend-common'; +import type { ICredentialsHelper, IWorkflowExecuteAdditionalData } from 'n8n-workflow'; +import { mock } from 'vitest-mock-extended'; + +import { EvalMockedCredentialsHelper } from '../eval-mocked-credentials-helper'; +import { createLlmMockHandler } from '../mock-handler'; +import { createEvalThreadRunConfigurer } from '../thread-run-mock'; + +vi.mock('../mock-handler', () => ({ + createLlmMockHandler: vi.fn(() => vi.fn()), +})); + +describe('createEvalThreadRunConfigurer', () => { + beforeEach(() => { + vi.mocked(createLlmMockHandler).mockClear(); + }); + + it('routes the run through the HTTP mock steered by the given hints', async () => { + const additionalData = { + credentialsHelper: mock(), + } as unknown as IWorkflowExecuteAdditionalData; + + await createEvalThreadRunConfigurer( + 'the HTTP node returns 500', + mock(), + )(additionalData); + + expect(createLlmMockHandler).toHaveBeenCalledWith({ + scenarioHints: 'the HTTP node returns 500', + }); + expect(additionalData.evalLlmMockHandler).toBe( + vi.mocked(createLlmMockHandler).mock.results[0].value, + ); + }); + + it('synthesizes missing credentials the way a scenario execution does', async () => { + const additionalData = { + credentialsHelper: mock(), + } as unknown as IWorkflowExecuteAdditionalData; + + await createEvalThreadRunConfigurer(undefined, mock())(additionalData); + + expect(createLlmMockHandler).toHaveBeenCalledWith({ scenarioHints: undefined }); + expect(additionalData.credentialsHelper).toBeInstanceOf(EvalMockedCredentialsHelper); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/thread-credential-allowlist.service.ts b/packages/cli/src/modules/instance-ai/eval/thread-credential-allowlist.service.ts index 064f6420409..dc10de5c255 100644 --- a/packages/cli/src/modules/instance-ai/eval/thread-credential-allowlist.service.ts +++ b/packages/cli/src/modules/instance-ai/eval/thread-credential-allowlist.service.ts @@ -4,7 +4,8 @@ import { Service } from '@n8n/di'; * Per-thread credential visibility for evaluation runs. The eval harness * declares which credentials a build thread may see; the builder context's * credential `list()` is filtered to that set. Threads without an entry see - * the unfiltered instance listing. + * the unfiltered instance listing. An entry also marks the thread as an eval + * thread, so the agent's own workflow runs go through the mock layer. */ @Service() export class EvalThreadCredentialAllowlistService { @@ -13,6 +14,8 @@ export class EvalThreadCredentialAllowlistService { * without contacting the provider — see `bypassCredentialTest` on * `InstanceAiEvalCredentialAllowlistRequest`. */ private readonly testBypassByThread = new Map(); + /** Mock hints for the agent's own runs of seeded workflows, by workflow id. */ + private readonly executionMockHintsByThread = new Map>(); set(threadId: string, credentialIds: string[], bypassCredentialTest?: string[]): void { this.byThread.set(threadId, [...credentialIds]); @@ -30,6 +33,10 @@ export class EvalThreadCredentialAllowlistService { return this.byThread.get(threadId); } + isEvalThread(threadId: string): boolean { + return this.byThread.has(threadId); + } + /** * Whether this thread should treat `credentialId`'s connection test as passing. * Queried per test rather than snapshotted into the credential adapter: the @@ -41,8 +48,21 @@ export class EvalThreadCredentialAllowlistService { return this.testBypassByThread.get(threadId)?.includes(credentialId) ?? false; } + setExecutionMockHints(threadId: string, hintsByWorkflowId: Record): void { + const hints = this.executionMockHintsByThread.get(threadId) ?? new Map(); + for (const [workflowId, hint] of Object.entries(hintsByWorkflowId)) { + hints.set(workflowId, hint); + } + this.executionMockHintsByThread.set(threadId, hints); + } + + getExecutionMockHints(threadId: string, workflowId: string): string | undefined { + return this.executionMockHintsByThread.get(threadId)?.get(workflowId); + } + clearThread(threadId: string): void { this.byThread.delete(threadId); this.testBypassByThread.delete(threadId); + this.executionMockHintsByThread.delete(threadId); } } diff --git a/packages/cli/src/modules/instance-ai/eval/thread-run-mock.ts b/packages/cli/src/modules/instance-ai/eval/thread-run-mock.ts new file mode 100644 index 00000000000..26dad4a8d06 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/thread-run-mock.ts @@ -0,0 +1,30 @@ +import type { Logger } from '@n8n/backend-common'; +import type { IWorkflowExecutionDataProcess } from 'n8n-workflow'; + +import { EvalMockedCredentialsHelper } from './eval-mocked-credentials-helper'; +import { createLlmMockHandler } from './mock-handler'; + +type ConfigureAdditionalData = NonNullable< + IWorkflowExecutionDataProcess['configureAdditionalData'] +>; + +/** + * Mocks a run the agent starts itself inside an eval thread the way a scenario + * execution is mocked: HTTP goes to the LLM mock, missing credentials are + * synthesized. `hints` come from the case's prior runs of the workflow. + */ +export function createEvalThreadRunConfigurer( + hints: string | undefined, + logger: Logger, +): ConfigureAdditionalData { + // ponytail: no Phase-1 hint pass, so cross-node consistency rests on `hints`; add it if agent reruns drift. + const mockHandler = createLlmMockHandler({ scenarioHints: hints }); + return (additionalData) => { + additionalData.credentialsHelper = new EvalMockedCredentialsHelper( + additionalData.credentialsHelper, + undefined, + logger, + ); + additionalData.evalLlmMockHandler = mockHandler; + }; +} diff --git a/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts index 124b8fb258e..1179f4ee647 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts @@ -187,6 +187,8 @@ import { WorkflowHistoryService } from '@/workflows/workflow-history/workflow-hi import { WorkflowService } from '@/workflows/workflow.service'; import { EnterpriseWorkflowService } from '@/workflows/workflow.service.ee'; +import { EvalThreadCredentialAllowlistService } from './eval/thread-credential-allowlist.service'; +import { createEvalThreadRunConfigurer } from './eval/thread-run-mock'; import { extractResolvedNodeParameters } from './extract-resolved-node-parameters'; import { waitForInstanceAiExecution } from './instance-ai-execution-wait'; import { @@ -410,6 +412,7 @@ export class InstanceAiAdapterService { private readonly folderFinderService?: FolderFinderService, private readonly instanceContext?: InstanceContextService, private readonly executeNodeService?: ExecuteNodeService, + private readonly evalThreadAllowlists?: EvalThreadCredentialAllowlistService, ) { this.logger = logger.scoped('instance-ai'); this.allowSendingParameterValues = globalConfig.ai.allowSendingParameterValues; @@ -507,7 +510,7 @@ export class InstanceAiAdapterService { : {}), mcpService: mcpConnectionsAvailable ? this.createMcpAdapter(user) : undefined, executeNodeService: this.executeNodeService - ? this.createExecuteNodeAdapter(this.executeNodeService, user, projectId) + ? this.createExecuteNodeAdapter(this.executeNodeService, user, projectId, threadId) : undefined, conversationHistoryService: conversationHistory, // The tool and context block use the same instance gate result. @@ -713,12 +716,14 @@ export class InstanceAiAdapterService { executeNodeService: ExecuteNodeService, user: User, boundProjectId?: string, + threadId?: string, ): InstanceAiExecuteNodeService { const { resolveProjectId } = this.createProjectScopeHelpers(user, boundProjectId); return { execute: async (request) => { this.assertInstanceNotReadOnly('executions'); const projectId = await resolveProjectId(['workflow:execute']); + const configureAdditionalData = this.evalRunConfigurer(threadId); const result = await executeNodeService.run(user, { type: request.type, version: request.version, @@ -729,12 +734,31 @@ export class InstanceAiAdapterService { input: request.input as Array<{ json: IDataObject }> | undefined, timeoutMs: request.timeoutMs, projectId, + ...(configureAdditionalData ? { configureAdditionalData } : {}), }); return redactExecuteNodeResult(result, this.allowSendingParameterValues); }, }; } + /** + * Eval threads run seeded and built workflows against the mock layer, like + * scenario executions do. Production threads never have an allowlist entry. + */ + private evalRunConfigurer( + threadId: string | undefined, + workflowId?: string, + ): IWorkflowExecutionDataProcess['configureAdditionalData'] { + if (!threadId || !this.evalThreadAllowlists?.isEvalThread(threadId)) return undefined; + // The closure does not survive queue serialization (see EvalExecutionService). + if (this.globalConfig.executions?.mode === 'queue') return undefined; + const hints = + workflowId === undefined + ? undefined + : this.evalThreadAllowlists.getExecutionMockHints(threadId, workflowId); + return createEvalThreadRunConfigurer(hints, this.logger); + } + /** Reads the connection rows rather than resolving them into loadable servers: * resolving decrypts credentials per connection, and a row that fails to resolve * is still connected (one connection per user+slug). */ @@ -1810,6 +1834,7 @@ export class InstanceAiAdapterService { globalConfig, } = this; const assertNotReadOnly = () => this.assertInstanceNotReadOnly('executions'); + const evalRunConfigurer = (workflowId: string) => this.evalRunConfigurer(threadId, workflowId); const DEFAULT_TIMEOUT_MS = 5 * Time.minutes.toMilliseconds; const MAX_TIMEOUT_MS = 10 * Time.minutes.toMilliseconds; @@ -1950,6 +1975,8 @@ export class InstanceAiAdapterService { // it is dispatched by the execution lifecycle, not by a node. ...(options?.isVerificationRun ? { suppressErrorWorkflow: true } : {}), }; + const configureAdditionalData = evalRunConfigurer(workflowId); + if (configureAdditionalData) runData.configureAdditionalData = configureAdditionalData; const pinDataPlan = buildInstanceAiRunPinDataPlan({ workflowPinData: workflow.pinData ?? {}, @@ -2202,6 +2229,8 @@ export class InstanceAiAdapterService { dirtyNodeNames: plan.dirtyNodeNames, source: 'instance_ai', }; + const configureAdditionalData = evalRunConfigurer(workflowId); + if (configureAdditionalData) runData.configureAdditionalData = configureAdditionalData; // A trigger has no upstream to run, so the chain and the partial paths // are the same thing. Naming it keeps the engine from auto-detecting a diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index 1ed00a634aa..b5b1149a6fc 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -1223,6 +1223,16 @@ export class InstanceAiController { await this.evalThreadRestore.deleteFolders(folders, folderIdMap, projectId, req.user); throw error; } + // After the rollback-able block, so a failed restore leaves no hints behind. + const executionMockHints = Object.fromEntries( + workflows.flatMap( + (workflow): Array<[string, string]> => + workflow.executionMockHints ? [[workflow.id, workflow.executionMockHints]] : [], + ), + ); + if (Object.keys(executionMockHints).length > 0) { + this.evalCredentialAllowlists.setExecutionMockHints(payload.threadId, executionMockHints); + } return { ok: true, threadId: payload.threadId, diff --git a/packages/cli/src/node-execution/execute-node.service.ts b/packages/cli/src/node-execution/execute-node.service.ts index 514b52da2de..e7d98a7556e 100644 --- a/packages/cli/src/node-execution/execute-node.service.ts +++ b/packages/cli/src/node-execution/execute-node.service.ts @@ -13,6 +13,7 @@ import type { INodeExecutionData, INodeParameters, INodeType, + IWorkflowExecutionDataProcess, } from 'n8n-workflow'; import { v4 as uuid } from 'uuid'; @@ -42,6 +43,8 @@ export interface ExecuteNodeRequest { input?: Array<{ json: IDataObject }>; timeoutMs?: number; projectId: string; + /** Eval-only additionalData decoration (e.g. HTTP mock handler) — never set on production paths. */ + configureAdditionalData?: IWorkflowExecutionDataProcess['configureAdditionalData']; } export interface ExecuteNodeOutputItem { @@ -107,7 +110,14 @@ export class ExecuteNodeService { const workflow = await this.createTemporaryWorkflow(request.projectId, node, timeoutMs); try { - return await this.execute(user, workflow, node, inputItems, timeoutMs); + return await this.execute( + user, + workflow, + node, + inputItems, + timeoutMs, + request.configureAdditionalData, + ); } catch (error) { return { status: 'error', error: this.toErrorResponse(error) }; } finally { @@ -237,6 +247,7 @@ export class ExecuteNodeService { node: INode, inputItems: INodeExecutionData[], timeoutMs: number, + configureAdditionalData?: IWorkflowExecutionDataProcess['configureAdditionalData'], ): Promise { const executionData = createRunExecutionData({ startData: {}, @@ -257,6 +268,7 @@ export class ExecuteNodeService { workflowData, executionData, userId: user.id, + ...(configureAdditionalData ? { configureAdditionalData } : {}), }); const timedOut = await this.waitForCompletion(executionId, timeoutMs);