diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index 5396d475282..7aa9d9354fd 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -201,8 +201,7 @@ suite. Test cases are pulled at run time from the LangTracer suite keep `--source disk` for authoring). To re-run after pushing a fix, dispatch `ci-instance-ai-evals.yml` with the PR number (optionally `tier: full` for broader coverage) — results post back to -the PR. The lighter `test-evals-discovery.yml` still runs on every push as part -of `ci-pull-requests.yml`. +the PR. **`ci-instance-ai-evals.yml` is the PR gate; `test-evals-instance-ai.yml` is the lab bench.** The gate deliberately exposes only PR re-runs. Anything that diff --git a/.github/workflows/ci-pull-requests.yml b/.github/workflows/ci-pull-requests.yml index d93bd7aa947..e6dc7f6fff7 100644 --- a/.github/workflows/ci-pull-requests.yml +++ b/.github/workflows/ci-pull-requests.yml @@ -176,7 +176,6 @@ jobs: packages/@n8n/instance-ai/evaluations/** packages/cli/src/modules/instance-ai/** packages/core/src/execution-engine/eval-mock-helpers.ts - .github/workflows/test-evals-discovery.yml db: packages/cli/src/databases/** packages/cli/src/modules/*/database/** @@ -510,25 +509,6 @@ jobs: ref: ${{ needs.install-and-build.outputs.commit_sha }} secrets: inherit - # In-process discovery eval — asserts the orchestrator reaches for browser/computer-use - # tools at OAuth/screenshot moments. Lightweight (no Docker), runs in parallel with the - # heavy workflow eval. Non-blocking initially; promote to required after stability. - instance-ai-discovery-evals: - name: Instance AI Discovery Evals - needs: [install-and-build, format-check] - if: >- - !cancelled() && - needs.install-and-build.result == 'success' && - needs.format-check.result == 'success' && - needs.install-and-build.outputs.instance_ai_workflow_eval == 'true' && - github.repository == 'n8n-io/n8n' && - github.event_name != 'merge_group' && - (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) - uses: ./.github/workflows/test-evals-discovery.yml - with: - branch: ${{ needs.install-and-build.outputs.commit_sha }} - secrets: inherit - # This job is required by GitHub branch protection rules. # PRs cannot be merged unless this job passes. required-checks: diff --git a/.github/workflows/test-evals-discovery.yml b/.github/workflows/test-evals-discovery.yml deleted file mode 100644 index 3c97253a232..00000000000 --- a/.github/workflows/test-evals-discovery.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: 'Test: Instance AI Discovery Evals' -run-name: "Test: Instance AI Discovery Evals (branch=${{ inputs.branch || 'master' }}, filter=${{ inputs.filter || 'all' }}, trials=${{ inputs.trials }})" - -on: - workflow_call: - inputs: - branch: - description: 'GitHub branch to test' - required: false - type: string - default: 'master' - filter: - description: 'Filter scenarios by id (e.g. "slack-oauth")' - required: false - type: string - default: '' - trials: - description: 'Trials per scenario' - required: false - type: number - default: 3 - workflow_dispatch: - inputs: - branch: - description: 'GitHub branch to test' - required: false - default: 'master' - filter: - description: 'Filter scenarios by id (e.g. "slack-oauth")' - required: false - default: '' - trials: - description: 'Trials per scenario' - required: false - default: '3' - -jobs: - run-discovery-evals: - name: 'Run Discovery Evals' - runs-on: blacksmith-4vcpu-ubuntu-2204 - timeout-minutes: 15 - env: - NODE_OPTIONS: '--max-old-space-size=7168' - permissions: - contents: read - pull-requests: write - - steps: - - name: Checkout - uses: useblacksmith/checkout@bcec731f1eb1367240608d1c889a75b96db6ec53 # v1.5.0 - with: - ref: ${{ inputs.branch || github.ref }} - fetch-depth: 1 - - - name: Setup Environment - uses: ./.github/actions/setup-nodejs - - - name: Export Node Types - run: | - ./packages/cli/bin/n8n export:nodes --output ./packages/@n8n/ai-workflow-builder.ee/evaluations/nodes.json - - - name: Run Discovery Evals - id: eval - working-directory: packages/@n8n/instance-ai - env: - ANTHROPIC_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }} - N8N_ENABLED_MODULES: agents,instance-ai - LANGSMITH_TRACING: 'true' - LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }} - LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }} - LANGSMITH_REVISION_ID: ${{ github.sha }} - LANGSMITH_BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }} - FILTER: ${{ inputs.filter }} - TRIALS: ${{ inputs.trials || 3 }} - run: | - set -o pipefail - EVAL_ARGS=(--trials "$TRIALS" --fail-on-zero-pass) - if [ -n "$FILTER" ]; then - EVAL_ARGS+=(--filter "$FILTER") - fi - pnpm eval:discovery "${EVAL_ARGS[@]}" 2>&1 | tee discovery-eval-output.txt - - - name: Post eval results to PR - if: ${{ always() && github.event.pull_request.number && hashFiles('packages/@n8n/instance-ai/discovery-eval-output.txt') != '' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - EVAL_OUTCOME: ${{ steps.eval.outcome }} - HEAD_REF: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }} - COMMIT_SHA: ${{ github.sha }} - run: | - if [ "$EVAL_OUTCOME" = "success" ]; then - STATUS_ICON="✅" - else - STATUS_ICON="❌" - fi - { - echo "### Instance AI Discovery Eval ${STATUS_ICON}" - echo "" - echo "Branch: \`${HEAD_REF}\` · Commit: \`${COMMIT_SHA}\`" - echo "" - echo "
Eval output" - echo "" - echo '```' - cat packages/@n8n/instance-ai/discovery-eval-output.txt - echo '```' - echo "" - echo "
" - } > /tmp/discovery-comment.md - - COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ - --jq '.[] | select(.body | startswith("### Instance AI Discovery Eval")) | .id' | tail -1) - - if [ -n "$COMMENT_ID" ]; then - gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/discovery-comment.md - else - gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/discovery-comment.md - fi - - - name: Upload Results - if: ${{ always() }} - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: instance-ai-discovery-eval-results - path: packages/@n8n/instance-ai/discovery-eval-output.txt - retention-days: 14 diff --git a/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts b/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts index 10f53beebbf..a0e50285304 100644 --- a/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts +++ b/packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts @@ -34,7 +34,6 @@ export type PubSubEventName = | 'resume-agent-workflow-tool' | 'relay-chat-stream-event' | 'relay-instance-ai-event' - | 'relay-instance-ai-task-control' | 'relay-chat-human-message' | 'relay-chat-message-edit' | 'reload-sso-provisioning-configuration' diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 0a678f2a78f..df7997f76b8 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -9,7 +9,6 @@ Six harnesses live here: - **`eval:instance-ai`** — end-to-end build + mocked execution + LLM verification (drives a running n8n instance) - **`eval:agents`** — standalone Agent build cases from the LangTracer `agents` suite (author new ones in `data/agents/`) - **`eval:subagent`** — legacy command name for the workflow-build compatibility corpus; it drives the live orchestrator/skill build path, scored by binary checks -- **`eval:discovery`** — orchestrator in-process, scored against required or forbidden tool/dispatch events (no n8n server) - **`eval:pairwise`** — live orchestrator workflow builds, scored by an LLM judge panel against do/don't lists. Intended for head-to-head comparison with `ai-workflow-builder.ee` on the same dataset - **`eval:computer-use`** — grades the computer-use agent (file / OAuth / doc-reading tasks) against fixtures; see [`computer-use/README.md`](computer-use/README.md) @@ -20,7 +19,6 @@ Sections: - [Running e2e + workflow-build evals](#running-evals) - [Regression detection](#regression-detection) - [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows) -- [Running discovery evals](#discovery-evals) - [Running pairwise evals](#pairwise-evals) - [How the e2e harness works](#how-the-e2e-harness-works) - [How the workflow-build harness works](#how-the-workflow-build-harness-works) @@ -546,24 +544,6 @@ lanes flat-out at `lanes * 4` (each lane's per-lane build cap) starves it, surfacing as verifier/MCP timeouts. Treat `lanes * 4` as an aggressive upper bound for keys with rate-limit headroom, not the starting point. -## Discovery evals - -Discovery evals run the orchestrator in-process and assert first-hop tool or -background-agent routing from captured `tool-call`, `tool-result`, `tool-error`, and -`agent-spawned` events. Use them when a regression is about which path the -agent chooses, not whether a generated workflow executes. - -To inspect runtime skill loading, run a focused verbose pass: - -```bash -pnpm eval:discovery --filter data-table-skill-loading --trials 3 --verbose --fail-on-zero-pass -``` - -Verbose output lists each trial's completed tool calls with argument previews. -For data-table routing, look for `load_skill(skillId="data-table-manager")` -and `data-tables(action="list")`, and verify there are no planning, -workflow-builder, or spawned-agent entries in the spawned-agent section. - ## Pairwise evals Pairwise evals score a built workflow against the dataset's `dos` / `donts` diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts deleted file mode 100644 index bd443fefc42..00000000000 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Smoke + shape tests for the discovery scenario loader. - * - * Loads every JSON in evaluations/data/discovery/ off disk, parses it, and - * asserts the shape required by `runExpectedToolsInvokedCheck`. Catches - * malformed scenario files at unit-test time before they reach the eval - * runner. - */ - -import { discoveryTestCaseSchema, loadDiscoveryTestCasesWithFiles } from '../data/discovery'; -import { runExpectedToolsInvokedCheck } from '../discovery/expected-tools-invoked'; - -describe('loadDiscoveryTestCasesWithFiles', () => { - const cases = loadDiscoveryTestCasesWithFiles(); - - it('finds at least the v1 scenario set', () => { - const slugs = cases.map((c) => c.fileSlug).sort(); - - expect(slugs).toEqual( - expect.arrayContaining([ - 'slack-oauth-credential-setup', - 'google-oauth-credential-setup', - 'screenshot-dashboard', - 'http-node-config-no-browser', - 'oauth-with-computer-use-disabled', - 'workflow-builder-no-credential-ask', - 'folder-scoped-workflow-list', - ]), - ); - }); - - it.each([ - 'slack-oauth-credential-setup', - 'google-oauth-credential-setup', - 'screenshot-dashboard', - 'http-node-config-no-browser', - 'oauth-with-computer-use-disabled', - 'workflow-builder-no-credential-ask', - 'folder-scoped-workflow-list', - ])('%s parses with a valid expectedToolInvocations rule', (slug) => { - const entry = cases.find((c) => c.fileSlug === slug); - expect(entry).toBeDefined(); - - const { testCase } = entry!; - - expect(testCase.id).toBe(slug); - expect(testCase.userMessage.length).toBeGreaterThan(0); - - // Calling the check with an empty outcome must not throw on rule validation; - // it should simply report pass/fail. This pins the JSON shape. - expect(() => - runExpectedToolsInvokedCheck(testCase, { - workflowIds: [], - executionIds: [], - dataTableIds: [], - artifactRefs: [], - finalText: '', - toolCalls: [], - agentActivities: [], - }), - ).not.toThrow(); - }); - - it('keeps positive and negative scenarios in the set (so we test both directions)', () => { - const positive = cases.filter((c) => Boolean(c.testCase.expectedToolInvocations.anyOf?.length)); - const negative = cases.filter((c) => - Boolean(c.testCase.expectedToolInvocations.noneOf?.length), - ); - - expect(positive.length).toBeGreaterThan(0); - expect(negative.length).toBeGreaterThan(0); - }); -}); - -describe('discoveryTestCaseSchema', () => { - const valid = { - id: 'my-scenario', - userMessage: 'do the thing', - expectedToolInvocations: { anyOf: ['build-workflow'] }, - }; - - it('accepts a minimal valid case', () => { - expect(discoveryTestCaseSchema.safeParse(valid).success).toBe(true); - }); - - it('rejects a typo-d key instead of passing vacuously', () => { - const typo = { ...valid, expectedToolInvocation: { anyOf: ['x'] } }; - expect(discoveryTestCaseSchema.safeParse(typo).success).toBe(false); - }); - - it('rejects empty expectations — a case must assert something', () => { - const empty = { ...valid, expectedToolInvocations: {} }; - expect(discoveryTestCaseSchema.safeParse(empty).success).toBe(false); - }); - - it('rejects an empty expectation list — dead config must fail at load time', () => { - const emptyList = { ...valid, expectedToolInvocations: { allOfToolCalls: [] } }; - expect(discoveryTestCaseSchema.safeParse(emptyList).success).toBe(false); - }); - - it.each([ - [ - 'connected with tool categories', - { status: 'connected', toolCategories: ['screenshot'] }, - true, - ], - ['connected without tool categories', { status: 'connected' }, false], - ['disconnected', { status: 'disconnected' }, true], - ['disabledByUser', { status: 'disabledByUser' }, true], - ['unavailable', { status: 'unavailable' }, true], - ['an unknown status', { status: 'on-fire' }, false], - ])('validates each computerUse channel strictly: %s', (_name, channel, ok) => { - const withChannel = { - ...valid, - instanceState: { - computerUse: { localComputer: channel, browser: { status: 'unavailable' } }, - }, - }; - expect(discoveryTestCaseSchema.safeParse(withChannel).success).toBe(ok); - }); - - it('requires both channels — a half-filled state is dead config', () => { - const halfFilled = { - ...valid, - instanceState: { computerUse: { localComputer: { status: 'disconnected' } } }, - }; - expect(discoveryTestCaseSchema.safeParse(halfFilled).success).toBe(false); - }); - - it('rejects the retired localGateway key instead of passing vacuously', () => { - const retired = { ...valid, instanceState: { localGateway: { status: 'disconnected' } } }; - expect(discoveryTestCaseSchema.safeParse(retired).success).toBe(false); - }); -}); diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/config-evals-skill-loading.json b/packages/@n8n/instance-ai/evaluations/data/discovery/config-evals-skill-loading.json deleted file mode 100644 index c1a2ffa17e0..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/config-evals-skill-loading.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "id": "config-evals-skill-loading", - "userMessage": "Set up an evaluation on my existing workflow that scores the AI Agent's answers against a test dataset. Use a config-based evaluation — do not add any evaluation nodes to the canvas.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["config-evals"] - } - ], - "noneOf": ["evals", "spawn_sub_agent:eval-setup", "eval-setup-with-agent"] - }, - "rationale": "Regression coverage for runtime skill loading of the config-evals skill. Config-based evaluation work must load the config-evals skill, and must never wire on-canvas evaluation nodes via the node-based evals tool or the eval-setup sub-agent." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-natural-list-skill-loading.json b/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-natural-list-skill-loading.json deleted file mode 100644 index 646a10bf95d..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-natural-list-skill-loading.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id": "data-table-natural-list-skill-loading", - "userMessage": "What data tables do I have?", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["data-table-manager"] - }, - { - "toolName": "data-tables", - "argsContainAny": ["list"] - } - ], - "noneOf": ["create-tasks", "build-workflow", "spawn_sub_agent:workflow-builder"] - }, - "rationale": "Regression coverage for natural standalone data-table requests. A plain request to list existing Data Tables should load the data-table-manager skill and use the direct data-tables list action, without routing through planning, workflow-building, or background agents." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-skill-loading.json b/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-skill-loading.json deleted file mode 100644 index 4c767feed20..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-skill-loading.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id": "data-table-skill-loading", - "userMessage": "Use the data-table-manager skill to list my n8n Data Tables and summarize what tables are available. This is standalone data-table work; do not build or modify a workflow.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["data-table-manager"] - }, - { - "toolName": "data-tables", - "argsContainAny": ["list"] - } - ], - "noneOf": ["create-tasks", "build-workflow", "spawn_sub_agent:workflow-builder"] - }, - "rationale": "Regression coverage for runtime skill loading. Standalone data-table work must load the data-table-manager skill and call data-tables directly, without routing through planning, workflow-building, or task creation." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-workflow-skill-loading.json b/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-workflow-skill-loading.json deleted file mode 100644 index ff671d1d5b9..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/data-table-workflow-skill-loading.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "data-table-workflow-skill-loading", - "userMessage": "Create me a workflow that implements an n8n form to capture responses into a data table. The form should be a decklist submission form for a MTG tournament, asking for player name and a deck list as text fields they'll fill.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["data-table-manager"] - }, - { - "toolName": "load_skill", - "argsContainAny": ["workflow-builder"] - }, - { - "toolName": "build-workflow" - } - ], - "noneOf": ["create-tasks"] - }, - "rationale": "Regression coverage for single-workflow build prompts that depend on Data Tables. The orchestrator should load both the data-table-manager skill and the workflow-builder skill, include workflow-local table requirements in the build, and avoid task planning or delegation.", - "timeoutMs": 180000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/folder-scoped-workflow-list.json b/packages/@n8n/instance-ai/evaluations/data/discovery/folder-scoped-workflow-list.json deleted file mode 100644 index d1114fc7bfa..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/folder-scoped-workflow-list.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "folder-scoped-workflow-list", - "userMessage": "Which workflows are in the Clients/Acme folder? Just list them, do not change anything.", - "instanceState": { "folderExploration": true }, - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "workflows", - "args": { "action": "list" }, - "argsContainAny": ["folderPath"] - } - ], - "noneOf": ["build-workflow", "create-tasks", "spawn_sub_agent:workflow-builder"] - }, - "rationale": "When the user names a FOLDER, the agent must address it with folderPath on workflows(list) rather than guessing a name filter or dumping the inventory. Folder membership is stored, not encoded in workflow names; a query prefix returns exactly the wrong set. Guards the discoverability of the folder-exploration capability." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/google-oauth-credential-setup.json b/packages/@n8n/instance-ai/evaluations/data/discovery/google-oauth-credential-setup.json deleted file mode 100644 index fa2879372ed..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/google-oauth-credential-setup.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "id": "google-oauth-credential-setup", - "userMessage": "I need to set up a Google OAuth credential so I can read my calendar. Walk me through the Google Cloud Console so I can find my Client ID and Client Secret to enter into n8n.", - "instanceState": { - "computerUse": { - "localComputer": { "status": "disconnected" }, - "browser": { "status": "connected", "toolCategories": ["browser"] } - } - }, - "expectedToolInvocations": { - "anyOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["credential-setup-with-computer-use"] - }, - { "toolName": "browser_connect" }, - { "toolName": "browser_navigate" }, - { "toolName": "browser_snapshot" }, - { "toolName": "browser_screenshot" } - ] - }, - "rationale": "Google OAuth involves multiple console screens; the orchestrator must reach for the Computer Use credential skill or browser_* tools rather than asking the user to copy values into chat.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/http-node-config-no-browser.json b/packages/@n8n/instance-ai/evaluations/data/discovery/http-node-config-no-browser.json deleted file mode 100644 index 777cbb73338..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/http-node-config-no-browser.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "id": "http-node-config-no-browser", - "userMessage": "How do I set the request timeout on my HTTP Request node?", - "instanceState": { - "computerUse": { - "localComputer": { "status": "disconnected" }, - "browser": { "status": "connected", "toolCategories": ["browser"] } - } - }, - "expectedToolInvocations": { - "noneOf": ["browser_navigate", "browser_snapshot", "browser_screenshot"] - }, - "rationale": "Negative control. A pure node-config question must NOT trigger Computer Use browser tooling. Guards against an over-eager regression that would bias every credential-adjacent question into browser mode." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/index.ts b/packages/@n8n/instance-ai/evaluations/data/discovery/index.ts deleted file mode 100644 index f01d33a77fc..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/index.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { readFileSync, readdirSync } from 'fs'; -import { basename, join } from 'path'; -import { z } from 'zod'; - -import type { ComputerUseChannelState, ComputerUseState } from '../../../src/types'; -import type { DiscoveryMcpState } from '../../discovery/stub-mcp-registry'; -import type { DiscoveryConfirmations, DiscoveryTestCase } from '../../discovery/types'; - -const forbiddenToolCallSchema = z - .object({ - toolName: z.string().min(1), - args: z - .record(z.string(), z.unknown()) - .refine((pattern) => Object.keys(pattern).length > 0, { message: 'args must not be empty' }) - .optional(), - argsContainAny: z.array(z.string().min(1)).min(1).optional(), - declined: z.boolean().optional(), - }) - .strict(); - -/** Mirrors `ComputerUseState` (src/types.ts) — the annotation makes tsc flag - * these schemas when the source union drifts. */ -const computerUseChannelStateSchema: z.ZodType = z.discriminatedUnion( - 'status', - [ - z.object({ status: z.literal('unavailable') }).strict(), - z.object({ status: z.literal('disconnected') }).strict(), - z.object({ status: z.literal('disabledByUser') }).strict(), - z.object({ status: z.literal('connected'), toolCategories: z.array(z.string()) }).strict(), - ], -); - -const computerUseStateSchema: z.ZodType = z - .object({ - localComputer: computerUseChannelStateSchema, - browser: computerUseChannelStateSchema, - }) - .strict(); - -const mcpStateSchema: z.ZodType = z - .object({ - registry: z.array(z.string().min(1)).optional(), - connected: z - .array( - z - .object({ slug: z.string().min(1), tools: z.array(z.string().min(1)).optional() }) - .strict(), - ) - .optional(), - }) - .strict(); - -const confirmationDecisionSchema = z.enum(['approve', 'deny']); - -/** An empty map is dead config: the default (approve everything) already applies. */ -const confirmationsSchema: z.ZodType = z - .record( - z.string().min(1), - z.union([ - confirmationDecisionSchema, - z - .object({ - decision: confirmationDecisionSchema, - resumeWith: z - .record(z.string(), z.unknown()) - .refine((fields) => Object.keys(fields).length > 0, { - message: 'resumeWith must not be empty', - }) - .refine((fields) => !('approved' in fields), { - message: 'resumeWith must not set `approved` — use `decision` instead', - }) - .optional(), - }) - .strict(), - ]), - ) - .refine((entries) => Object.keys(entries).length > 0, { - message: 'confirmations must not be empty', - }); - -/** Strict authoring schema for discovery cases — a typo'd key or an empty - * expectation must fail at load time, not pass vacuously at run time (the - * workflow-case loader has had this guarantee for a while; discovery cases - * were a blind JSON.parse cast until TRUST-261's cleanup). */ -export const discoveryTestCaseSchema = z - .object({ - id: z.string().min(1), - userMessage: z.string().min(1), - instanceState: z - .object({ - computerUse: computerUseStateSchema.optional(), - mcp: mcpStateSchema.optional(), - folderExploration: z.boolean().optional(), - }) - .strict() - .optional(), - confirmations: confirmationsSchema.optional(), - expectedToolInvocations: z - .object({ - // min(1) on every list: an empty expectation array is dead config that - // would otherwise pass here and only surface as a run-time failure. - anyOf: z.array(z.string().min(1)).min(1).optional(), - noneOf: z.array(z.string().min(1)).min(1).optional(), - anyOfToolCalls: z.array(forbiddenToolCallSchema).min(1).optional(), - allOfToolCalls: z.array(forbiddenToolCallSchema).min(1).optional(), - noneOfToolCalls: z.array(forbiddenToolCallSchema).min(1).optional(), - }) - .strict() - .refine((expectations) => Object.values(expectations).some((v) => v !== undefined), { - message: 'expectedToolInvocations needs at least one expectation key', - }), - rationale: z.string().optional(), - maxSteps: z.number().int().positive().optional(), - timeoutMs: z.number().int().positive().optional(), - }) - .strict(); - -export interface DiscoveryTestCaseWithFile { - testCase: DiscoveryTestCase; - /** Filename without extension, e.g. "slack-oauth-credential-setup" */ - fileSlug: string; -} - -function parseTestCaseFile(filePath: string): DiscoveryTestCase { - const content = readFileSync(filePath, 'utf-8'); - let raw: unknown; - try { - raw = JSON.parse(content); - } catch (error) { - throw new Error( - `Failed to parse discovery test case ${filePath}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - const parsed = discoveryTestCaseSchema.safeParse(raw); - if (!parsed.success) { - const issues = parsed.error.issues - .map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`) - .join('; '); - throw new Error(`Invalid discovery test case ${filePath}: ${issues}`); - } - return parsed.data; -} - -function parseSubstringList(value: string | undefined): string[] { - if (!value) return []; - return value - .split(',') - .map((s) => s.trim().toLowerCase()) - .filter((s) => s.length > 0); -} - -function getJsonFiles(filter?: string, exclude?: string): string[] { - const dir = __dirname; - let files = readdirSync(dir).filter((f) => f.endsWith('.json')); - - const includeTokens = parseSubstringList(filter); - if (includeTokens.length > 0) { - files = files.filter((f) => { - const lower = f.toLowerCase(); - return includeTokens.some((t) => lower.includes(t)); - }); - } - - const excludeTokens = parseSubstringList(exclude); - if (excludeTokens.length > 0) { - files = files.filter((f) => { - const lower = f.toLowerCase(); - return !excludeTokens.some((t) => lower.includes(t)); - }); - } - - return files.map((f) => join(dir, f)); -} - -export function loadDiscoveryTestCasesWithFiles( - filter?: string, - exclude?: string, -): DiscoveryTestCaseWithFile[] { - return getJsonFiles(filter, exclude).map((f) => ({ - testCase: parseTestCaseFile(f), - fileSlug: basename(f, '.json'), - })); -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-ambiguous-alert-response-clarifies.json b/packages/@n8n/instance-ai/evaluations/data/discovery/intent-ambiguous-alert-response-clarifies.json deleted file mode 100644 index ad1006618cc..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-ambiguous-alert-response-clarifies.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "id": "intent-ambiguous-alert-response-clarifies", - "userMessage": "I need to ingest a cybersecurity alert, analyze it, and take actions depending on the finding.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["intent-recognition"] - }, - { - "toolName": "ask-user", - "argsContainAny": [ - "fixed rules", - "predefined rules", - "human judgment", - "AI judgment", - "case-by-case", - "decide which action" - ] - } - ] - }, - "rationale": "The request does not say whether fixed rules or judgment choose each action. The assistant must clarify this anchor-defining choice before it selects an artifact.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-creative-publisher.json b/packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-creative-publisher.json deleted file mode 100644 index 70c4450f9c0..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-creative-publisher.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "intent-explicit-agent-creative-publisher", - "userMessage": "Build me an agent that decides what faceless YouTube videos to make and drafts each script.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["agent-builder"] - }, - { - "toolName": "build-agent" - } - ], - "noneOfToolCalls": [ - { - "toolName": "build-workflow" - } - ] - }, - "rationale": "An explicit Agent request with open-ended topic selection and creative work must use the Agent build path. This trims publishing integrations from the sourced failure so the case tests routing only.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-fixed-duty.json b/packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-fixed-duty.json deleted file mode 100644 index d5c3fc7c1d7..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-fixed-duty.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "intent-explicit-agent-fixed-duty", - "userMessage": "Build me an agent that fetches new GitHub stars each night and sends me a digest.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["agent-builder"] - }, - { - "toolName": "build-agent" - } - ], - "noneOfToolCalls": [ - { - "toolName": "build-workflow" - } - ] - }, - "rationale": "The user explicitly selected an Agent artifact. A fixed scheduled duty can be an Agent task, so the assistant must not override that selection with a workflow.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-fixed-scheduled-market-summary.json b/packages/@n8n/instance-ai/evaluations/data/discovery/intent-fixed-scheduled-market-summary.json deleted file mode 100644 index 79a5f5f8812..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-fixed-scheduled-market-summary.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "intent-fixed-scheduled-market-summary", - "userMessage": "Every weekday at 8am, fetch the latest closing prices for the S&P 500 and Nasdaq, format them into a two-line summary, and email it to me.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["workflow-builder"] - }, - { - "toolName": "build-workflow" - } - ], - "noneOfToolCalls": [ - { - "toolName": "build-agent" - }, - { - "toolName": "load_skill", - "argsContainAny": ["agent-builder"] - } - ] - }, - "rationale": "This scheduled report has fixed inputs, formatting, and delivery. It guards against routing every recurring role-like task to an Agent.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-role-shaped-scheduled-analyst.json b/packages/@n8n/instance-ai/evaluations/data/discovery/intent-role-shaped-scheduled-analyst.json deleted file mode 100644 index 3fafc89dc53..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/intent-role-shaped-scheduled-analyst.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "intent-role-shaped-scheduled-analyst", - "userMessage": "Set up a daily market analyst. It should research market changes, decide which developments matter, and send me a briefing each morning.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["intent-recognition"] - }, - { - "toolName": "load_skill", - "argsContainAny": ["agent-builder"] - }, - { - "toolName": "build-agent" - } - ], - "noneOfToolCalls": [ - { - "toolName": "build-workflow" - } - ] - }, - "rationale": "A persistent delegated role owns an open-ended research and judgment duty. The schedule is an Agent task and does not justify a workflow shell.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-broken-connection-reconnect.json b/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-broken-connection-reconnect.json deleted file mode 100644 index 2f6c0c3f07b..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-broken-connection-reconnect.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id": "mcp-broken-connection-reconnect", - "userMessage": "Search my Notion for the Q3 roadmap page and tell me what's in it.", - "instanceState": { - "mcp": { - "registry": ["notion", "linear"], - "connected": [{ "slug": "notion", "tools": [] }] - } - }, - "expectedToolInvocations": { - "allOfToolCalls": [ - { "toolName": "mcp-servers", "args": { "action": "connect", "serverSlugs": ["notion"] } } - ], - "noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "search" } }] - }, - "rationale": "A connected server serving no tools is a broken connection: the agent must reconnect the slug it already has, not fall back to a registry search that filters connected servers out and dead-ends.", - "maxSteps": 12 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-connect-unconnected-service.json b/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-connect-unconnected-service.json deleted file mode 100644 index 61dc0e23e10..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-connect-unconnected-service.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "id": "mcp-connect-unconnected-service", - "userMessage": "Pull up the notes from my last product review in Notion and give me a short summary.", - "instanceState": { - "mcp": { "registry": ["notion", "linear"], "connected": [] } - }, - "expectedToolInvocations": { - "allOfToolCalls": [ - { "toolName": "mcp-servers", "args": { "action": "search" } }, - { "toolName": "mcp-servers", "args": { "action": "connect", "serverSlugs": ["notion"] } } - ], - "noneOfToolCalls": [ - { "toolName": "mcp-servers", "args": { "action": "connect", "serverSlugs": ["linear"] } } - ] - }, - "rationale": "Nothing is connected: the agent must find Notion in the registry and offer it, not declare it unreachable or recite manual setup steps. Linear is a decoy.", - "maxSteps": 12 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-declined-tool-call-no-reconnect.json b/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-declined-tool-call-no-reconnect.json deleted file mode 100644 index 3936154ad46..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-declined-tool-call-no-reconnect.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "id": "mcp-declined-tool-call-no-reconnect", - "userMessage": "Find the 'Launch checklist' page in my Notion and summarise what's still open.", - "instanceState": { - "mcp": { - "registry": ["notion", "linear"], - "connected": [{ "slug": "notion", "tools": ["notion-search"] }] - } - }, - "confirmations": { "mcp_notion_notion-search": "deny" }, - "expectedToolInvocations": { - "allOfToolCalls": [{ "toolName": "mcp_notion_notion-search", "declined": true }], - "noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "connect" } }] - }, - "rationale": "A refused tool call is the user withholding permission, not a broken connection: the agent must report back rather than reconnect a server that is already connected.", - "maxSteps": 12 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-no-registry-match.json b/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-no-registry-match.json deleted file mode 100644 index 85bcdfd2551..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-no-registry-match.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "id": "mcp-no-registry-match", - "userMessage": "Pull up my latest deals from HubSpot and give me a short summary.", - "instanceState": { - "mcp": { "registry": ["notion", "linear"], "connected": [] } - }, - "expectedToolInvocations": { - "allOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "search" } }], - "noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "connect" } }] - }, - "rationale": "HubSpot is not in the registry: the agent must search before declaring it unavailable, and must not offer an invented slug or a near-miss server.", - "maxSteps": 12 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-not-offered-for-workflow-build.json b/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-not-offered-for-workflow-build.json deleted file mode 100644 index 8e5856f5663..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-not-offered-for-workflow-build.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "id": "mcp-not-offered-for-workflow-build", - "userMessage": "Build me a workflow that posts a standup reminder to our Slack channel every weekday at 9am.", - "instanceState": { - "mcp": { "registry": ["slack", "notion"], "connected": [] } - }, - "expectedToolInvocations": { - "allOfToolCalls": [{ "toolName": "build-workflow" }], - "noneOfToolCalls": [ - { "toolName": "mcp-servers", "args": { "action": "search" } }, - { "toolName": "mcp-servers", "args": { "action": "connect" } } - ] - }, - "rationale": "A workflow talks to a service through its node and credential, not an MCP connection. Slack is in the registry to make the wrong move available. `connected` is allowed — only offering a connection is the regression.", - "maxSteps": 24, - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-uses-connected-server-tools.json b/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-uses-connected-server-tools.json deleted file mode 100644 index 8127700d78b..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/mcp-uses-connected-server-tools.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "id": "mcp-uses-connected-server-tools", - "userMessage": "Pull up the notes from my last product review in Notion and give me a short summary.", - "instanceState": { - "mcp": { "registry": ["notion", "linear"], "connected": [{ "slug": "notion" }] } - }, - "expectedToolInvocations": { - "anyOfToolCalls": [ - { "toolName": "mcp_notion_notion-search" }, - { "toolName": "mcp_notion_notion-fetch" } - ], - "noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "connect" } }] - }, - "rationale": "Notion is already connected: the agent must use its tools instead of re-offering a connection the user already has.", - "maxSteps": 12 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/oauth-with-computer-use-disabled.json b/packages/@n8n/instance-ai/evaluations/data/discovery/oauth-with-computer-use-disabled.json deleted file mode 100644 index 5f42f431522..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/oauth-with-computer-use-disabled.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "id": "oauth-with-computer-use-disabled", - "userMessage": "I need to set up a Slack OAuth credential. Walk me through the Slack admin so I can find my Client ID and Client Secret to enter into n8n.", - "instanceState": { - "computerUse": { - "localComputer": { "status": "unavailable" }, - "browser": { "status": "unavailable" } - } - }, - "expectedToolInvocations": { - "noneOf": ["browser_connect", "browser_navigate", "browser_snapshot", "browser_screenshot"] - }, - "rationale": "When Computer Use is disabled globally the orchestrator must not pretend any browser tool exists. It should explain the standard credential setup path instead. Pairs with the OAuth-positive scenarios to prove discovery is gated on gateway state, not blindly triggered by the user message." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/screenshot-dashboard.json b/packages/@n8n/instance-ai/evaluations/data/discovery/screenshot-dashboard.json deleted file mode 100644 index 0a0fe49c395..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/screenshot-dashboard.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "id": "screenshot-dashboard", - "userMessage": "Take a screenshot of the Stripe dashboard I have open in my browser so I can attach it to this workflow.", - "instanceState": { - "computerUse": { - "localComputer": { "status": "disconnected" }, - "browser": { "status": "connected", "toolCategories": ["browser"] } - } - }, - "expectedToolInvocations": { - "anyOf": ["browser_screenshot", "browser_snapshot", "browser_navigate"] - }, - "rationale": "Direct browser-tool discovery — the user names a browser action verbatim. The orchestrator must reach for any browser tool (snapshot/screenshot/navigate are all valid first moves to inspect the page) rather than falling back to research." -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/slack-oauth-credential-setup.json b/packages/@n8n/instance-ai/evaluations/data/discovery/slack-oauth-credential-setup.json deleted file mode 100644 index 69b9b4252aa..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/slack-oauth-credential-setup.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "id": "slack-oauth-credential-setup", - "userMessage": "I need to set up a Slack OAuth credential. Walk me through the Slack admin so I can find my Client ID and Client Secret to enter into n8n.", - "instanceState": { - "computerUse": { - "localComputer": { "status": "disconnected" }, - "browser": { "status": "connected", "toolCategories": ["browser"] } - } - }, - "expectedToolInvocations": { - "anyOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["credential-setup-with-computer-use"] - }, - { "toolName": "browser_connect" }, - { "toolName": "browser_navigate" }, - { "toolName": "browser_snapshot" }, - { "toolName": "browser_screenshot" } - ] - }, - "rationale": "OAuth credential setup is the canonical browser-discovery moment. The orchestrator must reach for the Computer Use credential skill or browser_* tools directly; falling through to research/ask-user without browser involvement is the regression.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-agent-builder-leak.json b/packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-agent-builder-leak.json deleted file mode 100644 index 7ecc57911f9..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-agent-builder-leak.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "id": "workflow-builder-no-agent-builder-leak", - "userMessage": "Build a workflow that checks an RSS feed every hour and posts new items to a Slack channel.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["workflow-builder"] - } - ], - "noneOfToolCalls": [ - { - "toolName": "build-agent" - } - ] - }, - "rationale": "Regression gate for the agent-builder path into Instance AI. A plain workflow-building prompt must route through the workflow-builder skill and must never invoke the build-agent orchestration tool. Meaningful in both module modes; the strongest signal is a run with N8N_ENABLED_MODULES including 'agents', where build-agent is registered (it is always-loaded, not deferred) yet must still not be selected.", - "timeoutMs": 120000 -} diff --git a/packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-credential-ask.json b/packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-credential-ask.json deleted file mode 100644 index 70e72e90443..00000000000 --- a/packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-credential-ask.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "id": "workflow-builder-no-credential-ask", - "userMessage": "Build a workflow that lets customers request a meeting time in natural language, parses the preferred time with OpenAI, creates a Google Calendar event, and sends a Gmail confirmation email.", - "expectedToolInvocations": { - "allOfToolCalls": [ - { - "toolName": "load_skill", - "argsContainAny": ["workflow-builder"] - }, - { - "toolName": "build-workflow" - } - ], - "noneOf": ["create-tasks"], - "noneOfToolCalls": [ - { - "toolName": "ask-user", - "argsContainAny": ["credential", "credentials", "account", "auth", "api key", "token"] - }, - { - "toolName": "ask-user", - "argsContainAny": ["timezone", "time zone"] - } - ] - }, - "rationale": "Regression coverage for INS-204. A clear single-workflow build should route through the workflow-builder skill directly, may use credentials(action=\"list\") for discovery, and must not ask the user which credential/account to use when the builder can auto-select or mock unresolved credentials. It should also use contextual timezone data rather than asking for it.", - "maxSteps": 24, - "timeoutMs": 150000 -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/confirmation-policy.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/confirmation-policy.test.ts deleted file mode 100644 index 74b4826b05f..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/confirmation-policy.test.ts +++ /dev/null @@ -1,269 +0,0 @@ -import type { SuspensionInfo } from '../../../src/utils/stream-helpers'; -import { discoveryTestCaseSchema } from '../../data/discovery'; -import { - buildConfirmationPolicy, - resolveConfirmation, - unmatchedConfirmations, -} from '../confirmation-policy'; -import { credentialAutoSetupResponder } from '../credential-approval'; -import { createMcpConnectResponder, createStubMcpRegistry } from '../stub-mcp-registry'; -import type { DiscoveryTestCase } from '../types'; - -function scenario(overrides: Partial = {}): DiscoveryTestCase { - return { - id: 'test', - userMessage: 'do the thing', - expectedToolInvocations: { anyOf: ['mcp-servers'] }, - ...overrides, - }; -} - -function suspension( - toolName: string, - suspendPayload: Record = {}, -): SuspensionInfo { - return { toolCallId: 'call-1', requestId: 'req-1', toolName, suspendPayload }; -} - -const connectPayload = { - mcpConnectRequest: { servers: [{ serverSlug: 'notion' }, { serverSlug: 'linear' }] }, -}; - -const credentialPayload = { - credentialRequests: [{ credentialType: 'slackOAuth2Api' }], -}; - -function mcpResponders() { - const registry = createStubMcpRegistry({ registry: ['notion', 'linear'], connected: [] }); - return { registry, responders: [createMcpConnectResponder(registry)] }; -} - -describe('buildConfirmationPolicy', () => { - it('is empty when the scenario declares nothing', () => { - expect(buildConfirmationPolicy(scenario()).size).toBe(0); - }); - - it('normalizes a bare decision into an answer', () => { - const policy = buildConfirmationPolicy(scenario({ confirmations: { 'mcp-servers': 'deny' } })); - expect(policy.get('mcp-servers')).toEqual({ decision: 'deny' }); - }); - - it('keeps an answer object as declared', () => { - const answer = { decision: 'approve' as const, resumeWith: { connectedSlugs: ['notion'] } }; - const policy = buildConfirmationPolicy(scenario({ confirmations: { 'mcp-servers': answer } })); - expect(policy.get('mcp-servers')).toEqual(answer); - }); -}); - -describe('resolveConfirmation', () => { - const empty = buildConfirmationPolicy(scenario()); - - it('approves a suspension no policy entry covers', () => { - expect(resolveConfirmation(suspension('some-tool'), empty)).toEqual({ approved: true }); - }); - - it('approves when the suspension is unknown', () => { - expect(resolveConfirmation(undefined, empty)).toEqual({ approved: true }); - }); - - it('denies when the scenario says so', async () => { - const policy = buildConfirmationPolicy(scenario({ confirmations: { 'mcp-servers': 'deny' } })); - const { responders, registry } = mcpResponders(); - expect( - resolveConfirmation(suspension('mcp-servers', connectPayload), policy, responders), - ).toEqual({ approved: false }); - await expect(registry.service.listConnections()).resolves.toEqual([]); - }); - - it('denies an mcp tool call by its prefixed name', () => { - const policy = buildConfirmationPolicy( - scenario({ confirmations: { 'mcp_notion_notion-search': 'deny' } }), - ); - const approval = { type: 'approval', toolName: 'mcp_notion_notion-search', args: {} }; - expect(resolveConfirmation(suspension('mcp_notion_notion-search', approval), policy)).toEqual({ - approved: false, - }); - expect(resolveConfirmation(suspension('mcp_linear_list_issues', approval), policy)).toEqual({ - approved: true, - }); - }); - - it('falls back to the tool name the approval gate echoes into the payload', () => { - const policy = buildConfirmationPolicy( - scenario({ confirmations: { 'mcp_notion_notion-search': 'deny' } }), - ); - const gated: SuspensionInfo = { - toolCallId: 'call-1', - requestId: 'req-1', - suspendPayload: { type: 'approval', toolName: 'mcp_notion_notion-search', args: {} }, - }; - expect(resolveConfirmation(gated, policy)).toEqual({ approved: false }); - }); - - it('takes the first responder that recognises the payload', () => { - const { responders } = mcpResponders(); - const all = [credentialAutoSetupResponder, ...responders]; - expect(resolveConfirmation(suspension('mcp-servers', connectPayload), empty, all)).toEqual({ - approved: true, - connectedSlugs: ['notion', 'linear'], - }); - expect(resolveConfirmation(suspension('credentials', credentialPayload), empty, all)).toEqual({ - approved: true, - autoSetup: { credentialType: 'slackOAuth2Api' }, - }); - }); - - it('lets the scenario override responder fields without losing their side effects', async () => { - const { registry, responders } = mcpResponders(); - const policy = buildConfirmationPolicy( - scenario({ - confirmations: { - 'mcp-servers': { decision: 'approve', resumeWith: { connectedSlugs: ['notion'] } }, - }, - }), - ); - expect( - resolveConfirmation(suspension('mcp-servers', connectPayload), policy, responders), - ).toEqual({ approved: true, connectedSlugs: ['notion'] }); - await expect(registry.service.listConnections()).resolves.toEqual([ - { slug: 'notion' }, - { slug: 'linear' }, - ]); - }); - - it('keeps a deny denied even when resumeWith claims approval', () => { - const policy = buildConfirmationPolicy( - scenario({ - confirmations: { 'mcp-servers': { decision: 'deny', resumeWith: { approved: true } } }, - }), - ); - const { responders } = mcpResponders(); - expect( - resolveConfirmation(suspension('mcp-servers', connectPayload), policy, responders), - ).toEqual({ approved: false }); - }); - - it('keeps an approve approved even when resumeWith claims refusal', () => { - const policy = buildConfirmationPolicy( - scenario({ - confirmations: { 'ask-user': { decision: 'approve', resumeWith: { approved: false } } }, - }), - ); - expect(resolveConfirmation(suspension('ask-user'), policy)).toEqual({ approved: true }); - }); - - it('carries resumeWith when no responder recognises the payload', () => { - const policy = buildConfirmationPolicy( - scenario({ - confirmations: { 'ask-user': { decision: 'approve', resumeWith: { answer: 'yes' } } }, - }), - ); - expect(resolveConfirmation(suspension('ask-user'), policy)).toEqual({ - approved: true, - answer: 'yes', - }); - }); -}); - -describe('unmatchedConfirmations', () => { - const policy = buildConfirmationPolicy( - scenario({ - confirmations: { - 'mcp_notion_notion-search': 'deny', - credentials: { decision: 'approve', resumeWith: { autoSetup: { credentialType: 'x' } } }, - }, - }), - ); - - it('reports every behaviour-changing answer no suspension asked for', () => { - expect(unmatchedConfirmations(policy, [])).toEqual(['mcp_notion_notion-search', 'credentials']); - }); - - it('reports nothing once each declared tool has suspended', () => { - const asked = [suspension('mcp_notion_notion-search'), suspension('credentials')]; - expect(unmatchedConfirmations(policy, asked)).toEqual([]); - }); - - it('exempts a bare approve, which only asks for the default', () => { - const approveOnly = buildConfirmationPolicy( - scenario({ confirmations: { 'ask-user': 'approve' } }), - ); - expect(unmatchedConfirmations(approveOnly, [])).toEqual([]); - }); - - it('matches the tool name echoed into the suspend payload', () => { - const gated: SuspensionInfo = { - toolCallId: 'call-1', - requestId: 'req-1', - suspendPayload: { type: 'approval', toolName: 'mcp_notion_notion-search' }, - }; - expect(unmatchedConfirmations(policy, [gated, suspension('credentials')])).toEqual([]); - }); - - it('ignores suspensions the scenario never declared', () => { - expect( - unmatchedConfirmations(buildConfirmationPolicy(scenario()), [suspension('nodes')]), - ).toEqual([]); - }); -}); - -describe('credentialAutoSetupResponder', () => { - it('requests browser auto-setup for the first credential', () => { - expect(credentialAutoSetupResponder(credentialPayload)).toEqual({ - autoSetup: { credentialType: 'slackOAuth2Api' }, - }); - }); - - it.each([ - ['a payload with no credential requests', {}], - ['an empty request list', { credentialRequests: [] }], - ['a request with no credential type', { credentialRequests: [{}] }], - ])('passes on %s', (_label, payload) => { - expect(credentialAutoSetupResponder(payload)).toBeUndefined(); - }); -}); - -describe('createMcpConnectResponder', () => { - it('marks the offered slugs live on the registry', async () => { - const { registry, responders } = mcpResponders(); - expect(responders[0](connectPayload)).toEqual({ connectedSlugs: ['notion', 'linear'] }); - await expect(registry.service.listConnections()).resolves.toEqual([ - { slug: 'notion' }, - { slug: 'linear' }, - ]); - }); - - it('passes on a payload carrying no connect request', () => { - const { responders } = mcpResponders(); - expect(responders[0]({ credentialRequests: [] })).toBeUndefined(); - }); -}); - -describe('discoveryTestCaseSchema', () => { - const base = { id: 'x', userMessage: 'y', expectedToolInvocations: { anyOf: ['a'] } }; - - it.each([ - ['a bare decision', { 'mcp-servers': 'deny' }], - ['an answer object', { 'mcp-servers': { decision: 'approve' } }], - ['an answer with resume data', { credentials: { decision: 'approve', resumeWith: { a: 1 } } }], - ])('accepts %s', (_label, confirmations) => { - expect(discoveryTestCaseSchema.safeParse({ ...base, confirmations }).success).toBe(true); - }); - - it.each([ - ['an unknown decision', { confirmations: { 'mcp-servers': 'maybe' } }], - ['an empty map', { confirmations: {} }], - ['a missing decision', { confirmations: { 'mcp-servers': { resumeWith: { a: 1 } } } }], - [ - 'an empty resumeWith', - { confirmations: { 'mcp-servers': { decision: 'approve', resumeWith: {} } } }, - ], - ['a misspelled key', { confirmation: { 'mcp-servers': 'deny' } }], - [ - 'a resumeWith setting the reserved `approved` field', - { confirmations: { 'mcp-servers': { decision: 'deny', resumeWith: { approved: true } } } }, - ], - ])('rejects %s', (_label, invalid) => { - expect(discoveryTestCaseSchema.safeParse({ ...base, ...invalid }).success).toBe(false); - }); -}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/expected-tools-invoked.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/expected-tools-invoked.test.ts deleted file mode 100644 index 19070525a86..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/expected-tools-invoked.test.ts +++ /dev/null @@ -1,696 +0,0 @@ -import type { AgentActivity, CapturedToolCall, EventOutcome } from '../../types'; -import { evaluateDiscoveryTrial, runExpectedToolsInvokedCheck } from '../expected-tools-invoked'; -import type { DiscoveryTestCase, DiscoveryTrialFacts } from '../types'; - -type ToolCallInput = Pick & - Partial>; - -function makeToolCall(tc: ToolCallInput, i: number): CapturedToolCall { - return { - toolCallId: `call-${i}`, - toolName: tc.toolName, - args: tc.args ?? {}, - ...(tc.result === undefined ? {} : { result: tc.result }), - durationMs: 0, - }; -} - -function makeOutcome(opts: { - toolCalls?: ToolCallInput[]; - agents?: Array & { toolCalls?: ToolCallInput[] }>; -}): EventOutcome { - return { - workflowIds: [], - executionIds: [], - dataTableIds: [], - artifactRefs: [], - finalText: '', - toolCalls: (opts.toolCalls ?? []).map(makeToolCall), - agentActivities: (opts.agents ?? []).map((a, i) => ({ - agentId: `agent-${i}`, - role: a.role, - tools: a.tools, - toolCalls: (a.toolCalls ?? []).map(makeToolCall), - textContent: '', - reasoning: '', - status: 'completed', - })), - }; -} - -const slackOauthScenario: DiscoveryTestCase = { - id: 'test', - userMessage: 'Help me set up Slack credentials', - expectedToolInvocations: { - anyOf: ['load_skill', 'browser_navigate'], - }, -}; - -describe('runExpectedToolsInvokedCheck', () => { - describe('anyOf — positive cases', () => { - it('passes when the expected top-level tool was invoked', () => { - const result = runExpectedToolsInvokedCheck( - slackOauthScenario, - makeOutcome({ toolCalls: [{ toolName: 'load_skill' }] }), - ); - - expect(result.pass).toBe(true); - expect(result.invokedTools).toContain('load_skill'); - }); - - it('passes when a named sub-agent was spawned (matched via spawn_sub_agent: prefix)', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: '', - expectedToolInvocations: { anyOf: ['spawn_sub_agent:researcher'] }, - }, - makeOutcome({ - agents: [{ role: 'researcher', tools: ['credentials'] }], - }), - ); - - expect(result.pass).toBe(true); - expect(result.spawnedAgents).toContain('spawn_sub_agent:researcher'); - }); - - it('passes when the spawned sub-agent had the expected tool attached (sub-agent tools list)', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: '', - expectedToolInvocations: { anyOf: ['credentials'] }, - }, - makeOutcome({ - agents: [{ role: 'researcher', tools: ['credentials'] }], - }), - ); - - expect(result.pass).toBe(true); - expect(result.invokedTools).toContain('credentials'); - }); - }); - - describe('anyOf — negative cases', () => { - it('fails when none of the expected tools or sub-agents appear', () => { - const result = runExpectedToolsInvokedCheck( - slackOauthScenario, - makeOutcome({ toolCalls: [{ toolName: 'research' }] }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('Expected at least one of'); - expect(result.comment).toContain('browser_navigate'); - }); - - it('fails when the orchestrator only ran research with no browser dispatch', () => { - const result = runExpectedToolsInvokedCheck( - slackOauthScenario, - makeOutcome({ - toolCalls: [{ toolName: 'research' }, { toolName: 'ask-user' }], - }), - ); - - expect(result.pass).toBe(false); - }); - }); - - describe('noneOf — negative scenarios (over-eager invocation guard)', () => { - const httpNodeConfigScenario: DiscoveryTestCase = { - id: 'test', - userMessage: 'How do I set the timeout on my HTTP node?', - expectedToolInvocations: { - noneOf: ['browser_navigate', 'spawn_sub_agent:credential-helper'], - }, - }; - - it('passes when the forbidden tool was not invoked', () => { - const result = runExpectedToolsInvokedCheck( - httpNodeConfigScenario, - makeOutcome({ toolCalls: [{ toolName: 'nodes' }] }), - ); - - expect(result.pass).toBe(true); - }); - - it('fails when the forbidden tool was invoked', () => { - const result = runExpectedToolsInvokedCheck( - httpNodeConfigScenario, - makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('Expected none of'); - expect(result.comment).toContain('browser_navigate'); - }); - - it('fails when the forbidden sub-agent was spawned', () => { - const result = runExpectedToolsInvokedCheck( - httpNodeConfigScenario, - makeOutcome({ - agents: [{ role: 'credential-helper', tools: ['browser_navigate'] }], - }), - ); - - expect(result.pass).toBe(false); - }); - }); - - describe('combined anyOf + noneOf', () => { - const combinedScenario: DiscoveryTestCase = { - id: 'test', - userMessage: '', - expectedToolInvocations: { - anyOf: ['browser_navigate'], - noneOf: ['delegate'], - }, - }; - - it('passes when anyOf matches and noneOf is not violated', () => { - const result = runExpectedToolsInvokedCheck( - combinedScenario, - makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }), - ); - - expect(result.pass).toBe(true); - }); - - it('fails when noneOf is violated even if anyOf matched', () => { - const result = runExpectedToolsInvokedCheck( - combinedScenario, - makeOutcome({ - toolCalls: [{ toolName: 'browser_navigate' }, { toolName: 'delegate' }], - }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('Expected none of'); - }); - }); - - describe('noneOfToolCalls — actual tool-call guard', () => { - const planningScenario: DiscoveryTestCase = { - id: 'test', - userMessage: 'Build a Gmail and Calendar workflow', - expectedToolInvocations: { - anyOf: ['create-tasks'], - noneOfToolCalls: [{ toolName: 'ask-user', argsContainAny: ['credential'] }], - }, - }; - - it('passes when ask-user is available to a spawned agent but is not called', () => { - const result = runExpectedToolsInvokedCheck( - planningScenario, - makeOutcome({ - toolCalls: [{ toolName: 'create-tasks' }], - agents: [{ role: 'delegate', tools: ['credentials', 'ask-user'] }], - }), - ); - - expect(result.pass).toBe(true); - expect(result.invokedTools).toContain('ask-user'); - }); - - it('fails when the forbidden tool call happens with matching args', () => { - const result = runExpectedToolsInvokedCheck( - planningScenario, - makeOutcome({ - toolCalls: [ - { toolName: 'create-tasks' }, - { - toolName: 'ask-user', - args: { question: 'Which Google Calendar credential should I use?' }, - }, - ], - agents: [{ role: 'delegate', tools: ['credentials', 'ask-user'] }], - }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('Expected no actual tool call matching'); - expect(result.comment).toContain('credential'); - }); - - it('passes when the same tool is called for unrelated args', () => { - const result = runExpectedToolsInvokedCheck( - planningScenario, - makeOutcome({ - toolCalls: [ - { toolName: 'create-tasks' }, - { toolName: 'ask-user', args: { question: 'Which failure branch should run?' } }, - ], - agents: [{ role: 'delegate', tools: ['credentials', 'ask-user'] }], - }), - ); - - expect(result.pass).toBe(true); - }); - }); - - describe('anyOfToolCalls — actual tool-call alternatives', () => { - const credentialSetupScenario: DiscoveryTestCase = { - id: 'test', - userMessage: 'Help me set up Slack credentials', - expectedToolInvocations: { - anyOfToolCalls: [ - { toolName: 'load_skill', argsContainAny: ['credential-setup-with-computer-use'] }, - { toolName: 'browser_navigate' }, - ], - }, - }; - - it('passes when one expected actual tool call happened with matching args', () => { - const result = runExpectedToolsInvokedCheck( - credentialSetupScenario, - makeOutcome({ - toolCalls: [ - { - toolName: 'load_skill', - args: { skillId: 'credential-setup-with-computer-use' }, - }, - ], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('passes when a browser fallback tool call happened', () => { - const result = runExpectedToolsInvokedCheck( - credentialSetupScenario, - makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }), - ); - - expect(result.pass).toBe(true); - }); - - it('fails when only an unrelated skill was loaded', () => { - const result = runExpectedToolsInvokedCheck( - credentialSetupScenario, - makeOutcome({ toolCalls: [{ toolName: 'load_skill', args: { skillId: 'other-skill' } }] }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('credential-setup-with-computer-use'); - }); - }); - - describe('allOfToolCalls — actual tool-call requirements', () => { - const dataTableScenario: DiscoveryTestCase = { - id: 'test', - userMessage: 'List my n8n Data Tables.', - expectedToolInvocations: { - allOfToolCalls: [ - { toolName: 'load_skill', argsContainAny: ['data-table-manager'] }, - { toolName: 'data-tables', argsContainAny: ['list'] }, - ], - }, - }; - - it('passes when every expected actual tool call happened with matching args', () => { - const result = runExpectedToolsInvokedCheck( - dataTableScenario, - makeOutcome({ - toolCalls: [ - { toolName: 'load_skill', args: { skillId: 'data-table-manager' } }, - { toolName: 'data-tables', args: { action: 'list' } }, - ], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('fails when a tool is only available to a spawned agent but was not called', () => { - const result = runExpectedToolsInvokedCheck( - dataTableScenario, - makeOutcome({ - toolCalls: [{ toolName: 'load_skill', args: { skillId: 'data-table-manager' } }], - agents: [{ role: 'workflow-builder', tools: ['data-tables'] }], - }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('Expected actual tool call matching'); - expect(result.comment).toContain('data-tables'); - }); - - it('fails when the tool call args do not match the expectation', () => { - const result = runExpectedToolsInvokedCheck( - dataTableScenario, - makeOutcome({ - toolCalls: [ - { toolName: 'load_skill', args: { skillId: 'data-table-manager' } }, - { toolName: 'data-tables', args: { action: 'schema' } }, - ], - }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('list'); - }); - }); - - describe('args — structured argument matching', () => { - const connectNotion: DiscoveryTestCase = { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - anyOfToolCalls: [ - { toolName: 'mcp-servers', args: { action: 'connect', serverSlugs: ['notion'] } }, - ], - }, - }; - - it('passes on a deep-partial match, ignoring unlisted keys', () => { - const result = runExpectedToolsInvokedCheck( - connectNotion, - makeOutcome({ - toolCalls: [ - { - toolName: 'mcp-servers', - args: { action: 'connect', serverSlugs: ['notion'], reason: 'unlocks search' }, - }, - ], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('matches an array as a subset, not by length or order', () => { - const result = runExpectedToolsInvokedCheck( - connectNotion, - makeOutcome({ - toolCalls: [ - { - toolName: 'mcp-servers', - args: { action: 'connect', serverSlugs: ['linear', 'notion'] }, - }, - ], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('fails when a listed key differs, even though a substring match would pass', () => { - const result = runExpectedToolsInvokedCheck( - connectNotion, - makeOutcome({ - toolCalls: [ - { - toolName: 'mcp-servers', - args: { - action: 'connect', - serverSlugs: ['linear'], - reason: 'Linear covers this instead of Notion', - }, - }, - ], - }), - ); - - expect(result.pass).toBe(false); - }); - - it('fails when the action differs', () => { - const result = runExpectedToolsInvokedCheck( - connectNotion, - makeOutcome({ - toolCalls: [{ toolName: 'mcp-servers', args: { action: 'search', queries: ['notion'] } }], - }), - ); - - expect(result.pass).toBe(false); - }); - }); - - describe('declined tool calls', () => { - const declined = { - declined: true, - message: 'Tool "mcp_notion_notion-search" was not approved', - }; - - it('does not count a declined call as a violation', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - noneOfToolCalls: [{ toolName: 'mcp_notion_notion-search' }], - }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('does not count a declined call as satisfying a positive expectation', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - anyOfToolCalls: [{ toolName: 'mcp_notion_notion-search' }], - }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }], - }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('declined'); - }); - - it('still counts a call that was approved and ran', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - anyOfToolCalls: [{ toolName: 'mcp_notion_notion-search' }], - }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: { ok: true } }], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('matches a refusal when the expectation asks for one', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - allOfToolCalls: [{ toolName: 'mcp_notion_notion-search', declined: true }], - }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('does not match a call that ran when the expectation asks for a refusal', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - allOfToolCalls: [{ toolName: 'mcp_notion_notion-search', declined: true }], - }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: { ok: true } }], - }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('a declined result'); - }); - - it('does not count a sub-agent call the user refused as invoked', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { noneOf: ['mcp_notion_notion-search'] }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }], - agents: [ - { - role: 'researcher', - tools: [], - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }], - }, - ], - }), - ); - - expect(result.pass).toBe(true); - expect(result.invokedTools).not.toContain('mcp_notion_notion-search'); - }); - - it('counts a sub-agent call that ran as invoked', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { anyOf: ['mcp_notion_notion-search'] }, - }, - makeOutcome({ - agents: [ - { - role: 'researcher', - tools: [], - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: { ok: true } }], - }, - ], - }), - ); - - expect(result.pass).toBe(true); - }); - - it('forbids a refusal when noneOfToolCalls asks for one', () => { - const result = runExpectedToolsInvokedCheck( - { - id: 'test', - userMessage: 'Search my Notion.', - expectedToolInvocations: { - noneOfToolCalls: [{ toolName: 'mcp_notion_notion-search', declined: true }], - }, - }, - makeOutcome({ - toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }], - }), - ); - - expect(result.pass).toBe(false); - }); - }); - - describe('rule validation', () => { - it('throws when neither anyOf nor noneOf is provided', () => { - expect(() => - runExpectedToolsInvokedCheck( - { id: 'x', userMessage: '', expectedToolInvocations: {} }, - makeOutcome({}), - ), - ).toThrow(/anyOf.*noneOf/); - }); - - it('throws when both anyOf and noneOf are empty', () => { - expect(() => - runExpectedToolsInvokedCheck( - { id: 'x', userMessage: '', expectedToolInvocations: { anyOf: [], noneOf: [] } }, - makeOutcome({}), - ), - ).toThrow(); - }); - }); -}); - -describe('evaluateDiscoveryTrial', () => { - const trial = (overrides: Partial = {}): DiscoveryTrialFacts => ({ - streamStatus: 'completed', - timeoutMs: 60_000, - unmatchedConfirmations: [], - ...overrides, - }); - - const negativeOnly: DiscoveryTestCase = { - id: 'test', - userMessage: 'Set up a Slack credential.', - expectedToolInvocations: { noneOf: ['browser_navigate'] }, - }; - - const positiveOnly: DiscoveryTestCase = { - id: 'test', - userMessage: 'Screenshot my dashboard.', - expectedToolInvocations: { anyOf: ['browser_navigate'] }, - }; - - const satisfied = makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }); - - it('passes a satisfied expectation on a completed run', () => { - expect(evaluateDiscoveryTrial(negativeOnly, makeOutcome({}), trial()).pass).toBe(true); - expect(evaluateDiscoveryTrial(positiveOnly, satisfied, trial()).pass).toBe(true); - }); - - it.each([ - ['negative-only', negativeOnly, makeOutcome({})], - ['positive-only', positiveOnly, satisfied], - ])('fails a satisfied %s expectation when the run stopped at its step cap', (_l, s, outcome) => { - const result = evaluateDiscoveryTrial(s, outcome, trial({ streamStatus: 'step-exhausted' })); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('step cap'); - }); - - it.each(['errored', 'suspended'] as const)( - 'fails a satisfied expectation when the run %s', - (streamStatus) => { - const result = evaluateDiscoveryTrial(positiveOnly, satisfied, trial({ streamStatus })); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('Run did not complete'); - }, - ); - - it('fails a satisfied expectation when the run exceeded its budget', () => { - const result = evaluateDiscoveryTrial( - positiveOnly, - satisfied, - trial({ streamStatus: 'timed-out', timeoutMs: 150_000 }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('exceeded its 150000ms budget'); - }); - - it('reports the run error alongside the invalid trial', () => { - const result = evaluateDiscoveryTrial( - negativeOnly, - makeOutcome({}), - trial({ streamStatus: 'errored', runError: 'overloaded_error' }), - ); - - expect(result.comment).toContain('errored: overloaded_error'); - }); - - it('keeps the expectation diagnostic when an invalid trial also failed its expectation', () => { - const result = evaluateDiscoveryTrial( - positiveOnly, - makeOutcome({ toolCalls: [{ toolName: 'nodes' }] }), - trial({ streamStatus: 'timed-out' }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('budget and was abandoned'); - expect(result.comment).toContain('Expected at least one of'); - }); - - it('fails when a declared confirmation answer was never asked for', () => { - const result = evaluateDiscoveryTrial( - negativeOnly, - makeOutcome({}), - trial({ unmatchedConfirmations: ['mcp_notion_notion-search'] }), - ); - - expect(result.pass).toBe(false); - expect(result.comment).toContain('mcp_notion_notion-search'); - expect(result.comment).toContain('never ran'); - }); -}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts b/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts deleted file mode 100644 index 7fcf994fac0..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { ExecuteResumableStreamResult } from '../../../src/runtime/resumable-stream-executor'; -import { resolveStreamStatus } from '../stream-status'; - -function streamResult( - overrides: Partial = {}, -): ExecuteResumableStreamResult { - return { - status: 'completed', - agentRunId: 'run-1', - workSummary: { - toolCalls: [], - totalToolCalls: 0, - totalToolErrors: 0, - askedClarifyingQuestion: false, - }, - ...overrides, - }; -} - -describe('resolveStreamStatus', () => { - it('reports a clean finish as completed', () => { - expect(resolveStreamStatus(streamResult({ finishReason: 'stop' }), false)).toBe('completed'); - }); - - it('reports a run that hit its step cap as step-exhausted', () => { - expect(resolveStreamStatus(streamResult({ finishReason: 'max-iterations' }), false)).toBe( - 'step-exhausted', - ); - }); - - it.each(['length', 'content-filter', 'error', 'other', 'tool-calls'] as const)( - 'reports a run cut short by %s as errored', - (finishReason) => { - expect(resolveStreamStatus(streamResult({ finishReason }), false)).toBe('errored'); - }, - ); - - it('never reads a missing finish chunk as a clean finish', () => { - expect(resolveStreamStatus(streamResult(), false)).toBe('errored'); - }); - - it('prefers timed-out over the step cap', () => { - expect(resolveStreamStatus(streamResult({ finishReason: 'max-iterations' }), true)).toBe( - 'timed-out', - ); - }); - - it('prefers errored over the step cap', () => { - expect( - resolveStreamStatus( - streamResult({ status: 'errored', finishReason: 'max-iterations' }), - false, - ), - ).toBe('errored'); - }); - - it('reports the budget sentinel as timed-out', () => { - expect(resolveStreamStatus('timed-out', false)).toBe('timed-out'); - }); - - it.each([streamResult({ status: 'suspended' }), streamResult({ finishReason: 'paused' })])( - 'reports a suspended or paused run as suspended', - (result) => { - expect(resolveStreamStatus(result, false)).toBe('suspended'); - }, - ); - - it('never reads a cancelled run as a clean finish', () => { - expect(resolveStreamStatus(streamResult({ status: 'cancelled' }), false)).toBe('timed-out'); - }); -}); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/cli.ts b/packages/@n8n/instance-ai/evaluations/discovery/cli.ts deleted file mode 100644 index 163b436bc8a..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/cli.ts +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env node -// --------------------------------------------------------------------------- -// CLI for browser/computer-use discovery evaluation. -// -// Usage: -// pnpm eval:discovery # run all scenarios, 3 trials each -// pnpm eval:discovery --filter slack-oauth --verbose -// pnpm eval:discovery --trials 5 -// -// Loads scenarios from evaluations/data/discovery/, runs each scenario × N -// trials via the in-process runner, reports per-scenario pass-rates, exits -// non-zero on any scenario below threshold, or on any scenario with zero passes -// when --fail-on-zero-pass is set. -// --------------------------------------------------------------------------- - -import { runDiscoveryScenario, type DiscoveryRunResult } from './runner'; -import type { DiscoveryTestCase } from './types'; -import { loadDiscoveryTestCasesWithFiles } from '../data/discovery'; - -// --------------------------------------------------------------------------- -// Args -// --------------------------------------------------------------------------- - -interface CliArgs { - filter?: string; - verbose: boolean; - trials: number; - passThreshold: number; - timeoutMs: number; - /** Optional iteration cap; unset uses the SDK default. */ - maxSteps?: number; - modelId: string; - concurrency: number; - nodesJsonPath?: string; - failOnZeroPass: boolean; -} - -const DEFAULT_MODEL = process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6'; - -function previewArgs(args: Record): string { - const entries = Object.entries(args).slice(0, 3); - if (entries.length === 0) return ''; - return entries - .map(([k, v]) => { - let preview: string; - if (typeof v === 'string') { - preview = v.length > 40 ? `"${v.slice(0, 40)}…"` : `"${v}"`; - } else if (typeof v === 'number' || typeof v === 'boolean' || v === null) { - preview = String(v); - } else if (Array.isArray(v)) { - preview = `[${String(v.length)}]`; - } else { - preview = '{…}'; - } - return `${k}=${preview}`; - }) - .join(', '); -} - -function requirePositiveInt(raw: string | undefined, flag: string): number { - const n = Number(raw); - if (!Number.isFinite(n) || n < 1 || !Number.isInteger(n)) { - console.error(`Invalid value for ${flag}: ${String(raw)} (expected a positive integer)`); - process.exit(1); - } - return n; -} - -function requireNumberInRange( - raw: string | undefined, - flag: string, - lo: number, - hi: number, -): number { - const n = Number(raw); - if (!Number.isFinite(n) || n < lo || n > hi) { - console.error(`Invalid value for ${flag}: ${String(raw)} (expected number in [${lo}, ${hi}])`); - process.exit(1); - } - return n; -} - -function parseArgs(argv: string[]): CliArgs { - const args: CliArgs = { - verbose: false, - trials: 3, - passThreshold: 2 / 3, - timeoutMs: 60_000, - modelId: DEFAULT_MODEL, - concurrency: 3, - failOnZeroPass: false, - }; - - for (let i = 0; i < argv.length; i++) { - const arg = argv[i]; - switch (arg) { - case '--verbose': - case '-v': - args.verbose = true; - break; - case '--filter': - args.filter = argv[++i]; - break; - case '--trials': - args.trials = requirePositiveInt(argv[++i], '--trials'); - break; - case '--pass-threshold': - args.passThreshold = requireNumberInRange(argv[++i], '--pass-threshold', 0, 1); - break; - case '--timeout': - args.timeoutMs = requirePositiveInt(argv[++i], '--timeout'); - break; - case '--max-steps': - args.maxSteps = requirePositiveInt(argv[++i], '--max-steps'); - break; - case '--model': - args.modelId = argv[++i]; - break; - case '--concurrency': - args.concurrency = requirePositiveInt(argv[++i], '--concurrency'); - break; - case '--nodes-json': - args.nodesJsonPath = argv[++i]; - break; - case '--fail-on-zero-pass': - args.failOnZeroPass = true; - break; - default: - break; - } - } - - return args; -} - -// --------------------------------------------------------------------------- -// Local mode -// --------------------------------------------------------------------------- - -interface ScenarioAggregate { - scenario: DiscoveryTestCase; - results: DiscoveryRunResult[]; - passCount: number; - passRate: number; -} - -async function runLocalMode(args: CliArgs): Promise { - const cases = loadDiscoveryTestCasesWithFiles(args.filter); - - if (cases.length === 0) { - console.log('No discovery scenarios found.'); - return; - } - - console.log( - `Running ${String(cases.length)} discovery scenario(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`, - ); - - const aggregates: ScenarioAggregate[] = []; - - // Per scenario, run trials in parallel up to `concurrency`. Across scenarios, run sequentially - // — keeps the load profile predictable and per-scenario reports atomic. - for (const { testCase, fileSlug } of cases) { - process.stdout.write(`▸ ${fileSlug} ... `); - const trialResults: DiscoveryRunResult[] = []; - - for (let i = 0; i < args.trials; i += args.concurrency) { - const batchSize = Math.min(args.concurrency, args.trials - i); - const batch = await Promise.all( - Array.from( - { length: batchSize }, - async () => - await runDiscoveryScenario({ - scenario: testCase, - modelId: args.modelId, - maxSteps: args.maxSteps, - timeoutMs: args.timeoutMs, - ...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}), - }), - ), - ); - trialResults.push(...batch); - } - - const passCount = trialResults.filter((r) => r.check.pass).length; - const passRate = passCount / trialResults.length; - const status = passRate >= args.passThreshold ? '✓' : '✗'; - console.log( - `${status} ${String(passCount)}/${String(trialResults.length)} passed (${(passRate * 100).toFixed(0)}%)`, - ); - - if (args.verbose) { - for (const [idx, r] of trialResults.entries()) { - const icon = r.check.pass ? ' ✓' : ' ✗'; - const trial = `trial ${String(idx + 1)}`; - console.log(`${icon} ${trial} (${(r.durationMs / 1000).toFixed(1)}s) — ${r.check.comment}`); - if (r.runError) console.log(` run error: ${r.runError}`); - if (r.outcome.toolCalls.length > 0) { - console.log(' tool calls:'); - for (const tc of r.outcome.toolCalls) { - const argsPreview = previewArgs(tc.args); - console.log(` • ${tc.toolName}(${argsPreview})`); - } - } - if (r.outcome.agentActivities.length > 0) { - console.log(' spawned sub-agents:'); - for (const a of r.outcome.agentActivities) { - console.log(` • role=${a.role}, tools=[${a.tools.join(', ')}]`); - } - } - } - } - - aggregates.push({ scenario: testCase, results: trialResults, passCount, passRate }); - } - - printSummary(aggregates, args); - - const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold); - const zeroPassScenarios = aggregates.filter((a) => a.passCount === 0); - const shouldFail = args.failOnZeroPass - ? zeroPassScenarios.length > 0 - : failingScenarios.length > 0; - if (shouldFail) { - process.exitCode = 1; - } -} - -function printSummary(aggregates: ScenarioAggregate[], args: CliArgs): void { - console.log('\n=== Summary ==='); - const totalTrials = aggregates.reduce((sum, a) => sum + a.results.length, 0); - const totalPasses = aggregates.reduce((sum, a) => sum + a.passCount, 0); - const passingScenarios = aggregates.filter((a) => a.passRate >= args.passThreshold).length; - const totalDurationMs = aggregates - .flatMap((a) => a.results) - .reduce((sum, r) => sum + r.durationMs, 0); - - console.log( - `Scenarios: ${String(passingScenarios)}/${String(aggregates.length)} above threshold (${(args.passThreshold * 100).toFixed(0)}%)`, - ); - console.log( - `Trials: ${String(totalPasses)}/${String(totalTrials)} passed (${((totalPasses / totalTrials) * 100).toFixed(0)}%)`, - ); - console.log(`Total time: ${(totalDurationMs / 1000).toFixed(1)}s`); - - const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold); - if (failingScenarios.length > 0) { - console.log('\nFailing scenarios:'); - for (const a of failingScenarios) { - console.log(` ✗ ${a.scenario.id} (${(a.passRate * 100).toFixed(0)}%)`); - const firstFailure = a.results.find((r) => !r.check.pass); - if (firstFailure) { - console.log(` ${firstFailure.check.comment}`); - } - } - } -} - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - -async function main(): Promise { - const args = parseArgs(process.argv.slice(2)); - - if (!process.env.ANTHROPIC_API_KEY) { - console.error( - 'Error: ANTHROPIC_API_KEY is required to run discovery evaluations (the runner calls Anthropic in-process).', - ); - process.exit(1); - } - - await runLocalMode(args); -} - -main() - .then(() => { - process.stdout.write('', () => process.exit(process.exitCode ?? 0)); - }) - .catch((error) => { - console.error('Fatal error:', error); - process.exit(1); - }); diff --git a/packages/@n8n/instance-ai/evaluations/discovery/confirmation-policy.ts b/packages/@n8n/instance-ai/evaluations/discovery/confirmation-policy.ts deleted file mode 100644 index 68d419098ac..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/confirmation-policy.ts +++ /dev/null @@ -1,70 +0,0 @@ -// --------------------------------------------------------------------------- -// How the harness answers confirmation / HITL suspensions. -// -// Production waits for the user; the harness answers from the scenario, keyed on -// the suspending tool's name and defaulting to approve, so a scenario only -// declares the answers it wants to differ. A bare approval resumes -// `{ approved: true }`, which is the whole contract for an `approval` card — an -// MCP tool call included. -// -// Cards that need the user to *supply* something (a credential choice, a -// connect card's slugs, a Q&A wizard's answers) are filled in by an -// ApprovalResponder living with that domain — `credential-approval.ts`, -// `stub-mcp-registry.ts`. This module stays payload-agnostic: to cover a new -// card kind, write a responder next to its domain and register it in the -// runner. A scenario can always bypass the lot with `resumeWith`. -// --------------------------------------------------------------------------- - -import { isRecord } from '@n8n/utils/is-record'; - -import type { ConfirmationAnswer, DiscoveryTestCase } from './types'; -import type { SuspensionInfo } from '../../src/utils/stream-helpers'; - -export type ConfirmationPolicy = Map; - -// Supplies what an approving user would have filled in for one kind of card. -export type ApprovalResponder = ( - payload: Record, -) => Record | undefined; - -export function buildConfirmationPolicy(scenario: DiscoveryTestCase): ConfirmationPolicy { - const policy: ConfirmationPolicy = new Map(); - for (const [toolName, answer] of Object.entries(scenario.confirmations ?? {})) { - policy.set(toolName, typeof answer === 'string' ? { decision: answer } : answer); - } - return policy; -} - -export function resolveConfirmation( - suspension: SuspensionInfo | undefined, - policy: ConfirmationPolicy, - responders: ApprovalResponder[] = [], -): Record { - const answer = policy.get(suspendedToolName(suspension)); - if (answer?.decision === 'deny') return { ...answer.resumeWith, approved: false }; - - const payload = suspension?.suspendPayload ?? {}; - for (const responder of responders) { - const responderAnswer = responder(payload); - if (responderAnswer) return { ...responderAnswer, ...answer?.resumeWith, approved: true }; - } - return { ...answer?.resumeWith, approved: true }; -} - -export function unmatchedConfirmations( - policy: ConfirmationPolicy, - suspensions: Iterable, -): string[] { - const asked = new Set([...suspensions].map(suspendedToolName)); - return [...policy] - .filter( - ([tool, answer]) => !asked.has(tool) && (answer.decision === 'deny' || answer.resumeWith), - ) - .map(([tool]) => tool); -} - -function suspendedToolName(suspension: SuspensionInfo | undefined): string { - if (suspension?.toolName) return suspension.toolName; - const payload = suspension?.suspendPayload; - return isRecord(payload) && typeof payload.toolName === 'string' ? payload.toolName : ''; -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/credential-approval.ts b/packages/@n8n/instance-ai/evaluations/discovery/credential-approval.ts deleted file mode 100644 index 039680081a8..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/credential-approval.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { isRecord } from '@n8n/utils/is-record'; - -import type { ApprovalResponder } from './confirmation-policy'; - -/** Approving a credential card means "Auto-setup with browser". A bare approval - * takes the "user picked an existing credential" branch, so the orchestrator never - * sees `needsBrowserSetup=true` — the flag that wires it to the Computer Use - * credential setup skill. The card offers one credential at a time once browser - * setup is in play, so the first request is the one `autoSetup` names. */ -export const credentialAutoSetupResponder: ApprovalResponder = (payload) => { - const requests = payload.credentialRequests; - if (!Array.isArray(requests)) return undefined; - - const first: unknown = requests[0]; - if (!isRecord(first) || typeof first.credentialType !== 'string' || !first.credentialType) { - return undefined; - } - return { autoSetup: { credentialType: first.credentialType } }; -}; diff --git a/packages/@n8n/instance-ai/evaluations/discovery/expected-tools-invoked.ts b/packages/@n8n/instance-ai/evaluations/discovery/expected-tools-invoked.ts deleted file mode 100644 index e385daeb249..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/expected-tools-invoked.ts +++ /dev/null @@ -1,250 +0,0 @@ -// --------------------------------------------------------------------------- -// Discovery check — assert the orchestrator reached for the expected tool(s). -// -// Reads the captured event outcome (`toolCalls` + `agentActivities`) and -// compares against a `DiscoveryTestCase.expectedToolInvocations` rule. -// -// "Invoked" means either: -// - a top-level `tool-call` event with that tool name, OR -// - an `agent-spawned` event whose payload `tools` array contains that name -// (the sub-agent had access — even if it has not yet called it), OR -// - the rule names `spawn_sub_agent:` and a sub-agent with that role -// was spawned. -// -// The asymmetry (sub-agent existence counts as discovery) lets dispatch checks -// assert that a specialized background agent was reached even before it emits -// its own tool calls. -// --------------------------------------------------------------------------- - -import { isRecord } from '@n8n/utils/is-record'; - -import type { EventOutcome } from '../types'; -import type { - DiscoveryCheckResult, - DiscoveryTestCase, - DiscoveryTrialFacts, - ExpectedToolInvocations, - ForbiddenToolCall, -} from './types'; - -const SPAWN_PREFIX = 'spawn_sub_agent:'; - -function collectInvokedTools(outcome: EventOutcome): string[] { - const tools = new Set(); - for (const tc of outcome.toolCalls) { - if (tc.toolName && !wasDeclined(tc)) tools.add(tc.toolName); - } - for (const agent of outcome.agentActivities) { - for (const t of agent.tools) tools.add(t); - for (const tc of agent.toolCalls) { - if (tc.toolName && !wasDeclined(tc)) tools.add(tc.toolName); - } - } - return [...tools]; -} - -function collectSpawnedAgents(outcome: EventOutcome): string[] { - return outcome.agentActivities - .filter((a) => a.role.length > 0) - .map((a) => `${SPAWN_PREFIX}${a.role}`); -} - -function matches(name: string, invokedTools: string[], spawnedAgents: string[]): boolean { - if (name.startsWith(SPAWN_PREFIX)) { - return spawnedAgents.includes(name); - } - return invokedTools.includes(name); -} - -function validateRule(rule: ExpectedToolInvocations): void { - const hasAnyOf = Array.isArray(rule.anyOf) && rule.anyOf.length > 0; - const hasNoneOf = Array.isArray(rule.noneOf) && rule.noneOf.length > 0; - const hasAnyOfToolCalls = Array.isArray(rule.anyOfToolCalls) && rule.anyOfToolCalls.length > 0; - const hasAllOfToolCalls = Array.isArray(rule.allOfToolCalls) && rule.allOfToolCalls.length > 0; - const hasNoneOfToolCalls = Array.isArray(rule.noneOfToolCalls) && rule.noneOfToolCalls.length > 0; - if (!hasAnyOf && !hasNoneOf && !hasAnyOfToolCalls && !hasAllOfToolCalls && !hasNoneOfToolCalls) { - throw new Error( - 'expectedToolInvocations must specify a non-empty `anyOf`, `noneOf`, `anyOfToolCalls`, `allOfToolCalls`, or `noneOfToolCalls` list', - ); - } -} - -function wasDeclined(toolCall: EventOutcome['toolCalls'][number]): boolean { - return isRecord(toolCall.result) && toolCall.result.declined === true; -} - -function toolCallMatchesExpectation( - toolCall: EventOutcome['toolCalls'][number], - expectation: ForbiddenToolCall, -): boolean { - if (wasDeclined(toolCall) !== (expectation.declined ?? false)) return false; - if (toolCall.toolName !== expectation.toolName) return false; - if (expectation.args && !matchesArgPattern(expectation.args, toolCall.args)) return false; - - const argsContainAny = expectation.argsContainAny ?? []; - if (argsContainAny.length === 0) return true; - - const argsText = JSON.stringify(toolCall.args).toLowerCase(); - return argsContainAny.some((term) => argsText.includes(term.toLowerCase())); -} - -function matchesArgPattern(pattern: unknown, actual: unknown): boolean { - if (Array.isArray(pattern)) { - return ( - Array.isArray(actual) && - pattern.every((item) => actual.some((candidate) => matchesArgPattern(item, candidate))) - ); - } - if (isRecord(pattern)) { - return ( - isRecord(actual) && - Object.entries(pattern).every(([key, value]) => matchesArgPattern(value, actual[key])) - ); - } - return pattern === actual; -} - -function describeActualToolCalls(outcome: EventOutcome): string { - return ( - outcome.toolCalls - .map((tc) => (wasDeclined(tc) ? `${tc.toolName} (declined)` : tc.toolName)) - .join(', ') || '∅' - ); -} - -function formatToolCallExpectation(expectation: ForbiddenToolCall): string { - const clauses: string[] = []; - if (expectation.declined) clauses.push('a declined result'); - if (expectation.args) clauses.push(`args matching ${JSON.stringify(expectation.args)}`); - if (expectation.argsContainAny && expectation.argsContainAny.length > 0) { - clauses.push(`args containing one of [${expectation.argsContainAny.join(', ')}]`); - } - return clauses.length > 0 - ? `${expectation.toolName} with ${clauses.join(' and ')}` - : expectation.toolName; -} - -export function runExpectedToolsInvokedCheck( - scenario: DiscoveryTestCase, - outcome: EventOutcome, -): DiscoveryCheckResult { - validateRule(scenario.expectedToolInvocations); - - const invokedTools = collectInvokedTools(outcome); - const spawnedAgents = collectSpawnedAgents(outcome); - - const { anyOf, noneOf, anyOfToolCalls, allOfToolCalls, noneOfToolCalls } = - scenario.expectedToolInvocations; - - if (anyOf && anyOf.length > 0) { - const matched = anyOf.find((name) => matches(name, invokedTools, spawnedAgents)); - if (!matched) { - return { - pass: false, - comment: `Expected at least one of [${anyOf.join(', ')}] to be invoked. Invoked: [${invokedTools.join(', ') || '∅'}]; spawned: [${spawnedAgents.join(', ') || '∅'}].`, - invokedTools, - spawnedAgents, - }; - } - } - - if (noneOf && noneOf.length > 0) { - const violated = noneOf.find((name) => matches(name, invokedTools, spawnedAgents)); - if (violated) { - return { - pass: false, - comment: `Expected none of [${noneOf.join(', ')}] to be invoked, but [${violated}] was reached.`, - invokedTools, - spawnedAgents, - }; - } - } - - if (anyOfToolCalls && anyOfToolCalls.length > 0) { - const matched = anyOfToolCalls.find((expectation) => - outcome.toolCalls.some((toolCall) => toolCallMatchesExpectation(toolCall, expectation)), - ); - if (!matched) { - const actualToolCalls = describeActualToolCalls(outcome); - return { - pass: false, - comment: `Expected at least one actual tool call matching [${anyOfToolCalls.map(formatToolCallExpectation).join(', ')}]. Actual tool calls: [${actualToolCalls}].`, - invokedTools, - spawnedAgents, - }; - } - } - - if (allOfToolCalls && allOfToolCalls.length > 0) { - for (const expectation of allOfToolCalls) { - const matched = outcome.toolCalls.find((toolCall) => - toolCallMatchesExpectation(toolCall, expectation), - ); - if (!matched) { - const actualToolCalls = describeActualToolCalls(outcome); - return { - pass: false, - comment: `Expected actual tool call matching [${formatToolCallExpectation(expectation)}]. Actual tool calls: [${actualToolCalls}].`, - invokedTools, - spawnedAgents, - }; - } - } - } - - if (noneOfToolCalls && noneOfToolCalls.length > 0) { - for (const expectation of noneOfToolCalls) { - const violated = outcome.toolCalls.find((toolCall) => - toolCallMatchesExpectation(toolCall, expectation), - ); - if (violated) { - return { - pass: false, - comment: `Expected no actual tool call matching [${formatToolCallExpectation(expectation)}], but saw ${violated.toolName} with args ${JSON.stringify(violated.args)}.`, - invokedTools, - spawnedAgents, - }; - } - } - } - - return { - pass: true, - comment: 'Discovery expectation satisfied.', - invokedTools, - spawnedAgents, - }; -} - -/** - * Only a run that finished on its own terms can settle an expectation, so anything else - * fails whichever way the expectation points — a truncated run that satisfied a positive - * expectation still means the agent never got to finish what it was doing. - */ -export function evaluateDiscoveryTrial( - scenario: DiscoveryTestCase, - outcome: EventOutcome, - trial: DiscoveryTrialFacts, -): DiscoveryCheckResult { - const check = runExpectedToolsInvokedCheck(scenario, outcome); - const invalid = invalidTrialReason(trial); - if (!invalid) return check; - - return { ...check, pass: false, comment: check.pass ? invalid : `${invalid} ${check.comment}` }; -} - -function invalidTrialReason(trial: DiscoveryTrialFacts): string | undefined { - switch (trial.streamStatus) { - case 'timed-out': - return `Run exceeded its ${String(trial.timeoutMs)}ms budget and was abandoned.`; - case 'step-exhausted': - return 'Run stopped at its step cap instead of finishing.'; - case 'errored': - case 'suspended': - return `Run did not complete (${trial.runError ? `${trial.streamStatus}: ${trial.runError}` : trial.streamStatus}).`; - case 'completed': - return trial.unmatchedConfirmations.length > 0 - ? `Scenario declared confirmation answers for [${trial.unmatchedConfirmations.join(', ')}] that no suspension asked for, so those decisions never ran.` - : undefined; - } -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/runner.ts b/packages/@n8n/instance-ai/evaluations/discovery/runner.ts deleted file mode 100644 index a714b2e4669..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/runner.ts +++ /dev/null @@ -1,359 +0,0 @@ -// --------------------------------------------------------------------------- -// In-process discovery runner. -// -// Drives the orchestrator with a scenario's userMessage + instanceState, -// captures InstanceAi events into the CapturedEvent[] shape that -// extractOutcomeFromEvents consumes, then runs the discovery check. No -// Docker, no n8n server — the orchestrator runs in-process against -// stubbed services. -// -// What's tested: the orchestrator's first dispatch decision. Tools are NOT -// stubbed — when the orchestrator loads a runtime skill or reaches for a -// Computer Use browser tool, the tool-call event fires before any downstream -// failure, so the discovery check still sees the dispatch intent. The wall-clock -// timeout bounds the trial, not the run: at the budget the trial stops and fails, -// and the abandoned stream is left to unwind on its own. Scenarios or --max-steps -// can additionally opt into an iteration cap. -// --------------------------------------------------------------------------- - -import type { InstanceAiEvent, TaskList } from '@n8n/api-types'; -import { nanoid } from 'nanoid'; - -import { - buildConfirmationPolicy, - resolveConfirmation, - unmatchedConfirmations, - type ApprovalResponder, -} from './confirmation-policy'; -import { credentialAutoSetupResponder } from './credential-approval'; -import { evaluateDiscoveryTrial } from './expected-tools-invoked'; -import { resolveStreamStatus } from './stream-status'; -import { createStubLocalMcpServer } from './stub-local-mcp'; -import { - createMcpConnectResponder, - createStubMcpRegistry, - createStubMcpToolRegistry, - StubMcpClientManager, - stubMcpServerConfigs, - type StubMcpRegistry, -} from './stub-mcp-registry'; -import type { DiscoveryCheckResult, DiscoveryStreamStatus, DiscoveryTestCase } from './types'; -import { createInstanceAgent } from '../../src/agent/instance-agent'; -import type { InstanceAiEventBus } from '../../src/event-bus'; -import type { Logger } from '../../src/logger'; -import { - executeResumableStream, - normalizeStreamSource, -} from '../../src/runtime/resumable-stream-executor'; -import { loadInstanceAiRuntimeSkillSource } from '../../src/skills/runtime-skills'; -import type { - BuilderTurnStream, - InstanceAiContext, - InstanceAiBuilderDelegate, - ComputerUseState, - ModelConfig, - OrchestrationContext, - TaskStorage, -} from '../../src/types'; -import { isAgentFeatureEnabled } from '../../src/utils/agent-feature-enabled'; -import { asResumable, type SuspensionInfo } from '../../src/utils/stream-helpers'; -import { createInMemoryEventBus, wrapEventBusWithObserver } from '../harness/in-memory-event-bus'; -import { createStubServices, defaultNodesJsonPath } from '../harness/stub-services'; -import { createStubWorkspace, stubWorkspaceRoot } from '../harness/stub-workspace'; -import { extractOutcomeFromEvents } from '../outcome/event-parser'; -import type { CapturedEvent, EventOutcome } from '../types'; - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -export interface DiscoveryRunOptions { - scenario: DiscoveryTestCase; - modelId: ModelConfig; - /** Defaults to `defaultNodesJsonPath()`. */ - nodesJsonPath?: string; - /** Hard cap on agent steps. Unset leaves the SDK's own 30-iteration ceiling in place. */ - maxSteps?: number; - /** Per-trial timeout in ms. */ - timeoutMs?: number; -} - -export interface DiscoveryRunResult { - scenario: DiscoveryTestCase; - check: DiscoveryCheckResult; - events: CapturedEvent[]; - outcome: EventOutcome; - durationMs: number; - /** Final agent status — useful for diagnosing why noop / unexpected loops happened. */ - streamStatus: DiscoveryStreamStatus; - /** Populated when the run errored before reaching the check. */ - runError?: string; -} - -export async function runDiscoveryScenario( - options: DiscoveryRunOptions, -): Promise { - const started = Date.now(); - // Unset by default: the orchestrator legitimately explores past any small fixed cap - // (data-table-workflow needs >8 iterations). Unset still lands on the SDK's own - // 30-iteration ceiling, which reports `step-exhausted`. - const maxSteps = options.scenario?.maxSteps ?? options.maxSteps; - const timeoutMs = options.scenario?.timeoutMs ?? options.timeoutMs ?? 60_000; - const nodesJsonPath = options.nodesJsonPath ?? defaultNodesJsonPath(); - - const events: CapturedEvent[] = []; - - let streamStatus: DiscoveryRunResult['streamStatus'] = 'completed'; - let runError: string | undefined; - - const confirmationPolicy = buildConfirmationPolicy(options.scenario); - const suspensions = new Map(); - - const abortController = new AbortController(); - let mcpManager: StubMcpClientManager | undefined; - let timeoutHandle: NodeJS.Timeout | undefined; - const budgetExpired = new Promise<'timed-out'>((resolve) => { - timeoutHandle = setTimeout(() => { - abortController.abort(); - resolve('timed-out'); - }, timeoutMs); - }); - - try { - const services = await createStubServices({ nodesJsonPath }); - const mcpState = options.scenario.instanceState?.mcp; - const mcpRegistry = mcpState ? createStubMcpRegistry(mcpState) : undefined; - const context: InstanceAiContext = { - ...applyInstanceState(services.context, options.scenario, mcpRegistry), - ...(isAgentFeatureEnabled() ? { builderDelegate: createStubBuilderDelegate() } : {}), - workspace: createStubWorkspace(), - workspaceRoot: stubWorkspaceRoot, - }; - - mcpManager = new StubMcpClientManager(createStubMcpToolRegistry(mcpState ?? {})); - const threadId = 'discovery-thread-' + nanoid(6); - const runId = 'discovery-run-' + nanoid(6); - - const approvalResponders: ApprovalResponder[] = [ - credentialAutoSetupResponder, - ...(mcpRegistry ? [createMcpConnectResponder(mcpRegistry)] : []), - ]; - - const eventBus = wrapEventBusWithObserver(createInMemoryEventBus(), (event) => { - events.push(toCapturedEvent(event)); - }); - - // `OrchestrationContext` is required for the orchestrator to receive tools like - // `create-tasks` and runtime skills. Discovery scenarios measure first-step - // tool-call decisions, not background execution. - const orchestrationContext = createStubOrchestrationContext({ - context, - modelId: options.modelId, - eventBus, - threadId, - runId, - abortSignal: abortController.signal, - }); - - const { agent } = await createInstanceAgent({ - modelId: options.modelId, - context, - orchestrationContext, - mcpServers: stubMcpServerConfigs(mcpState ?? {}), - mcpManager, - // No memory: discovery measures stateless first-step tool dispatch. - memoryConfig: {}, - thinkingEnabled: false, - }); - - const streamSource = normalizeStreamSource( - await agent.stream(options.scenario.userMessage, { - maxIterations: maxSteps, - abortSignal: abortController.signal, - providerOptions: { - anthropic: { cacheControl: { type: 'ephemeral' as const } }, - }, - }), - ); - - const run = executeResumableStream({ - agent: asResumable(agent), - stream: streamSource, - context: { - threadId, - runId, - agentId: 'n8n-instance-agent', - eventBus, - signal: abortController.signal, - logger: silentLogger(), - }, - control: { - mode: 'auto', - onSuspension: (suspension) => suspensions.set(suspension.requestId, suspension), - waitForConfirmation: async (requestId: string): Promise> => - await Promise.resolve( - resolveConfirmation(suspensions.get(requestId), confirmationPolicy, approvalResponders), - ), - }, - }); - void run.catch(() => {}); - const result = await Promise.race([run, budgetExpired]); - - streamStatus = resolveStreamStatus(result, abortController.signal.aborted); - } catch (error) { - runError = error instanceof Error ? error.message : String(error); - streamStatus = abortController.signal.aborted ? 'timed-out' : 'errored'; - } finally { - clearTimeout(timeoutHandle); - await mcpManager?.disconnect(); - } - - const observedEvents = [...events]; - const outcome = extractOutcomeFromEvents(observedEvents); - const check = evaluateDiscoveryTrial(options.scenario, outcome, { - streamStatus, - timeoutMs, - ...(runError ? { runError } : {}), - unmatchedConfirmations: unmatchedConfirmations(confirmationPolicy, suspensions.values()), - }); - - return { - scenario: options.scenario, - check, - // An abandoned stream can still publish, so hand back what the verdict saw. - events: [...events], - outcome, - durationMs: Date.now() - started, - streamStatus, - ...(runError ? { runError } : {}), - }; -} - -// --------------------------------------------------------------------------- -// Internal helpers -// --------------------------------------------------------------------------- - -function applyInstanceState( - base: InstanceAiContext, - scenario: DiscoveryTestCase, - mcpRegistry: StubMcpRegistry | undefined, -): InstanceAiContext { - const state = scenario.instanceState; - if (!state) return base; - - const computerUse: ComputerUseState | undefined = state.computerUse; - // Both channels can serve tools, so the stub server gets the union. - const liveToolCategories = computerUse - ? Object.values(computerUse).flatMap((channel) => - channel.status === 'connected' ? channel.toolCategories : [], - ) - : []; - - const localMcpServer = - liveToolCategories.length > 0 - ? createStubLocalMcpServer({ - capabilities: liveToolCategories.filter( - (c): c is 'browser' | 'filesystem' | 'shell' => - c === 'browser' || c === 'filesystem' || c === 'shell', - ), - }) - : base.localMcpServer; - - return { - ...base, - ...(computerUse ? { computerUseState: computerUse } : {}), - ...(localMcpServer ? { localMcpServer } : {}), - ...(mcpRegistry ? { mcpService: mcpRegistry.service } : {}), - ...(state.folderExploration !== undefined - ? { folderExplorationEnabled: state.folderExploration } - : {}), - }; -} - -function silentLogger(): Logger { - return { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} }; -} - -function completedBuilderTurn(): BuilderTurnStream { - return { - fullStream: (async function* () { - await Promise.resolve(); - yield { - type: 'tool-call', - toolCallId: 'discovery-write', - toolName: 'write_config', - input: {}, - }; - yield { type: 'tool-result', toolCallId: 'discovery-write', output: { ok: true } }; - })(), - text: Promise.resolve('Agent configured for discovery evaluation.'), - }; -} - -function createStubBuilderDelegate(): InstanceAiBuilderDelegate { - return { - createAgent: async (name) => - await Promise.resolve({ - agentId: 'discovery-agent', - projectId: 'discovery-project', - name, - }), - streamBuild: async () => await Promise.resolve(completedBuilderTurn()), - resumeBuild: async () => await Promise.resolve(completedBuilderTurn()), - findOpenSuspensions: async () => await Promise.resolve([]), - cancelOpenSuspension: async () => await Promise.resolve(), - resolveAgentName: async () => await Promise.resolve(undefined), - }; -} - -interface StubOrchestrationContextOptions { - context: InstanceAiContext; - modelId: ModelConfig; - eventBus: InstanceAiEventBus; - threadId: string; - runId: string; - abortSignal: AbortSignal; -} - -function createStubOrchestrationContext( - opts: StubOrchestrationContextOptions, -): OrchestrationContext { - const taskStorage: TaskStorage = { - // eslint-disable-next-line @typescript-eslint/require-await - get: async (): Promise => null, - - save: async (): Promise => {}, - }; - - return { - threadId: opts.threadId, - runId: opts.runId, - userId: opts.context.userId, - orchestratorAgentId: 'n8n-instance-agent', - modelId: opts.modelId, - eventBus: opts.eventBus, - logger: silentLogger(), - runtimeSkills: loadInstanceAiRuntimeSkillSource(), - abortSignal: opts.abortSignal, - taskStorage, - // Surface the localMcpServer so Computer Use browser tools are available to the - // orchestrator. - ...(opts.context.localMcpServer ? { localMcpServer: opts.context.localMcpServer } : {}), - // Registers the `workspace_*` file tools for build-workflow - ...(opts.context.workspace ? { workspace: opts.context.workspace } : {}), - ...(opts.context.workspaceRoot ? { workspaceRoot: opts.context.workspaceRoot } : {}), - // Used for the orchestrator's untrusted-content doctrine and other domain references. - // Provide the same context the orchestrator sees. - domainContext: opts.context, - }; -} - -function toCapturedEvent(event: InstanceAiEvent): CapturedEvent { - return { - timestamp: Date.now(), - type: event.type, - // `extractOutcomeFromEvents` reads `data.payload.toolName` etc. — our - // InstanceAiEvent already has that shape, so we pass it through directly. - data: event, - }; -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/stream-status.ts b/packages/@n8n/instance-ai/evaluations/discovery/stream-status.ts deleted file mode 100644 index 04a975d509f..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/stream-status.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { FinishReason } from '@n8n/agents'; - -import type { DiscoveryStreamStatus } from './types'; -import type { ExecuteResumableStreamResult } from '../../src/runtime/resumable-stream-executor'; - -export function resolveStreamStatus( - result: ExecuteResumableStreamResult | 'timed-out', - aborted: boolean, -): DiscoveryStreamStatus { - if (result === 'timed-out' || aborted) return 'timed-out'; - if (result.status === 'errored') return 'errored'; - if (result.status === 'suspended') return 'suspended'; - if (result.status === 'cancelled') return 'timed-out'; - if (result.finishReason === undefined) return 'errored'; - return statusForFinishReason(result.finishReason); -} - -function statusForFinishReason(finishReason: FinishReason): DiscoveryStreamStatus { - switch (finishReason) { - case 'stop': - return 'completed'; - case 'max-iterations': - return 'step-exhausted'; - case 'paused': - return 'suspended'; - case 'tool-calls': - case 'length': - case 'content-filter': - case 'error': - case 'guardrail': - case 'other': - return 'errored'; - default: { - const unhandled: never = finishReason; - throw new Error(`Unhandled finish reason: ${String(unhandled)}`); - } - } -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/stub-local-mcp.ts b/packages/@n8n/instance-ai/evaluations/discovery/stub-local-mcp.ts deleted file mode 100644 index 820e31b72aa..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/stub-local-mcp.ts +++ /dev/null @@ -1,88 +0,0 @@ -// --------------------------------------------------------------------------- -// Stub LocalMcpServer for discovery evals. -// -// A case declares its connected channels through `instanceState.computerUse`, and -// the runner plugs in this stub so the agent holds the matching tools without -// spinning up the real computer-use daemon or the Chrome extension. The stub -// advertises tools by name and category. It never receives actual `callTool` invocations during -// discovery scenarios — the orchestrator's first dispatch decision is what we -// measure, not downstream tool execution. -// --------------------------------------------------------------------------- - -import type { McpTool, McpToolCallRequest, McpToolCallResult } from '@n8n/api-types'; - -import type { LocalMcpServer } from '../../src/types'; - -const STUB_BROWSER_TOOL_NAMES = [ - 'browser_connect', - 'browser_tab_open', - 'browser_navigate', - 'browser_snapshot', - 'browser_content', - 'browser_click', - 'browser_type', - 'browser_screenshot', - 'browser_capture_secret', - 'browser_create_credential', -]; - -const STUB_FILESYSTEM_TOOL_NAMES = ['fs_read_file', 'fs_search_files', 'fs_list_dir']; - -const STUB_SHELL_TOOL_NAMES = ['shell_run']; - -function makeStubTool(name: string, category: string): McpTool { - return { - name, - description: `Stub ${category} tool — no-op for discovery evals.`, - inputSchema: { type: 'object', properties: {} }, - annotations: { category }, - }; -} - -export interface CreateStubLocalMcpServerOptions { - /** Capabilities to advertise. Each capability publishes a small set of stub tool names. */ - capabilities: ReadonlyArray<'browser' | 'filesystem' | 'shell'>; -} - -export function createStubLocalMcpServer(options: CreateStubLocalMcpServerOptions): LocalMcpServer { - const tools: McpTool[] = []; - if (options.capabilities.includes('browser')) { - for (const name of STUB_BROWSER_TOOL_NAMES) tools.push(makeStubTool(name, 'browser')); - } - if (options.capabilities.includes('filesystem')) { - for (const name of STUB_FILESYSTEM_TOOL_NAMES) tools.push(makeStubTool(name, 'filesystem')); - } - if (options.capabilities.includes('shell')) { - for (const name of STUB_SHELL_TOOL_NAMES) tools.push(makeStubTool(name, 'shell')); - } - - return { - getAvailableTools: () => tools, - getToolsByCategory: (category: string) => - tools.filter((t) => { - const annotations = t.annotations; - return ( - annotations !== undefined && - typeof annotations === 'object' && - (annotations as Record).category === category - ); - }), - // eslint-disable-next-line @typescript-eslint/require-await - callTool: async (req: McpToolCallRequest): Promise => { - // Return a normal tool-error result rather than throwing. native agent logs an - // error stack trace for thrown tool errors; an `isError: true` result is - // treated as expected and surfaced as a `tool-error` event without spam. - // The discovery check still records the underlying `tool-call` event, so - // dispatch is captured either way. - return { - content: [ - { - type: 'text' as const, - text: `Stub: ${req.name} would have executed in production. Discovery evals measure dispatch, not execution.`, - }, - ], - isError: true, - }; - }, - }; -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/stub-mcp-registry.ts b/packages/@n8n/instance-ai/evaluations/discovery/stub-mcp-registry.ts deleted file mode 100644 index 5752cd39f87..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/stub-mcp-registry.ts +++ /dev/null @@ -1,201 +0,0 @@ -// --------------------------------------------------------------------------- -// Stub MCP registry + connections for discovery evals. Fakes the registry -// service; `stub-local-mcp.ts` fakes the computer-use gateway. -// -// --------------------------------------------------------------------------- - -import { sanitizeToolName, wrapToolForApproval, type BuiltTool } from '@n8n/agents'; -import { DEFAULT_INSTANCE_AI_PERMISSIONS, type InstanceAiPermissions } from '@n8n/api-types'; -import { isRecord } from '@n8n/utils/is-record'; - -import type { ApprovalResponder } from './confirmation-policy'; -import type { Logger } from '../../src/logger'; -import { McpClientManager } from '../../src/mcp/mcp-client-manager'; -import { createToolRegistry } from '../../src/tool-registry'; -import type { - InstanceAiMcpService, - InstanceAiToolRegistry, - McpRegistryConnectServerSummary, - McpRegistryServerSummary, - McpServerConfig, -} from '../../src/types'; - -/** What production derives for cli's shared e2e fixtures (`registry/mock-servers.ts`). - * Search below drops credential options and real relevance scoring. */ -const CATALOGUE: McpRegistryConnectServerSummary[] = [ - { - slug: 'notion', - title: 'Notion', - description: 'Connect to the Notion MCP Server', - usesCredentials: [ - { credentialType: 'notionMcpOAuth2Api', name: 'OAuth2', value: 'oAuth2' }, - ], - tools: ['notion-search', 'notion-fetch', 'notion-create-pages'], - }, - { - slug: 'linear', - title: 'Linear', - description: 'Connect to the Linear MCP Server', - usesCredentials: [ - { credentialType: 'linearMcpOAuth2Api', name: 'OAuth2', value: 'oAuth2' }, - ], - tools: ['list_issues', 'get_issue', 'save_issue'], - }, - { - slug: 'slack', - title: 'Slack', - description: 'Connect to the Slack MCP Server', - usesCredentials: [ - { credentialType: 'slackMcpOAuth2Api', name: 'OAuth2', value: 'oAuth2' }, - ], - tools: [], - }, -]; - -export interface StubMcpConnection { - slug: string; - /** Defaults to the catalogue's tools; empty models a broken connection. */ - tools?: string[]; -} - -export interface DiscoveryMcpState { - registry?: string[]; - connected?: StubMcpConnection[]; -} - -export interface StubMcpRegistry { - service: InstanceAiMcpService; - markConnected: (slugs: string[]) => void; -} - -function catalogueServer(slug: string): McpRegistryConnectServerSummary { - const server = CATALOGUE.find((entry) => entry.slug === slug); - if (!server) { - const known = CATALOGUE.map((entry) => entry.slug).join(', '); - throw new Error(`Unknown MCP registry slug "${slug}" — expected one of ${known}.`); - } - return server; -} - -export function createStubMcpRegistry(state: DiscoveryMcpState): StubMcpRegistry { - const servers = (state.registry ?? []).map(catalogueServer); - const connected = new Set((state.connected ?? []).map((entry) => entry.slug)); - - const matches = (server: McpRegistryServerSummary, query: string): boolean => - `${server.slug} ${server.title} ${server.description}` - .toLowerCase() - .includes(query.trim().toLowerCase()); - - return { - markConnected: (slugs) => slugs.forEach((slug) => connected.add(slug)), - service: { - // Connected servers are filtered out, as in the adapter. - search: async (queries) => - await Promise.resolve( - servers.filter( - (server) => - !connected.has(server.slug) && queries.some((query) => matches(server, query)), - ).map(({ slug, title, description, tools }) => ({ slug, title, description, tools })), - ), - getServers: async (slugs) => - await Promise.resolve(servers.filter((server) => slugs.includes(server.slug))), - listConnections: async () => await Promise.resolve([...connected].map((slug) => ({ slug }))), - }, - }; -} - -/** Approving a connect card claims the offered slugs on resume *and* makes them - * live here — `resumeConnect` counts a slug only when both hold. */ -export function createMcpConnectResponder(registry: StubMcpRegistry): ApprovalResponder { - return (payload) => { - const request = payload.mcpConnectRequest; - if (!isRecord(request) || !Array.isArray(request.servers)) return undefined; - - const slugs = request.servers - .map((server) => (isRecord(server) ? server.serverSlug : undefined)) - .filter((slug): slug is string => typeof slug === 'string' && slug.length > 0); - if (slugs.length === 0) return undefined; - - registry.markConnected(slugs); - return { connectedSlugs: slugs }; - }; -} - -/** `listConnectedMcpServices` pairs a tool with its slug through this name. */ -function serverName(slug: string): string { - return `mcp_${slug}`; -} - -export function stubMcpServerConfigs(state: DiscoveryMcpState): McpServerConfig[] { - return (state.connected ?? []).map(({ slug }) => ({ - name: serverName(slug), - url: `https://${slug}.mcp.invalid/mcp`, - metadata: { serverSlug: slug }, - })); -} - -/** Mirrors `mcp-tool-resolver`'s output, down to the server-prefixed name. */ -function stubMcpTool(rawName: string, slug: string): BuiltTool { - return { - name: sanitizeToolName(`${serverName(slug)}_${rawName}`), - description: `${slug} MCP tool.`, - inputSchema: { type: 'object', properties: {} }, - mcpTool: true, - mcpServerName: serverName(slug), - mcpToolName: rawName, - // An empty or apologetic result reads to the agent as an expired connection - // and sends it to `connect` — a stub artefact that looks like a regression. - handler: async () => - await Promise.resolve({ - ok: true, - items: [{ id: `${slug}-eval-stub-1`, title: 'Eval harness result' }], - }), - }; -} - -export function createStubMcpToolRegistry(state: DiscoveryMcpState): InstanceAiToolRegistry { - const registry = createToolRegistry(); - for (const { slug, tools } of state.connected ?? []) { - for (const rawName of tools ?? catalogueServer(slug).tools) { - const tool = stubMcpTool(rawName, slug); - registry.set(tool.name, tool); - } - } - return registry; -} - -/** Production wraps MCP tools in the same gate whenever the caller asks for - * approval; the real path goes through `McpConnection.listTools`, which this stub - * bypasses. */ -function withApprovalGate(tools: InstanceAiToolRegistry): InstanceAiToolRegistry { - const gated = createToolRegistry(); - for (const [name, tool] of tools) { - gated.set(name, wrapToolForApproval(tool, { requireApproval: true })); - } - return gated; -} - -/** `CreateInstanceAgentOptions.mcpManager` is the concrete class, so this - * subclasses rather than structurally fakes it. */ -export class StubMcpClientManager extends McpClientManager { - constructor(private readonly stubTools: InstanceAiToolRegistry) { - super(); - } - - override async getRegularTools( - _configs: McpServerConfig[], - _logger?: Logger, - { - mcpRead, - mcpWrite, - }: Pick = DEFAULT_INSTANCE_AI_PERMISSIONS, - ) { - const requireApproval = mcpRead !== 'always_allow' || mcpWrite !== 'always_allow'; - const tools = requireApproval ? withApprovalGate(this.stubTools) : this.stubTools; - return await Promise.resolve({ tools, connectionFailures: [] }); - } - - override async disconnect() { - await Promise.resolve(); - } -} diff --git a/packages/@n8n/instance-ai/evaluations/discovery/types.ts b/packages/@n8n/instance-ai/evaluations/discovery/types.ts deleted file mode 100644 index 8f97f82f80b..00000000000 --- a/packages/@n8n/instance-ai/evaluations/discovery/types.ts +++ /dev/null @@ -1,119 +0,0 @@ -// --------------------------------------------------------------------------- -// Tool-discovery scenario types — guards browser/computer-use discoverability. -// -// Discovery scenarios run the orchestrator against a user message that should -// (or should not) cause a specific tool / sub-agent to be reached for, and -// assert the captured events match the expectation. Workflow-build scenarios -// in evaluations/data/workflows/ assert the *output workflow*; discovery -// scenarios assert the *agent's tool/dispatch behavior*. -// --------------------------------------------------------------------------- - -import type { DiscoveryMcpState } from './stub-mcp-registry'; -import type { ComputerUseState } from '../../src/types'; - -/** - * Pass condition for tool invocations. - * - * - `anyOf` — pass if at least one of the listed tool names was invoked - * (top-level orchestrator call, or via a spawned sub-agent's tool list). - * - `noneOf` — pass only if NONE of the listed tool names was invoked. - * Used for negative scenarios that guard against over-eager invocation. - * - `anyOfToolCalls` — pass if at least one listed actual tool call happened. - * Unlike `anyOf`, this can check serialized tool args. - * - `allOfToolCalls` — pass only if EVERY listed actual tool call happened. - * Unlike `anyOf`, this checks completed/errored tool calls only; a spawned - * sub-agent having the tool available does not count as a match. - * - `noneOfToolCalls` — pass only if NONE of the listed tool calls happened. - * Unlike `noneOf`, this checks actual tool calls only; a spawned sub-agent - * having the tool available does not count as a violation. - * - * Both forms accept a sub-agent role prefix `spawn_sub_agent:` to match - * an `agent-spawned` event whose role equals ``. - */ -export interface ForbiddenToolCall { - toolName: string; - /** Deep-partial match: listed keys must match, extra keys ignored, pattern - * arrays matched as subsets. Prefer this for schema'd fields — a substring - * can't tell a value from the same word in a free-text sibling. */ - args?: Record; - /** Case-insensitive substrings over the serialized args, for free-text - * fields. Combines with `args`; both must hold on the same call. */ - argsContainAny?: string[]; - /** Match a call the user refused (an approval-gated tool resuming with - * `{ declined: true }`) instead of one that ran. Defaults to false, so an - * expectation never matches a refusal by accident. */ - declined?: boolean; -} - -export interface ExpectedToolInvocations { - anyOf?: string[]; - noneOf?: string[]; - anyOfToolCalls?: ForbiddenToolCall[]; - allOfToolCalls?: ForbiddenToolCall[]; - noneOfToolCalls?: ForbiddenToolCall[]; -} - -/** - * Optional instance configuration overrides for a discovery scenario. - * Mirrors a subset of `SystemPromptOptions` so a scenario can pin the gateway - * status (e.g. `disabledGlobally` to test the "explain how to enable" branch). - */ -export interface DiscoveryInstanceState { - computerUse?: ComputerUseState; - mcp?: DiscoveryMcpState; - /** Turn the folder-exploration capability on for the run (default off). */ - folderExploration?: boolean; -} - -export type ConfirmationDecision = 'approve' | 'deny'; - -export interface ConfirmationAnswer { - decision: ConfirmationDecision; - resumeWith?: Record; -} - -// Keyed by the suspending tool's name -export type DiscoveryConfirmations = Record; - -export interface DiscoveryTestCase { - /** Unique scenario identifier — also used as the scenario filename (without .json). */ - id: string; - /** The user message sent to the orchestrator. */ - userMessage: string; - /** Optional instance state overrides applied when constructing the agent. */ - instanceState?: DiscoveryInstanceState; - /** Answers for the tools listed; every other suspension is approved. */ - confirmations?: DiscoveryConfirmations; - /** Pass condition. At least one expectation key is required. */ - expectedToolInvocations: ExpectedToolInvocations; - /** Free-form note explaining what regression this scenario protects against. */ - rationale?: string; - /** Override the runner step cap when orchestrator-inline work needs more iterations. */ - maxSteps?: number; - /** Override the runner timeout when a scenario needs more time. */ - timeoutMs?: number; -} - -export type DiscoveryStreamStatus = - | 'completed' - | 'errored' - | 'timed-out' - | 'suspended' - | 'step-exhausted'; - -export interface DiscoveryTrialFacts { - streamStatus: DiscoveryStreamStatus; - timeoutMs: number; - runError?: string; - unmatchedConfirmations: string[]; -} - -export interface DiscoveryCheckResult { - pass: boolean; - /** Human-readable reason — included in failure reports. */ - comment: string; - /** Tool names actually invoked during the run (top-level + via sub-agents). */ - invokedTools: string[]; - /** `spawn_sub_agent:` markers for every spawned sub-agent. */ - spawnedAgents: string[]; -} diff --git a/packages/@n8n/instance-ai/package.json b/packages/@n8n/instance-ai/package.json index b846e314fa1..ee47dc5e6cc 100644 --- a/packages/@n8n/instance-ai/package.json +++ b/packages/@n8n/instance-ai/package.json @@ -23,7 +23,6 @@ "eval:pairwise:compare": "tsx evaluations/cli/compare-pairwise.ts", "eval:subagent": "tsx evaluations/subagent/cli.ts", "eval:computer-use": "tsx evaluations/computer-use/cli.ts", - "eval:discovery": "tsx evaluations/discovery/cli.ts", "prompts:print": "tsx scripts/print-prompts.ts" }, "main": "dist/index.js", diff --git a/packages/@n8n/instance-ai/skills/planned-task-runtime/SKILL.md b/packages/@n8n/instance-ai/skills/planned-task-runtime/SKILL.md index 48ba8200e8b..d45eaae5a46 100644 --- a/packages/@n8n/instance-ai/skills/planned-task-runtime/SKILL.md +++ b/packages/@n8n/instance-ai/skills/planned-task-runtime/SKILL.md @@ -45,12 +45,6 @@ If a task appears stuck, tell the user and stop; do not try to detect completion yourself. Do not re-dispatch a build whose task ID is already visible in ``. -When `` context is present, use it only to reference active task -IDs for cancellation or corrections. - -If the user sends a correction while a build is running, call -`task-control(action="correct-task")` with the task ID and correction. - ## Synthesize follow-up When `` is present, all planned tasks diff --git a/packages/@n8n/instance-ai/src/__tests__/index.test.ts b/packages/@n8n/instance-ai/src/__tests__/index.test.ts index 9d5412503a2..00356bff782 100644 --- a/packages/@n8n/instance-ai/src/__tests__/index.test.ts +++ b/packages/@n8n/instance-ai/src/__tests__/index.test.ts @@ -66,7 +66,6 @@ vi.mock('../storage', () => ({ ThreadTaskStorage: class ThreadTaskStorage {}, PlannedTaskStorage: class PlannedTaskStorage {}, getThread: () => ({ id: 'thread-1' }), - TerminalOutcomeStorage: class TerminalOutcomeStorage {}, patchThread: () => ({ id: 'thread-1' }), WorkflowLoopStorage: class WorkflowLoopStorage {}, })); @@ -108,40 +107,10 @@ vi.mock('../workspace/sandbox-setup', () => ({ setupSandboxWorkspace: () => undefined, })); vi.mock('../workspace/snapshot-manager', () => ({ SnapshotManager: class SnapshotManager {} })); -vi.mock('../runtime/background-task-manager', () => ({ - BackgroundTaskManager: class BackgroundTaskManager {}, - enrichMessageWithRunningTasks: () => ({ content: 'message' }), -})); vi.mock('../runtime/run-state-registry', () => ({ RunStateRegistry: class RunStateRegistry {} })); vi.mock('../runtime/terminal-response-guard', () => ({ InstanceAiTerminalResponseGuard: class InstanceAiTerminalResponseGuard {}, })); -vi.mock('../runtime/resumable-stream-executor', () => ({ - executeResumableStream: () => ({ status: 'done' }), -})); -vi.mock('../runtime/stream-runner', () => ({ - resumeAgentRun: () => ({ status: 'resumed' }), - streamAgentRun: () => ({ status: 'streamed' }), -})); -vi.mock('../runtime/liveness-policy', () => { - const INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG = { - confirmationTimeoutMs: 1, - backgroundTaskIdleTimeoutMs: 1, - backgroundTaskMaxLifetimeMs: 1, - activeRunIdleTimeoutMs: 0, - activeRunMaxLifetimeMs: 1, - }; - class InstanceAiLivenessPolicy { - hasEnabledTimeouts() { - return true; - } - } - return { - INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, - createInstanceAiLivenessPolicyConfig: () => INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, - InstanceAiLivenessPolicy, - }; -}); vi.mock('../workflow-loop', () => ({ workflowBuildOutcomeSchema: { safeParse: () => ({ success: true }) }, attemptRecordSchema: { safeParse: () => ({ success: false }) }, @@ -188,7 +157,6 @@ describe('@n8n/instance-ai public entrypoint', () => { expect(entrypoint.getParseableAttachmentMimeTypes).toEqual(expect.any(Function)); expect(entrypoint.mapAgentChunkToEvent).toEqual(expect.any(Function)); expect(entrypoint.wrapUntrustedData).toEqual(expect.any(Function)); - expect(entrypoint.BackgroundTaskManager).toEqual(expect.any(Function)); expect(entrypoint.IdRemapper).toEqual(expect.any(Function)); expect(entrypoint.McpClientManager).toEqual(expect.any(Function)); expect(entrypoint.UnsupportedAttachmentError).toEqual(expect.any(Function)); @@ -242,9 +210,6 @@ describe('@n8n/instance-ai public entrypoint', () => { expect(construct(entrypoint.ThreadTaskStorage)).toBeInstanceOf(entrypoint.ThreadTaskStorage); expect(construct(entrypoint.PlannedTaskStorage)).toBeInstanceOf(entrypoint.PlannedTaskStorage); expect(call(entrypoint.getThread)).toEqual({ id: 'thread-1' }); - expect(construct(entrypoint.TerminalOutcomeStorage)).toBeInstanceOf( - entrypoint.TerminalOutcomeStorage, - ); expect(call(entrypoint.patchThread)).toEqual({ id: 'thread-1' }); expect(construct(entrypoint.WorkflowLoopStorage)).toBeInstanceOf( entrypoint.WorkflowLoopStorage, @@ -277,25 +242,10 @@ describe('@n8n/instance-ai public entrypoint', () => { expect(call(entrypoint.createWorkspace)).toEqual({ type: 'workspace' }); expect(construct(entrypoint.SnapshotManager)).toBeInstanceOf(entrypoint.SnapshotManager); - expect(construct(entrypoint.BackgroundTaskManager)).toBeInstanceOf( - entrypoint.BackgroundTaskManager, - ); - expect(call(entrypoint.enrichMessageWithRunningTasks)).toEqual({ content: 'message' }); - expect(call(entrypoint.enrichMessageWithBackgroundTasks)).toEqual({ content: 'message' }); expect(construct(entrypoint.RunStateRegistry)).toBeInstanceOf(entrypoint.RunStateRegistry); expect(construct(entrypoint.InstanceAiTerminalResponseGuard)).toBeInstanceOf( entrypoint.InstanceAiTerminalResponseGuard, ); - expect(call(entrypoint.executeResumableStream)).toEqual({ status: 'done' }); - expect(call(entrypoint.resumeAgentRun)).toEqual({ status: 'resumed' }); - expect(call(entrypoint.streamAgentRun)).toEqual({ status: 'streamed' }); - - expect(call(entrypoint.createInstanceAiLivenessPolicyConfig)).toEqual( - entrypoint.INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, - ); - expect(construct(entrypoint.InstanceAiLivenessPolicy)).toBeInstanceOf( - entrypoint.InstanceAiLivenessPolicy, - ); expect(construct(entrypoint.WorkflowTaskCoordinator)).toBeInstanceOf( entrypoint.WorkflowTaskCoordinator, ); diff --git a/packages/@n8n/instance-ai/src/debug/__tests__/run-debug-buffer.test.ts b/packages/@n8n/instance-ai/src/debug/__tests__/run-debug-buffer.test.ts deleted file mode 100644 index b0b3a78ff1c..00000000000 --- a/packages/@n8n/instance-ai/src/debug/__tests__/run-debug-buffer.test.ts +++ /dev/null @@ -1,282 +0,0 @@ -import type { GenerateTextStepEndEvent, GenerateTextStepStartEvent } from 'ai'; -import { describe, expect, it } from 'vitest'; -import { mock } from 'vitest-mock-extended'; -import { z } from 'zod'; - -import type { Logger } from '../../logger'; -import { - RunDebugBuffer, - buildRunDebugLabel, - createRunDebugStepHooks, - sanitizeStepFinish, - sanitizeStepStart, -} from '../run-debug-buffer'; -import { sanitizeDebugSnapshotValue } from '../sanitize-debug-snapshot'; - -function makeStepEndEvent(text: string): GenerateTextStepEndEvent { - return { - stepNumber: 0, - text, - toolCalls: [], - finishReason: 'tool-calls', - usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 }, - response: { - modelId: 'test-model', - timestamp: new Date('2026-01-01T00:00:00.000Z'), - messages: [{ role: 'assistant', content: text }], - }, - } as unknown as GenerateTextStepEndEvent; -} - -function makeStepStartEvent(): GenerateTextStepStartEvent { - return { - stepNumber: 0, - system: 'You are helpful', - messages: [{ role: 'user', content: 'hello' }], - tools: { search: { description: 'search', inputSchema: { type: 'object' } } }, - toolChoice: 'auto', - activeTools: ['search'], - abortSignal: new AbortController().signal, - } as unknown as GenerateTextStepStartEvent; -} - -describe('RunDebugBuffer', () => { - it('pairs step start and finish by run-scoped step index', () => { - const buffer = new RunDebugBuffer(); - buffer.ensure('run-1', 'thread-1'); - - buffer.recordStepStart('run-1', 0, makeStepStartEvent()); - buffer.recordStepFinish('run-1', 0, makeStepEndEvent('done')); - - const record = buffer.get('run-1'); - expect(record?.steps).toHaveLength(1); - expect(record?.steps[0]?.stepNumber).toBe(0); - expect(record?.steps[0]?.input?.messages).toBeDefined(); - expect(record?.steps[0]?.output?.text).toBe('done'); - }); - - it('records multiple orchestrator iterations even when SDK stepNumber stays 0', () => { - const buffer = new RunDebugBuffer(); - buffer.ensure('run-1', 'thread-1'); - const hooks = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' }); - - for (const label of ['first', 'second', 'third']) { - hooks.onStepStart(makeStepStartEvent()); - hooks.onStepEnd(makeStepEndEvent(label)); - } - - const record = buffer.get('run-1'); - expect(record?.steps).toHaveLength(3); - expect(record?.steps.map((step) => step.stepNumber)).toEqual([0, 1, 2]); - expect(record?.steps.map((step) => step.output?.text)).toEqual(['first', 'second', 'third']); - }); - - it('continues step numbering after resume hooks are recreated', () => { - const buffer = new RunDebugBuffer(); - buffer.ensure('run-1', 'thread-1'); - - const firstPass = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' }); - firstPass.onStepStart(makeStepStartEvent()); - firstPass.onStepEnd(makeStepEndEvent('before suspend')); - - const resumePass = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' }); - resumePass.onStepStart(makeStepStartEvent()); - resumePass.onStepEnd(makeStepEndEvent('after resume')); - - const record = buffer.get('run-1'); - expect(record?.steps).toHaveLength(2); - expect(record?.steps[1]?.stepNumber).toBe(1); - expect(record?.steps[1]?.output?.text).toBe('after resume'); - }); - - it('keeps full tool definitions and strips abortSignal from step start', () => { - const sanitized = sanitizeStepStart( - { - stepNumber: 1, - system: 'system prompt', - messages: [{ role: 'user', content: 'hello' }], - tools: { - search: { description: 'search', inputSchema: { type: 'object', properties: { q: {} } } }, - }, - toolChoice: 'auto', - activeTools: ['search'], - abortSignal: new AbortController().signal, - } as unknown as GenerateTextStepStartEvent, - 4, - ); - - expect(sanitized.stepNumber).toBe(4); - expect(sanitized.sdkStepNumber).toBe(1); - expect(sanitized.activeTools).toEqual(['search']); - expect(sanitized.tools).toEqual({ - search: { description: 'search', inputSchema: { type: 'object', properties: { q: {} } } }, - }); - expect(sanitized).not.toHaveProperty('abortSignal'); - }); - - it('drops step-start data that repeats other captured fields', () => { - const stepTools = [{ type: 'function', name: 'search', inputSchema: { type: 'object' } }]; - const sanitized = sanitizeStepStart( - { - stepNumber: 0, - instructions: 'system prompt', - messages: [{ role: 'user', content: 'hello' }], - tools: { search: { description: 'search', inputSchema: z.object({ q: z.string() }) } }, - stepTools, - promptMessages: [{ role: 'system', content: 'system prompt' }], - steps: [{ stepNumber: 0, text: 'earlier step' }], - modelId: 'claude', - } as unknown as GenerateTextStepStartEvent, - 0, - ); - - expect(sanitized.stepTools).toEqual(stepTools); - expect(sanitized.modelId).toBe('claude'); - for (const key of ['tools', 'promptMessages', 'steps']) { - expect(sanitized).not.toHaveProperty(key); - } - }); - - it('drops empty step-start containers and keeps populated ones', () => { - const sanitized = sanitizeStepStart( - { - stepNumber: 0, - runtimeContext: {}, - toolsContext: { search: { region: 'eu' } }, - activeTools: [], - } as unknown as GenerateTextStepStartEvent, - 0, - ); - - expect(sanitized).not.toHaveProperty('runtimeContext'); - expect(sanitized).not.toHaveProperty('activeTools'); - expect(sanitized.toolsContext).toEqual({ search: { region: 'eu' } }); - }); - - it('keeps tool descriptions without Zod internals when stepTools is missing', () => { - const sanitized = sanitizeStepStart( - { - stepNumber: 0, - tools: { search: { description: 'search', inputSchema: z.object({ q: z.string() }) } }, - } as unknown as GenerateTextStepStartEvent, - 0, - ); - - expect(sanitized.tools).toEqual({ search: { description: 'search' } }); - }); - - it('does not truncate long captured strings', () => { - const longSystem = 'x'.repeat(5_000); - const sanitized = sanitizeDebugSnapshotValue(longSystem); - expect(sanitized).toHaveLength(5_000); - }); - - it('captures full tool call inputs and outputs on step finish', () => { - const sanitized = sanitizeStepFinish( - { - stepNumber: 0, - text: '', - toolCalls: [ - { - toolCallId: 'tc-1', - toolName: 'build-workflow', - input: { code: 'full workflow code payload' }, - }, - ], - toolResults: [ - { - toolCallId: 'tc-1', - toolName: 'build-workflow', - output: { success: true, workflowId: 'wf-1' }, - }, - ], - finishReason: 'tool-calls', - usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 }, - response: { - modelId: 'test-model', - timestamp: new Date('2026-01-01T00:00:00.000Z'), - messages: [{ role: 'assistant', content: '' }], - body: { secret: 'raw-provider-body' }, - }, - } as unknown as GenerateTextStepEndEvent, - 0, - ); - - expect(sanitized.toolCalls).toEqual([ - { - toolCallId: 'tc-1', - toolName: 'build-workflow', - input: { code: 'full workflow code payload' }, - }, - ]); - expect(sanitized.toolResults).toEqual([ - { - toolCallId: 'tc-1', - toolName: 'build-workflow', - output: { success: true, workflowId: 'wf-1' }, - }, - ]); - expect(sanitized.response).not.toHaveProperty('body'); - }); - - it('evicts oldest run when cap is exceeded', () => { - const logger = mock(); - const buffer = new RunDebugBuffer(logger); - - for (let index = 0; index < 51; index++) { - const runId = `run-${index}`; - buffer.ensure(runId, 'thread-1'); - buffer.recordStepStart(runId, 0, { - stepNumber: 0, - messages: [], - } as unknown as GenerateTextStepStartEvent); - } - - expect(buffer.get('run-0')).toBeUndefined(); - expect(buffer.get('run-50')).toBeDefined(); - expect(logger.warn).toHaveBeenCalled(); - }); - - it('createRunDebugStepHooks writes into the buffer', () => { - const buffer = new RunDebugBuffer(); - buffer.ensure('run-1', 'thread-1'); - const hooks = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' }); - - hooks.onStepStart({ - stepNumber: 2, - messages: [{ role: 'user', content: 'ping' }], - } as unknown as GenerateTextStepStartEvent); - - expect(buffer.get('run-1')?.steps[0]?.stepNumber).toBe(0); - expect(buffer.get('run-1')?.steps[0]?.input?.sdkStepNumber).toBe(2); - }); - - it('exposes onStepEnd and keeps onStepFinish as a compatibility alias', () => { - const buffer = new RunDebugBuffer(); - const hooks = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' }); - - expect(typeof hooks.onStepEnd).toBe('function'); - expect(hooks.onStepFinish).toBe(hooks.onStepEnd); - }); - - it('stores a run label on first ensure', () => { - const buffer = new RunDebugBuffer(); - buffer.ensure('run-1', 'thread-1', 'build a weather workflow'); - expect(buffer.get('run-1')?.label).toBe('build a weather workflow'); - buffer.ensure('run-1', 'thread-1', 'ignored'); - expect(buffer.get('run-1')?.label).toBe('build a weather workflow'); - }); -}); - -describe('buildRunDebugLabel', () => { - it('uses the user message when no resume reason is provided', () => { - expect(buildRunDebugLabel({ message: 'build a simple workflow' })).toBe( - 'build a simple workflow', - ); - }); - - it('maps resume reasons to action labels', () => { - expect(buildRunDebugLabel({ resumeReason: 'approval' })).toBe('Resume · approval'); - expect(buildRunDebugLabel({ resumeReason: 'replan' })).toBe('Follow-up · replan'); - }); -}); diff --git a/packages/@n8n/instance-ai/src/debug/__tests__/sanitize-debug-snapshot.test.ts b/packages/@n8n/instance-ai/src/debug/__tests__/sanitize-debug-snapshot.test.ts deleted file mode 100644 index c645957aaa0..00000000000 --- a/packages/@n8n/instance-ai/src/debug/__tests__/sanitize-debug-snapshot.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - sanitizeDebugSnapshotRecord, - sanitizeDebugSnapshotValue, -} from '../sanitize-debug-snapshot'; - -describe('sanitizeDebugSnapshotValue', () => { - it('redacts secrets and omits runtime-only payloads', () => { - expect( - sanitizeDebugSnapshotRecord({ - apiKey: 'plain-value', - message: 'token sk-abcdefghijklmnop', - abortSignal: new AbortController().signal, - response: { body: 'raw provider response', status: 500 }, - }), - ).toEqual({ - apiKey: '[redacted]', - message: '[REDACTED]', - response: { status: 500 }, - }); - }); - - it('scrubs error messages', () => { - expect(sanitizeDebugSnapshotValue(new Error('failed with sk-abcdefghijklmnop'))).toEqual({ - name: 'Error', - message: 'failed with [REDACTED]', - }); - }); - - it('handles circular values', () => { - const value: Record = {}; - value.self = value; - - expect(sanitizeDebugSnapshotValue(value)).toEqual({ self: '[Circular]' }); - }); - - it('serializes a shared reference in each place it appears', () => { - const shared = { anthropic: { eagerInputStreaming: false } }; - - expect( - sanitizeDebugSnapshotValue({ - first: { providerOptions: shared }, - second: [{ providerOptions: shared }], - }), - ).toEqual({ - first: { providerOptions: shared }, - second: [{ providerOptions: shared }], - }); - }); -}); diff --git a/packages/@n8n/instance-ai/src/debug/run-debug-buffer.ts b/packages/@n8n/instance-ai/src/debug/run-debug-buffer.ts deleted file mode 100644 index 2fdf46a7437..00000000000 --- a/packages/@n8n/instance-ai/src/debug/run-debug-buffer.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { isRecord } from '@n8n/utils/is-record'; -import { scrubSecretsInText } from '@n8n/utils/scrub-secrets'; -import type { GenerateTextStepEndEvent, GenerateTextStepStartEvent } from 'ai'; - -import type { Logger } from '../logger'; -import { sanitizeDebugSnapshotRecord, sanitizeDebugSnapshotValue } from './sanitize-debug-snapshot'; - -const MAX_RUNS = 50; -const MAX_STEPS_PER_RUN = 200; -const MAX_WORKFLOW_SNAPSHOTS_PER_RUN = 100; - -export interface WorkflowCodeSnapshotInput { - code: string; - source: 'full-code' | 'patch'; - patches?: unknown; - workflowId?: string; - toolCallId?: string; - success: boolean; - errors?: string[]; - capturedAt: number; -} - -export type SanitizedStepStart = { - stepNumber: number; - sdkStepNumber?: number; -} & Record; - -export type SanitizedStepFinish = { - stepNumber: number; - sdkStepNumber?: number; -} & Record; - -export type WorkflowCodeSnapshot = WorkflowCodeSnapshotInput; - -export interface RunDebugStep { - stepNumber: number; - input?: SanitizedStepStart; - output?: SanitizedStepFinish; -} - -export interface RunDebugRecord { - threadId: string; - runId: string; - startedAt: number; - label?: string; - /** Next run-scoped step index; survives step-cap eviction. */ - nextStepIndex: number; - steps: RunDebugStep[]; - workflowCode: WorkflowCodeSnapshot[]; -} - -export interface RunDebugStepHookOptions { - runId: string; - threadId: string; -} - -/** - * Step-start keys that repeat data captured elsewhere: - * - `tools` holds live Zod schemas; `stepTools` has the JSON Schema the model received. - * - `promptMessages` repeats `instructions` and `messages`. - * - `steps` repeats earlier steps, which the buffer records on their own. - */ -const DUPLICATE_STEP_START_KEYS = new Set(['tools', 'promptMessages', 'steps']); - -function isEmptyContainer(value: unknown): boolean { - if (Array.isArray(value)) return value.length === 0; - return isRecord(value) && Object.keys(value).length === 0; -} - -function captureStepStartPayload(event: GenerateTextStepStartEvent): Record { - const hasStepTools = 'stepTools' in event && Array.isArray(event.stepTools); - const payload: Record = {}; - for (const [key, value] of Object.entries(event)) { - if (key === 'tools' && !hasStepTools) { - payload.tools = summarizeToolSet(value); - continue; - } - if (DUPLICATE_STEP_START_KEYS.has(key) || isEmptyContainer(value)) continue; - payload[key] = value; - } - return sanitizeDebugSnapshotRecord(payload); -} - -/** - * Keeps tool descriptions and plain JSON schemas when the SDK does not report - * `stepTools`. Zod schemas are dropped: serialized, they are only method stubs. - */ -function summarizeToolSet(tools: unknown): unknown { - if (!isRecord(tools)) return undefined; - const summary: Record = {}; - for (const [name, tool] of Object.entries(tools)) { - if (!isRecord(tool)) continue; - summary[name] = { - ...(typeof tool.description === 'string' ? { description: tool.description } : {}), - ...(tool.inputSchema !== undefined && !isZodSchema(tool.inputSchema) - ? { inputSchema: tool.inputSchema } - : {}), - }; - } - return summary; -} - -/** Matches Zod 3 (`_def`) and Zod 4 (`_zod`) schemas without depending on one Zod copy. */ -function isZodSchema(value: unknown): boolean { - return isRecord(value) && ('_def' in value || '_zod' in value); -} - -function captureStepFinishPayload(event: GenerateTextStepEndEvent): Record { - return sanitizeDebugSnapshotRecord(event); -} - -export function sanitizeStepStart( - event: GenerateTextStepStartEvent, - stepNumber: number, -): SanitizedStepStart { - return { - ...captureStepStartPayload(event), - stepNumber, - sdkStepNumber: event.stepNumber, - }; -} - -export function sanitizeStepFinish( - event: GenerateTextStepEndEvent, - stepNumber: number, -): SanitizedStepFinish { - return { - ...captureStepFinishPayload(event), - stepNumber, - sdkStepNumber: event.stepNumber, - }; -} - -export function createRunDebugStepHooks( - buffer: RunDebugBuffer, - options: RunDebugStepHookOptions, -): { - onStepStart: (event: GenerateTextStepStartEvent) => void; - onStepEnd: (event: GenerateTextStepEndEvent) => void; - /** @deprecated Use `onStepEnd` instead. */ - onStepFinish: (event: GenerateTextStepEndEvent) => void; -} { - // The agent runtime calls streamText/generateText once per loop iteration. The AI SDK - // resets stepNumber to 0 on each call, so we allocate a run-scoped sequence instead. - let stepIndex = buffer.getNextStepIndex(options.runId); - - const onStepEnd = (event: GenerateTextStepEndEvent) => { - buffer.recordStepFinish(options.runId, stepIndex, event); - stepIndex++; - }; - - return { - onStepStart: (event) => { - buffer.recordStepStart(options.runId, stepIndex, event); - }, - onStepEnd, - onStepFinish: onStepEnd, - }; -} - -export function buildRunDebugLabel(options: { - message?: string; - resumeReason?: string; -}): string { - const resumeLabels: Record = { - approval: 'Resume · approval', - background_task_completed: 'Follow-up · background task completed', - workflow_verification: 'Follow-up · workflow verification', - workflow_setup: 'Follow-up · workflow setup', - planned_checkpoint: 'Follow-up · planned checkpoint', - replan: 'Follow-up · replan', - synthesize: 'Follow-up · synthesize', - }; - - if (options.resumeReason && resumeLabels[options.resumeReason]) { - return resumeLabels[options.resumeReason]; - } - - const trimmed = options.message?.trim(); - if (trimmed) { - return trimmed.length > 80 ? `${trimmed.slice(0, 80)}…` : trimmed; - } - - return 'Orchestrator run'; -} - -export class RunDebugBuffer { - private readonly records = new Map(); - - constructor(private readonly logger?: Logger) {} - - ensure(runId: string, threadId: string, label?: string): void { - if (this.records.has(runId)) return; - - this.evictOldestRunIfNeeded(); - this.records.set(runId, { - threadId, - runId, - startedAt: Date.now(), - label: label ? scrubSecretsInText(label.trim()) : undefined, - nextStepIndex: 0, - steps: [], - workflowCode: [], - }); - } - - getNextStepIndex(runId: string): number { - return this.records.get(runId)?.nextStepIndex ?? 0; - } - - recordStepStart(runId: string, stepIndex: number, event: GenerateTextStepStartEvent): void { - const record = this.records.get(runId); - if (!record) return; - - const existing = record.steps.find((step) => step.stepNumber === stepIndex); - if (existing) { - existing.input = sanitizeStepStart(event, stepIndex); - return; - } - - this.evictOldestStepIfNeeded(record); - record.steps.push({ - stepNumber: stepIndex, - input: sanitizeStepStart(event, stepIndex), - }); - record.steps.sort((a, b) => a.stepNumber - b.stepNumber); - } - - recordStepFinish(runId: string, stepIndex: number, event: GenerateTextStepEndEvent): void { - const record = this.records.get(runId); - if (!record) return; - - const existing = record.steps.find((step) => step.stepNumber === stepIndex); - if (existing) { - existing.output = sanitizeStepFinish(event, stepIndex); - record.nextStepIndex = stepIndex + 1; - return; - } - - this.evictOldestStepIfNeeded(record); - record.steps.push({ - stepNumber: stepIndex, - output: sanitizeStepFinish(event, stepIndex), - }); - record.steps.sort((a, b) => a.stepNumber - b.stepNumber); - record.nextStepIndex = stepIndex + 1; - } - - recordWorkflowCode(runId: string, snapshot: WorkflowCodeSnapshotInput): void { - const record = this.records.get(runId); - if (!record) return; - - if (record.workflowCode.length >= MAX_WORKFLOW_SNAPSHOTS_PER_RUN) { - record.workflowCode.shift(); - this.logger?.warn('Evicted oldest workflow code snapshot from run debug buffer', { - runId, - maxSnapshots: MAX_WORKFLOW_SNAPSHOTS_PER_RUN, - }); - } - - record.workflowCode.push({ - ...snapshot, - code: scrubSecretsInText(snapshot.code), - patches: sanitizeDebugSnapshotValue(snapshot.patches), - errors: snapshot.errors?.map((error) => scrubSecretsInText(error)), - }); - } - - get(runId: string): RunDebugRecord | undefined { - const record = this.records.get(runId); - if (!record) return undefined; - return structuredClone(record); - } - - listByThread(threadId: string): RunDebugRecord[] { - return [...this.records.values()] - .filter((record) => record.threadId === threadId) - .sort((a, b) => a.startedAt - b.startedAt) - .map((record) => structuredClone(record)); - } - - private evictOldestRunIfNeeded(): void { - if (this.records.size < MAX_RUNS) return; - - const oldest = [...this.records.values()].sort((a, b) => a.startedAt - b.startedAt)[0]; - if (!oldest) return; - - this.records.delete(oldest.runId); - this.logger?.warn('Evicted oldest run from debug buffer', { - runId: oldest.runId, - threadId: oldest.threadId, - maxRuns: MAX_RUNS, - }); - } - - private evictOldestStepIfNeeded(record: RunDebugRecord): void { - if (record.steps.length < MAX_STEPS_PER_RUN) return; - - const removed = record.steps.shift(); - this.logger?.warn('Evicted oldest step from run debug buffer', { - runId: record.runId, - stepNumber: removed?.stepNumber, - maxSteps: MAX_STEPS_PER_RUN, - }); - } -} diff --git a/packages/@n8n/instance-ai/src/debug/sanitize-debug-snapshot.ts b/packages/@n8n/instance-ai/src/debug/sanitize-debug-snapshot.ts deleted file mode 100644 index 85b87b9a2d0..00000000000 --- a/packages/@n8n/instance-ai/src/debug/sanitize-debug-snapshot.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { isRecord } from '@n8n/utils/is-record'; -import { isSensitiveKey } from '@n8n/utils/redaction/sensitive-key'; -import { scrubSecretsInText } from '@n8n/utils/scrub-secrets'; - -const OMIT_KEYS = new Set(['abortSignal']); - -function shouldOmitKey(key: string, parentKey?: string): boolean { - if (OMIT_KEYS.has(key)) { - return true; - } - - // Raw provider HTTP payloads are not useful in the debug buffer. - if (key === 'body' && parentKey === 'response') { - return true; - } - - return false; -} - -function redactSensitiveKey(key: string, value: unknown, seen: WeakSet): unknown { - if (isSensitiveKey(key) && typeof value === 'string') { - return '[redacted]'; - } - - return sanitizeDebugSnapshotValue(value, key, seen); -} - -/** - * Full-fidelity JSON-safe snapshot for the in-memory run debug buffer. - * Unlike trace sanitization, this does not truncate strings, arrays, or object keys. - */ -export function sanitizeDebugSnapshotValue( - value: unknown, - keyHint?: string, - seen?: WeakSet, -): unknown { - const ancestors = seen ?? new WeakSet(); - - if (value === undefined || value === null) { - return value; - } - - if (typeof value === 'string') { - if (keyHint && isSensitiveKey(keyHint)) { - return '[redacted]'; - } - return scrubSecretsInText(value); - } - - if (typeof value === 'number' || typeof value === 'boolean') { - return value; - } - - if (typeof value === 'bigint') { - return value.toString(); - } - - if (typeof value === 'function') { - return `[function ${value.name || 'anonymous'}]`; - } - - if (value instanceof AbortSignal) { - return '[AbortSignal]'; - } - - if (value instanceof Date) { - return value.toISOString(); - } - - if (value instanceof Error) { - return { - name: value.name, - message: scrubSecretsInText(value.message), - }; - } - - if (value instanceof Uint8Array) { - return `[binary ${value.byteLength} bytes]`; - } - - // Only objects on the current path count as circular. A shared reference that - // appears in several sibling branches is serialized in each of them. - if (Array.isArray(value)) { - if (ancestors.has(value)) { - return '[Circular]'; - } - ancestors.add(value); - const sanitized = value.map((entry) => sanitizeDebugSnapshotValue(entry, keyHint, ancestors)); - ancestors.delete(value); - return sanitized; - } - - if (isRecord(value)) { - if (ancestors.has(value)) { - return '[Circular]'; - } - ancestors.add(value); - const sanitized: Record = {}; - for (const [key, entryValue] of Object.entries(value)) { - if (shouldOmitKey(key, keyHint)) { - continue; - } - sanitized[key] = redactSensitiveKey(key, entryValue, ancestors); - } - ancestors.delete(value); - return sanitized; - } - - if (typeof value === 'symbol') { - return value.toString(); - } - - return '[unsupported value]'; -} - -export function sanitizeDebugSnapshotRecord(value: unknown): Record { - const sanitized = sanitizeDebugSnapshotValue(value); - return isRecord(sanitized) ? sanitized : { value: sanitized }; -} diff --git a/packages/@n8n/instance-ai/src/index.ts b/packages/@n8n/instance-ai/src/index.ts index c40028344b4..bc25737b4a4 100644 --- a/packages/@n8n/instance-ai/src/index.ts +++ b/packages/@n8n/instance-ai/src/index.ts @@ -13,12 +13,8 @@ import type * as ValidateAttachmentsMod from './parsers/validate-attachments'; import type * as PlannedTaskPermissionsMod from './planned-tasks/planned-task-permissions'; import type * as PlannedTaskServiceMod from './planned-tasks/planned-task-service'; import type * as PromptProfilesMod from './prompts/prompt-profiles'; -import type * as BackgroundTaskManagerMod from './runtime/background-task-manager'; import type * as InstanceContextStateMod from './runtime/instance-context-state'; -import type * as LivenessPolicyMod from './runtime/liveness-policy'; -import type * as ResumableStreamExecutorMod from './runtime/resumable-stream-executor'; import type * as RunStateRegistryMod from './runtime/run-state-registry'; -import type * as StreamRunnerMod from './runtime/stream-runner'; import type * as TerminalResponseGuardMod from './runtime/terminal-response-guard'; import type * as MaterializeRuntimeSkillsMod from './skills/materialize-runtime-skills'; import type * as RuntimeSkillsMod from './skills/runtime-skills'; @@ -170,21 +166,9 @@ const loadSnapshotManager = lazyModule( const loadRunStateRegistry = lazyModule( () => require('./runtime/run-state-registry') as typeof RunStateRegistryMod, ); -const loadBackgroundTaskManager = lazyModule( - () => require('./runtime/background-task-manager') as typeof BackgroundTaskManagerMod, -); const loadTerminalResponseGuard = lazyModule( () => require('./runtime/terminal-response-guard') as typeof TerminalResponseGuardMod, ); -const loadResumableStreamExecutor = lazyModule( - () => require('./runtime/resumable-stream-executor') as typeof ResumableStreamExecutorMod, -); -const loadStreamRunner = lazyModule( - () => require('./runtime/stream-runner') as typeof StreamRunnerMod, -); -const loadLivenessPolicy = lazyModule( - () => require('./runtime/liveness-policy') as typeof LivenessPolicyMod, -); const loadWorkflowLoop = lazyModule(() => require('./workflow-loop') as typeof WorkflowLoopMod); const loadWorkflowLoopRuntime = lazyModule( () => require('./workflow-loop/runtime') as typeof WorkflowLoopRuntimeMod, @@ -405,10 +389,6 @@ export const PlannedTaskStorage: typeof StorageMod.PlannedTaskStorage = lazyClas () => loadStorage().PlannedTaskStorage, ); export const getThread: typeof StorageMod.getThread = lazyFunction(() => loadStorage().getThread); -export type TerminalOutcomeStorage = StorageMod.TerminalOutcomeStorage; -export const TerminalOutcomeStorage: typeof StorageMod.TerminalOutcomeStorage = lazyClass( - () => loadStorage().TerminalOutcomeStorage, -); export const patchThread: typeof StorageMod.patchThread = lazyFunction( () => loadStorage().patchThread, ); @@ -422,7 +402,6 @@ export type { IterationLog, PatchableThreadMemory, ThreadPatch, - TerminalOutcome, WorkflowSetupRoutingClaim, WorkflowLoopWorkItemRecord, } from './storage'; @@ -488,10 +467,6 @@ defineLazyExport( 'N8N_WORKSPACE_DIR_ENV', () => loadMaterializeRuntimeSkills().N8N_WORKSPACE_DIR_ENV, ); -defineLazyExport( - 'INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG', - () => loadLivenessPolicy().INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, -); defineLazyExport('workflowBuildOutcomeSchema', () => loadWorkflowLoop().workflowBuildOutcomeSchema); defineLazyExport( 'workflowVerificationEvidenceSchema', @@ -538,20 +513,8 @@ export const SnapshotManager: typeof SnapshotManagerMod.SnapshotManager = lazyCl () => loadSnapshotManager().SnapshotManager, ); export type { InstanceAiEventBus, StoredEvent } from './event-bus'; -export type BackgroundTaskManager = BackgroundTaskManagerMod.BackgroundTaskManager; -export const BackgroundTaskManager: typeof BackgroundTaskManagerMod.BackgroundTaskManager = - lazyClass(() => loadBackgroundTaskManager().BackgroundTaskManager); -export const enrichMessageWithRunningTasks: typeof BackgroundTaskManagerMod.enrichMessageWithRunningTasks = - lazyFunction(() => loadBackgroundTaskManager().enrichMessageWithRunningTasks); -export const enrichMessageWithBackgroundTasks: typeof BackgroundTaskManagerMod.enrichMessageWithRunningTasks = - enrichMessageWithRunningTasks; -export type { - BackgroundTaskStatus, - ManagedBackgroundTask, - SpawnManagedBackgroundTaskOptions, -} from './runtime/background-task-manager'; export { MemoryTaskRegistry } from './runtime/memory-task-registry'; -export type RunStateRegistry = RunStateRegistryMod.RunStateRegistry; +export type RunStateRegistry = RunStateRegistryMod.RunStateRegistry; export const RunStateRegistry: typeof RunStateRegistryMod.RunStateRegistry = lazyClass( () => loadRunStateRegistry().RunStateRegistry, ); @@ -562,21 +525,7 @@ export { formatWorkflowSetupStateNote, observeWorkflowSetupStates, } from './tools/workflows/setup-panel-state'; -export type { RunDebugRecord } from './debug/run-debug-buffer'; -export { - RunDebugBuffer, - buildRunDebugLabel, - createRunDebugStepHooks, -} from './debug/run-debug-buffer'; -export type { - ActiveRunState, - BackgroundTaskStatusSnapshot, - ConfirmationData, - PendingConfirmation, - RunStateTimeoutDetails, - StartedRunState, - SuspendedRunState, -} from './runtime/run-state-registry'; +export type { ConfirmationData } from './runtime/run-state-registry'; export type InstanceAiTerminalResponseGuard = TerminalResponseGuardMod.InstanceAiTerminalResponseGuard; export const InstanceAiTerminalResponseGuard: typeof TerminalResponseGuardMod.InstanceAiTerminalResponseGuard = @@ -586,18 +535,6 @@ export type { TerminalResponseStatus, TerminalVisibilitySource, } from './runtime/terminal-response-guard'; -export const executeResumableStream: typeof ResumableStreamExecutorMod.executeResumableStream = - lazyFunction(() => loadResumableStreamExecutor().executeResumableStream); -export type { - AutoResumeControl, - ExecuteResumableStreamOptions, - ExecuteResumableStreamResult, - ManualSuspensionControl, - ResumableStreamContext, - ResumableStreamControl, - ResumableStreamSource, - TraceStatus, -} from './runtime/resumable-stream-executor'; export type { WorkSummary, ToolCallSummary } from './stream/work-summary-accumulator'; export type { AgentChunkPublisherOptions, @@ -617,30 +554,6 @@ export const mergeInstanceContextReach: typeof InstanceContextReachMod.mergeInst export type { RunTokenUsage, BuilderUsageItem } from './stream/usage-accumulator'; export const tokenUsageToBuilderUsageItems: typeof UsageAccumulatorMod.tokenUsageToBuilderUsageItems = lazyFunction(() => loadUsageAccumulator().tokenUsageToBuilderUsageItems); -export const resumeAgentRun: typeof StreamRunnerMod.resumeAgentRun = lazyFunction( - () => loadStreamRunner().resumeAgentRun, -); -export const streamAgentRun: typeof StreamRunnerMod.streamAgentRun = lazyFunction( - () => loadStreamRunner().streamAgentRun, -); -export const createInstanceAiLivenessPolicyConfig: typeof LivenessPolicyMod.createInstanceAiLivenessPolicyConfig = - lazyFunction(() => loadLivenessPolicy().createInstanceAiLivenessPolicyConfig); -export declare const INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG: typeof LivenessPolicyMod.INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG; -export type InstanceAiLivenessPolicy = LivenessPolicyMod.InstanceAiLivenessPolicy; -export const InstanceAiLivenessPolicy: typeof LivenessPolicyMod.InstanceAiLivenessPolicy = - lazyClass(() => loadLivenessPolicy().InstanceAiLivenessPolicy); -export type { - InstanceAiLivenessDecision, - InstanceAiLivenessInput, - InstanceAiLivenessPolicyConfig, - InstanceAiLivenessSurface, - InstanceAiLivenessTimeoutReason, -} from './runtime/liveness-policy'; -export type { - StreamableAgent, - StreamRunOptions, - StreamRunResult, -} from './runtime/stream-runner'; export type WorkflowTaskCoordinator = WorkflowLoopMod.WorkflowTaskCoordinator; export const WorkflowTaskCoordinator: typeof WorkflowLoopMod.WorkflowTaskCoordinator = lazyClass( () => loadWorkflowLoop().WorkflowTaskCoordinator, @@ -728,7 +641,7 @@ export type { PlannedTaskSchedulerAction, PlannedTaskService, OrchestrationContext, - BackgroundTaskResult, + TraceStatus, InstanceAiToolTraceOptions, InstanceAiTraceContext, InstanceAiTraceRun, diff --git a/packages/@n8n/instance-ai/src/runtime/__tests__/background-task-manager.test.ts b/packages/@n8n/instance-ai/src/runtime/__tests__/background-task-manager.test.ts deleted file mode 100644 index 2c582daf620..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/__tests__/background-task-manager.test.ts +++ /dev/null @@ -1,852 +0,0 @@ -import type { BackgroundTaskResult } from '../../types'; -import { BackgroundTaskManager, enrichMessageWithRunningTasks } from '../background-task-manager'; -import type { - ManagedBackgroundTask, - SpawnManagedBackgroundTaskOptions, -} from '../background-task-manager'; -import { InstanceAiLivenessPolicy } from '../liveness-policy'; - -const day = 24 * 60 * 60_000; - -function makeSpawnOptions( - overrides: Partial = {}, -): SpawnManagedBackgroundTaskOptions { - return { - taskId: 'task-1', - threadId: 'thread-1', - runId: 'run-1', - role: 'builder', - agentId: 'agent-1', - run: vi.fn().mockResolvedValue('done'), - ...overrides, - }; -} - -describe('BackgroundTaskManager', () => { - let manager: BackgroundTaskManager; - - beforeEach(() => { - manager = new BackgroundTaskManager(3); - }); - - describe('liveness timeouts', () => { - const policy = new InstanceAiLivenessPolicy({ - confirmationTimeoutMs: day, - backgroundTaskIdleTimeoutMs: 10_000, - backgroundTaskMaxLifetimeMs: 30_000, - activeRunIdleTimeoutMs: 10_000, - activeRunMaxLifetimeMs: 30_000, - }); - - it('fails and settles idle running tasks', async () => { - const onFailed = vi.fn((_task: ManagedBackgroundTask) => undefined); - const onSettled = vi.fn((_task: ManagedBackgroundTask) => undefined); - let signal: AbortSignal | undefined; - - manager.spawn( - makeSpawnOptions({ - run: async (abortSignal) => { - signal = abortSignal; - await new Promise(() => {}); - return 'never'; - }, - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' }, - onFailed, - onSettled, - }), - ); - - const task = manager.getRunningTasks('thread-1')[0]; - task.startedAt = 0; - task.lastActivityAt = 0; - - const timedOut = await manager.timeoutTimedOutTasks(policy, 10_000); - - expect(timedOut).toHaveLength(1); - expect(signal?.aborted).toBe(true); - expect(onFailed).toHaveBeenCalledWith( - expect.objectContaining({ - status: 'failed', - timeoutReason: 'idle_timeout', - }), - ); - expect(onFailed.mock.calls[0]?.[0].error).toEqual(expect.stringContaining('timed out')); - expect(onSettled).toHaveBeenCalledWith( - expect.objectContaining({ status: 'failed', timeoutReason: 'idle_timeout' }), - ); - expect(manager.getRunningTasks('thread-1')).toHaveLength(0); - - const next = manager.spawn( - makeSpawnOptions({ - taskId: 'task-2', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' }, - }), - ); - - expect(next.status).toBe('started'); - }); - - it('keeps touched tasks alive until they exceed the idle timeout', async () => { - manager.spawn( - makeSpawnOptions({ - run: async () => await new Promise(() => {}), - }), - ); - const task = manager.getRunningTasks('thread-1')[0]; - task.startedAt = 0; - manager.touchTask('thread-1', 'task-1', 9_000); - - const timedOut = await manager.timeoutTimedOutTasks(policy, 10_000); - - expect(timedOut).toEqual([]); - expect(manager.getRunningTasks('thread-1')).toHaveLength(1); - }); - - it('skips timeout checks for tasks currently waiting on HITL', async () => { - manager.spawn( - makeSpawnOptions({ - run: async () => await new Promise(() => {}), - }), - ); - const task = manager.getRunningTasks('thread-1')[0]; - task.startedAt = 0; - task.lastActivityAt = 0; - - const timedOut = await manager.timeoutTimedOutTasks(policy, 30_000, { - shouldSkipTask: (candidate) => candidate.threadId === 'thread-1', - }); - - expect(timedOut).toEqual([]); - expect(manager.getRunningTasks('thread-1')).toHaveLength(1); - }); - }); - - describe('spawn', () => { - it('spawns a task and tracks it as running', () => { - const result = manager.spawn(makeSpawnOptions()); - - expect(result.status).toBe('started'); - expect(manager.getRunningTasks('thread-1')).toHaveLength(1); - expect(manager.getRunningTasks('thread-1')[0].taskId).toBe('task-1'); - }); - - it('rejects spawn when concurrent limit is reached', () => { - const onLimitReached = vi.fn(); - const createTraceContext = vi.fn(); - - manager.spawn( - makeSpawnOptions({ taskId: 't1', run: async () => await new Promise(() => {}) }), - ); - manager.spawn( - makeSpawnOptions({ taskId: 't2', run: async () => await new Promise(() => {}) }), - ); - manager.spawn( - makeSpawnOptions({ taskId: 't3', run: async () => await new Promise(() => {}) }), - ); - - const result = manager.spawn( - makeSpawnOptions({ taskId: 't4', onLimitReached, createTraceContext }), - ); - - expect(result.status).toBe('limit-reached'); - expect(onLimitReached).toHaveBeenCalledWith(expect.stringContaining('limit of 3')); - expect(createTraceContext).not.toHaveBeenCalled(); - }); - - describe('instance-wide ceiling', () => { - const forever = async () => await new Promise(() => {}); - - it('rejects a spawn once the total across threads is reached', () => { - // Per-thread 5 so it can't fire; the total of 2 is what we're exercising. - const capped = new BackgroundTaskManager(5, 2); - const onLimitReached = vi.fn(); - - capped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1', run: forever })); - capped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2', run: forever })); - - const result = capped.spawn( - makeSpawnOptions({ taskId: 't3', threadId: 'thread-3', onLimitReached }), - ); - - expect(result.status).toBe('limit-reached'); - expect(onLimitReached).toHaveBeenCalledWith(expect.stringContaining('this n8n instance')); - }); - - it('is unlimited when the ceiling is -1', () => { - const uncapped = new BackgroundTaskManager(5, -1); - - uncapped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1', run: forever })); - uncapped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2', run: forever })); - const result = uncapped.spawn(makeSpawnOptions({ taskId: 't3', threadId: 'thread-3' })); - - expect(result.status).toBe('started'); - }); - - it('frees a slot when a task settles', async () => { - const capped = new BackgroundTaskManager(5, 1); - - capped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1' })); - await vi.waitFor(() => expect(capped.runningTaskCount()).toBe(0)); - - const result = capped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2' })); - - expect(result.status).toBe('started'); - }); - - it('counts running tasks across every thread', () => { - const capped = new BackgroundTaskManager(5, 10); - - capped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1', run: forever })); - capped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2', run: forever })); - - expect(capped.runningTaskCount()).toBe(2); - }); - }); - - it('creates lazy trace context only after a task is accepted', async () => { - const traceContext = { projectName: 'instance-ai' } as never; - const createTraceContext = vi.fn().mockResolvedValue(traceContext); - const run = vi.fn().mockResolvedValue('done'); - - manager.spawn(makeSpawnOptions({ createTraceContext, run })); - await flushPromises(); - - expect(createTraceContext).toHaveBeenCalledTimes(1); - expect(run).toHaveBeenCalledWith( - expect.any(AbortSignal), - expect.any(Function), - expect.any(Function), - { traceContext }, - ); - }); - - it('calls onCompleted and onSettled when run resolves with string', async () => { - const onCompleted = vi.fn(); - const onSettled = vi.fn(); - const { promise, resolve } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async () => await promise, - onCompleted, - onSettled, - }), - ); - - resolve('result-text'); - await flushPromises(); - - expect(onCompleted).toHaveBeenCalledWith( - expect.objectContaining({ status: 'completed', result: 'result-text' }), - ); - expect(onSettled).toHaveBeenCalled(); - }); - - it('calls onCompleted with structured result', async () => { - const onCompleted = vi.fn(); - const { promise, resolve } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async () => await promise, - onCompleted, - }), - ); - - resolve({ text: 'summary', outcome: { workflowId: 'wf-1' } }); - await flushPromises(); - - expect(onCompleted).toHaveBeenCalledWith( - expect.objectContaining({ - status: 'completed', - result: 'summary', - outcome: { workflowId: 'wf-1' }, - }), - ); - }); - - it('calls onFailed and onSettled when run rejects', async () => { - const onFailed = vi.fn(); - const onSettled = vi.fn(); - const { promise, reject } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async () => await promise, - onFailed, - onSettled, - }), - ); - - reject(new Error('boom')); - await flushPromises(); - - expect(onFailed).toHaveBeenCalledWith( - expect.objectContaining({ status: 'failed', error: 'boom' }), - ); - expect(onSettled).toHaveBeenCalled(); - }); - - it('does not call onFailed when aborted', async () => { - const onFailed = vi.fn(); - const { promise, reject } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async () => await promise, - onFailed, - }), - ); - - manager.cancelTask('thread-1', 'task-1'); - reject(new Error('aborted')); - await flushPromises(); - - expect(onFailed).not.toHaveBeenCalled(); - }); - - it('does not call onSettled when aborted', async () => { - const onSettled = vi.fn(); - const { promise, reject } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async () => await promise, - onSettled, - }), - ); - - manager.cancelTask('thread-1', 'task-1'); - reject(new Error('aborted')); - await flushPromises(); - - expect(onSettled).not.toHaveBeenCalled(); - }); - - it('removes task from map after settlement', async () => { - const { promise, resolve } = createDeferred(); - - manager.spawn(makeSpawnOptions({ run: async () => await promise })); - expect(manager.getTaskSnapshots('thread-1')).toHaveLength(1); - - resolve('done'); - await flushPromises(); - - expect(manager.getTaskSnapshots('thread-1')).toHaveLength(0); - }); - }); - - describe('single-flight dedupe', () => { - it('returns duplicate when plannedTaskId matches a running task', () => { - const first = manager.spawn( - makeSpawnOptions({ - taskId: 'first', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' }, - }), - ); - expect(first.status).toBe('started'); - - const run = vi.fn(async (): Promise => await new Promise(() => {})); - const second = manager.spawn( - makeSpawnOptions({ - taskId: 'second', - run, - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' }, - }), - ); - - expect(second.status).toBe('duplicate'); - if (second.status === 'duplicate') { - expect(second.existing.taskId).toBe('first'); - } - expect(run).not.toHaveBeenCalled(); - expect(manager.getRunningTasks('thread-1')).toHaveLength(1); - }); - - it('does not create lazy trace context for duplicate spawns', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 'first', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-trace' }, - }), - ); - const createTraceContext = vi.fn(); - - const second = manager.spawn( - makeSpawnOptions({ - taskId: 'second', - createTraceContext, - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-trace' }, - }), - ); - - expect(second.status).toBe('duplicate'); - expect(createTraceContext).not.toHaveBeenCalled(); - }); - - it('allows a new spawn once the first planned-task settles', async () => { - const { promise, resolve } = createDeferred(); - manager.spawn( - makeSpawnOptions({ - taskId: 'first', - run: async () => await promise, - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-2' }, - }), - ); - - resolve('done'); - await flushPromises(); - - const second = manager.spawn( - makeSpawnOptions({ - taskId: 'second', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-2' }, - }), - ); - expect(second.status).toBe('started'); - }); - - it('returns duplicate when workflowId + role matches a running task without plannedTaskId', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 'first', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', workflowId: 'wf-1' }, - }), - ); - - const run = vi.fn(async (): Promise => await new Promise(() => {})); - const second = manager.spawn( - makeSpawnOptions({ - taskId: 'second', - run, - dedupeKey: { role: 'workflow-builder', workflowId: 'wf-1' }, - }), - ); - - expect(second.status).toBe('duplicate'); - expect(run).not.toHaveBeenCalled(); - }); - - it('does not collapse two distinct plannedTaskIds that target the same workflowId', () => { - // A plan may include two work items for the same workflow — e.g., - // initial build (planned-A) followed by a patch (planned-B). They are - // distinct planned tasks and must both run; collapsing them on - // workflowId would skip work the user approved. - const first = manager.spawn( - makeSpawnOptions({ - taskId: 'task-A', - run: async () => await new Promise(() => {}), - dedupeKey: { - role: 'workflow-builder', - plannedTaskId: 'planned-A', - workflowId: 'wf-shared', - }, - }), - ); - expect(first.status).toBe('started'); - - const run = vi.fn(async (): Promise => await new Promise(() => {})); - const second = manager.spawn( - makeSpawnOptions({ - taskId: 'task-B', - run, - dedupeKey: { - role: 'workflow-builder', - plannedTaskId: 'planned-B', - workflowId: 'wf-shared', - }, - }), - ); - - expect(second.status).toBe('started'); - expect(run).toHaveBeenCalledTimes(1); - expect(manager.getRunningTasks('thread-1')).toHaveLength(2); - }); - - it('does not dedupe across roles for the same workflowId', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 'builder', - role: 'workflow-builder', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', workflowId: 'wf-1' }, - }), - ); - - const other = manager.spawn( - makeSpawnOptions({ - taskId: 'data-table-manager', - role: 'data-table-manager', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'data-table-manager', workflowId: 'wf-1' }, - }), - ); - - expect(other.status).toBe('started'); - expect(manager.getRunningTasks('thread-1')).toHaveLength(2); - }); - - it('limit-reached still fires even when dedupe would have passed', () => { - const filler = makeSpawnOptions({ run: async () => await new Promise(() => {}) }); - manager.spawn({ ...filler, taskId: 't1' }); - manager.spawn({ ...filler, taskId: 't2' }); - manager.spawn({ ...filler, taskId: 't3' }); - - const onLimitReached = vi.fn(); - const result = manager.spawn( - makeSpawnOptions({ - taskId: 't4', - onLimitReached, - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-fresh' }, - }), - ); - expect(result.status).toBe('limit-reached'); - expect(onLimitReached).toHaveBeenCalled(); - }); - - it('cancelTask releases dedupe indices so a fresh spawn is allowed', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 'first', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-3' }, - }), - ); - manager.cancelTask('thread-1', 'first'); - - const second = manager.spawn( - makeSpawnOptions({ - taskId: 'second', - run: async () => await new Promise(() => {}), - dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-3' }, - }), - ); - expect(second.status).toBe('started'); - }); - }); - - describe('queueCorrection', () => { - it('queues correction for running task', () => { - manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) })); - - expect(manager.queueCorrection('thread-1', 'task-1', 'fix this')).toBe('queued'); - }); - - it('returns task-not-found for unknown task', () => { - expect(manager.queueCorrection('thread-1', 'unknown', 'fix')).toBe('task-not-found'); - }); - - it('returns task-not-found for wrong thread', () => { - manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) })); - - expect(manager.queueCorrection('thread-2', 'task-1', 'fix')).toBe('task-not-found'); - }); - - it('drains corrections during run execution', async () => { - const drainedCorrections: string[][] = []; - const { promise, resolve } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async (_signal, drain) => { - drainedCorrections.push(drain()); - return await promise; - }, - }), - ); - - // Queue correction before the run resolves - await flushPromises(); - manager.queueCorrection('thread-1', 'task-1', 'correction-1'); - - resolve('done'); - await flushPromises(); - - expect(drainedCorrections[0]).toEqual([]); - }); - - it('notifies waitForCorrection when a correction is queued', async () => { - let waitForCorrection: (() => Promise) | undefined; - const { promise, resolve } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async (_signal, _drain, waitFn) => { - waitForCorrection = waitFn; - return await promise; - }, - }), - ); - - await flushPromises(); - - const correctionPromise = waitForCorrection!(); - let resolved = false; - void correctionPromise.then(() => { - resolved = true; - }); - - await flushPromises(); - expect(resolved).toBe(false); - - manager.queueCorrection('thread-1', 'task-1', 'use openrouter node'); - await flushPromises(); - - expect(resolved).toBe(true); - - resolve('done'); - await flushPromises(); - }); - - it('resolves waitForCorrection immediately when corrections are already queued', async () => { - let waitForCorrection: (() => Promise) | undefined; - const { promise, resolve } = createDeferred(); - - manager.spawn( - makeSpawnOptions({ - run: async (_signal, _drain, waitFn) => { - waitForCorrection = waitFn; - return await promise; - }, - }), - ); - - await flushPromises(); - - manager.queueCorrection('thread-1', 'task-1', 'pending correction'); - const correctionPromise = waitForCorrection!(); - let resolved = false; - void correctionPromise.then(() => { - resolved = true; - }); - - await flushPromises(); - expect(resolved).toBe(true); - - resolve('done'); - await flushPromises(); - }); - }); - - describe('cancelTask', () => { - it('cancels a running task and aborts its signal', () => { - manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) })); - - const cancelled = manager.cancelTask('thread-1', 'task-1'); - - expect(cancelled).toBeDefined(); - expect(cancelled!.status).toBe('cancelled'); - expect(cancelled!.abortController.signal.aborted).toBe(true); - expect(manager.getRunningTasks('thread-1')).toHaveLength(0); - }); - - it('returns undefined for non-existent task', () => { - expect(manager.cancelTask('thread-1', 'unknown')).toBeUndefined(); - }); - - it('returns undefined for wrong thread', () => { - manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) })); - - expect(manager.cancelTask('thread-2', 'task-1')).toBeUndefined(); - }); - }); - - describe('cancelThread', () => { - it('cancels all running tasks for a thread', () => { - manager.spawn( - makeSpawnOptions({ taskId: 't1', run: async () => await new Promise(() => {}) }), - ); - manager.spawn( - makeSpawnOptions({ taskId: 't2', run: async () => await new Promise(() => {}) }), - ); - - const cancelled = manager.cancelThread('thread-1'); - - expect(cancelled).toHaveLength(2); - expect(manager.getRunningTasks('thread-1')).toHaveLength(0); - }); - - it('does not cancel tasks for other threads', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 't1', - threadId: 'thread-1', - run: async () => await new Promise(() => {}), - }), - ); - manager.spawn( - makeSpawnOptions({ - taskId: 't2', - threadId: 'thread-2', - run: async () => await new Promise(() => {}), - }), - ); - - manager.cancelThread('thread-1'); - - expect(manager.getRunningTasks('thread-2')).toHaveLength(1); - }); - }); - - describe('cancelAll', () => { - it('cancels all tasks across all threads', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 't1', - threadId: 'thread-1', - run: async () => await new Promise(() => {}), - }), - ); - manager.spawn( - makeSpawnOptions({ - taskId: 't2', - threadId: 'thread-2', - run: async () => await new Promise(() => {}), - }), - ); - - const cancelled = manager.cancelAll(); - - expect(cancelled).toHaveLength(2); - expect(cancelled.every((t) => t.status === 'cancelled')).toBe(true); - expect(manager.getRunningTasks('thread-1')).toHaveLength(0); - expect(manager.getRunningTasks('thread-2')).toHaveLength(0); - }); - }); - - describe('getTaskSnapshots / getRunningTasks', () => { - it('returns empty array for unknown thread', () => { - expect(manager.getTaskSnapshots('unknown')).toEqual([]); - expect(manager.getRunningTasks('unknown')).toEqual([]); - }); - - it('getRunningTasks excludes non-running tasks', async () => { - const { promise, resolve } = createDeferred(); - manager.spawn(makeSpawnOptions({ run: async () => await promise })); - - resolve('done'); - await flushPromises(); - - expect(manager.getRunningTasks('thread-1')).toHaveLength(0); - }); - }); - - describe('getRunningTasksByParentCheckpoint', () => { - it('returns running tasks tagged with the given checkpoint id', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 'child-1', - run: async () => await new Promise(() => {}), - parentCheckpointId: 'cp-verify-1', - }), - ); - manager.spawn( - makeSpawnOptions({ - taskId: 'child-2', - run: async () => await new Promise(() => {}), - parentCheckpointId: 'cp-verify-1', - }), - ); - manager.spawn( - makeSpawnOptions({ - taskId: 'unrelated', - run: async () => await new Promise(() => {}), - }), - ); - - const children = manager.getRunningTasksByParentCheckpoint('thread-1', 'cp-verify-1'); - expect(children.map((c) => c.taskId).sort()).toEqual(['child-1', 'child-2']); - }); - - it('excludes tasks tagged under a different checkpoint', () => { - manager.spawn( - makeSpawnOptions({ - taskId: 'child-a', - run: async () => await new Promise(() => {}), - parentCheckpointId: 'cp-A', - }), - ); - manager.spawn( - makeSpawnOptions({ - taskId: 'child-b', - run: async () => await new Promise(() => {}), - parentCheckpointId: 'cp-B', - }), - ); - - const childrenA = manager.getRunningTasksByParentCheckpoint('thread-1', 'cp-A'); - expect(childrenA.map((c) => c.taskId)).toEqual(['child-a']); - }); - - it('excludes tasks that have already settled', async () => { - const { promise, resolve } = createDeferred(); - manager.spawn( - makeSpawnOptions({ - taskId: 'child-done', - parentCheckpointId: 'cp-verify-1', - run: async () => await promise, - }), - ); - - resolve('done'); - await flushPromises(); - - expect(manager.getRunningTasksByParentCheckpoint('thread-1', 'cp-verify-1')).toHaveLength(0); - }); - }); -}); - -describe('enrichMessageWithRunningTasks', () => { - it('returns original message when no tasks', async () => { - const result = await enrichMessageWithRunningTasks('Hello', []); - expect(result).toBe('Hello'); - }); - - it('prepends running-tasks block with default format', async () => { - const tasks = [{ taskId: 'task-1', role: 'builder' } as ManagedBackgroundTask]; - - const result = await enrichMessageWithRunningTasks('Hello', tasks); - - expect(result).toContain(''); - expect(result).toContain('builder'); - expect(result).toContain('task-1'); - expect(result).toContain('Hello'); - }); - - it('uses custom formatTask when provided', async () => { - const tasks = [{ taskId: 'task-1', role: 'builder' } as ManagedBackgroundTask]; - - const result = await enrichMessageWithRunningTasks('Hello', tasks, { - formatTask: (t) => `Custom: ${t.role}`, - }); - - expect(result).toContain('Custom: builder'); - }); -}); - -// --- Helpers --- - -async function flushPromises(): Promise { - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); -} - -function createDeferred(): { - promise: Promise; - resolve: (value: T) => void; - reject: (reason: unknown) => void; -} { - let resolve!: (value: T) => void; - let reject!: (reason: unknown) => void; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - return { promise, resolve, reject }; -} diff --git a/packages/@n8n/instance-ai/src/runtime/__tests__/liveness-policy.test.ts b/packages/@n8n/instance-ai/src/runtime/__tests__/liveness-policy.test.ts deleted file mode 100644 index aee17fa5858..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/__tests__/liveness-policy.test.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { - createInstanceAiLivenessPolicyConfig, - INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, - InstanceAiLivenessPolicy, -} from '../liveness-policy'; - -const minute = 60_000; -const day = 24 * 60 * minute; - -function createPolicy() { - return new InstanceAiLivenessPolicy(INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG); -} - -describe('InstanceAiLivenessPolicy', () => { - it('keeps default liveness limits centralized and allows the existing confirmation override', () => { - expect(INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG).toEqual({ - confirmationTimeoutMs: day, - backgroundTaskIdleTimeoutMs: 10 * minute, - backgroundTaskMaxLifetimeMs: 30 * minute, - activeRunIdleTimeoutMs: 0, - activeRunMaxLifetimeMs: day, - }); - - expect(createInstanceAiLivenessPolicyConfig({ confirmationTimeoutMs: 42_000 })).toEqual({ - ...INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, - confirmationTimeoutMs: 42_000, - }); - }); - - it('keeps active work alive while it is still reporting activity', () => { - const decision = createPolicy().evaluate({ - surface: 'background-task', - startedAt: 0, - lastActivityAt: 9 * minute, - now: 10 * minute, - }); - - expect(decision).toEqual({ action: 'keep-alive' }); - }); - - it('times out background work after the idle limit', () => { - const decision = createPolicy().evaluate({ - surface: 'background-task', - startedAt: 0, - lastActivityAt: 0, - now: 10 * minute, - }); - - expect(decision).toMatchObject({ - action: 'timeout', - reason: 'idle_timeout', - surface: 'background-task', - timeoutMs: 10 * minute, - }); - }); - - it('times out active work at the max lifetime even with recent activity', () => { - const decision = createPolicy().evaluate({ - surface: 'active-run', - startedAt: 0, - lastActivityAt: day + minute, - now: day + minute, - }); - - expect(decision).toMatchObject({ - action: 'timeout', - reason: 'max_lifetime', - surface: 'active-run', - timeoutMs: day, - }); - }); - - it('does not idle-timeout an active run that is waiting on another surface', () => { - const decision = createPolicy().evaluate({ - surface: 'active-run', - startedAt: 0, - lastActivityAt: 0, - now: 6 * 60 * minute, - }); - - expect(decision).toEqual({ action: 'keep-alive' }); - }); - - it('uses the one-day confirmation timeout for suspended runs and pending confirmations by default', () => { - const policy = createPolicy(); - const shorterWorkTimeout = 30 * minute; - - expect( - policy.evaluate({ - surface: 'suspended-run', - startedAt: 0, - lastActivityAt: 0, - now: shorterWorkTimeout, - }), - ).toEqual({ action: 'keep-alive' }); - - expect( - policy.evaluate({ - surface: 'pending-confirmation', - startedAt: 0, - lastActivityAt: 0, - now: shorterWorkTimeout, - }), - ).toEqual({ action: 'keep-alive' }); - - expect( - policy.evaluate({ - surface: 'suspended-run', - startedAt: 0, - lastActivityAt: 0, - now: day, - }), - ).toMatchObject({ action: 'timeout', reason: 'idle_timeout', timeoutMs: day }); - - expect( - policy.evaluate({ - surface: 'pending-confirmation', - startedAt: 0, - lastActivityAt: 0, - now: day, - }), - ).toMatchObject({ action: 'timeout', reason: 'idle_timeout', timeoutMs: day }); - }); - - it('uses a configured confirmation timeout for suspended runs and pending confirmations', () => { - const policy = new InstanceAiLivenessPolicy( - createInstanceAiLivenessPolicyConfig({ confirmationTimeoutMs: 10 * minute }), - ); - - expect( - policy.evaluate({ - surface: 'suspended-run', - startedAt: 0, - lastActivityAt: 0, - now: 10 * minute, - }), - ).toMatchObject({ action: 'timeout', reason: 'idle_timeout' }); - - expect( - policy.evaluate({ - surface: 'pending-confirmation', - startedAt: 0, - lastActivityAt: 0, - now: 10 * minute, - }), - ).toMatchObject({ action: 'timeout', reason: 'idle_timeout' }); - }); - - it('keeps a surface alive when its timeout is disabled', () => { - const policy = new InstanceAiLivenessPolicy({ - confirmationTimeoutMs: 0, - backgroundTaskIdleTimeoutMs: 0, - backgroundTaskMaxLifetimeMs: 0, - activeRunIdleTimeoutMs: 0, - activeRunMaxLifetimeMs: 0, - }); - - expect( - policy.evaluate({ - surface: 'pending-confirmation', - startedAt: 0, - lastActivityAt: 0, - now: 60 * minute, - }), - ).toEqual({ action: 'keep-alive' }); - }); -}); diff --git a/packages/@n8n/instance-ai/src/runtime/__tests__/resumable-stream-executor.test.ts b/packages/@n8n/instance-ai/src/runtime/__tests__/resumable-stream-executor.test.ts deleted file mode 100644 index 420ca3afb17..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/__tests__/resumable-stream-executor.test.ts +++ /dev/null @@ -1,772 +0,0 @@ -import type { InstanceAiEvent } from '@n8n/api-types'; - -import type { SuspensionInfo } from '../../utils/stream-helpers'; -import { executeResumableStream, normalizeStreamSource } from '../resumable-stream-executor'; - -function createEventBus() { - return { - publish: vi.fn(), - subscribe: vi.fn(), - getEventsAfter: vi.fn(), - getEventsForRun: vi.fn().mockReturnValue([]), - getEventsForRuns: vi.fn().mockReturnValue([]), - }; -} - -function createLogger() { - return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; -} - -async function* fromChunks(chunks: unknown[]) { - for (const chunk of chunks) { - await Promise.resolve(); - yield chunk; - } -} - -function readableFromChunks(chunks: unknown[]) { - return new ReadableStream({ - start(controller) { - for (const chunk of chunks) { - controller.enqueue(chunk); - } - controller.close(); - }, - }); -} - -function textChunk(text: string) { - return { type: 'text-delta', delta: text }; -} - -function errorChunk(error: unknown) { - return { type: 'error', error }; -} - -function suspensionChunk(input: { - toolCallId: string; - toolName?: string; - suspendPayload?: Record; - input?: Record; -}) { - return { - type: 'tool-call-suspended', - toolCallId: input.toolCallId, - ...(input.toolName ? { toolName: input.toolName } : {}), - ...(input.input ? { input: input.input } : {}), - suspendPayload: input.suspendPayload ?? {}, - }; -} - -function createDeferred() { - let resolve!: (value: T | PromiseLike) => void; - let reject!: (reason?: unknown) => void; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - - return { promise, resolve, reject }; -} - -interface PublishedEvent { - type: string; - responseId?: string; - payload?: { - requestId?: string; - toolCallId?: string; - text?: string; - }; -} - -describe('normalizeStreamSource', () => { - it('normalizes native @n8n/agents streams', async () => { - const source = normalizeStreamSource({ - runId: 'agent-run-1', - stream: readableFromChunks([ - textChunk('Hello'), - { - type: 'message', - message: { - role: 'assistant', - content: [{ type: 'text', text: ' world' }], - }, - }, - ]), - getState: vi.fn(), - }); - - expect(source.runId).toBe('agent-run-1'); - await expect(source.text).resolves.toBe('Hello'); - }); -}); - -describe('executeResumableStream', () => { - it('buffers the confirmation event in manual mode', async () => { - const eventBus = createEventBus(); - - const result = await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - textChunk('Working...'), - suspensionChunk({ - toolCallId: 'tool-call-1', - toolName: 'ask-user', - suspendPayload: { - requestId: 'request-1', - message: 'Need approval', - }, - }), - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - expect(result).toEqual( - expect.objectContaining({ - status: 'suspended', - agentRunId: 'agent-run-1', - suspension: { - toolCallId: 'tool-call-1', - requestId: 'request-1', - toolName: 'ask-user', - suspendPayload: { requestId: 'request-1', message: 'Need approval' }, - }, - }), - ); - expect(eventBus.publish).toHaveBeenCalledWith( - 'thread-1', - expect.objectContaining({ type: 'text-delta', runId: 'run-1', agentId: 'agent-1' }), - ); - expect(eventBus.publish).not.toHaveBeenCalledWith( - 'thread-1', - expect.objectContaining({ type: 'confirmation-request' }), - ); - expect(result.confirmationEvent?.type).toBe('confirmation-request'); - expect(result.confirmationEvent?.payload.requestId).toBe('request-1'); - }); - - it('returns errored status when stream contains an error chunk', async () => { - const error = new Error('Not Found'); - const result = await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([textChunk('Working...'), errorChunk(error)]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus: createEventBus(), - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - expect(result.status).toBe('errored'); - expect(result.agentRunId).toBe('agent-run-1'); - expect(result.error).toBe(error); - }); - - it('reports the terminal finish reason of a completed run', async () => { - const result = await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - textChunk('Working...'), - { type: 'finish', finishReason: 'max-iterations' }, - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus: createEventBus(), - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - expect(result.status).toBe('completed'); - expect(result.finishReason).toBe('max-iterations'); - }); - - it('reports the finish reason of the resumed stream, not the suspended one', async () => { - const resume = vi.fn().mockResolvedValue({ - runId: 'agent-run-2', - stream: readableFromChunks([ - textChunk('Done.'), - { type: 'finish', finishReason: 'max-iterations' }, - ]), - }); - - const result = await executeResumableStream({ - agent: { resume }, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - suspensionChunk({ - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: { requestId: 'request-1', message: 'Please confirm' }, - }), - { type: 'finish', finishReason: 'tool-calls' }, - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus: createEventBus(), - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'auto', waitForConfirmation: vi.fn().mockResolvedValue({ approved: true }) }, - }); - - expect(result.status).toBe('completed'); - expect(result.finishReason).toBe('max-iterations'); - }); - - it('publishes only the quota error when a follow-on error chunk arrives', async () => { - const eventBus = createEventBus(); - const quotaError = Object.assign(new Error('Have reached end of quota'), { - statusCode: 403, - errorCode: 'quota_exhausted', - }); - const followOn = new Error('No output generated. Check the stream for errors.'); - - const result = await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([errorChunk(quotaError), errorChunk(followOn)]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - const errorEvents = eventBus.publish.mock.calls - .map(([, event]) => event as InstanceAiEvent) - .filter((event) => event.type === 'error'); - - expect(errorEvents).toHaveLength(1); - expect(errorEvents[0].payload).toMatchObject({ code: 'quota_exhausted' }); - // The swallowed follow-on chunk must not overwrite result.error, or telemetry - // would log the generic "No output generated" while the user saw the quota error. - expect(result.error).toBe(quotaError); - }); - - it('captures terminal usage on an aborted run so cancelled runs are billed', async () => { - const controller = new AbortController(); - - // Yield a text chunk, abort mid-stream (as a user "stop" does), then let - // the agent emit its terminal finish chunk carrying the run's usage. - async function* abortingStream() { - await Promise.resolve(); - yield textChunk('partial'); - controller.abort(); - yield { - type: 'finish', - finishReason: 'error', - usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 }, - }; - } - - const result = await executeResumableStream({ - agent: {}, - stream: { runId: 'agent-run-1', fullStream: abortingStream() }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus: createEventBus(), - signal: controller.signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - expect(result.status).toBe('cancelled'); - expect(result.usage).toMatchObject({ - promptTokens: 10, - completionTokens: 5, - totalTokens: 15, - }); - }); - - it('reports liveness activity for each consumed chunk', async () => { - const onActivity = vi.fn(); - - await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([textChunk('Working...'), textChunk('Done.')]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus: createEventBus(), - signal: new AbortController().signal, - logger: createLogger(), - onActivity, - }, - control: { mode: 'manual' }, - }); - - expect(onActivity).toHaveBeenCalledTimes(2); - }); - - it('assigns stable response IDs from native start-step chunks', async () => { - const eventBus = createEventBus(); - - await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - { type: 'start-step' }, - textChunk('First'), - { - type: 'tool-call', - toolCallId: 'tool-call-1', - toolName: 'test-tool', - input: {}, - }, - textChunk(' step'), - { type: 'finish-step' }, - { type: 'start-step' }, - textChunk('Second step'), - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent); - const firstText = publishedEvents.find((event) => event.payload?.text === 'First'); - const toolCall = publishedEvents.find((event) => event.type === 'tool-call'); - const firstStepContinuation = publishedEvents.find((event) => event.payload?.text === ' step'); - const secondText = publishedEvents.find((event) => event.payload?.text === 'Second step'); - - expect(firstText?.responseId).toBe('agent-run-1:step:1'); - expect(toolCall?.responseId).toBe('agent-run-1:step:1'); - expect(firstStepContinuation?.responseId).toBe('agent-run-1:step:1'); - expect(secondText?.responseId).toBe('agent-run-1:step:2'); - }); - - it('mints a synthetic per-segment responseId when the provider supplies none', async () => { - const eventBus = createEventBus(); - - // No start-step chunks at all; some providers never emit them. Deltas - // must still carry a responseId (segment identity keys the run reducer's - // block replace semantics), one per contiguous delta run. - await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - textChunk('First'), - textChunk(' segment'), - { - type: 'tool-call', - toolCallId: 'tool-call-1', - toolName: 'test-tool', - input: {}, - }, - textChunk('Second segment'), - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent); - // Match the segment's first delta by prefix; deltas publish per chunk. - const first = publishedEvents.find((event) => event.payload?.text?.startsWith('First')); - const toolCall = publishedEvents.find((event) => event.type === 'tool-call'); - const second = publishedEvents.find((event) => event.payload?.text === 'Second segment'); - - expect(first?.responseId).toBeDefined(); - // Sticky for the whole segment (the closing structural fact inherits it, - // mirroring provider-supplied step ids)... - expect(toolCall?.responseId).toBe(first?.responseId); - // ...and rolled after it, so two segments never share an id. - expect(second?.responseId).toBeDefined(); - expect(second?.responseId).not.toBe(first?.responseId); - }); - - it('rolls the synthetic responseId at a finish-step boundary that maps to no event', async () => { - const eventBus = createEventBus(); - - // finish-step publishes nothing, but it still separates two segments; a - // sticky synthetic id across it would let the reducer's id-keyed replace - // pair blocks from different steps. - await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - textChunk('First step text'), - { type: 'finish-step' }, - textChunk('Second step text'), - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { mode: 'manual' }, - }); - - const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent); - // Match each segment's delta by prefix. - const first = publishedEvents.find((event) => event.payload?.text?.startsWith('First')); - const second = publishedEvents.find((event) => event.payload?.text?.startsWith('Second')); - - expect(first?.responseId).toBeDefined(); - expect(second?.responseId).toBeDefined(); - expect(second?.responseId).not.toBe(first?.responseId); - }); - - it('stops consuming after a requested handoff while publishing the current tool result', async () => { - const eventBus = createEventBus(); - let shouldStop = false; - eventBus.publish.mockImplementation((_: string, event: PublishedEvent) => { - if (event.type === 'tool-result') { - shouldStop = true; - } - }); - - const result = await executeResumableStream({ - agent: {}, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - { - type: 'tool-call', - toolCallId: 'tool-call-1', - toolName: 'create-tasks', - input: {}, - }, - { - type: 'tool-result', - toolCallId: 'tool-call-1', - output: { result: 'Plan approved. Started 1 task.', taskCount: 1 }, - }, - textChunk('This inline continuation should not be published.'), - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - stopSignal: () => (shouldStop ? { reason: 'planned-tasks-scheduled' } : undefined), - }, - control: { mode: 'manual' }, - }); - - const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent); - - expect(result.status).toBe('completed'); - expect(result.stopReason).toBe('planned-tasks-scheduled'); - expect(publishedEvents.some((event) => event.type === 'tool-call')).toBe(true); - expect(publishedEvents.some((event) => event.type === 'tool-result')).toBe(true); - expect( - publishedEvents.some( - (event) => event.payload?.text === 'This inline continuation should not be published.', - ), - ).toBe(false); - }); - - it('auto-resumes suspended streams and passes drained corrections to resume data', async () => { - const eventBus = createEventBus(); - const resume = vi.fn().mockResolvedValue({ - runId: 'agent-run-2', - stream: readableFromChunks([textChunk('Done.')]), - }); - const waitForConfirmation = vi.fn().mockResolvedValue({ approved: true }); - let hasDrainedCorrection = false; - - const result = await executeResumableStream({ - agent: { resume }, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - suspensionChunk({ - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: { - requestId: 'request-1', - message: 'Please confirm', - }, - }), - ]), - text: Promise.resolve('Initial text'), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { - mode: 'auto', - waitForConfirmation, - drainCorrections: () => { - if (hasDrainedCorrection) { - return []; - } - - hasDrainedCorrection = true; - return ['Prefer Slack instead of email']; - }, - }, - }); - - expect(waitForConfirmation).toHaveBeenCalledWith('request-1'); - expect(resume).toHaveBeenCalledWith( - 'stream', - expect.objectContaining({ - __correctionOverride: true, - corrections: ['Prefer Slack instead of email'], - }), - { runId: 'agent-run-1', toolCallId: 'tool-call-1' }, - ); - expect(result.status).toBe('completed'); - expect(result.agentRunId).toBe('agent-run-2'); - await expect(result.text ?? Promise.resolve('')).resolves.toBe('Done.'); - expect(eventBus.publish).toHaveBeenCalledWith( - 'thread-1', - expect.objectContaining({ - type: 'text-delta', - // Correction lines are their own segments: each carries a unique - // synthetic responseId so the reducer never merges them. - responseId: expect.stringContaining(':correction:') as string, - payload: { text: '\n[USER CORRECTION]: Prefer Slack instead of email\n' }, - }), - ); - }); - - it('passes resume options from the control hook', async () => { - const resume = vi.fn().mockResolvedValue({ - runId: 'agent-run-2', - stream: readableFromChunks([textChunk('Done.')]), - }); - - await executeResumableStream({ - agent: { resume }, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - suspensionChunk({ - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: { requestId: 'request-1' }, - }), - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus: createEventBus(), - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { - mode: 'auto', - waitForConfirmation: async () => await Promise.resolve({ approved: true }), - buildResumeOptions: ({ agentRunId, suspension }) => ({ - runId: agentRunId, - toolCallId: suspension.toolCallId, - maxIterations: 5, - }), - }, - }); - - expect(resume).toHaveBeenCalledWith( - 'stream', - { approved: true }, - { runId: 'agent-run-1', toolCallId: 'tool-call-1', maxIterations: 5 }, - ); - }); - - it('registers auto confirmations before the stream finishes draining', async () => { - const eventBus = createEventBus(); - const finishGate = createDeferred(); - const approval = createDeferred>(); - const waitStarted = createDeferred(); - const resume = vi.fn().mockResolvedValue({ - runId: 'agent-run-2', - stream: readableFromChunks([textChunk('Done.')]), - }); - const waitForConfirmation = vi.fn().mockImplementation(async () => { - waitStarted.resolve(undefined); - return await approval.promise; - }); - - const execution = executeResumableStream({ - agent: { resume }, - stream: { - runId: 'agent-run-1', - fullStream: (async function* () { - yield suspensionChunk({ - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: { - requestId: 'request-1', - message: 'Please confirm', - }, - }); - await finishGate.promise; - yield { type: 'finish', finishReason: 'tool-calls' }; - })(), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { - mode: 'auto', - waitForConfirmation, - }, - }); - - await waitStarted.promise; - - expect(waitForConfirmation).toHaveBeenCalledWith('request-1'); - expect(resume).not.toHaveBeenCalled(); - const publishCalls = eventBus.publish.mock.calls as Array<[string, PublishedEvent]>; - const confirmationEvent = publishCalls.find( - ([, event]) => event.type === 'confirmation-request', - ); - expect(confirmationEvent?.[0]).toBe('thread-1'); - expect(confirmationEvent?.[1].payload?.requestId).toBe('request-1'); - - approval.resolve({ approved: true }); - finishGate.resolve(undefined); - - await expect(execution).resolves.toEqual( - expect.objectContaining({ - status: 'completed', - agentRunId: 'agent-run-2', - }), - ); - }); - - it('surfaces only the first actionable suspension in a drain', async () => { - const eventBus = createEventBus(); - const resume = vi.fn().mockResolvedValue({ - runId: 'agent-run-2', - stream: readableFromChunks([textChunk('Done.')]), - }); - const waitForConfirmation = vi.fn().mockResolvedValue({ approved: true }); - const onSuspension = vi.fn((_: SuspensionInfo) => undefined); - - await executeResumableStream({ - agent: { resume }, - stream: { - runId: 'agent-run-1', - fullStream: fromChunks([ - suspensionChunk({ - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: { - requestId: 'request-1', - message: 'First confirmation', - }, - }), - suspensionChunk({ - toolCallId: 'tool-call-2', - toolName: 'pause-for-user', - suspendPayload: { - requestId: 'request-2', - message: 'Second confirmation', - }, - }), - { type: 'finish', finishReason: 'tool-calls' }, - ]), - }, - context: { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - eventBus, - signal: new AbortController().signal, - logger: createLogger(), - }, - control: { - mode: 'auto', - waitForConfirmation, - onSuspension, - }, - }); - - expect(onSuspension).toHaveBeenCalledTimes(1); - expect(onSuspension).toHaveBeenCalledWith({ - requestId: 'request-1', - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: { requestId: 'request-1', message: 'First confirmation' }, - }); - expect(waitForConfirmation).toHaveBeenCalledTimes(1); - expect(waitForConfirmation).toHaveBeenCalledWith('request-1'); - - const confirmationEvents = (eventBus.publish.mock.calls as Array<[string, PublishedEvent]>) - .map(([, event]) => event) - .filter((event) => event.type === 'confirmation-request'); - expect(confirmationEvents).toHaveLength(1); - expect(confirmationEvents[0].payload?.requestId).toBe('request-1'); - }); -}); diff --git a/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts b/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts index 63ce18981be..3ef6ad386f2 100644 --- a/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts +++ b/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts @@ -1,1437 +1,51 @@ -import { nanoid } from 'nanoid'; - -import type { InstanceAiTraceContext } from '../../types'; -import { InstanceAiLivenessPolicy } from '../liveness-policy'; -import type { - BackgroundTaskStatusSnapshot, - ConfirmationData, - PendingConfirmation, - SuspendedRunState, -} from '../run-state-registry'; import { RunStateRegistry } from '../run-state-registry'; -vi.mock('nanoid', () => ({ - nanoid: vi.fn(), -})); - -const mockedNanoid = vi.mocked(nanoid); -const day = 24 * 60 * 60_000; - -interface TestUser { - id: string; - name: string; -} - -function createSuspendedRunState( - overrides: Partial> = {}, -): SuspendedRunState { - return { - runId: 'run_abc', - abortController: new AbortController(), - agentRunId: 'agent-run-1', - agent: {}, - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - toolCallId: 'tool-call-1', - requestId: 'request-1', - createdAt: Date.now(), - ...overrides, - }; -} - -function createBackgroundTask( - overrides: Partial = {}, -): BackgroundTaskStatusSnapshot { - return { - taskId: 'task-1', - role: 'builder', - agentId: 'agent-1', - status: 'running', - startedAt: Date.now(), - runId: 'run_abc', - threadId: 'thread-1', - ...overrides, - }; -} - describe('RunStateRegistry', () => { - let registry: RunStateRegistry; - let nanoidCounter: number; - const policy = new InstanceAiLivenessPolicy({ - confirmationTimeoutMs: 30_000, - backgroundTaskIdleTimeoutMs: 30_000, - backgroundTaskMaxLifetimeMs: 90_000, - activeRunIdleTimeoutMs: 30_000, - activeRunMaxLifetimeMs: 90_000, + it('indexes runs by message group without duplicates', () => { + const registry = new RunStateRegistry(); + + registry.indexRunInGroup('thread-1', 'mg-1', 'run-1'); + registry.indexRunInGroup('thread-1', 'mg-1', 'run-2'); + registry.indexRunInGroup('thread-1', 'mg-1', 'run-1'); + + expect(registry.getMessageGroupId('thread-1')).toBe('mg-1'); + expect(registry.getRunIdsForMessageGroup('mg-1')).toEqual(['run-1', 'run-2']); + expect(registry.getRunIdsForMessageGroup('unknown')).toEqual([]); }); - beforeEach(() => { - registry = new RunStateRegistry((user) => user.id); - nanoidCounter = 0; - mockedNanoid.mockReset(); - mockedNanoid.mockImplementation(() => `id-${++nanoidCounter}`); - }); + it('keeps per-thread request options and clears omitted values', () => { + const registry = new RunStateRegistry(); - it('retains mode across internal runs and clears it on a control request', () => { + registry.setTimeZone('thread-1', 'Europe/Helsinki'); registry.setBuildMode('thread-1', 'progressive'); - registry.startRun({ threadId: 'thread-1', user: { id: 'user-1', name: 'Alice' } }); + registry.setPromptVersion('thread-1', 'concise@1'); + registry.setObserverThresholdTokens('thread-1', 1000); + registry.setSetupPanelEnabled('thread-1', true); + + expect(registry.getTimeZone('thread-1')).toBe('Europe/Helsinki'); expect(registry.getBuildMode('thread-1')).toBe('progressive'); + expect(registry.getPromptVersion('thread-1')).toBe('concise@1'); + expect(registry.getObserverThresholdTokens('thread-1')).toBe(1000); + expect(registry.isSetupPanelEnabled('thread-1')).toBe(true); + registry.setBuildMode('thread-1', undefined); - expect(registry.getBuildMode('thread-1')).toBeUndefined(); - }); - - it('removes retained mode when a thread is cleared', () => { - registry.setBuildMode('thread-1', 'progressive'); - registry.clearThread('thread-1'); - expect(registry.getBuildMode('thread-1')).toBeUndefined(); - }); - - it('scopes the observer threshold override to its own thread and clears it', () => { - registry.setObserverThresholdTokens('thread-1', 8000); - // A second thread on the same process keeps the instance default. - expect(registry.getObserverThresholdTokens('thread-1')).toBe(8000); - expect(registry.getObserverThresholdTokens('thread-2')).toBeUndefined(); - registry.setObserverThresholdTokens('thread-1', undefined); + + expect(registry.getBuildMode('thread-1')).toBeUndefined(); expect(registry.getObserverThresholdTokens('thread-1')).toBeUndefined(); }); - it('drops the observer threshold override when a thread is cleared', () => { - registry.setObserverThresholdTokens('thread-1', 8000); + it('removes all state for a cleared thread', () => { + const registry = new RunStateRegistry(); + registry.indexRunInGroup('thread-1', 'mg-1', 'run-1'); + registry.setTimeZone('thread-1', 'UTC'); + registry.indexRunInGroup('thread-2', 'mg-2', 'run-2'); + registry.clearThread('thread-1'); - expect(registry.getObserverThresholdTokens('thread-1')).toBeUndefined(); - }); - it('drops every observer threshold override on shutdown', () => { - registry.setObserverThresholdTokens('thread-1', 8000); - registry.shutdown(); - expect(registry.getObserverThresholdTokens('thread-1')).toBeUndefined(); - }); - - it('retains the selected prompt version and metadata until the next explicit selection', () => { - const metadata = { - version: 'progressive@1', - systemPromptVersion: 'instance-agent@1', - skillVariants: ['progressive-building@1'], - skillsHash: 'selected-skills', - }; - registry.setPromptConfiguration('thread-1', metadata); - registry.startRun({ threadId: 'thread-1', user: { id: 'user-1', name: 'Alice' } }); - expect(registry.getPromptVersion('thread-1')).toBe('progressive@1'); - expect(registry.getPromptConfiguration('thread-1')).toEqual(metadata); - registry.setPromptVersion('thread-1', 'default@1'); - expect(registry.getPromptConfiguration('thread-1')).toBeUndefined(); - expect(registry.getPromptVersion('thread-1')).toBe('default@1'); - registry.clearThread('thread-1'); - expect(registry.getPromptVersion('thread-1')).toBeUndefined(); - }); - - // ── startRun ────────────────────────────────────────────────────────────── - - describe('startRun', () => { - it('creates run with generated runId and messageGroupId', () => { - const result = registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - }); - - expect(result.runId).toBe('run_id-1'); - expect(result.messageGroupId).toBe('mg_id-2'); - expect(result.abortController).toBeInstanceOf(AbortController); - }); - - it('stores user for the thread', () => { - registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - }); - - expect(registry.getThreadUser('thread-1')).toEqual({ id: 'user-1', name: 'Alice' }); - }); - - it('reuses provided messageGroupId instead of generating one', () => { - const result = registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - messageGroupId: 'mg_existing', - }); - - // Only one nanoid call for runId, not for messageGroupId - expect(result.messageGroupId).toBe('mg_existing'); - expect(result.runId).toBe('run_id-1'); - expect(mockedNanoid).toHaveBeenCalledTimes(1); - }); - - it('cleans up previous message group mapping when no messageGroupId is provided', () => { - // Start first run - generates mg_id-2 - registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - }); - expect(registry.getRunIdsForMessageGroup('mg_id-2')).toEqual(['run_id-1']); - - // Start second run on same thread without messageGroupId - should clean up old group - const secondResult = registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - }); - - // Old message group should be cleaned up - expect(registry.getRunIdsForMessageGroup('mg_id-2')).toEqual([]); - // New message group should have the new run - expect(registry.getRunIdsForMessageGroup(secondResult.messageGroupId!)).toEqual(['run_id-3']); - }); - - it('does not clean up previous group when messageGroupId is provided (reuse)', () => { - // Start first run - generates mg_id-2 - registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - }); - - // Start second run reusing the same group - registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - messageGroupId: 'mg_id-2', - }); - - // Both runs should be tracked under the same group - expect(registry.getRunIdsForMessageGroup('mg_id-2')).toEqual(['run_id-1', 'run_id-3']); - }); - - it('tracks multiple runIds per message group', () => { - const sharedGroupId = 'mg_shared'; - - registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - messageGroupId: sharedGroupId, - }); - - registry.startRun({ - threadId: 'thread-2', - user: { id: 'user-2', name: 'Bob' }, - messageGroupId: sharedGroupId, - }); - - expect(registry.getRunIdsForMessageGroup(sharedGroupId)).toEqual(['run_id-1', 'run_id-2']); - }); - }); - - // ── State queries ───────────────────────────────────────────────────────── - - describe('state queries', () => { - describe('hasActiveRun', () => { - it('returns true when thread has an active run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - expect(registry.hasActiveRun('thread-1')).toBe(true); - }); - - it('returns false when thread has no active run', () => { - expect(registry.hasActiveRun('thread-1')).toBe(false); - }); - - it('returns false when thread only has a suspended run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1' })); - - expect(registry.hasActiveRun('thread-1')).toBe(false); - }); - }); - - describe('hasSuspendedRun', () => { - it('returns true when thread has a suspended run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1' })); - - expect(registry.hasSuspendedRun('thread-1')).toBe(true); - }); - - it('returns false when thread has no suspended run', () => { - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - }); - - it('returns false when thread only has an active run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - }); - }); - - describe('hasLiveRun', () => { - it('returns true when thread has an active run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - expect(registry.hasLiveRun('thread-1')).toBe(true); - }); - - it('returns true when thread has a suspended run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1' })); - - expect(registry.hasLiveRun('thread-1')).toBe(true); - }); - - it('returns false when thread has no runs', () => { - expect(registry.hasLiveRun('thread-1')).toBe(false); - }); - }); - - describe('getActiveRun', () => { - it('returns the active run state when present', () => { - const started = registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - }); - const activeRun = registry.getActiveRun('thread-1'); - - expect(activeRun).toBeDefined(); - expect(activeRun!.runId).toBe(started.runId); - expect(activeRun!.abortController).toBe(started.abortController); - }); - - it('returns undefined when no active run', () => { - expect(registry.getActiveRun('thread-1')).toBeUndefined(); - }); - }); - - describe('getActiveRunId', () => { - it('returns the runId of the active run', () => { - const started = registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - }); - - expect(registry.getActiveRunId('thread-1')).toBe(started.runId); - }); - - it('returns undefined when no active run', () => { - expect(registry.getActiveRunId('nonexistent')).toBeUndefined(); - }); - }); - - describe('getSuspendedRun', () => { - it('returns the suspended run state when present', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - registry.suspendRun('thread-1', suspendedState); - - const result = registry.getSuspendedRun('thread-1'); - expect(result).toBe(suspendedState); - }); - - it('returns undefined when no suspended run', () => { - expect(registry.getSuspendedRun('thread-1')).toBeUndefined(); - }); - }); - - describe('getThreadUser', () => { - it('returns the user stored for the thread', () => { - registry.startRun({ - threadId: 'thread-1', - user: { id: 'user-1', name: 'Alice' }, - }); - - expect(registry.getThreadUser('thread-1')).toEqual({ id: 'user-1', name: 'Alice' }); - }); - - it('returns undefined for unknown thread', () => { - expect(registry.getThreadUser('unknown')).toBeUndefined(); - }); - }); - }); - - // ── getThreadStatus ─────────────────────────────────────────────────────── - - describe('getThreadStatus', () => { - it('reflects active run state', () => { - const started = registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - const status = registry.getThreadStatus('thread-1', []); - - expect(status.hasActiveRun).toBe(true); - expect(status.isSuspended).toBe(false); - expect(status.runId).toBe(started.runId); - expect(status.backgroundTasks).toEqual([]); - }); - - it('reflects suspended run state', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - runId: 'run-suspended', - messageGroupId: 'mg-1', - }); - registry.suspendRun('thread-1', suspendedState); - - const status = registry.getThreadStatus('thread-1', []); - - expect(status.hasActiveRun).toBe(false); - expect(status.isSuspended).toBe(true); - expect(status.runId).toBe('run-suspended'); - }); - - it('reflects idle state with no runs', () => { - const status = registry.getThreadStatus('thread-1', []); - - expect(status.hasActiveRun).toBe(false); - expect(status.isSuspended).toBe(false); - expect(status.backgroundTasks).toEqual([]); - }); - - it('filters background tasks by threadId', () => { - const tasks: BackgroundTaskStatusSnapshot[] = [ - createBackgroundTask({ taskId: 'task-1', threadId: 'thread-1' }), - createBackgroundTask({ taskId: 'task-2', threadId: 'thread-2' }), - createBackgroundTask({ taskId: 'task-3', threadId: 'thread-1' }), - ]; - - const status = registry.getThreadStatus('thread-1', tasks); - - expect(status.backgroundTasks).toHaveLength(2); - expect(status.backgroundTasks.map((t) => t.taskId)).toEqual(['task-1', 'task-3']); - }); - - it('maps background task fields correctly, stripping threadId', () => { - const task = createBackgroundTask({ - taskId: 'task-1', - role: 'builder', - agentId: 'agent-1', - status: 'running', - startedAt: 1000, - runId: 'run_x', - messageGroupId: 'mg_y', - threadId: 'thread-1', - }); - - const status = registry.getThreadStatus('thread-1', [task]); - - expect(status.backgroundTasks[0]).toEqual({ - taskId: 'task-1', - role: 'builder', - agentId: 'agent-1', - status: 'running', - startedAt: 1000, - runId: 'run_x', - messageGroupId: 'mg_y', - }); - // threadId should not be in the mapped output - expect(status.backgroundTasks[0]).not.toHaveProperty('threadId'); - }); - }); - - // ── Message group tracking ──────────────────────────────────────────────── - - describe('message group tracking', () => { - describe('getMessageGroupId', () => { - it('returns the stored message group for the thread', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - expect(registry.getMessageGroupId('thread-1')).toBe('mg_id-2'); - }); - - it('returns undefined for unknown thread', () => { - expect(registry.getMessageGroupId('unknown')).toBeUndefined(); - }); - }); - - describe('getLiveMessageGroupId', () => { - it('returns thread message group when there is an active run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - const result = registry.getLiveMessageGroupId('thread-1', []); - - expect(result).toBe('mg_id-2'); - }); - - it('returns thread message group when there is a suspended run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1' })); - - const result = registry.getLiveMessageGroupId('thread-1', []); - - expect(result).toBe('mg_id-2'); - }); - - it('falls back to the most recent running background task messageGroupId', () => { - // No live run - const tasks = [ - createBackgroundTask({ - threadId: 'thread-1', - status: 'running', - startedAt: 1000, - messageGroupId: 'mg_older', - }), - createBackgroundTask({ - threadId: 'thread-1', - status: 'running', - startedAt: 2000, - messageGroupId: 'mg_newer', - }), - ]; - - const result = registry.getLiveMessageGroupId('thread-1', tasks); - - expect(result).toBe('mg_newer'); - }); - - it('falls back to stored threadMessageGroupId when no running background tasks match', () => { - // Start and then clear the active run, but threadMessageGroupId remains - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.clearActiveRun('thread-1'); - - const completedTasks = [ - createBackgroundTask({ - threadId: 'thread-1', - status: 'completed', - messageGroupId: 'mg_completed', - }), - ]; - - const result = registry.getLiveMessageGroupId('thread-1', completedTasks); - - // No live run, no running background tasks, falls through to threadMessageGroupId - expect(result).toBe('mg_id-2'); - }); - - it('returns undefined when no state exists at all', () => { - const result = registry.getLiveMessageGroupId('unknown', []); - - expect(result).toBeUndefined(); - }); - }); - - describe('getRunIdsForMessageGroup', () => { - it('returns run IDs tracked under a message group', () => { - const groupId = 'mg_shared'; - registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - messageGroupId: groupId, - }); - registry.startRun({ - threadId: 'thread-2', - user: { id: 'u2', name: 'B' }, - messageGroupId: groupId, - }); - - expect(registry.getRunIdsForMessageGroup(groupId)).toEqual(['run_id-1', 'run_id-2']); - }); - - it('returns empty array for unknown group', () => { - expect(registry.getRunIdsForMessageGroup('unknown')).toEqual([]); - }); - }); - - describe('deleteMessageGroup', () => { - it('removes the message group entry from runIdsByMessageGroup', () => { - const groupId = 'mg_shared'; - registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - messageGroupId: groupId, - }); - expect(registry.getRunIdsForMessageGroup(groupId)).toHaveLength(1); - - registry.deleteMessageGroup(groupId); - - expect(registry.getRunIdsForMessageGroup(groupId)).toEqual([]); - }); - - it('does nothing when group does not exist', () => { - expect(() => registry.deleteMessageGroup('nonexistent')).not.toThrow(); - }); - }); - }); - - // ── Run lifecycle ───────────────────────────────────────────────────────── - - describe('run lifecycle', () => { - describe('attachTracing', () => { - it('updates the active run with tracing context', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - const tracing = { - projectName: 'test', - } as unknown as InstanceAiTraceContext; - registry.attachTracing('thread-1', tracing); - - const activeRun = registry.getActiveRun('thread-1'); - expect(activeRun!.tracing).toBe(tracing); - }); - - it('does nothing when no active run exists', () => { - const tracing = { - projectName: 'test', - } as unknown as InstanceAiTraceContext; - - // Should not throw - expect(() => registry.attachTracing('nonexistent', tracing)).not.toThrow(); - }); - - it('preserves other active run properties', () => { - const started = registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - }); - - const tracing = { - projectName: 'test', - } as unknown as InstanceAiTraceContext; - registry.attachTracing('thread-1', tracing); - - const activeRun = registry.getActiveRun('thread-1'); - expect(activeRun!.runId).toBe(started.runId); - expect(activeRun!.messageGroupId).toBe(started.messageGroupId); - }); - }); - - describe('clearActiveRun', () => { - it('removes the active run for the thread', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - expect(registry.hasActiveRun('thread-1')).toBe(true); - - registry.clearActiveRun('thread-1'); - - expect(registry.hasActiveRun('thread-1')).toBe(false); - expect(registry.getActiveRun('thread-1')).toBeUndefined(); - }); - - it('does nothing when no active run exists', () => { - expect(() => registry.clearActiveRun('nonexistent')).not.toThrow(); - }); - - it('does not clear an active run owned by a newer executor', () => { - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - const currentExecution = Symbol('current-execution'); - registry.suspendRun('thread-1', suspendedState); - registry.activateSuspendedRun('thread-1', currentExecution); - - registry.clearActiveRun('thread-1', Symbol('stale-execution')); - - expect(registry.getActiveRun('thread-1')?.executionToken).toBe(currentExecution); - }); - - it('does not let an unowned finalizer clear a newer tokenized executor', () => { - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - const currentExecution = Symbol('current-execution'); - registry.suspendRun('thread-1', suspendedState); - registry.activateSuspendedRun('thread-1', currentExecution); - - registry.clearActiveRun('thread-1'); - - expect(registry.getActiveRun('thread-1')?.executionToken).toBe(currentExecution); - }); - - it('clears the active run owned by the matching executor', () => { - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - const executionToken = Symbol('execution'); - registry.suspendRun('thread-1', suspendedState); - registry.activateSuspendedRun('thread-1', executionToken); - - registry.clearActiveRun('thread-1', executionToken); - - expect(registry.hasActiveRun('thread-1')).toBe(false); - }); - }); - - describe('cancelActiveRun', () => { - it('removes and returns only the active run for the thread', () => { - const started = registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - }); - - const result = registry.cancelActiveRun('thread-1'); - - expect(result?.runId).toBe(started.runId); - expect(registry.hasActiveRun('thread-1')).toBe(false); - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - }); - - it('returns undefined when no active run exists', () => { - expect(registry.cancelActiveRun('thread-1')).toBeUndefined(); - }); - }); - - describe('suspendRun', () => { - it('moves run from active to suspended', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - - registry.suspendRun('thread-1', suspendedState); - - expect(registry.hasActiveRun('thread-1')).toBe(false); - expect(registry.hasSuspendedRun('thread-1')).toBe(true); - expect(registry.getSuspendedRun('thread-1')).toBe(suspendedState); - }); - - it('rehydrates the message-group indexes when they start empty (restart-resumed orphan)', () => { - // Simulate a restart: no prior startRun, so threadMessageGroupId / - // runIdsByMessageGroup are empty when the orphan is re-suspended. - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - runId: 'run_orphan', - messageGroupId: 'mg_orphan', - }); - - registry.suspendRun('thread-1', suspendedState); - - expect(registry.getMessageGroupId('thread-1')).toBe('mg_orphan'); - expect(registry.getRunIdsForMessageGroup('mg_orphan')).toEqual(['run_orphan']); - }); - - it('does not duplicate the runId in the group when suspending a run started normally', () => { - const started = registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - runId: started.runId, - messageGroupId: started.messageGroupId, - }); - - registry.suspendRun('thread-1', suspendedState); - - expect(registry.getRunIdsForMessageGroup(started.messageGroupId!)).toEqual([started.runId]); - }); - }); - - describe('activateSuspendedRun', () => { - it('moves run from suspended to active and returns suspended state', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - runId: 'run_suspended', - messageGroupId: 'mg_suspended', - }); - registry.suspendRun('thread-1', suspendedState); - - const result = registry.activateSuspendedRun('thread-1'); - - expect(result).toBe(suspendedState); - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - expect(registry.hasActiveRun('thread-1')).toBe(true); - - const activeRun = registry.getActiveRun('thread-1'); - expect(activeRun!.runId).toBe('run_suspended'); - expect(activeRun!.messageGroupId).toBe('mg_suspended'); - }); - - it('copies tracing from the suspended state into the active run', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const tracing = { - projectName: 'test', - } as unknown as InstanceAiTraceContext; - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - tracing, - }); - registry.suspendRun('thread-1', suspendedState); - - registry.activateSuspendedRun('thread-1'); - - const activeRun = registry.getActiveRun('thread-1'); - expect(activeRun!.tracing).toBe(tracing); - }); - - it('returns undefined when no suspended run exists', () => { - const result = registry.activateSuspendedRun('nonexistent'); - - expect(result).toBeUndefined(); - }); - - it('rehydrates the message-group indexes on activation when they start empty', () => { - // Restart-resume path: orphan re-suspended then reactivated on a main - // that never ran startRun for this thread. - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - runId: 'run_orphan', - messageGroupId: 'mg_orphan', - }); - registry.suspendRun('thread-1', suspendedState); - // Clear the group indexes to isolate activation's own rehydration. - registry.deleteMessageGroup('mg_orphan'); - - registry.activateSuspendedRun('thread-1'); - - expect(registry.getMessageGroupId('thread-1')).toBe('mg_orphan'); - expect(registry.getRunIdsForMessageGroup('mg_orphan')).toEqual(['run_orphan']); - }); - }); - - describe('cancelSuspendedRun', () => { - it('removes and returns only the suspended run for the thread', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - registry.suspendRun('thread-1', suspendedState); - - const result = registry.cancelSuspendedRun('thread-1'); - - expect(result).toBe(suspendedState); - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - expect(registry.hasActiveRun('thread-1')).toBe(false); - }); - - it('returns undefined when no suspended run exists', () => { - expect(registry.cancelSuspendedRun('thread-1')).toBeUndefined(); - }); - }); - - describe('findSuspendedByRequestId', () => { - it('finds a suspended run by its requestId', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ - threadId: 'thread-1', - requestId: 'req-123', - }); - registry.suspendRun('thread-1', suspendedState); - - const result = registry.findSuspendedByRequestId('req-123'); - - expect(result).toBe(suspendedState); - }); - - it('searches across multiple suspended runs', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun( - 'thread-1', - createSuspendedRunState({ threadId: 'thread-1', requestId: 'req-1' }), - ); - - registry.startRun({ threadId: 'thread-2', user: { id: 'u2', name: 'B' } }); - const target = createSuspendedRunState({ threadId: 'thread-2', requestId: 'req-2' }); - registry.suspendRun('thread-2', target); - - expect(registry.findSuspendedByRequestId('req-2')).toBe(target); - }); - - it('returns undefined when no match', () => { - expect(registry.findSuspendedByRequestId('nonexistent')).toBeUndefined(); - }); - }); - }); - - // ── Confirmation flow ───────────────────────────────────────────────────── - - describe('confirmation flow', () => { - describe('registerPendingConfirmation + resolvePendingConfirmation', () => { - it('resolves pending confirmation with matching userId', () => { - const resolve = vi.fn(); - const pending: PendingConfirmation = { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }; - - registry.registerPendingConfirmation('req-1', pending); - - const data: ConfirmationData = { approved: true, userInput: 'looks good' }; - const result = registry.resolvePendingConfirmation('user-1', 'req-1', data); - - expect(result).toBe(true); - expect(resolve).toHaveBeenCalledWith(data); - }); - - it('removes the confirmation after resolving', () => { - const resolve = vi.fn(); - const pending: PendingConfirmation = { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }; - - registry.registerPendingConfirmation('req-1', pending); - registry.resolvePendingConfirmation('user-1', 'req-1', { approved: true }); - - // Second resolve should fail - confirmation already consumed - const secondResult = registry.resolvePendingConfirmation('user-1', 'req-1', { - approved: true, - }); - expect(secondResult).toBe(false); - }); - }); - - it('returns false when userId does not match', () => { - const resolve = vi.fn(); - const pending: PendingConfirmation = { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }; - - registry.registerPendingConfirmation('req-1', pending); - - const result = registry.resolvePendingConfirmation('wrong-user', 'req-1', { - approved: true, - }); - - expect(result).toBe(false); - expect(resolve).not.toHaveBeenCalled(); - }); - - it('returns false when requestId is unknown', () => { - const result = registry.resolvePendingConfirmation('user-1', 'unknown', { - approved: true, - }); - - expect(result).toBe(false); - }); - - describe('pending confirmation queries', () => { - it('returns pending confirmation metadata without consuming it', () => { - const resolve = vi.fn(); - const pending: PendingConfirmation = { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }; - - registry.registerPendingConfirmation('req-1', pending); - - expect(registry.getPendingConfirmation('req-1')).toMatchObject({ - threadId: 'thread-1', - userId: 'user-1', - }); - expect(registry.hasPendingConfirmationForThread('thread-1')).toBe(true); - expect(registry.resolvePendingConfirmation('user-1', 'req-1', { approved: true })).toBe( - true, - ); - }); - - it('reports no pending confirmation for unknown requests or threads', () => { - expect(registry.getPendingConfirmation('req-1')).toBeUndefined(); - expect(registry.hasPendingConfirmationForThread('thread-1')).toBe(false); - }); - }); - - describe('rejectPendingConfirmation', () => { - it('auto-rejects with { approved: false }', () => { - const resolve = vi.fn(); - const pending: PendingConfirmation = { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }; - - registry.registerPendingConfirmation('req-1', pending); - - const result = registry.rejectPendingConfirmation('req-1'); - - expect(result).toBe(true); - expect(resolve).toHaveBeenCalledWith({ approved: false }); - }); - - it('removes the confirmation after rejecting', () => { - const resolve = vi.fn(); - registry.registerPendingConfirmation('req-1', { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - - registry.rejectPendingConfirmation('req-1'); - - // Second reject should return false - expect(registry.rejectPendingConfirmation('req-1')).toBe(false); - }); - - it('returns false for unknown requestId', () => { - expect(registry.rejectPendingConfirmation('unknown')).toBe(false); - }); - }); - }); - - // ── Cancellation ────────────────────────────────────────────────────────── - - describe('cancelThread', () => { - it('resolves pending confirmations for the thread and returns active/suspended runs', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - const suspendedState = createSuspendedRunState({ threadId: 'thread-1' }); - registry.suspendRun('thread-1', suspendedState); - - // Re-add an active run (to test both active and suspended) - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - const resolve = vi.fn(); - registry.registerPendingConfirmation('req-1', { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - - const result = registry.cancelThread('thread-1'); - - expect(resolve).toHaveBeenCalledWith({ approved: false }); - expect(result.active).toBeDefined(); - expect(result.suspended).toBeDefined(); - }); - - it('uses custom cancellation data when provided', () => { - const resolve = vi.fn(); - registry.registerPendingConfirmation('req-1', { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - - const customData: ConfirmationData = { approved: false, userInput: 'cancelled by user' }; - registry.cancelThread('thread-1', customData); - - expect(resolve).toHaveBeenCalledWith(customData); - }); - - it('removes suspended run from the registry', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1' })); - - registry.cancelThread('thread-1'); - - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - }); - - it('returns empty object when no runs exist for the thread', () => { - const result = registry.cancelThread('nonexistent'); - - expect(result.active).toBeUndefined(); - expect(result.suspended).toBeUndefined(); - }); - - it('only resolves confirmations belonging to the target thread', () => { - const resolveThread1 = vi.fn(); - const resolveThread2 = vi.fn(); - - registry.registerPendingConfirmation('req-1', { - resolve: resolveThread1, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - registry.registerPendingConfirmation('req-2', { - resolve: resolveThread2, - threadId: 'thread-2', - userId: 'user-2', - createdAt: Date.now(), - }); - - registry.cancelThread('thread-1'); - - expect(resolveThread1).toHaveBeenCalledWith({ approved: false }); - expect(resolveThread2).not.toHaveBeenCalled(); - }); - }); - - // ── Cleanup ─────────────────────────────────────────────────────────────── - - describe('clearThread', () => { - it('removes all per-thread state', () => { - registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'Alice' }, - }); - - const resolve = vi.fn(); - registry.registerPendingConfirmation('req-1', { - resolve, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - - registry.setSetupPanelEnabled('thread-1', true); - expect(registry.isSetupPanelEnabled('thread-1')).toBe(true); - expect(registry.isSetupPanelEnabled('thread-2')).toBe(false); - const result = registry.clearThread('thread-1'); - expect(registry.isSetupPanelEnabled('thread-1')).toBe(false); - - // Confirmations resolved - expect(resolve).toHaveBeenCalledWith({ approved: false }); - - // Active run returned and deleted - expect(result.active).toBeDefined(); - expect(registry.hasActiveRun('thread-1')).toBe(false); - - // User deleted - expect(registry.getThreadUser('thread-1')).toBeUndefined(); - - // Message group mappings deleted - expect(registry.getMessageGroupId('thread-1')).toBeUndefined(); - expect(registry.getRunIdsForMessageGroup('mg_id-2')).toEqual([]); - }); - - it('clears both active and suspended runs', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1' })); - // Re-add active - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - - const result = registry.clearThread('thread-1'); - - expect(result.active).toBeDefined(); - expect(result.suspended).toBeDefined(); - expect(registry.hasActiveRun('thread-1')).toBe(false); - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - }); - - it('returns empty when thread has no state', () => { - const result = registry.clearThread('nonexistent'); - - expect(result.active).toBeUndefined(); - expect(result.suspended).toBeUndefined(); - }); - }); - - describe('shutdown', () => { - it('clears everything and returns all active and suspended runs', () => { - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.setPromptConfiguration('thread-1', { - version: 'progressive@1', - systemPromptVersion: 'instance-agent@1', - skillVariants: ['progressive-building@1'], - skillsHash: 'selected-skills', - }); - registry.startRun({ threadId: 'thread-2', user: { id: 'u2', name: 'B' } }); - registry.suspendRun( - 'thread-2', - createSuspendedRunState({ threadId: 'thread-2', runId: 'run_suspended' }), - ); - registry.startRun({ threadId: 'thread-3', user: { id: 'u3', name: 'C' } }); - registry.setPromptVersion('thread-3', 'default@1'); - expect(registry.getPromptConfiguration('thread-3')).toBeUndefined(); - - const result = registry.shutdown(); - - expect(result.activeRuns).toHaveLength(2); - expect(result.activeRuns[0].runId).toBe('run_id-1'); - expect(result.activeRuns[0].promptVersion).toBe('progressive@1'); - expect(result.activeRuns.find((run) => run.threadId === 'thread-3')?.promptVersion).toBe( - 'default@1', - ); - expect(registry.getPromptConfiguration('thread-1')).toBeUndefined(); - expect(registry.getPromptVersion('thread-3')).toBeUndefined(); - expect(result.suspendedRuns).toHaveLength(1); - expect(result.suspendedRuns[0].runId).toBe('run_suspended'); - }); - - it('leaves pending confirmation resolvers untouched and returns their thread ids', () => { - // Auto-resolving inline confirmations on shutdown causes the awaiting - // tool to run to completion as "denied" and mutate the snapshot - // mid-shutdown; we intentionally let them dangle so the user sees - // the original confirmation card on reload. - const resolve1 = vi.fn(); - const resolve2 = vi.fn(); - - registry.registerPendingConfirmation('req-1', { - resolve: resolve1, - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - registry.registerPendingConfirmation('req-2', { - resolve: resolve2, - threadId: 'thread-2', - userId: 'user-2', - createdAt: Date.now(), - }); - - const result = registry.shutdown(); - - expect(resolve1).not.toHaveBeenCalled(); - expect(resolve2).not.toHaveBeenCalled(); - expect(result.pendingThreadIds).toEqual(expect.arrayContaining(['thread-1', 'thread-2'])); - }); - - it('deduplicates pendingThreadIds when one thread has multiple confirmations', () => { - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-shared', - userId: 'user-1', - createdAt: Date.now(), - }); - registry.registerPendingConfirmation('req-2', { - resolve: vi.fn(), - threadId: 'thread-shared', - userId: 'user-1', - createdAt: Date.now(), - }); - - const result = registry.shutdown(); - - expect(result.pendingThreadIds).toEqual(['thread-shared']); - }); - - it('leaves registry fully empty after shutdown', () => { - registry.startRun({ - threadId: 'thread-1', - user: { id: 'u1', name: 'A' }, - }); - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'user-1', - createdAt: Date.now(), - }); - - registry.shutdown(); - - expect(registry.hasActiveRun('thread-1')).toBe(false); - expect(registry.hasSuspendedRun('thread-1')).toBe(false); - expect(registry.getThreadUser('thread-1')).toBeUndefined(); - expect(registry.getMessageGroupId('thread-1')).toBeUndefined(); - expect(registry.getRunIdsForMessageGroup('mg_id-2')).toEqual([]); - }); - }); - - // ── sweepTimedOut ───────────────────────────────────────────────────────── - - describe('sweepTimedOut', () => { - it('identifies active runs that stop reporting activity', () => { - registry.startRun({ threadId: 'thread-old', user: { id: 'u1', name: 'A' } }); - registry.touchActiveRun('thread-old', 0); - - registry.startRun({ threadId: 'thread-new', user: { id: 'u2', name: 'B' } }); - registry.touchActiveRun('thread-new', 20_000); - - const result = registry.sweepTimedOut(policy, 30_000); - - expect(result.activeThreadIds).toEqual(['thread-old']); - expect(result.activeTimeouts['thread-old']).toMatchObject({ - reason: 'idle_timeout', - surface: 'active-run', - timeoutMs: 30_000, - idleMs: 30_000, - }); - }); - - it('identifies suspended runs older than maxAgeMs', () => { - const now = Date.now(); - registry.startRun({ threadId: 'thread-old', user: { id: 'u1', name: 'A' } }); - registry.suspendRun( - 'thread-old', - createSuspendedRunState({ threadId: 'thread-old', createdAt: now - 60_000 }), - ); - - registry.startRun({ threadId: 'thread-new', user: { id: 'u2', name: 'B' } }); - registry.suspendRun( - 'thread-new', - createSuspendedRunState({ threadId: 'thread-new', createdAt: now - 10_000 }), - ); - - const result = registry.sweepTimedOut(policy, now); - - expect(result.suspendedThreadIds).toEqual(['thread-old']); - }); - - it('identifies pending confirmations older than maxAgeMs', () => { - const now = Date.now(); - - registry.registerPendingConfirmation('req-old', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'user-1', - createdAt: now - 60_000, - }); - - registry.registerPendingConfirmation('req-new', { - resolve: vi.fn(), - threadId: 'thread-2', - userId: 'user-2', - createdAt: now - 10_000, - }); - - const result = registry.sweepTimedOut(policy, now); - - expect(result.confirmationRequestIds).toEqual(['req-old']); - }); - - it('does not time out an active run while a pending confirmation owns the wait', () => { - const confirmationPolicy = new InstanceAiLivenessPolicy({ - confirmationTimeoutMs: day, - backgroundTaskIdleTimeoutMs: 0, - backgroundTaskMaxLifetimeMs: 0, - activeRunIdleTimeoutMs: 10_000, - activeRunMaxLifetimeMs: 0, - }); - - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.touchActiveRun('thread-1', 0); - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'user-1', - createdAt: 20_000, - }); - - const result = registry.sweepTimedOut(confirmationPolicy, 30_000); - - expect(result.activeThreadIds).toEqual([]); - expect(result.confirmationRequestIds).toEqual([]); - }); - - it('does NOT mutate state', () => { - const now = Date.now(); - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun( - 'thread-1', - createSuspendedRunState({ threadId: 'thread-1', createdAt: now - 60_000 }), - ); - - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'user-1', - createdAt: now - 60_000, - }); - - registry.sweepTimedOut(policy, now); - - // State should still be intact - expect(registry.hasSuspendedRun('thread-1')).toBe(true); - // The confirmation should still be resolvable - expect(registry.rejectPendingConfirmation('req-1')).toBe(true); - }); - - it('returns empty arrays when nothing is timed out', () => { - const now = Date.now(); - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun( - 'thread-1', - createSuspendedRunState({ threadId: 'thread-1', createdAt: now }), - ); - - const result = registry.sweepTimedOut(policy, now); - - expect(result.activeThreadIds).toEqual([]); - expect(result.suspendedThreadIds).toEqual([]); - expect(result.confirmationRequestIds).toEqual([]); - }); - - it('includes items exactly at the maxAge boundary', () => { - const now = Date.now(); - registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); - registry.suspendRun( - 'thread-1', - createSuspendedRunState({ threadId: 'thread-1', createdAt: now - 30_000 }), - ); - - const result = registry.sweepTimedOut(policy, now); - - // now - createdAt === maxAgeMs, so >= matches - expect(result.suspendedThreadIds).toEqual(['thread-1']); - }); - }); - - // ── activeRunCountForUser ───────────────────────────────────────────────── - - describe('activeRunCountForUser', () => { - const alice: TestUser = { id: 'alice', name: 'Alice' }; - const bob: TestUser = { id: 'bob', name: 'Bob' }; - - it("counts only the given user's executing runs", () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - registry.startRun({ threadId: 'thread-2', user: alice }); - registry.startRun({ threadId: 'thread-3', user: bob }); - - expect(registry.activeRunCountForUser('alice')).toBe(2); - expect(registry.activeRunCountForUser('bob')).toBe(1); - expect(registry.activeRunCount()).toBe(3); - }); - - it('returns zero for a user with no runs', () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - - expect(registry.activeRunCountForUser('carol')).toBe(0); - }); - - it('stops counting a run once it is cleared', () => { - const { executionToken } = registry.startRun({ threadId: 'thread-1', user: alice }); - registry.clearActiveRun('thread-1', executionToken); - - expect(registry.activeRunCountForUser('alice')).toBe(0); - }); - - // A suspended run spends nothing while it waits, so counting it would lock the user - // out for the whole confirmation timeout after a few abandoned HITL cards. - it('excludes suspended runs, and counts them again on resume', () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - registry.suspendRun( - 'thread-1', - createSuspendedRunState({ threadId: 'thread-1', user: alice }), - ); - - expect(registry.activeRunCountForUser('alice')).toBe(0); - - registry.activateSuspendedRun('thread-1'); - - expect(registry.activeRunCountForUser('alice')).toBe(1); - }); - - // A run restored from the DB after a restart was never stamped by `startRun`, so the - // owner has to be recoverable from the suspended state's `user`. - it('attributes a resumed run with no stamped userId via its user', () => { - registry.suspendRun('thread-1', createSuspendedRunState({ threadId: 'thread-1', user: bob })); - - registry.activateSuspendedRun('thread-1'); - - expect(registry.activeRunCountForUser('bob')).toBe(1); - }); - - // An inline approval card leaves the run in `activeRuns` (it is blocked on a Promise - // inside `waitForConfirmation`), so without this the user would be walled out of - // starting anything else until they answered or the card timed out. - it('excludes a run parked on an inline confirmation', () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'alice', - createdAt: Date.now(), - }); - - expect(registry.activeRunCountForUser('alice')).toBe(0); - // The instance cap still counts it: parked or not, the run holds its memory. - expect(registry.activeRunCount()).toBe(1); - }); - - it('counts the run again once the confirmation resolves', () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'alice', - createdAt: Date.now(), - }); - registry.resolvePendingConfirmation('alice', 'req-1', { approved: true }); - - expect(registry.activeRunCountForUser('alice')).toBe(1); - }); - - it('only discounts the thread that is actually parked', () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - registry.startRun({ threadId: 'thread-2', user: alice }); - registry.registerPendingConfirmation('req-1', { - resolve: vi.fn(), - threadId: 'thread-1', - userId: 'alice', - createdAt: Date.now(), - }); - - expect(registry.activeRunCountForUser('alice')).toBe(1); - }); - - it('keeps attribution across attachTracing', () => { - registry.startRun({ threadId: 'thread-1', user: alice }); - registry.attachTracing('thread-1', {} as never); - - expect(registry.activeRunCountForUser('alice')).toBe(1); - }); + expect(registry.getMessageGroupId('thread-1')).toBeUndefined(); + expect(registry.getRunIdsForMessageGroup('mg-1')).toEqual([]); + expect(registry.getTimeZone('thread-1')).toBeUndefined(); + expect(registry.getRunIdsForMessageGroup('mg-2')).toEqual(['run-2']); }); }); diff --git a/packages/@n8n/instance-ai/src/runtime/__tests__/stream-runner.test.ts b/packages/@n8n/instance-ai/src/runtime/__tests__/stream-runner.test.ts deleted file mode 100644 index e007a8b6dbe..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/__tests__/stream-runner.test.ts +++ /dev/null @@ -1,282 +0,0 @@ -import type { WorkSummary } from '../../stream/work-summary-accumulator'; -import type * as ResumableStreamExecutor from '../resumable-stream-executor'; -import { executeResumableStream } from '../resumable-stream-executor'; -import { streamAgentRun } from '../stream-runner'; - -vi.mock('../resumable-stream-executor', async () => { - const actual = await vi.importActual( - '../resumable-stream-executor', - ); - - return { - ...actual, - executeResumableStream: vi.fn(), - }; -}); - -const emptyWorkSummary: WorkSummary = { - toolCalls: [], - totalToolCalls: 0, - totalToolErrors: 0, - askedClarifyingQuestion: false, -}; - -function createLogger() { - return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; -} - -function createEventBus() { - return { - publish: vi.fn(), - subscribe: vi.fn(), - getEventsAfter: vi.fn(), - getEventsForRun: vi.fn().mockReturnValue([]), - getEventsForRuns: vi.fn().mockReturnValue([]), - }; -} - -async function* fromChunks(chunks: unknown[]) { - for (const chunk of chunks) { - await Promise.resolve(); - yield chunk; - } -} - -async function* emptyStream() { - await Promise.resolve(); - yield* []; -} - -async function collectAsyncIterable(stream: AsyncIterable) { - const chunks: unknown[] = []; - for await (const chunk of stream) { - chunks.push(chunk); - } - return chunks; -} - -describe('streamAgentRun', () => { - it('returns errored status when agent stream contains an error chunk', async () => { - vi.mocked(executeResumableStream).mockResolvedValue({ - status: 'errored', - agentRunId: 'agent-run-1', - workSummary: emptyWorkSummary, - }); - const eventBus = createEventBus(); - const agent = { - stream: vi.fn().mockResolvedValue({ - runId: 'agent-run-1', - fullStream: fromChunks([ - { type: 'text-delta', delta: 'Hello' }, - { type: 'error', error: new Error('Not Found') }, - ]), - }), - }; - - const result = await streamAgentRun( - agent, - 'test input', - {}, - { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - signal: new AbortController().signal, - eventBus, - logger: createLogger(), - }, - ); - - expect(result.status).toBe('errored'); - expect(result.agentRunId).toBe('agent-run-1'); - expect(result.workSummary).toBe(emptyWorkSummary); - }); - - it('returns completed status for successful streams', async () => { - vi.mocked(executeResumableStream).mockResolvedValue({ - status: 'completed', - agentRunId: 'agent-run-1', - workSummary: emptyWorkSummary, - }); - const eventBus = createEventBus(); - const agent = { - stream: vi.fn().mockResolvedValue({ - runId: 'agent-run-1', - fullStream: fromChunks([{ type: 'text-delta', delta: 'All good' }]), - }), - }; - - const result = await streamAgentRun( - agent, - 'test input', - {}, - { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - signal: new AbortController().signal, - eventBus, - logger: createLogger(), - }, - ); - - expect(result.status).toBe('completed'); - expect(result.workSummary).toBe(emptyWorkSummary); - }); - - it('passes through the buffered manual confirmation event', async () => { - const mockedExecuteResumableStream = vi.mocked(executeResumableStream); - const agent = { - stream: vi.fn().mockResolvedValue({ - runId: 'agent-run-1', - fullStream: emptyStream(), - }), - }; - const eventBus = createEventBus(); - - mockedExecuteResumableStream.mockResolvedValue({ - status: 'suspended', - agentRunId: 'agent-run-1', - workSummary: emptyWorkSummary, - suspension: { - requestId: 'request-1', - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - suspendPayload: {}, - }, - confirmationEvent: { - type: 'confirmation-request', - runId: 'run-1', - agentId: 'agent-1', - payload: { - requestId: 'request-1', - toolCallId: 'tool-call-1', - toolName: 'pause-for-user', - args: {}, - severity: 'warning', - message: 'Please confirm', - }, - }, - }); - - const result = await streamAgentRun( - agent, - 'hello', - {}, - { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - signal: new AbortController().signal, - eventBus, - logger: createLogger(), - }, - ); - - expect(result.status).toBe('suspended'); - expect(result.agentRunId).toBe('agent-run-1'); - expect(result.workSummary).toBe(emptyWorkSummary); - expect(result.suspension?.requestId).toBe('request-1'); - expect(result.confirmationEvent?.type).toBe('confirmation-request'); - expect(result.confirmationEvent?.payload.requestId).toBe('request-1'); - expect(mockedExecuteResumableStream).toHaveBeenCalledWith( - expect.objectContaining({ - control: { mode: 'manual' }, - }), - ); - }); - - it('passes an already-normalized native stream source through to the resumable executor', async () => { - const mockedExecuteResumableStream = vi.mocked(executeResumableStream); - const streamResult = { - runId: 'agent-run-2', - fullStream: emptyStream(), - text: Promise.resolve('done'), - steps: Promise.resolve([{ text: 'done' }]), - response: Promise.resolve({ id: 'response-1' }), - usage: Promise.resolve({ inputTokens: 1, outputTokens: 2, totalTokens: 3 }), - }; - const agent = { - stream: vi.fn().mockResolvedValue(streamResult), - }; - const eventBus = createEventBus(); - - mockedExecuteResumableStream.mockResolvedValue({ - status: 'completed', - agentRunId: 'agent-run-2', - text: Promise.resolve('done'), - workSummary: emptyWorkSummary, - }); - - await streamAgentRun( - agent, - 'hello', - {}, - { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - signal: new AbortController().signal, - eventBus, - logger: createLogger(), - }, - ); - - expect(mockedExecuteResumableStream).toHaveBeenCalledWith( - expect.objectContaining({ - stream: streamResult, - }), - ); - }); - - it('normalizes native agent readable streams for the resumable executor', async () => { - const mockedExecuteResumableStream = vi.mocked(executeResumableStream); - mockedExecuteResumableStream.mockClear(); - const nativeChunk = { type: 'text-delta', delta: 'All good' }; - const readable = new ReadableStream({ - start(controller) { - controller.enqueue(nativeChunk); - controller.close(); - }, - }); - const agent = { - stream: vi.fn().mockResolvedValue({ - runId: 'agent-run-1', - stream: readable, - getState: vi.fn(), - }), - }; - const eventBus = createEventBus(); - - mockedExecuteResumableStream.mockResolvedValue({ - status: 'completed', - agentRunId: 'agent-run-1', - workSummary: emptyWorkSummary, - }); - - await streamAgentRun( - agent, - 'hello', - {}, - { - threadId: 'thread-1', - runId: 'run-1', - agentId: 'agent-1', - signal: new AbortController().signal, - eventBus, - logger: createLogger(), - }, - ); - - const call = mockedExecuteResumableStream.mock.calls[0]; - expect(call).toBeDefined(); - const source = call?.[0].stream; - expect(source).toEqual( - expect.objectContaining({ - runId: 'agent-run-1', - }), - ); - await expect(collectAsyncIterable(source.fullStream)).resolves.toEqual([nativeChunk]); - await expect(source.text).resolves.toBe('All good'); - }); -}); diff --git a/packages/@n8n/instance-ai/src/runtime/background-task-manager.ts b/packages/@n8n/instance-ai/src/runtime/background-task-manager.ts deleted file mode 100644 index ff03d378d73..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/background-task-manager.ts +++ /dev/null @@ -1,468 +0,0 @@ -import type { BackgroundTaskResult, InstanceAiTraceContext } from '../types'; -import type { - InstanceAiLivenessDecision, - InstanceAiLivenessPolicy, - InstanceAiLivenessTimeoutReason, -} from './liveness-policy'; - -export type BackgroundTaskStatus = 'running' | 'completed' | 'failed' | 'cancelled'; - -export interface ManagedBackgroundTask { - taskId: string; - threadId: string; - runId: string; - role: string; - agentId: string; - status: BackgroundTaskStatus; - result?: string; - error?: string; - startedAt: number; - lastActivityAt: number; - timeoutReason?: InstanceAiLivenessTimeoutReason; - abortController: AbortController; - corrections: string[]; - /** Callback resolved when a new correction is queued. Re-created by the consumer after each notification. */ - onCorrectionQueued?: () => void; - messageGroupId?: string; - outcome?: Record; - plannedTaskId?: string; - workItemId?: string; - traceContext?: InstanceAiTraceContext; - createTraceContext?: () => Promise; - /** Identity used for single-flight dedupe lookups; copied from the spawn options. */ - dedupeKey?: BackgroundTaskDedupeKey; - /** - * The checkpoint task id this background task was spawned under, when the - * orchestrator called a detached sub-agent tool inside a - * `` turn. The checkpoint safety - * net uses this to tell "orchestrator exited silently" apart from - * "orchestrator handed off to an in-flight patch builder". - */ - parentCheckpointId?: string; -} - -export interface BackgroundTaskDedupeKey { - /** Planned-task graph ID this background task is dispatched for. Primary dedupe key. */ - plannedTaskId?: string; - /** Target workflow ID for this background task. Fallback dedupe key when there is no planned task. */ - workflowId?: string; - /** Agent role (e.g. 'workflow-builder'). Scopes the workflowId fallback so different roles against the same workflow don't collide. */ - role: string; -} - -export interface SpawnManagedBackgroundTaskOptions { - taskId: string; - threadId: string; - runId: string; - role: string; - agentId: string; - messageGroupId?: string; - plannedTaskId?: string; - workItemId?: string; - traceContext?: InstanceAiTraceContext; - createTraceContext?: () => Promise; - /** - * Identity for single-flight dedupe. When supplied, a spawn with the same `plannedTaskId` - * (primary) or `role + workflowId` (fallback) as a currently-running task returns - * `{ status: 'duplicate', existing }` instead of launching a second task. - */ - dedupeKey?: BackgroundTaskDedupeKey; - /** - * Link this background task to a running checkpoint in the planned-task - * graph. Set when the orchestrator spawns a detached sub-agent (builder, - * delegate) from inside a - * `` turn. The post-run safety - * net defers failing the checkpoint while any child with this id is still - * running, and the settlement path re-emits the checkpoint follow-up when - * the last child settles. - */ - parentCheckpointId?: string; - run: ( - signal: AbortSignal, - drainCorrections: () => string[], - waitForCorrection: () => Promise, - taskContext: { traceContext?: InstanceAiTraceContext }, - ) => Promise; - onLimitReached?: (errorMessage: string) => void; - onCompleted?: (task: ManagedBackgroundTask) => void | Promise; - onFailed?: (task: ManagedBackgroundTask) => void | Promise; - onSettled?: (task: ManagedBackgroundTask) => void | Promise; -} - -export type SpawnManagedBackgroundTaskResult = - | { status: 'started'; task: ManagedBackgroundTask } - | { status: 'limit-reached' } - | { status: 'duplicate'; existing: ManagedBackgroundTask }; - -export interface BackgroundTaskTimeoutOptions { - shouldSkipTask?: (task: ManagedBackgroundTask) => boolean; -} - -export interface BackgroundTaskMessageOptions< - TTask extends ManagedBackgroundTask = ManagedBackgroundTask, -> { - formatTask?: (task: TTask) => Promise | string; -} - -export class BackgroundTaskManager { - private readonly tasks = new Map(); - private readonly callbacks = new Map< - string, - Pick - >(); - /** plannedTaskId → taskId for the currently-running task. Populated only when the caller provides a dedupeKey with plannedTaskId. */ - private readonly byPlannedTaskId = new Map(); - /** - * `${role}:${workflowId}` → taskId for the currently-running task. Only - * populated (and only consulted) when the caller provides a dedupeKey - * WITHOUT a plannedTaskId. When both keys are present we treat - * plannedTaskId as the canonical identity — two distinct planned tasks may - * legitimately target the same workflow (e.g., build + later patch) and - * must not collapse into each other. - */ - private readonly byRoleAndWorkflowId = new Map(); - - constructor( - private readonly maxConcurrentPerThread = 5, - /** - * Ceiling on sub-agents running across every thread on this process. Guards the - * fan-out the per-thread limit misses: a handful of runs each spawning their full - * complement. `-1` means unlimited, matching the execution concurrency limits. - */ - private readonly maxConcurrentTotal = -1, - ) {} - - private workflowKey(role: string, workflowId: string): string { - return `${role}:${workflowId}`; - } - - private findDuplicate( - dedupeKey: BackgroundTaskDedupeKey | undefined, - ): ManagedBackgroundTask | undefined { - if (!dedupeKey) return undefined; - if (dedupeKey.plannedTaskId) { - // plannedTaskId is the canonical identity when present — we must NOT - // fall back to the workflowId index, otherwise distinct planned tasks - // targeting the same (role, workflowId) would falsely collapse. - const existingId = this.byPlannedTaskId.get(dedupeKey.plannedTaskId); - if (existingId) { - const existing = this.tasks.get(existingId); - if (existing?.status === 'running') return existing; - } - return undefined; - } - if (dedupeKey.workflowId) { - const existingId = this.byRoleAndWorkflowId.get( - this.workflowKey(dedupeKey.role, dedupeKey.workflowId), - ); - if (existingId) { - const existing = this.tasks.get(existingId); - if (existing?.status === 'running') return existing; - } - } - return undefined; - } - - getTaskSnapshots(threadId: string): ManagedBackgroundTask[] { - return [...this.tasks.values()].filter((task) => task.threadId === threadId); - } - - getRunningTasks(threadId: string): ManagedBackgroundTask[] { - return [...this.tasks.values()].filter( - (task) => task.threadId === threadId && task.status === 'running', - ); - } - - /** Sub-agents running across every thread on this process. */ - runningTaskCount(): number { - let count = 0; - for (const task of this.tasks.values()) { - if (task.status === 'running') count++; - } - return count; - } - - /** - * Return all running background tasks on this thread that were spawned - * under the given checkpoint task id. Used by the checkpoint safety net to - * defer failing a checkpoint while a detached patch sub-agent it just - * launched is still in-flight. - */ - getRunningTasksByParentCheckpoint( - threadId: string, - checkpointTaskId: string, - ): ManagedBackgroundTask[] { - return [...this.tasks.values()].filter( - (task) => - task.threadId === threadId && - task.status === 'running' && - task.parentCheckpointId === checkpointTaskId, - ); - } - - queueCorrection( - threadId: string, - taskId: string, - correction: string, - ): 'queued' | 'task-completed' | 'task-not-found' { - const task = this.tasks.get(taskId); - if (task?.threadId !== threadId) return 'task-not-found'; - if (task.status !== 'running') return 'task-completed'; - this.touchTask(threadId, taskId); - task.corrections.push(correction); - if (task.onCorrectionQueued) { - const notify = task.onCorrectionQueued; - task.onCorrectionQueued = undefined; - notify(); - } - return 'queued'; - } - - cancelTask(threadId: string, taskId: string): ManagedBackgroundTask | undefined { - const task = this.tasks.get(taskId); - if (task?.threadId !== threadId || task.status !== 'running') return undefined; - - task.abortController.abort(); - task.status = 'cancelled'; - this.tasks.delete(taskId); - this.releaseDedupeIndices(task); - this.callbacks.delete(taskId); - return task; - } - - cancelThread(threadId: string): ManagedBackgroundTask[] { - const cancelled: ManagedBackgroundTask[] = []; - for (const [taskId, task] of this.tasks) { - if (task.threadId !== threadId || task.status !== 'running') continue; - task.abortController.abort(); - task.status = 'cancelled'; - cancelled.push(task); - this.tasks.delete(taskId); - this.releaseDedupeIndices(task); - this.callbacks.delete(taskId); - } - return cancelled; - } - - cancelAll(): ManagedBackgroundTask[] { - const cancelled: ManagedBackgroundTask[] = []; - for (const [taskId, task] of this.tasks) { - task.abortController.abort(); - task.status = 'cancelled'; - cancelled.push(task); - this.tasks.delete(taskId); - this.releaseDedupeIndices(task); - this.callbacks.delete(taskId); - } - return cancelled; - } - - touchTask(threadId: string, taskId: string, at = Date.now()): boolean { - const task = this.tasks.get(taskId); - if (task?.threadId !== threadId || task.status !== 'running') return false; - task.lastActivityAt = at; - return true; - } - - async timeoutTimedOutTasks( - policy: InstanceAiLivenessPolicy, - now = Date.now(), - options: BackgroundTaskTimeoutOptions = {}, - ): Promise { - const timedOut: ManagedBackgroundTask[] = []; - for (const task of [...this.tasks.values()]) { - if (task.status !== 'running') continue; - if (options.shouldSkipTask?.(task)) continue; - const decision = policy.evaluate({ - surface: 'background-task', - startedAt: task.startedAt, - lastActivityAt: task.lastActivityAt, - now, - }); - if (decision.action !== 'timeout') continue; - await this.timeoutTask(task, decision); - timedOut.push(task); - } - return timedOut; - } - - spawn(options: SpawnManagedBackgroundTaskOptions): SpawnManagedBackgroundTaskResult { - const duplicate = this.findDuplicate(options.dedupeKey); - if (duplicate) return { status: 'duplicate', existing: duplicate }; - - const runningCount = this.getRunningTasks(options.threadId).length; - if (runningCount >= this.maxConcurrentPerThread) { - options.onLimitReached?.( - `Cannot start background task: limit of ${this.maxConcurrentPerThread} concurrent tasks reached. Wait for existing tasks to complete.`, - ); - return { status: 'limit-reached' }; - } - - if (this.maxConcurrentTotal !== -1 && this.runningTaskCount() >= this.maxConcurrentTotal) { - options.onLimitReached?.( - `Cannot start background task: this n8n instance is at its limit of ${this.maxConcurrentTotal} concurrent tasks. Wait for existing tasks to complete.`, - ); - return { status: 'limit-reached' }; - } - - const now = Date.now(); - const task: ManagedBackgroundTask = { - taskId: options.taskId, - threadId: options.threadId, - runId: options.runId, - role: options.role, - agentId: options.agentId, - status: 'running', - startedAt: now, - lastActivityAt: now, - abortController: new AbortController(), - corrections: [], - messageGroupId: options.messageGroupId, - plannedTaskId: options.plannedTaskId, - workItemId: options.workItemId, - traceContext: options.traceContext, - dedupeKey: options.dedupeKey, - parentCheckpointId: options.parentCheckpointId, - }; - - this.tasks.set(options.taskId, task); - this.callbacks.set(options.taskId, { - onCompleted: options.onCompleted, - onFailed: options.onFailed, - onSettled: options.onSettled, - }); - if (options.dedupeKey?.plannedTaskId) { - this.byPlannedTaskId.set(options.dedupeKey.plannedTaskId, options.taskId); - } else if (options.dedupeKey?.workflowId) { - // Only index by (role, workflowId) when there is no plannedTaskId. - // Otherwise a later spawn for a different planned task targeting the - // same workflow would be wrongly matched against this one. - this.byRoleAndWorkflowId.set( - this.workflowKey(options.dedupeKey.role, options.dedupeKey.workflowId), - options.taskId, - ); - } - void this.executeTask(task, options); - return { status: 'started', task }; - } - - private releaseDedupeIndices(task: ManagedBackgroundTask): void { - const key = task.dedupeKey; - if (!key) return; - if (key.plannedTaskId) { - if (this.byPlannedTaskId.get(key.plannedTaskId) === task.taskId) { - this.byPlannedTaskId.delete(key.plannedTaskId); - } - return; - } - if (key.workflowId) { - const wfKey = this.workflowKey(key.role, key.workflowId); - if (this.byRoleAndWorkflowId.get(wfKey) === task.taskId) { - this.byRoleAndWorkflowId.delete(wfKey); - } - } - } - - private async executeTask( - task: ManagedBackgroundTask, - options: SpawnManagedBackgroundTaskOptions, - ): Promise { - const drainCorrections = (): string[] => { - const pending = [...task.corrections]; - task.corrections.length = 0; - return pending; - }; - - const waitForCorrection = async (): Promise => - await new Promise((resolve) => { - if (task.corrections.length > 0) { - resolve(); - return; - } - task.onCorrectionQueued = resolve; - }); - - try { - if (!task.traceContext && options.createTraceContext) { - task.traceContext = await options.createTraceContext(); - } - const raw = await options.run( - task.abortController.signal, - drainCorrections, - waitForCorrection, - { traceContext: task.traceContext }, - ); - if (task.status !== 'running') return; - task.status = 'completed'; - task.result = typeof raw === 'string' ? raw : raw.text; - task.outcome = typeof raw === 'string' ? undefined : raw.outcome; - await options.onCompleted?.(task); - } catch (error) { - if (task.status !== 'running') return; - if (task.abortController.signal.aborted) return; - task.status = 'failed'; - task.error = error instanceof Error ? error.message : String(error); - await options.onFailed?.(task); - } finally { - try { - if (!task.abortController.signal.aborted && task.status !== 'running') { - await options.onSettled?.(task); - } - } finally { - if (task.status !== 'running') { - this.tasks.delete(task.taskId); - this.releaseDedupeIndices(task); - this.callbacks.delete(task.taskId); - } - } - } - } - - private async timeoutTask( - task: ManagedBackgroundTask, - decision: Extract, - ): Promise { - if (task.status !== 'running') return; - - task.abortController.abort(); - task.status = 'failed'; - task.timeoutReason = decision.reason; - task.error = `Background ${task.role} task timed out after ${decision.timeoutMs}ms`; - - const callbacks = this.callbacks.get(task.taskId); - try { - await callbacks?.onFailed?.(task); - } finally { - try { - await callbacks?.onSettled?.(task); - } finally { - this.tasks.delete(task.taskId); - this.releaseDedupeIndices(task); - this.callbacks.delete(task.taskId); - } - } - } -} - -export async function enrichMessageWithRunningTasks< - TTask extends ManagedBackgroundTask = ManagedBackgroundTask, ->( - message: string, - runningTasks: TTask[], - options: BackgroundTaskMessageOptions = {}, -): Promise { - if (runningTasks.length === 0) return message; - - const parts: string[] = []; - - for (const task of runningTasks) { - if (options.formatTask) { - parts.push(await options.formatTask(task)); - continue; - } - - parts.push(`[Running task — ${task.role}, task: ${task.taskId}]`); - } - - return `\n${parts.join('\n')}\n\n\n${message}`; -} diff --git a/packages/@n8n/instance-ai/src/runtime/liveness-policy.ts b/packages/@n8n/instance-ai/src/runtime/liveness-policy.ts deleted file mode 100644 index 91314a6426e..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/liveness-policy.ts +++ /dev/null @@ -1,129 +0,0 @@ -export type InstanceAiLivenessSurface = - | 'active-run' - | 'suspended-run' - | 'pending-confirmation' - | 'background-task'; - -export type InstanceAiLivenessTimeoutReason = 'idle_timeout' | 'max_lifetime'; - -export interface InstanceAiLivenessPolicyConfig { - confirmationTimeoutMs: number; - backgroundTaskIdleTimeoutMs: number; - backgroundTaskMaxLifetimeMs: number; - activeRunIdleTimeoutMs: number; - activeRunMaxLifetimeMs: number; -} - -const MINUTE_MS = 60_000; -const DAY_MS = 24 * 60 * MINUTE_MS; - -export const INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG = { - confirmationTimeoutMs: DAY_MS, - backgroundTaskIdleTimeoutMs: 10 * MINUTE_MS, - backgroundTaskMaxLifetimeMs: 30 * MINUTE_MS, - // Left disabled on purpose. An orchestrator awaiting a long sub-agent can be - // legitimately quiet for a while, and whether `touchRun` fires across every such - // stretch is unverified -- enabling this would risk reaping live runs. - activeRunIdleTimeoutMs: 0, - // Matches `confirmationTimeoutMs` on purpose, and must not simply be lowered. - // `startedAt` survives suspend/resume (see `suspendRun` / `activateSuspendedRun`), so - // this bounds wall-clock since the run first began -- including time parked waiting on - // a human -- not execution time. Lowering it to, say, 2h would reap a run whose user - // approved a confirmation card three hours later, right as it resumed. - // - // Once `N8N_INSTANCE_AI_MAX_CONCURRENT_RUNS` is enabled a day is too long to hold a - // slot, because a wedged run then costs a share of instance capacity rather than just - // its own thread. The fix at that point is to measure from the last activation rather - // than the original start, not to shrink this number. - activeRunMaxLifetimeMs: DAY_MS, -} satisfies InstanceAiLivenessPolicyConfig; - -export function createInstanceAiLivenessPolicyConfig( - overrides: Partial> = {}, -): InstanceAiLivenessPolicyConfig { - return { - ...INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG, - ...overrides, - }; -} - -export interface InstanceAiLivenessInput { - surface: InstanceAiLivenessSurface; - startedAt: number; - lastActivityAt: number; - now?: number; -} - -export type InstanceAiLivenessDecision = - | { action: 'keep-alive' } - | { - action: 'timeout'; - reason: InstanceAiLivenessTimeoutReason; - surface: InstanceAiLivenessSurface; - timeoutMs: number; - elapsedMs: number; - idleMs: number; - }; - -export class InstanceAiLivenessPolicy { - constructor(private readonly config: InstanceAiLivenessPolicyConfig) {} - - hasEnabledTimeouts(): boolean { - return Object.values(this.config).some((value) => value > 0); - } - - evaluate(input: InstanceAiLivenessInput): InstanceAiLivenessDecision { - const now = input.now ?? Date.now(); - const elapsedMs = Math.max(0, now - input.startedAt); - const idleMs = Math.max(0, now - input.lastActivityAt); - const limits = this.getLimits(input.surface); - - if (limits.idleTimeoutMs > 0 && idleMs >= limits.idleTimeoutMs) { - return { - action: 'timeout', - reason: 'idle_timeout', - surface: input.surface, - timeoutMs: limits.idleTimeoutMs, - elapsedMs, - idleMs, - }; - } - - if (limits.maxLifetimeMs > 0 && elapsedMs >= limits.maxLifetimeMs) { - return { - action: 'timeout', - reason: 'max_lifetime', - surface: input.surface, - timeoutMs: limits.maxLifetimeMs, - elapsedMs, - idleMs, - }; - } - - return { action: 'keep-alive' }; - } - - private getLimits(surface: InstanceAiLivenessSurface): { - idleTimeoutMs: number; - maxLifetimeMs: number; - } { - if (surface === 'background-task') { - return { - idleTimeoutMs: this.config.backgroundTaskIdleTimeoutMs, - maxLifetimeMs: this.config.backgroundTaskMaxLifetimeMs, - }; - } - - if (surface === 'active-run') { - return { - idleTimeoutMs: this.config.activeRunIdleTimeoutMs, - maxLifetimeMs: this.config.activeRunMaxLifetimeMs, - }; - } - - return { - idleTimeoutMs: this.config.confirmationTimeoutMs, - maxLifetimeMs: 0, - }; - } -} diff --git a/packages/@n8n/instance-ai/src/runtime/resumable-stream-executor.ts b/packages/@n8n/instance-ai/src/runtime/resumable-stream-executor.ts deleted file mode 100644 index 5eec43157ca..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/resumable-stream-executor.ts +++ /dev/null @@ -1,689 +0,0 @@ -import { isFinishReason } from '@n8n/agents'; -import type { FinishReason, StreamResult } from '@n8n/agents'; -import type { InstanceAiEvent } from '@n8n/api-types'; -import { isRecord } from '@n8n/utils/is-record'; -import { randomUUID } from 'node:crypto'; - -import type { InstanceAiEventBus } from '../event-bus'; -import type { Logger } from '../logger'; -import type { - OrchestratorRunHandoffReason, - OrchestratorRunStopSignal, -} from './orchestrator-run-control'; -import { isQuotaExhaustedError, mapAgentChunkToEvent } from '../stream/map-chunk'; -import { UsageAccumulator, type RunTokenUsage } from '../stream/usage-accumulator'; -import { WorkSummaryAccumulator, type WorkSummary } from '../stream/work-summary-accumulator'; -import { parseSuspension, resumeAgentStream } from '../utils/stream-helpers'; -import type { SuspensionInfo } from '../utils/stream-helpers'; - -type ConfirmationRequestEvent = Extract; - -export interface ResumableStreamSource { - runId?: string; - fullStream: AsyncIterable; - text?: Promise; - steps?: Promise; - response?: Promise; - finishReason?: Promise; - request?: Promise; - usage?: Promise; - totalUsage?: Promise; -} - -export interface ResumableStreamContext { - threadId: string; - runId: string; - agentId: string; - eventBus: InstanceAiEventBus; - signal: AbortSignal; - logger: Logger; - onActivity?: () => void; - /** Stop consuming after the current chunk has been mapped and published. */ - stopSignal?: () => OrchestratorRunStopSignal | undefined; -} - -export interface ManualSuspensionControl { - mode: 'manual'; -} - -export interface AutoResumeControl { - mode: 'auto'; - waitForConfirmation: (requestId: string) => Promise>; - drainCorrections?: () => string[]; - /** Returns a promise that resolves when a new user correction is queued. Used to unblock - * HITL suspensions so the builder can incorporate the correction instead of waiting forever. */ - waitForCorrection?: () => Promise; - onSuspension?: (suspension: SuspensionInfo) => void; - buildResumeOptions?: (input: { - agentRunId: string; - suspension: SuspensionInfo; - }) => Record; -} - -export type ResumableStreamControl = ManualSuspensionControl | AutoResumeControl; - -export interface ExecuteResumableStreamOptions { - agent: unknown; - stream: ResumableStreamSource; - context: ResumableStreamContext; - control: ResumableStreamControl; - initialAgentRunId?: string; -} - -export type TraceStatus = 'completed' | 'cancelled' | 'suspended' | 'errored'; - -export interface ExecuteResumableStreamResult { - status: TraceStatus; - agentRunId: string; - text?: Promise; - error?: unknown; - suspension?: SuspensionInfo; - confirmationEvent?: ConfirmationRequestEvent; - /** Accumulated tool call outcomes observed during stream consumption. */ - workSummary: WorkSummary; - /** Accumulated token usage and cost, when the stream emitted usage. */ - usage?: RunTokenUsage; - /** Reason this stream stopped early after publishing the current chunk. */ - stopReason?: OrchestratorRunHandoffReason; - /** Terminal `finish` chunk's reason; `'max-iterations'` means the agent ran out of steps. */ - finishReason?: FinishReason; -} - -function isAsyncIterable(value: unknown): value is AsyncIterable { - return ( - value !== null && - typeof value === 'object' && - typeof Reflect.get(value, Symbol.asyncIterator) === 'function' - ); -} - -function isReadableStream(value: unknown): value is ReadableStream { - return ( - value !== null && - typeof value === 'object' && - typeof Reflect.get(value, 'getReader') === 'function' - ); -} - -function isResumableStreamSource(value: unknown): value is ResumableStreamSource { - return isRecord(value) && isAsyncIterable(value.fullStream); -} - -function isNativeStreamResult(value: unknown): value is StreamResult { - return isRecord(value) && isReadableStream(value.stream); -} - -function extractContentText(content: unknown): string { - if (typeof content === 'string') { - return content; - } - - if (!Array.isArray(content)) { - return ''; - } - - return content - .map((part) => { - if (!isRecord(part) || part.type !== 'text') { - return ''; - } - - return typeof part.text === 'string' ? part.text : ''; - }) - .join(''); -} - -async function collectNativeStreamText(stream: ReadableStream): Promise { - const reader = stream.getReader(); - let deltaText = ''; - let messageText = ''; - - try { - while (true) { - const { done, value } = await reader.read(); - if (done) return deltaText || messageText; - - if (!isRecord(value)) { - continue; - } - - if (value.type === 'text-delta' && typeof value.delta === 'string') { - deltaText += value.delta; - continue; - } - - if (value.type !== 'message' || !isRecord(value.message)) { - continue; - } - - if (value.message.role === 'assistant') { - messageText += extractContentText(value.message.content); - } - } - } finally { - reader.releaseLock(); - } -} - -async function* readableStreamToAsyncIterable(stream: ReadableStream) { - const reader = stream.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) return; - yield value; - } - } finally { - reader.releaseLock(); - } -} - -export function normalizeStreamSource(result: unknown): ResumableStreamSource { - if (isResumableStreamSource(result)) { - return result; - } - - if (isNativeStreamResult(result)) { - const [eventStream, textStream] = result.stream.tee(); - - return { - runId: result.runId, - fullStream: readableStreamToAsyncIterable(eventStream), - text: collectNativeStreamText(textStream), - }; - } - - throw new Error('Unsupported agent stream result'); -} - -function buildCancelledResult( - agentRunId: string, - text: Promise | undefined, - workSummaryAccumulator: WorkSummaryAccumulator, - usageAccumulator: UsageAccumulator, -): ExecuteResumableStreamResult { - return { - status: 'cancelled', - agentRunId, - text, - workSummary: workSummaryAccumulator.toSummary(), - usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined, - }; -} - -/** - * Record a parsed suspension. The first suspension is kept (and, in auto mode, - * its confirmation awaited); any further distinct suspension before resume is - * logged and deferred. - */ -function recordSuspension( - parsedSuspension: SuspensionInfo, - current: SuspensionInfo | undefined, - control: ResumableStreamControl, - context: ResumableStreamContext, -): { suspension: SuspensionInfo; pendingConfirmation?: Promise> } { - if (current) { - if (!isSameSuspension(parsedSuspension, current)) { - context.logger.warn('Additional HITL suspension encountered before resume; deferring', { - threadId: context.threadId, - runId: context.runId, - activeRequestId: current.requestId, - deferredRequestId: parsedSuspension.requestId, - activeToolCallId: current.toolCallId, - deferredToolCallId: parsedSuspension.toolCallId, - }); - } - return { suspension: current }; - } - - if (control.mode === 'auto') { - control.onSuspension?.(parsedSuspension); - return { - suspension: parsedSuspension, - pendingConfirmation: control.waitForConfirmation(parsedSuspension.requestId), - }; - } - return { suspension: parsedSuspension }; -} - -/** - * Publish events, holding back the primary confirmation-request event in - * manual mode and de-duplicating it. Returns the updated confirmation-tracking state. - */ -function publishEvents( - events: InstanceAiEvent[], - args: { - suspension: SuspensionInfo | undefined; - confirmationEvent: ConfirmationRequestEvent | undefined; - confirmationEventPublished: boolean; - control: ResumableStreamControl; - context: ResumableStreamContext; - workSummaryAccumulator: WorkSummaryAccumulator; - }, -): { confirmationEvent?: ConfirmationRequestEvent; confirmationEventPublished: boolean } { - const { suspension, control, context, workSummaryAccumulator } = args; - let confirmationEvent = args.confirmationEvent; - let confirmationEventPublished = args.confirmationEventPublished; - - for (const event of events) { - workSummaryAccumulator.observe(event); - let shouldPublishEvent = true; - - if (event.type === 'confirmation-request') { - const isPrimarySuspension = - event.payload.requestId === suspension?.requestId && - event.payload.toolCallId === suspension?.toolCallId; - if (!isPrimarySuspension || confirmationEventPublished || confirmationEvent) { - shouldPublishEvent = false; - } - - if (shouldPublishEvent && control.mode === 'manual') { - confirmationEvent = event; - shouldPublishEvent = false; - } - - if (shouldPublishEvent) { - confirmationEventPublished = true; - } - } - - if (shouldPublishEvent) { - context.eventBus.publish(context.threadId, event); - } - } - - return { confirmationEvent, confirmationEventPublished }; -} - -interface StreamPassResult { - cancelled: boolean; - finishReason?: FinishReason; - stopReason?: OrchestratorRunHandoffReason; - suspension?: SuspensionInfo; - hasError: boolean; - error?: unknown; - pendingConfirmation?: Promise>; - confirmationEvent?: ConfirmationRequestEvent; - drainedCorrectionsForResume: string[]; - currentResponseId: string | undefined; - nativeStepIndex: number; -} - -/** - * Consume one stream until it ends (or is cancelled), publishing events, - * accumulating usage/work, and capturing the first suspension. Returns the pass - * outcome plus the updated response-id / step counters for the next pass. - */ -async function consumeStreamPass(args: { - activeStream: AsyncIterable; - activeAgentRunId: string; - options: ExecuteResumableStreamOptions; - workSummaryAccumulator: WorkSummaryAccumulator; - usageAccumulator: UsageAccumulator; - currentResponseId: string | undefined; - nativeStepIndex: number; -}): Promise { - const { activeStream, activeAgentRunId, options, workSummaryAccumulator, usageAccumulator } = - args; - let currentResponseId = args.currentResponseId; - let nativeStepIndex = args.nativeStepIndex; - /** - * Segment id minted for deltas that arrive before any `start-step` supplied - * a provider response id (some providers never emit one). Every delta/block - * must carry a responseId: the shared run reducer keys block replace - * semantics on segment identity, and id-less adjacent segments would merge - * into one timeline entry. Sticky for the contiguous delta run, cleared at - * the next structural fact so blocks stay exactly 1:1 with segments. - */ - let syntheticSegmentId: string | undefined; - let suspension: SuspensionInfo | undefined; - let hasError = false; - let error: unknown; - // Once we've surfaced an out-of-credits error, drop any follow-on error chunks - // (e.g. the SDK's generic "no output generated") so the user sees one clear reason. - let quotaErrorPublished = false; - let pendingConfirmation: Promise> | undefined; - let confirmationEvent: ConfirmationRequestEvent | undefined; - let confirmationEventPublished = false; - let finishReason: FinishReason | undefined; - const drainedCorrectionsForResume: string[] = []; - - for await (const chunk of activeStream) { - if (options.context.signal.aborted) { - // A stop aborts the agent stream too: it stops generating and emits a - // terminal finish chunk carrying the run's usage. Drain the rest of the - // stream (without publishing further events) so that usage is observed - // and the cancelled run still gets billed for the tokens it consumed. - usageAccumulator.observe(chunk); - try { - for await (const remaining of activeStream) { - usageAccumulator.observe(remaining); - } - } catch (drainError) { - options.context.logger.debug('Instance AI abort drain ended early', { - threadId: options.context.threadId, - runId: options.context.runId, - error: drainError instanceof Error ? drainError.message : String(drainError), - }); - } - return { - cancelled: true, - hasError, - drainedCorrectionsForResume, - currentResponseId, - nativeStepIndex, - }; - } - - options.context.onActivity?.(); - usageAccumulator.observe(chunk); - if (isRecord(chunk) && chunk.type === 'finish' && isFinishReason(chunk.finishReason)) { - finishReason = chunk.finishReason; - } - - if (isRecord(chunk) && chunk.type === 'start-step') { - nativeStepIndex += 1; - const responseRunId = activeAgentRunId || options.context.runId; - currentResponseId = `${responseRunId}:step:${nativeStepIndex}`; - syntheticSegmentId = undefined; - } - - const parsedSuspension = parseSuspension(chunk); - if (parsedSuspension) { - const recorded = recordSuspension( - parsedSuspension, - suspension, - options.control, - options.context, - ); - suspension = recorded.suspension; - if (recorded.pendingConfirmation) pendingConfirmation = recorded.pendingConfirmation; - } - - if (isErrorChunk(chunk)) { - hasError = true; - // A quota error was already surfaced this run — swallow later error - // chunks (usage was still observed above) to avoid a confusing second - // callout. Do this before overwriting `error` so `result.error` stays the - // quota error the user saw, rather than the swallowed follow-on — otherwise - // telemetry would log a failure the user was never shown. - if (quotaErrorPublished) continue; - error = chunk.error; - if (isQuotaExhaustedError(chunk.error)) quotaErrorPublished = true; - } - - const isDeltaChunk = - isRecord(chunk) && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta'); - if (isDeltaChunk && !currentResponseId && !syntheticSegmentId) { - syntheticSegmentId = `${activeAgentRunId || options.context.runId}:seg:${randomUUID()}`; - } - - const mappedEvent = mapAgentChunkToEvent( - options.context.runId, - options.context.agentId, - chunk, - currentResponseId ?? syntheticSegmentId, - ); - - // A segment boundary ends the synthetic segment; the next delta mints a - // fresh id so two different segments can never share one. finish-step - // maps to no event but is still a boundary — left sticky, a provider - // that never emits start-step would leak one id across steps, and the - // reducer's id-keyed replace could then drop an earlier block on replay. - const isFinishStep = isRecord(chunk) && chunk.type === 'finish-step'; - if ((mappedEvent && !isDeltaChunk) || isFinishStep) syntheticSegmentId = undefined; - - const events = mappedEvent ? [mappedEvent] : []; - - const published = publishEvents(events, { - suspension, - confirmationEvent, - confirmationEventPublished, - control: options.control, - context: options.context, - workSummaryAccumulator, - }); - confirmationEvent = published.confirmationEvent; - confirmationEventPublished = published.confirmationEventPublished; - - if (options.control.mode === 'auto' && options.control.drainCorrections) { - const corrections = options.control.drainCorrections(); - publishCorrections(options.context, corrections); - drainedCorrectionsForResume.push(...corrections); - } - - const stopSignal = options.context.stopSignal?.(); - if (stopSignal) { - return { - cancelled: false, - stopReason: stopSignal.reason, - finishReason, - suspension, - hasError, - error, - pendingConfirmation, - confirmationEvent, - drainedCorrectionsForResume, - currentResponseId, - nativeStepIndex, - }; - } - } - - return { - cancelled: false, - finishReason, - suspension, - hasError, - error, - pendingConfirmation, - confirmationEvent, - drainedCorrectionsForResume, - currentResponseId, - nativeStepIndex, - }; -} - -export async function executeResumableStream( - options: ExecuteResumableStreamOptions, -): Promise { - let activeStream = options.stream.fullStream; - let activeAgentRunId = options.stream.runId ?? options.initialAgentRunId ?? ''; - let text = options.stream.text; - const workSummaryAccumulator = new WorkSummaryAccumulator(); - const usageAccumulator = new UsageAccumulator(); - - let currentResponseId: string | undefined; - let nativeStepIndex = 0; - - while (true) { - const pass = await consumeStreamPass({ - activeStream, - activeAgentRunId, - options, - workSummaryAccumulator, - usageAccumulator, - currentResponseId, - nativeStepIndex, - }); - currentResponseId = pass.currentResponseId; - nativeStepIndex = pass.nativeStepIndex; - - if (pass.cancelled) { - return buildCancelledResult(activeAgentRunId, text, workSummaryAccumulator, usageAccumulator); - } - - const { suspension, hasError, error, pendingConfirmation, confirmationEvent } = pass; - const { drainedCorrectionsForResume } = pass; - - if (options.context.signal.aborted) { - return buildCancelledResult(activeAgentRunId, text, workSummaryAccumulator, usageAccumulator); - } - - if (pass.stopReason) { - return { - status: hasError ? 'errored' : 'completed', - agentRunId: activeAgentRunId, - text, - ...(error !== undefined ? { error } : {}), - finishReason: pass.finishReason, - workSummary: workSummaryAccumulator.toSummary(), - usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined, - stopReason: pass.stopReason, - }; - } - - if (!suspension) { - return { - status: hasError ? 'errored' : 'completed', - agentRunId: activeAgentRunId, - text, - ...(error !== undefined ? { error } : {}), - finishReason: pass.finishReason, - workSummary: workSummaryAccumulator.toSummary(), - usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined, - }; - } - - if (options.control.mode === 'manual') { - return { - status: 'suspended', - agentRunId: activeAgentRunId, - text, - suspension, - workSummary: workSummaryAccumulator.toSummary(), - usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined, - ...(confirmationEvent ? { confirmationEvent } : {}), - }; - } - - const confirmationPromise = - pendingConfirmation ?? options.control.waitForConfirmation(suspension.requestId); - const resumeData = await waitForConfirmationOrCorrection( - options.context.signal, - confirmationPromise, - options.control, - options.context, - drainedCorrectionsForResume, - ); - const resumeOptions = options.control.buildResumeOptions?.({ - agentRunId: activeAgentRunId, - suspension, - }) ?? { - runId: activeAgentRunId, - toolCallId: suspension.toolCallId, - }; - const resumed = await resumeAgentStream(options.agent, resumeData, { - ...resumeOptions, - }); - const resumedSource = normalizeStreamSource(resumed); - - activeAgentRunId = - (typeof resumedSource.runId === 'string' ? resumedSource.runId : '') || activeAgentRunId; - activeStream = resumedSource.fullStream; - text = resumedSource.text; - } -} - -function publishCorrections(context: ResumableStreamContext, corrections: string[]): void { - for (const correction of corrections) { - context.eventBus.publish(context.threadId, { - type: 'text-delta', - runId: context.runId, - agentId: context.agentId, - // Each correction line is its own segment: a unique responseId keeps - // the run reducer's block replace semantics exact (two publishes - // sharing an id with different texts would read as one segment). - responseId: `${context.runId}:correction:${randomUUID()}`, - payload: { text: `\n[USER CORRECTION]: ${correction}\n` }, - }); - } -} - -function buildCorrectionResumeData(corrections: string[]): Record { - return { - __correctionOverride: true, - message: - 'The user sent a correction while this run was active. ' + - 'The confirmation has been skipped. Apply the correction and continue.', - corrections, - }; -} - -function isErrorChunk(chunk: unknown): chunk is { type: 'error'; error?: unknown } { - return isRecord(chunk) && chunk.type === 'error'; -} - -async function waitForConfirmation( - signal: AbortSignal, - confirmationPromise: Promise>, -): Promise> { - if (signal.aborted) { - throw new Error( - 'The assistant was interrupted while waiting for your response. Send a new message to continue.', - ); - } - - let abortHandler: (() => void) | undefined; - - try { - return await Promise.race([ - confirmationPromise, - new Promise((_, reject) => { - abortHandler = () => - reject( - new Error( - 'The assistant was interrupted while waiting for your response. Send a new message to continue.', - ), - ); - signal.addEventListener('abort', abortHandler, { once: true }); - }), - ]); - } finally { - if (abortHandler) { - signal.removeEventListener('abort', abortHandler); - } - } -} - -/** - * Race the HITL confirmation promise against an incoming user correction. - * When a correction arrives first, auto-confirm the suspended tool call with - * override data so the builder can resume and incorporate the correction. - */ -async function waitForConfirmationOrCorrection( - signal: AbortSignal, - confirmationPromise: Promise>, - control: AutoResumeControl, - context: ResumableStreamContext, - drainedCorrectionsForResume: string[], -): Promise> { - if (drainedCorrectionsForResume.length > 0) { - void confirmationPromise.catch(() => undefined); - return buildCorrectionResumeData(drainedCorrectionsForResume); - } - - if (!control.waitForCorrection) { - return await waitForConfirmation(signal, confirmationPromise); - } - - const correctionSentinel = Object.freeze({ __correctionOverride: true }); - const raced = Promise.race([ - confirmationPromise, - control.waitForCorrection().then(() => correctionSentinel as Record), - ]); - - const result = await waitForConfirmation(signal, raced); - - if (result === correctionSentinel) { - const corrections = control.drainCorrections?.() ?? []; - publishCorrections(context, corrections); - return buildCorrectionResumeData(corrections); - } - - return result; -} - -function isSameSuspension(left: SuspensionInfo, right: SuspensionInfo): boolean { - return left.requestId === right.requestId && left.toolCallId === right.toolCallId; -} diff --git a/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts b/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts index e4d92fbc3a5..0a93ed3a7fa 100644 --- a/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts +++ b/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts @@ -3,66 +3,7 @@ import type { InstanceAiBuildMode, InstanceAiPromptConfiguration, InstanceAiCredentialDestinationDecision, - InstanceAiThreadStatusResponse, } from '@n8n/api-types'; -import { nanoid } from 'nanoid'; - -import type { InstanceAiTraceContext, ModelConfig, OrchestrationContext } from '../types'; -import type { - InstanceAiLivenessPolicy, - InstanceAiLivenessSurface, - InstanceAiLivenessTimeoutReason, -} from './liveness-policy'; -import type { OrchestratorRunHandoffState } from './orchestrator-run-control'; -import type { WorkflowBuildOutcome } from '../workflow-loop/workflow-loop-state'; -import type { SuspendedInstanceContext } from './instance-context-state'; - -export interface ActiveRunState { - runId: string; - threadId: string; - abortController: AbortController; - /** Prevents an older finalizer from clearing a newer executor for the same thread. */ - executionToken?: symbol; - messageGroupId?: string; - tracing?: InstanceAiTraceContext; - modelId?: ModelConfig; - startedAt?: number; - lastActivityAt?: number; - /** Owner of the run, resolved at entry so concurrency can be counted per user. */ - userId?: string; -} - -export interface SuspendedRunState extends ActiveRunState { - agentRunId: string; - agent: unknown; - /** The orchestration context the agent's tools closed over. Stored so a - * resume can rebind `tracing` to the new resume trace — spans emitted - * through the suspended turn's shut-down runtime export nothing. */ - orchestrationContext?: OrchestrationContext; - threadId: string; - user: TUser; - toolCallId: string; - toolName?: string; - suspendPayload?: Record; - requestId: string; - createdAt: number; - /** Set when the suspended run was a planned-task checkpoint follow-up. - * Preserved across suspend/resume so the resumed run's finalizer can - * run the deadlock fallback and reschedule. */ - checkpoint?: { isCheckpointFollowUp: true; checkpointTaskId: string }; - /** Set when the suspended run was a planned build-workflow follow-up. */ - plannedBuild?: { - isPlannedBuildFollowUp: true; - buildTaskId: string; - workItemId: string; - isSupportingWorkflowTask?: boolean; - savedOutcome?: WorkflowBuildOutcome; - }; - /** Shared signal used to stop resumed orchestration after durable work is handed off. */ - runHandoff?: OrchestratorRunHandoffState; - /** Keep the injection and gate results across suspension. Resumed segments build no new block. */ - instanceContext?: SuspendedInstanceContext; -} /** * Flat confirmation payload consumed by native tool `resumeSchema`s and sub-agent HITL. @@ -99,53 +40,12 @@ export interface ConfirmationData { connectedSlugs?: string[]; } -export interface PendingConfirmation { - resolve: (data: ConfirmationData) => void; - threadId: string; - userId: string; - createdAt: number; - startedAt?: number; - lastActivityAt?: number; -} - -export interface BackgroundTaskStatusSnapshot { - taskId: string; - role: string; - agentId: string; - status: 'running' | 'completed' | 'failed' | 'cancelled'; - startedAt: number; - runId: string; - messageGroupId?: string; - threadId: string; -} - -export interface RunStateTimeoutDetails { - reason: InstanceAiLivenessTimeoutReason; - surface: InstanceAiLivenessSurface; - timeoutMs: number; - elapsedMs: number; - idleMs: number; -} - -export interface StartRunOptions { - threadId: string; - user: TUser; - messageGroupId?: string; -} - -export interface StartedRunState extends ActiveRunState { - messageGroupId?: string; -} - -export class RunStateRegistry { - private readonly activeRuns = new Map(); - - private readonly suspendedRuns = new Map>(); - - private readonly pendingConfirmations = new Map(); - - private readonly threadUsers = new Map(); - +/** + * Per-thread state that follow-up turns reuse: request options the client sent + * on the user turn, and the message-group index of runs. The Agents runtime + * owns the run lifecycle itself. + */ +export class RunStateRegistry { private readonly threadMessageGroupId = new Map(); private readonly runIdsByMessageGroup = new Map(); @@ -154,8 +54,8 @@ export class RunStateRegistry { private readonly threadTimeZones = new Map(); /** Computer Use entries the client reported, reused by follow-up runs. Only the - * client can see its own rollout and the device, and a resumed or background - * run has no request of its own to ask. */ + * client can see its own rollout and the device, and a follow-up run has no + * request of its own to ask. */ private readonly threadComputerUseChannels = new Map(); /** Build mode captured at user-run entry and reused by follow-up runs. */ @@ -167,51 +67,10 @@ export class RunStateRegistry { string, { version: string; metadata?: InstanceAiPromptConfiguration } >(); - /** - * Resolves a user id from the opaque `TUser` the registry is parameterised over. - * Required rather than optional: per-user concurrency counting depends on it, and a - * missing extractor would silently report every user as holding zero runs. - */ - constructor(private readonly getUserId: (user: TUser) => string) {} - - startRun(options: StartRunOptions): StartedRunState { - const runId = `run_${nanoid()}`; - const abortController = new AbortController(); - const messageGroupId = options.messageGroupId ?? `mg_${nanoid()}`; - const now = Date.now(); - - this.activeRuns.set(options.threadId, { - runId, - threadId: options.threadId, - abortController, - messageGroupId, - startedAt: now, - lastActivityAt: now, - userId: this.getUserId(options.user), - }); - this.threadUsers.set(options.threadId, options.user); - - // When creating a fresh message group (no reuse), clean up the previous - // one so runIdsByMessageGroup doesn't leak entries indefinitely. - if (!options.messageGroupId) { - const prevGroupId = this.threadMessageGroupId.get(options.threadId); - if (prevGroupId && prevGroupId !== messageGroupId) { - this.runIdsByMessageGroup.delete(prevGroupId); - } - } - - this.indexRunInGroup(options.threadId, messageGroupId, runId); - - return { runId, threadId: options.threadId, abortController, messageGroupId }; - } /** * Seed the message-group indexes for a run: map the thread to its current * group and record the run under that group. Idempotent. - * - * Called on `startRun` and re-applied on `suspendRun`/`activateSuspendedRun` - * so a run resumed after a restart (where these maps start empty) repopulates - * the group association the SSE bootstrap relies on. */ indexRunInGroup(threadId: string, messageGroupId: string, runId: string): void { this.threadMessageGroupId.set(threadId, messageGroupId); @@ -223,282 +82,14 @@ export class RunStateRegistry { if (!groupRunIds.includes(runId)) groupRunIds.push(runId); } - getThreadStatus( - threadId: string, - backgroundTasks: BackgroundTaskStatusSnapshot[], - ): InstanceAiThreadStatusResponse { - const activeRun = this.activeRuns.get(threadId); - const suspendedRun = this.suspendedRuns.get(threadId); - const liveRun = activeRun ?? suspendedRun; - - return { - hasActiveRun: activeRun !== undefined, - isSuspended: suspendedRun !== undefined, - ...(liveRun ? { runId: liveRun.runId } : {}), - backgroundTasks: backgroundTasks - .filter((task) => task.threadId === threadId) - .map((task) => ({ - taskId: task.taskId, - role: task.role, - agentId: task.agentId, - status: task.status, - startedAt: task.startedAt, - runId: task.runId, - messageGroupId: task.messageGroupId, - })), - }; - } - - hasLiveRun(threadId: string): boolean { - return this.activeRuns.has(threadId) || this.suspendedRuns.has(threadId); - } - - hasActiveRun(threadId: string): boolean { - return this.activeRuns.has(threadId); - } - - hasSuspendedRun(threadId: string): boolean { - return this.suspendedRuns.has(threadId); - } - getMessageGroupId(threadId: string): string | undefined { return this.threadMessageGroupId.get(threadId); } - getLiveMessageGroupId( - threadId: string, - backgroundTasks: BackgroundTaskStatusSnapshot[], - ): string | undefined { - if (this.hasLiveRun(threadId)) { - return this.threadMessageGroupId.get(threadId); - } - - const runningTask = backgroundTasks - .filter((task) => task.threadId === threadId && task.status === 'running') - .sort((a, b) => b.startedAt - a.startedAt)[0]; - - return runningTask?.messageGroupId ?? this.threadMessageGroupId.get(threadId); - } - getRunIdsForMessageGroup(messageGroupId: string): string[] { return this.runIdsByMessageGroup.get(messageGroupId) ?? []; } - getActiveRunId(threadId: string): string | undefined { - return this.activeRuns.get(threadId)?.runId; - } - - /** - * Runs holding a slot on this process, which is what the instance cap admits against. - * - * Includes a run parked on an inline approval card, which stays in `activeRuns`. - * Excludes a suspended run, which leaves `activeRuns` but keeps its agent in memory. - * So this bounds concurrent execution, not resident memory. - */ - activeRunCount(): number { - return this.activeRuns.size; - } - - /** - * Number of runs this user currently has executing, across all their threads. - * - * Excludes everything parked on a human, because none of it is spending anything and - * counting it would lock a user out for the whole confirmation timeout after a few - * abandoned cards. That means two things, not one: - * - * - suspended runs, which have left `activeRuns` entirely; and - * - runs blocked in `waitForConfirmation`, which stay in `activeRuns` while an inline - * approval card is open. `sweepTimedOut` skips these for the same reason. - * - * {@link activeRunCount} differs on the second case only. An inline-parked run is still - * in `activeRuns`, so it holds a slot there. A suspended run holds a slot in neither, - * although it still retains its agent -- see {@link activeRunCount}. - * - * The scan is linear in concurrently-executing runs. That stays small in practice - * regardless of the caps -- each run costs ~12-20MB, so a process cannot hold many -- - * and it is negligible next to the model call that follows. - */ - activeRunCountForUser(userId: string): number { - let count = 0; - for (const [threadId, run] of this.activeRuns) { - if (run.userId !== userId) continue; - if (this.hasPendingConfirmationForThread(threadId)) continue; - count++; - } - return count; - } - - getActiveRun(threadId: string): ActiveRunState | undefined { - return this.activeRuns.get(threadId); - } - - getSuspendedRun(threadId: string): SuspendedRunState | undefined { - return this.suspendedRuns.get(threadId); - } - - attachTracing(threadId: string, tracing: InstanceAiTraceContext): void { - const activeRun = this.activeRuns.get(threadId); - if (!activeRun) return; - - this.activeRuns.set(threadId, { - ...activeRun, - threadId, - tracing, - }); - } - - clearActiveRun(threadId: string, executionToken?: symbol): void { - if (this.activeRuns.get(threadId)?.executionToken !== executionToken) return; - this.activeRuns.delete(threadId); - } - - cancelActiveRun(threadId: string): ActiveRunState | undefined { - const active = this.activeRuns.get(threadId); - if (!active) return undefined; - - this.activeRuns.delete(threadId); - return active; - } - - suspendRun(threadId: string, state: SuspendedRunState): void { - const activeRun = this.activeRuns.get(threadId); - this.activeRuns.delete(threadId); - state.startedAt = state.startedAt ?? activeRun?.startedAt ?? state.createdAt; - state.lastActivityAt = state.lastActivityAt ?? state.createdAt; - state.userId = state.userId ?? activeRun?.userId; - this.suspendedRuns.set(threadId, state); - - // Re-seed group indexes: on a restart-resumed orphan these maps start - // empty, so without this the SSE bootstrap loses the group association. - if (state.messageGroupId) { - this.indexRunInGroup(threadId, state.messageGroupId, state.runId); - } - } - - findSuspendedByRequestId(requestId: string): SuspendedRunState | undefined { - for (const run of this.suspendedRuns.values()) { - if (run.requestId === requestId) return run; - } - return undefined; - } - - cancelSuspendedRun(threadId: string): SuspendedRunState | undefined { - const suspended = this.suspendedRuns.get(threadId); - if (!suspended) return undefined; - - this.suspendedRuns.delete(threadId); - return suspended; - } - - activateSuspendedRun( - threadId: string, - executionToken?: symbol, - ): SuspendedRunState | undefined { - const suspended = this.suspendedRuns.get(threadId); - if (!suspended) return undefined; - - this.suspendedRuns.delete(threadId); - const now = Date.now(); - this.activeRuns.set(threadId, { - runId: suspended.runId, - threadId, - abortController: suspended.abortController, - executionToken, - messageGroupId: suspended.messageGroupId, - tracing: suspended.tracing, - modelId: suspended.modelId, - startedAt: suspended.startedAt ?? suspended.createdAt, - lastActivityAt: now, - userId: suspended.userId ?? this.getUserId(suspended.user), - }); - - // Re-seed group indexes for the reactivated run (empty after a restart). - if (suspended.messageGroupId) { - this.indexRunInGroup(threadId, suspended.messageGroupId, suspended.runId); - } - return suspended; - } - - registerPendingConfirmation(requestId: string, pending: PendingConfirmation): void { - this.pendingConfirmations.set(requestId, { - ...pending, - startedAt: pending.startedAt ?? pending.createdAt, - lastActivityAt: pending.lastActivityAt ?? pending.createdAt, - }); - } - - getPendingConfirmation(requestId: string): PendingConfirmation | undefined { - return this.pendingConfirmations.get(requestId); - } - - hasPendingConfirmationForThread(threadId: string): boolean { - for (const pending of this.pendingConfirmations.values()) { - if (pending.threadId === threadId) return true; - } - - return false; - } - - touchActiveRun(threadId: string, at = Date.now()): boolean { - const active = this.activeRuns.get(threadId); - if (!active) return false; - active.lastActivityAt = at; - return true; - } - - touchSuspendedRun(threadId: string, at = Date.now()): boolean { - const suspended = this.suspendedRuns.get(threadId); - if (!suspended) return false; - suspended.lastActivityAt = at; - return true; - } - - touchPendingConfirmation(requestId: string, at = Date.now()): boolean { - const pending = this.pendingConfirmations.get(requestId); - if (!pending) return false; - pending.lastActivityAt = at; - return true; - } - - resolvePendingConfirmation( - requestingUserId: string, - requestId: string, - data: ConfirmationData, - ): boolean { - const pending = this.pendingConfirmations.get(requestId); - if (pending?.userId !== requestingUserId) return false; - - this.pendingConfirmations.delete(requestId); - pending.resolve(data); - return true; - } - - cancelThread( - threadId: string, - cancelledConfirmation: ConfirmationData = { approved: false }, - ): { - active?: ActiveRunState; - suspended?: SuspendedRunState; - } { - for (const [requestId, pending] of this.pendingConfirmations) { - if (pending.threadId !== threadId) continue; - pending.resolve(cancelledConfirmation); - this.pendingConfirmations.delete(requestId); - } - - const active = this.activeRuns.get(threadId); - const suspended = this.suspendedRuns.get(threadId); - if (suspended) { - this.suspendedRuns.delete(threadId); - } - - return { ...(active ? { active } : {}), ...(suspended ? { suspended } : {}) }; - } - - getThreadUser(threadId: string): TUser | undefined { - return this.threadUsers.get(threadId); - } - setTimeZone(threadId: string, timeZone: string): void { this.threadTimeZones.set(threadId, timeZone); } @@ -562,155 +153,8 @@ export class RunStateRegistry { return this.threadPromptSelections.get(threadId)?.metadata; } - /** - * Find active/suspended runs and pending confirmations that timed out under policy. - * Returns thread IDs and request IDs that should be cancelled/rejected. - * Does NOT mutate state — the caller is responsible for cancelling. - */ - sweepTimedOut(maxAgeMs: number): { - suspendedThreadIds: string[]; - confirmationRequestIds: string[]; - }; - sweepTimedOut( - policy: InstanceAiLivenessPolicy, - now?: number, - ): { - activeThreadIds: string[]; - suspendedThreadIds: string[]; - confirmationRequestIds: string[]; - activeTimeouts: Record; - suspendedTimeouts: Record; - confirmationTimeouts: Record; - }; - sweepTimedOut( - policyOrMaxAgeMs: InstanceAiLivenessPolicy | number, - now = Date.now(), - ): - | { - suspendedThreadIds: string[]; - confirmationRequestIds: string[]; - } - | { - activeThreadIds: string[]; - suspendedThreadIds: string[]; - confirmationRequestIds: string[]; - activeTimeouts: Record; - suspendedTimeouts: Record; - confirmationTimeouts: Record; - } { - if (typeof policyOrMaxAgeMs === 'number') { - const maxAgeMs = policyOrMaxAgeMs; - const suspendedThreadIds: string[] = []; - for (const [threadId, run] of this.suspendedRuns) { - if (now - run.createdAt >= maxAgeMs) { - suspendedThreadIds.push(threadId); - } - } - const confirmationRequestIds: string[] = []; - for (const [reqId, pending] of this.pendingConfirmations) { - if (now - pending.createdAt >= maxAgeMs) { - confirmationRequestIds.push(reqId); - } - } - return { suspendedThreadIds, confirmationRequestIds }; - } - - const policy = policyOrMaxAgeMs; - const activeThreadIds: string[] = []; - const activeTimeouts: Record = {}; - for (const [threadId, run] of this.activeRuns) { - if (this.hasPendingConfirmationForThread(threadId)) continue; - - const startedAt = run.startedAt ?? run.lastActivityAt ?? now; - const lastActivityAt = run.lastActivityAt ?? startedAt; - const decision = policy.evaluate({ - surface: 'active-run', - startedAt, - lastActivityAt, - now, - }); - if (decision.action === 'timeout') { - activeThreadIds.push(threadId); - activeTimeouts[threadId] = decision; - } - } - - const suspendedThreadIds: string[] = []; - const suspendedTimeouts: Record = {}; - for (const [threadId, run] of this.suspendedRuns) { - const startedAt = run.startedAt ?? run.createdAt; - const lastActivityAt = run.lastActivityAt ?? run.createdAt; - const decision = policy.evaluate({ - surface: 'suspended-run', - startedAt, - lastActivityAt, - now, - }); - if (decision.action === 'timeout') { - suspendedThreadIds.push(threadId); - suspendedTimeouts[threadId] = decision; - } - } - const confirmationRequestIds: string[] = []; - const confirmationTimeouts: Record = {}; - for (const [reqId, pending] of this.pendingConfirmations) { - const startedAt = pending.startedAt ?? pending.createdAt; - const lastActivityAt = pending.lastActivityAt ?? pending.createdAt; - const decision = policy.evaluate({ - surface: 'pending-confirmation', - startedAt, - lastActivityAt, - now, - }); - if (decision.action === 'timeout') { - confirmationRequestIds.push(reqId); - confirmationTimeouts[reqId] = decision; - } - } - return { - activeThreadIds, - suspendedThreadIds, - confirmationRequestIds, - activeTimeouts, - suspendedTimeouts, - confirmationTimeouts, - }; - } - - /** - * Auto-reject a pending confirmation by request ID. - * Returns true if the confirmation existed and was rejected. - */ - rejectPendingConfirmation(requestId: string): boolean { - const pending = this.pendingConfirmations.get(requestId); - if (!pending) return false; - this.pendingConfirmations.delete(requestId); - pending.resolve({ approved: false }); - return true; - } - - /** Remove a message-group entry from runIdsByMessageGroup. */ - deleteMessageGroup(groupId: string): void { - this.runIdsByMessageGroup.delete(groupId); - } - - /** - * Remove all per-thread state: active/suspended runs, confirmations, - * user, time zone, and message-group mappings. - * Returns the cancelled active/suspended runs so the caller can abort them. - */ - clearThread( - threadId: string, - cancelledConfirmation: ConfirmationData = { approved: false }, - ): { - active?: ActiveRunState; - suspended?: SuspendedRunState; - } { - const { active, suspended } = this.cancelThread(threadId, cancelledConfirmation); - - if (active) this.activeRuns.delete(threadId); - - this.threadUsers.delete(threadId); + /** Remove all per-thread state. */ + clearThread(threadId: string): void { this.threadTimeZones.delete(threadId); this.threadComputerUseChannels.delete(threadId); this.threadBuildModes.delete(threadId); @@ -721,49 +165,9 @@ export class RunStateRegistry { const groupId = this.threadMessageGroupId.get(threadId); if (groupId) this.runIdsByMessageGroup.delete(groupId); this.threadMessageGroupId.delete(threadId); - - return { ...(active ? { active } : {}), ...(suspended ? { suspended } : {}) }; } - /** - * Process-wide teardown. Returns the in-flight runs so the service can - * abort them and persist terminal snapshots where appropriate. - * - * Pending confirmations are intentionally NOT resolved — auto-resolving an - * inline HITL (`waitForConfirmation(...)`) with `{ approved: false }` - * causes the awaiting agent tool to run to completion as "denied" before - * the process exits, which then mutates the snapshot tree mid-shutdown - * and clobbers the plan/ask card the user would otherwise see on reload. - * Letting the Promises dangle is safe: the process is exiting, the - * abortController for each active run is aborted next, and the - * `instance_ai_pending_confirmations` row survives so the user can still - * see the confirmation card (and get a clear "lost on restart" error if - * they click confirm — see `handleOrphanedConfirmation`). - * - * `pendingThreadIds` is returned so the service can skip the - * publish-run-finish + terminal-snapshot treatment for runs that are - * only sitting in `activeRuns` because they're waiting on an inline - * confirmation Promise. - */ - shutdown(): { - activeRuns: Array; - suspendedRuns: Array>; - pendingThreadIds: string[]; - } { - // Capture the selected version before clearing per-thread state. - const activeRuns = [...this.activeRuns.values()].map((run) => ({ - ...run, - promptVersion: this.getPromptVersion(run.threadId), - })); - const suspendedRuns = [...this.suspendedRuns.values()]; - const pendingThreadIds = [ - ...new Set([...this.pendingConfirmations.values()].map((p) => p.threadId)), - ]; - - this.activeRuns.clear(); - this.suspendedRuns.clear(); - this.pendingConfirmations.clear(); - this.threadUsers.clear(); + clear(): void { this.threadTimeZones.clear(); this.threadComputerUseChannels.clear(); this.threadBuildModes.clear(); @@ -772,7 +176,5 @@ export class RunStateRegistry { this.threadPromptSelections.clear(); this.threadMessageGroupId.clear(); this.runIdsByMessageGroup.clear(); - - return { activeRuns, suspendedRuns, pendingThreadIds }; } } diff --git a/packages/@n8n/instance-ai/src/runtime/stream-runner.ts b/packages/@n8n/instance-ai/src/runtime/stream-runner.ts deleted file mode 100644 index b80f693e204..00000000000 --- a/packages/@n8n/instance-ai/src/runtime/stream-runner.ts +++ /dev/null @@ -1,119 +0,0 @@ -import type { InstanceAiEvent } from '@n8n/api-types'; - -import type { InstanceAiEventBus } from '../event-bus'; -import type { Logger } from '../logger'; -import type { - OrchestratorRunHandoffReason, - OrchestratorRunStopSignal, -} from './orchestrator-run-control'; -import { - executeResumableStream, - normalizeStreamSource, - type ResumableStreamSource, - type TraceStatus, -} from './resumable-stream-executor'; -import type { RunTokenUsage } from '../stream/usage-accumulator'; -import type { WorkSummary } from '../stream/work-summary-accumulator'; -import { resumeAgentStream } from '../utils/stream-helpers'; -import type { SuspensionInfo } from '../utils/stream-helpers'; - -export interface StreamableAgent { - stream: (input: unknown, options: Record) => Promise; -} - -export interface StreamRunOptions { - threadId: string; - runId: string; - agentId: string; - signal: AbortSignal; - eventBus: InstanceAiEventBus; - logger: Logger; - onActivity?: () => void; - stopSignal?: () => OrchestratorRunStopSignal | undefined; -} - -export interface StreamRunResult { - status: TraceStatus; - agentRunId: string; - text?: Promise; - error?: unknown; - workSummary: WorkSummary; - usage?: RunTokenUsage; - stopReason?: OrchestratorRunHandoffReason; - suspension?: SuspensionInfo; - confirmationEvent?: Extract; -} - -export async function streamAgentRun( - agent: StreamableAgent, - input: unknown, - streamOptions: Record, - options: StreamRunOptions, -): Promise { - const result = await agent.stream(input, streamOptions); - const stream = normalizeStreamSource(result); - const agentRunId = typeof stream.runId === 'string' ? stream.runId : ''; - return await consumeStream(agent, stream, { ...options, agentRunId }); -} - -export async function resumeAgentRun( - agent: unknown, - resumeData: Record, - resumeOptions: Record, - options: StreamRunOptions & { agentRunId: string }, -): Promise { - const resumed = await resumeAgentStream(agent, resumeData, resumeOptions); - const stream = normalizeStreamSource(resumed); - const agentRunId = (typeof stream.runId === 'string' && stream.runId) || options.agentRunId; - return await consumeStream(agent, stream, { ...options, agentRunId }); -} - -async function consumeStream( - agent: unknown, - stream: ResumableStreamSource, - options: StreamRunOptions & { agentRunId: string }, -): Promise { - const result = await executeResumableStream({ - agent, - stream, - context: { - threadId: options.threadId, - runId: options.runId, - agentId: options.agentId, - eventBus: options.eventBus, - signal: options.signal, - logger: options.logger, - onActivity: options.onActivity, - stopSignal: options.stopSignal, - }, - control: { mode: 'manual' }, - initialAgentRunId: options.agentRunId, - }); - - if (result.status === 'suspended' && result.suspension) { - return { - status: 'suspended', - agentRunId: result.agentRunId, - text: result.text, - workSummary: result.workSummary, - usage: result.usage, - suspension: result.suspension, - ...(result.confirmationEvent ? { confirmationEvent: result.confirmationEvent } : {}), - }; - } - - return { - status: - result.status === 'cancelled' - ? 'cancelled' - : result.status === 'errored' - ? 'errored' - : 'completed', - agentRunId: result.agentRunId, - text: result.text, - ...(result.error !== undefined ? { error: result.error } : {}), - workSummary: result.workSummary, - usage: result.usage, - ...(result.stopReason ? { stopReason: result.stopReason } : {}), - }; -} diff --git a/packages/@n8n/instance-ai/src/storage/__tests__/terminal-outcome-storage.test.ts b/packages/@n8n/instance-ai/src/storage/__tests__/terminal-outcome-storage.test.ts deleted file mode 100644 index 185fed8472f..00000000000 --- a/packages/@n8n/instance-ai/src/storage/__tests__/terminal-outcome-storage.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -vi.mock('../thread-patch', () => ({ - getThread: vi.fn(), - patchThread: vi.fn(), -})); - -import { TerminalOutcomeStorage, type TerminalOutcome } from '../terminal-outcome-storage'; -import { getThread, patchThread, type PatchableThreadMemory } from '../thread-patch'; - -const mockedGetThread = vi.mocked(getThread); -const mockedPatchThread = vi.mocked(patchThread); - -function makeMemory(): PatchableThreadMemory { - return {}; -} - -function makeOutcome(overrides: Partial = {}): TerminalOutcome { - return { - id: 'outcome-1', - threadId: 'thread-1', - runId: 'run-1', - taskId: 'task-1', - agentId: 'agent-1', - status: 'completed', - userFacingMessage: 'done', - createdAt: '2026-05-02T00:00:00.000Z', - ...overrides, - }; -} - -function makeThread(metadata: Record): Awaited> { - return { - id: 'thread-1', - title: 'Thread', - metadata, - resourceId: 'resource-1', - createdAt: new Date('2026-05-02T00:00:00.000Z'), - updatedAt: new Date('2026-05-02T00:00:00.000Z'), - }; -} - -describe('TerminalOutcomeStorage', () => { - let memory: PatchableThreadMemory; - let storage: TerminalOutcomeStorage; - - beforeEach(() => { - vi.clearAllMocks(); - memory = makeMemory(); - storage = new TerminalOutcomeStorage(memory); - }); - - describe('getUndelivered()', () => { - it('returns valid undelivered outcomes when one stored entry is malformed', async () => { - const valid = makeOutcome({ id: 'outcome-valid' }); - mockedGetThread.mockResolvedValue( - makeThread({ - instanceAiTerminalOutcomes: { - 'outcome-valid': valid, - 'outcome-broken': { id: 'outcome-broken' }, - }, - }), - ); - - const result = await storage.getUndelivered('thread-1'); - - expect(result).toEqual([valid]); - }); - - it('returns empty list when metadata is missing', async () => { - mockedGetThread.mockResolvedValue(makeThread({})); - - const result = await storage.getUndelivered('thread-1'); - - expect(result).toEqual([]); - }); - - it('skips delivered outcomes', async () => { - const undelivered = makeOutcome({ id: 'undelivered' }); - const delivered = makeOutcome({ - id: 'delivered', - deliveredAt: '2026-05-02T00:00:01.000Z', - }); - mockedGetThread.mockResolvedValue( - makeThread({ - instanceAiTerminalOutcomes: { - undelivered, - delivered, - }, - }), - ); - - const result = await storage.getUndelivered('thread-1'); - - expect(result).toEqual([undelivered]); - }); - }); - - describe('upsert()', () => { - it('preserves valid outcomes when an existing entry is malformed', async () => { - const valid = makeOutcome({ id: 'outcome-valid' }); - const next = makeOutcome({ id: 'outcome-new' }); - - let captured: Record | undefined; - mockedPatchThread.mockImplementation(async (_memory, args) => { - await Promise.resolve(); - const patch = args.update({ - metadata: { - instanceAiTerminalOutcomes: { - 'outcome-valid': valid, - 'outcome-broken': { not: 'an outcome' }, - }, - }, - } as unknown as Parameters[0]); - captured = patch?.metadata?.instanceAiTerminalOutcomes as Record; - return null; - }); - - await storage.upsert('thread-1', next); - - expect(captured).toEqual({ - 'outcome-valid': valid, - 'outcome-new': next, - }); - }); - }); -}); diff --git a/packages/@n8n/instance-ai/src/storage/index.ts b/packages/@n8n/instance-ai/src/storage/index.ts index 0d22ac1a43b..6bf30625c39 100644 --- a/packages/@n8n/instance-ai/src/storage/index.ts +++ b/packages/@n8n/instance-ai/src/storage/index.ts @@ -5,8 +5,6 @@ export { ThreadIterationLogStorage } from './thread-iteration-log-storage'; export { ThreadTaskStorage } from './thread-task-storage'; export { PlannedTaskStorage } from './planned-task-storage'; export { getThread, patchThread } from './thread-patch'; -export { TerminalOutcomeStorage } from './terminal-outcome-storage'; -export type { TerminalOutcome } from './terminal-outcome-storage'; export type { PatchableThreadMemory, ThreadPatch } from './thread-patch'; export { WorkflowLoopStorage } from './workflow-loop-storage'; export type { diff --git a/packages/@n8n/instance-ai/src/storage/terminal-outcome-storage.ts b/packages/@n8n/instance-ai/src/storage/terminal-outcome-storage.ts deleted file mode 100644 index 2c420e19443..00000000000 --- a/packages/@n8n/instance-ai/src/storage/terminal-outcome-storage.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { z } from 'zod'; - -import { getThread, patchThread, type PatchableThreadMemory } from './thread-patch'; - -const METADATA_KEY = 'instanceAiTerminalOutcomes'; - -const terminalOutcomeStatusSchema = z.enum(['completed', 'failed', 'cancelled']); - -const terminalOutcomeSchema = z.object({ - id: z.string(), - threadId: z.string(), - runId: z.string(), - messageGroupId: z.string().optional(), - correlationId: z.string().optional(), - taskId: z.string(), - agentId: z.string(), - status: terminalOutcomeStatusSchema, - userFacingMessage: z.string(), - createdAt: z.string(), - deliveredAt: z.string().optional(), -}); - -export type TerminalOutcome = z.infer; - -export class TerminalOutcomeStorage { - constructor(private readonly memory: PatchableThreadMemory) {} - - async upsert(threadId: string, outcome: TerminalOutcome): Promise { - await patchThread(this.memory, { - threadId, - update: ({ metadata = {} }) => { - const current = parseOutcomes(metadata[METADATA_KEY]); - return { - metadata: { - ...metadata, - [METADATA_KEY]: { - ...current, - [outcome.id]: outcome, - }, - }, - }; - }, - }); - } - - async markDelivered(threadId: string, outcomeId: string, deliveredAt: string): Promise { - await patchThread(this.memory, { - threadId, - update: ({ metadata = {} }) => { - const current = parseOutcomes(metadata[METADATA_KEY]); - const outcome = current[outcomeId]; - if (!outcome) return null; - - return { - metadata: { - ...metadata, - [METADATA_KEY]: { - ...current, - [outcomeId]: { - ...outcome, - deliveredAt, - }, - }, - }, - }; - }, - }); - } - - async getUndelivered(threadId: string): Promise { - const thread = await getThread(this.memory, threadId); - const outcomes = parseOutcomes(thread?.metadata?.[METADATA_KEY]); - return Object.values(outcomes).filter((outcome) => !outcome.deliveredAt); - } -} - -function parseOutcomes(raw: unknown): Record { - if (!raw || typeof raw !== 'object') return {}; - const outcomes: Record = {}; - for (const [key, value] of Object.entries(raw as Record)) { - const parsed = terminalOutcomeSchema.safeParse(value); - if (parsed.success) outcomes[key] = parsed.data; - } - return outcomes; -} diff --git a/packages/@n8n/instance-ai/src/stream/consume-with-hitl.ts b/packages/@n8n/instance-ai/src/stream/consume-with-hitl.ts index cf77d4da085..d755e2134b7 100644 --- a/packages/@n8n/instance-ai/src/stream/consume-with-hitl.ts +++ b/packages/@n8n/instance-ai/src/stream/consume-with-hitl.ts @@ -1,13 +1,14 @@ +import type { StreamResult } from '@n8n/agents'; import type { InstanceAiEvent } from '@n8n/api-types'; +import { isRecord } from '@n8n/utils/is-record'; +import { randomUUID } from 'node:crypto'; import type { InstanceAiEventBus } from '../event-bus/event-bus.interface'; import type { Logger } from '../logger'; -import { - executeResumableStream, - normalizeStreamSource, -} from '../runtime/resumable-stream-executor'; -import type { RunTokenUsage } from '../stream/usage-accumulator'; -import type { WorkSummary } from '../stream/work-summary-accumulator'; +import { isQuotaExhaustedError, mapAgentChunkToEvent } from './map-chunk'; +import { UsageAccumulator, type RunTokenUsage } from './usage-accumulator'; +import { WorkSummaryAccumulator, type WorkSummary } from './work-summary-accumulator'; +import { parseSuspension } from '../utils/stream-helpers'; import type { SuspensionInfo } from '../utils/stream-helpers'; type ConfirmationRequestEvent = Extract; @@ -41,6 +42,105 @@ export type ConsumeStreamCascadingResult = usage?: RunTokenUsage; }; +interface StreamSource { + runId?: string; + fullStream: AsyncIterable; + text?: Promise; +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return ( + value !== null && + typeof value === 'object' && + typeof Reflect.get(value, Symbol.asyncIterator) === 'function' + ); +} + +function isReadableStream(value: unknown): value is ReadableStream { + return ( + value !== null && + typeof value === 'object' && + typeof Reflect.get(value, 'getReader') === 'function' + ); +} + +function isStreamSource(value: unknown): value is StreamSource { + return isRecord(value) && isAsyncIterable(value.fullStream); +} + +function isNativeStreamResult(value: unknown): value is StreamResult { + return isRecord(value) && isReadableStream(value.stream); +} + +function extractContentText(content: unknown): string { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + return content + .map((part) => + isRecord(part) && part.type === 'text' && typeof part.text === 'string' ? part.text : '', + ) + .join(''); +} + +async function collectNativeStreamText(stream: ReadableStream): Promise { + const reader = stream.getReader(); + let deltaText = ''; + let messageText = ''; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return deltaText || messageText; + if (!isRecord(value)) continue; + + if (value.type === 'text-delta' && typeof value.delta === 'string') { + deltaText += value.delta; + continue; + } + + if (value.type === 'message' && isRecord(value.message)) { + if (value.message.role === 'assistant') { + messageText += extractContentText(value.message.content); + } + } + } + } finally { + reader.releaseLock(); + } +} + +async function* readableStreamToAsyncIterable(stream: ReadableStream) { + const reader = stream.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) return; + yield value; + } + } finally { + reader.releaseLock(); + } +} + +function normalizeStreamSource(result: unknown): StreamSource { + if (isStreamSource(result)) return result; + + if (isNativeStreamResult(result)) { + const [eventStream, textStream] = result.stream.tee(); + return { + runId: result.runId, + fullStream: readableStreamToAsyncIterable(eventStream), + text: collectNativeStreamText(textStream), + }; + } + + throw new Error('Unsupported agent stream result'); +} + +function isErrorChunk(chunk: unknown): chunk is { type: 'error'; error?: unknown } { + return isRecord(chunk) && chunk.type === 'error'; +} + /** * Consume a sub-agent stream and return cleanly when it either finishes or * hits a HITL suspension. This returns the suspension info to the caller so it can decide @@ -48,46 +148,139 @@ export type ConsumeStreamCascadingResult = * via `ctx.suspend(payload)`, so the parent's agent run is also checkpointed * and survives a process restart. * - * Uses `executeResumableStream`'s manual mode, which suppresses the - * sub-agent's `confirmation-request` event publish (returning it on the - * result instead) — the caller emits the card at whatever runId is - * meaningful at its level. + * The sub-agent's primary `confirmation-request` event is not published. It + * is returned on the result instead, so the caller emits the card at whatever + * runId is meaningful at its level. */ export async function consumeStreamCascading( options: ConsumeStreamCascadingOptions, ): Promise { const stream = normalizeStreamSource(options.stream); - const result = await executeResumableStream({ - agent: options.agent, - stream, - context: { - threadId: options.threadId, - runId: options.runId, - agentId: options.agentId, - eventBus: options.eventBus, - signal: options.abortSignal, - logger: options.logger, - }, - control: { mode: 'manual' }, - }); + const agentRunId = stream.runId ?? ''; + const workSummaryAccumulator = new WorkSummaryAccumulator(); + const usageAccumulator = new UsageAccumulator(); + const usage = () => (usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined); - if (result.status === 'suspended' && result.suspension) { + let currentResponseId: string | undefined; + let nativeStepIndex = 0; + // Segment id for deltas that arrive before a `start-step` (some providers + // never emit one). Cleared at the next structural chunk. + let syntheticSegmentId: string | undefined; + let suspension: SuspensionInfo | undefined; + let confirmationEvent: ConfirmationRequestEvent | undefined; + let hasError = false; + // Drop follow-on error chunks after a quota error, so the user sees one reason. + let quotaErrorPublished = false; + + for await (const chunk of stream.fullStream) { + if (options.abortSignal.aborted) { + // Drain the rest so the usage of the cancelled stream is still observed. + usageAccumulator.observe(chunk); + try { + for await (const remaining of stream.fullStream) usageAccumulator.observe(remaining); + } catch (drainError) { + options.logger.debug('Instance AI abort drain ended early', { + threadId: options.threadId, + runId: options.runId, + error: drainError instanceof Error ? drainError.message : String(drainError), + }); + } + return { + status: 'cancelled', + agentRunId, + text: stream.text ?? Promise.resolve(''), + workSummary: workSummaryAccumulator.toSummary(), + usage: usage(), + }; + } + + usageAccumulator.observe(chunk); + + if (isRecord(chunk) && chunk.type === 'start-step') { + nativeStepIndex += 1; + currentResponseId = `${agentRunId || options.runId}:step:${nativeStepIndex}`; + syntheticSegmentId = undefined; + } + + const parsedSuspension = parseSuspension(chunk); + if (parsedSuspension) { + if (!suspension) { + suspension = parsedSuspension; + } else if ( + parsedSuspension.requestId !== suspension.requestId || + parsedSuspension.toolCallId !== suspension.toolCallId + ) { + options.logger.warn('Additional HITL suspension encountered before resume; deferring', { + threadId: options.threadId, + runId: options.runId, + activeRequestId: suspension.requestId, + deferredRequestId: parsedSuspension.requestId, + }); + } + } + + if (isErrorChunk(chunk)) { + hasError = true; + if (quotaErrorPublished) continue; + if (isQuotaExhaustedError(chunk.error)) quotaErrorPublished = true; + } + + const isDeltaChunk = + isRecord(chunk) && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta'); + if (isDeltaChunk && !currentResponseId && !syntheticSegmentId) { + syntheticSegmentId = `${agentRunId || options.runId}:seg:${randomUUID()}`; + } + + const event = mapAgentChunkToEvent( + options.runId, + options.agentId, + chunk, + currentResponseId ?? syntheticSegmentId, + ); + + const isFinishStep = isRecord(chunk) && chunk.type === 'finish-step'; + if ((event && !isDeltaChunk) || isFinishStep) syntheticSegmentId = undefined; + if (!event) continue; + + workSummaryAccumulator.observe(event); + if (event.type === 'confirmation-request') { + // Hold back the primary card and drop duplicates or secondary cards. + const isPrimary = + event.payload.requestId === suspension?.requestId && + event.payload.toolCallId === suspension?.toolCallId; + if (isPrimary && !confirmationEvent) confirmationEvent = event; + continue; + } + options.eventBus.publish(options.threadId, event); + } + + if (options.abortSignal.aborted) { + return { + status: 'cancelled', + agentRunId, + text: stream.text ?? Promise.resolve(''), + workSummary: workSummaryAccumulator.toSummary(), + usage: usage(), + }; + } + + if (suspension) { return { status: 'suspended', - agentRunId: result.agentRunId, - suspension: result.suspension, - ...(result.confirmationEvent ? { confirmationEvent: result.confirmationEvent } : {}), - ...(result.text ? { text: result.text } : {}), - workSummary: result.workSummary, - ...(result.usage ? { usage: result.usage } : {}), + agentRunId, + suspension, + ...(confirmationEvent ? { confirmationEvent } : {}), + ...(stream.text ? { text: stream.text } : {}), + workSummary: workSummaryAccumulator.toSummary(), + usage: usage(), }; } return { - status: result.status === 'suspended' ? 'errored' : result.status, - agentRunId: result.agentRunId, - text: result.text ?? stream.text ?? Promise.resolve(''), - workSummary: result.workSummary, - ...(result.usage ? { usage: result.usage } : {}), + status: hasError ? 'errored' : 'completed', + agentRunId, + text: stream.text ?? Promise.resolve(''), + workSummary: workSummaryAccumulator.toSummary(), + usage: usage(), }; } diff --git a/packages/@n8n/instance-ai/src/tools/__tests__/task-control.tool.test.ts b/packages/@n8n/instance-ai/src/tools/__tests__/task-control.tool.test.ts index 8ac03cf395f..03c55968c92 100644 --- a/packages/@n8n/instance-ai/src/tools/__tests__/task-control.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/__tests__/task-control.tool.test.ts @@ -1,5 +1,3 @@ -import type { Mock } from 'vitest'; - import { executeTool, parseToolInput } from '../../__tests__/tool-test-utils'; import type { OrchestrationContext } from '../../types'; import { createTaskControlTool } from '../task-control.tool'; @@ -23,8 +21,6 @@ function createMockContext(overrides: Partial = {}): Orche get: vi.fn(), save: vi.fn(), }, - cancelBackgroundTask: vi.fn(), - sendCorrectionToTask: vi.fn(), ...overrides, } as unknown as OrchestrationContext; } @@ -83,134 +79,6 @@ describe('task-control tool', () => { expect(result).toEqual({ saved: true }); }); }); - - // ── cancel-task ──────────────────────────────────────────────────────── - - describe('cancel-task action', () => { - it('should call cancelBackgroundTask and return success message', async () => { - const context = createMockContext(); - - const tool = createTaskControlTool(context); - const result = await executeTool( - tool, - { action: 'cancel-task' as const, taskId: 'build-ABC123' }, - {} as never, - ); - - expect(context.cancelBackgroundTask).toHaveBeenCalledWith('build-ABC123'); - expect(result).toEqual({ result: 'Background task build-ABC123 cancelled.' }); - }); - - it('should return error when cancelBackgroundTask is not available', async () => { - const context = createMockContext({ - cancelBackgroundTask: undefined, - }); - - const tool = createTaskControlTool(context); - const result = await executeTool( - tool, - { action: 'cancel-task' as const, taskId: 'build-XYZ' }, - {} as never, - ); - - expect(result).toEqual({ - result: 'Error: background task cancellation not available.', - }); - }); - }); - - // ── correct-task ─────────────────────────────────────────────────────── - - describe('correct-task action', () => { - it('should send correction and return success message', async () => { - const context = createMockContext(); - (context.sendCorrectionToTask as Mock).mockReturnValue('queued'); - - const tool = createTaskControlTool(context); - const result = await executeTool( - tool, - { - action: 'correct-task' as const, - taskId: 'build-ABC', - correction: 'use the Projects database', - }, - {} as never, - ); - - expect(context.sendCorrectionToTask).toHaveBeenCalledWith( - 'build-ABC', - 'use the Projects database', - ); - expect(result).toEqual({ - result: - 'Correction sent to task build-ABC: "use the Projects database". ' + - 'The builder will see this on its next step.', - }); - }); - - it('should return task-not-found message when task does not exist', async () => { - const context = createMockContext(); - (context.sendCorrectionToTask as Mock).mockReturnValue('task-not-found'); - - const tool = createTaskControlTool(context); - const result = await executeTool( - tool, - { - action: 'correct-task' as const, - taskId: 'build-GONE', - correction: 'fix the trigger', - }, - {} as never, - ); - - expect(result).toEqual({ - result: 'Task build-GONE not found. It may have already been cleaned up.', - }); - }); - - it('should return task-completed message when task has finished', async () => { - const context = createMockContext(); - (context.sendCorrectionToTask as Mock).mockReturnValue('task-completed'); - - const tool = createTaskControlTool(context); - const result = await executeTool( - tool, - { - action: 'correct-task' as const, - taskId: 'build-DONE', - correction: 'add error handling', - }, - {} as never, - ); - - expect(result).toEqual({ - result: - 'Task build-DONE has already completed. The correction was not delivered. ' + - 'Incorporate "add error handling" into a new follow-up task instead.', - }); - }); - - it('should return error when sendCorrectionToTask is not available', async () => { - const context = createMockContext({ - sendCorrectionToTask: undefined, - }); - - const tool = createTaskControlTool(context); - const result = await executeTool( - tool, - { - action: 'correct-task' as const, - taskId: 'build-ABC', - correction: 'fix it', - }, - {} as never, - ); - - expect(result).toEqual({ - result: 'Error: correction delivery not available.', - }); - }); - }); }); describe('task-control tool — checklist item contract', () => { diff --git a/packages/@n8n/instance-ai/src/tools/task-control.tool.ts b/packages/@n8n/instance-ai/src/tools/task-control.tool.ts index 29719e39236..7eabf4537f2 100644 --- a/packages/@n8n/instance-ai/src/tools/task-control.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/task-control.tool.ts @@ -1,5 +1,5 @@ /** - * Consolidated task-control tool — update-checklist + cancel-task + correct-task. + * Task-control tool — writes the lightweight visible checklist. */ import { Tool } from '@n8n/agents'; import { taskItemSchema } from '@n8n/api-types'; @@ -33,22 +33,7 @@ const updateChecklistAction = z.object({ tasks: z.array(checklistItemSchema).describe('Ordered list of tasks'), }); -const cancelTaskAction = z.object({ - action: z.literal('cancel-task').describe('Cancel a running background task by its task ID'), - taskId: z.string().describe('Task ID (e.g. build-XXXXXXXX)'), -}); - -const correctTaskAction = z.object({ - action: z.literal('correct-task').describe('Send a correction to a running background task'), - taskId: z.string().describe('Task ID (e.g. build-XXXXXXXX)'), - correction: z - .string() - .describe("The correction message from the user (e.g. 'use the Projects database')"), -}); - -const inputSchema = sanitizeInputSchema( - z.discriminatedUnion('action', [updateChecklistAction, cancelTaskAction, correctTaskAction]), -); +const inputSchema = sanitizeInputSchema(z.discriminatedUnion('action', [updateChecklistAction])); type Input = z.infer; @@ -69,58 +54,18 @@ async function handleUpdateChecklist( return { saved: true }; } -async function handleCancelTask( - context: OrchestrationContext, - input: Extract, -) { - if (!context.cancelBackgroundTask) { - return { result: 'Error: background task cancellation not available.' }; - } - await context.cancelBackgroundTask(input.taskId); - return { result: `Background task ${input.taskId} cancelled.` }; -} - -async function handleCorrectTask( - context: OrchestrationContext, - input: Extract, -) { - if (!context.sendCorrectionToTask) { - return await Promise.resolve({ result: 'Error: correction delivery not available.' }); - } - const status = context.sendCorrectionToTask(input.taskId, input.correction); - if (status === 'task-not-found') { - return await Promise.resolve({ - result: `Task ${input.taskId} not found. It may have already been cleaned up.`, - }); - } - if (status === 'task-completed') { - return await Promise.resolve({ - result: - `Task ${input.taskId} has already completed. The correction was not delivered. ` + - `Incorporate "${input.correction}" into a new follow-up task instead.`, - }); - } - return await Promise.resolve({ - result: `Correction sent to task ${input.taskId}: "${input.correction}". The builder will see this on its next step.`, - }); -} - // ── Tool factory ──────────────────────────────────────────────────────────── export function createTaskControlTool(context: OrchestrationContext) { return new Tool('task-control') .description( - 'Manage tasks and background work. Use action="update-checklist" only for lightweight visible checklists that do not need scheduler-driven execution; for coordinated background tasks use create-tasks instead.', + 'Manage the visible task checklist. Use action="update-checklist" only for lightweight visible checklists that do not need scheduler-driven execution; for coordinated background tasks use create-tasks instead.', ) .input(inputSchema) .handler(async (input: Input) => { switch (input.action) { case 'update-checklist': return await handleUpdateChecklist(context, input); - case 'cancel-task': - return await handleCancelTask(context, input); - case 'correct-task': - return await handleCorrectTask(context, input); } }) .build(); diff --git a/packages/@n8n/instance-ai/src/types.ts b/packages/@n8n/instance-ai/src/types.ts index 871f605c2b4..4f4350cdafc 100644 --- a/packages/@n8n/instance-ai/src/types.ts +++ b/packages/@n8n/instance-ai/src/types.ts @@ -43,14 +43,12 @@ import type { z } from 'zod'; // Service interfaces — dependency inversion so the package stays decoupled from n8n internals. // The backend module provides concrete implementations via InstanceAiAdapterService. -import type { WorkflowCodeSnapshotInput } from './debug/run-debug-buffer'; import type { DomainAccessTracker } from './domain-access/domain-access-tracker'; import type { InstanceAiEventBus } from './event-bus/event-bus.interface'; import type { Logger } from './logger'; import type { AgentContextInput } from './tools/agent-context.tool'; import type { McpClientManager } from './mcp/mcp-client-manager'; import type { OrchestratorRunHandoffReason } from './runtime/orchestrator-run-control'; -import type { TraceStatus } from './runtime/resumable-stream-executor'; import type { IterationLog } from './storage/iteration-log'; import type { PatchableThreadMemory } from './storage/thread-patch'; import type { BuilderUsageItem } from './stream/usage-accumulator'; @@ -1686,8 +1684,6 @@ export interface InstanceAiContext { markCreated: (credentialType: string) => void; markCreateFailed: (credentialType: string, errorCode: string) => void; }; - /** Records workflow code snapshots for the run debug buffer (dev tooling). */ - recordWorkflowCodeSnapshot?: (snapshot: WorkflowCodeSnapshotInput) => void; /** * Setup panel v2 sink for durable `setup-items` snapshots. Wired by the host * only while the setup panel flag is on, so its presence is the package-side @@ -2149,16 +2145,6 @@ export interface InstanceAiTraceContext { traceWriter?: TraceWriter; } -// ── Background task spawning ───────────────────────────────────────────────── - -/** Structured result from a background task. The `text` field is the human-readable - * summary; `outcome` carries an optional typed payload consumed by the workflow - * loop controller. */ -export interface BackgroundTaskResult { - text: string; - outcome?: Record; -} - export interface WorkflowTaskService { reportBuildOutcome(outcome: WorkflowBuildOutcome): Promise; reportVerificationVerdict(verdict: VerificationResult): Promise; @@ -2184,6 +2170,9 @@ export interface WorkflowTaskService { // ── Orchestration context (plan tools) ────────────────────────────────────── +/** Terminal status of an agent stream, used to label usage claims and traces. */ +export type TraceStatus = 'completed' | 'cancelled' | 'suspended' | 'errored'; + export interface OrchestrationContext { threadId: string; runId: string; @@ -2241,8 +2230,6 @@ export interface OrchestrationContext { runtimeSkillCatalog?: RuntimeSkillSource; /** OAuth2 callback URL for the n8n instance (e.g. http://localhost:5678/rest/oauth2-credential/callback) */ oauth2CallbackUrl?: string; - /** Cancel a running background task by its ID */ - cancelBackgroundTask?: (taskId: string) => Promise; /** Persist and inspect dependency-aware planned tasks for this thread. */ plannedTaskService?: PlannedTaskService; /** Run one scheduler pass after plan/task state changes. */ @@ -2272,15 +2259,6 @@ export interface OrchestrationContext { domainContext?: InstanceAiContext; /** Thread-scoped iteration log for accumulating attempt history across retries */ iterationLog?: IterationLog; - /** Send a correction message to a running background task */ - sendCorrectionToTask?: ( - taskId: string, - correction: string, - ) => 'queued' | 'task-completed' | 'task-not-found'; - /** Mark the current orchestrator run as making progress. */ - touchRun?: () => boolean; - /** Mark a running background task as making progress. */ - touchBackgroundTask?: (taskId: string) => boolean; /** Shared workflow-task state service for build / verify / credential-finalize flows */ workflowTaskService?: WorkflowTaskService; /** IANA time zone for the current user (e.g. "Europe/Helsinki"). Propagated to sub-agents diff --git a/packages/@n8n/instance-ai/vite.config.ts b/packages/@n8n/instance-ai/vite.config.ts index 04902d39947..e6c40995ec7 100644 --- a/packages/@n8n/instance-ai/vite.config.ts +++ b/packages/@n8n/instance-ai/vite.config.ts @@ -27,13 +27,9 @@ const INDEX_LAZY_REQUIRE_TEST_SPECS = new Set([ './workspace/sandbox-setup', './workspace/snapshot-manager', './runtime/run-state-registry', - './runtime/background-task-manager', './runtime/terminal-response-guard', - './runtime/resumable-stream-executor', - './runtime/stream-runner', './parsers/structured-file-parser', './parsers/validate-attachments', - './runtime/liveness-policy', './workflow-loop', './workflow-loop/runtime', './planned-tasks/planned-task-service', diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-liveness.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-liveness.service.test.ts deleted file mode 100644 index 76b20c96c67..00000000000 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-liveness.service.test.ts +++ /dev/null @@ -1,272 +0,0 @@ -import type { InstanceAiEvent } from '@n8n/api-types'; -import type { Logger } from '@n8n/backend-common'; -import { mock } from 'vitest-mock-extended'; - -vi.mock('@n8n/instance-ai', async () => ({ - ...(await vi.importActual>( - '../../../../../@n8n/instance-ai/src/runtime/liveness-policy', - )), - orchestratorAgentId: (runId: string) => `orchestrator-${runId}`, -})); - -import { - createInstanceAiLivenessPolicyConfig, - InstanceAiLivenessPolicy, - type InstanceAiLivenessTimeoutReason, -} from '@n8n/instance-ai'; - -import { - INSTANCE_AI_RUN_TIMEOUT_REASON, - InstanceAiLivenessService, - type InstanceAiLivenessSweepResult, -} from '../liveness'; -import type { InstanceAiRunTimeoutDetails } from '../run-timeout-details'; - -type TestSuspendedRun = { - runId: string; - threadId: string; - abortController: AbortController; -}; - -function createLivenessService() { - const policyConfig = createInstanceAiLivenessPolicyConfig(); - const policy = new InstanceAiLivenessPolicy(policyConfig); - const runState = { - sweepTimedOut: vi.fn( - (_policy: InstanceAiLivenessPolicy, _now?: number): InstanceAiLivenessSweepResult => ({ - activeThreadIds: [] as string[], - suspendedThreadIds: [] as string[], - confirmationRequestIds: [] as string[], - }), - ), - cancelActiveRun: vi.fn( - (_threadId: string): { runId: string; abortController: AbortController } | undefined => - undefined, - ), - cancelSuspendedRun: vi.fn((_threadId: string): TestSuspendedRun | undefined => undefined), - getActiveRunId: vi.fn((_threadId: string): string | undefined => undefined), - getPendingConfirmation: vi.fn( - (_requestId: string): { threadId: string } | undefined => undefined, - ), - hasPendingConfirmationForThread: vi.fn((_threadId: string) => false), - rejectPendingConfirmation: vi.fn((_requestId: string) => true), - }; - const backgroundTasks = { - timeoutTimedOutTasks: vi.fn( - async ( - _policy: InstanceAiLivenessPolicy, - _now?: number, - _options?: { - shouldSkipTask?: (task: { - threadId: string; - taskId: string; - role: string; - timeoutReason?: InstanceAiLivenessTimeoutReason; - }) => boolean; - }, - ) => - [] as Array<{ - threadId: string; - taskId: string; - role: string; - timeoutReason?: InstanceAiLivenessTimeoutReason; - }>, - ), - }; - const eventBus = { - publish: vi.fn((_threadId: string, _event: InstanceAiEvent) => {}), - }; - const finalizeCancelledSuspendedRun = vi.fn( - (_suspended: TestSuspendedRun, _reason: string) => {}, - ); - const onPendingConfirmationRejected = vi.fn((_requestId: string) => {}); - const logger = mock(); - - const service = new InstanceAiLivenessService({ - policy, - backgroundTaskIdleTimeoutMs: policyConfig.backgroundTaskIdleTimeoutMs, - runState, - backgroundTasks, - eventBus, - finalizeCancelledSuspendedRun, - onPendingConfirmationRejected, - logger, - }); - - return { - service, - policy, - runState, - backgroundTasks, - eventBus, - finalizeCancelledSuspendedRun, - onPendingConfirmationRejected, - logger, - }; -} - -describe('InstanceAiLivenessService', () => { - it('cancels timed-out run surfaces without cascading into background tasks', async () => { - const { - service, - policy, - runState, - backgroundTasks, - eventBus, - finalizeCancelledSuspendedRun, - onPendingConfirmationRejected, - } = createLivenessService(); - const activeAbortController = new AbortController(); - const suspendedAbortController = new AbortController(); - const suspended = { - runId: 'run-suspended', - threadId: 'thread-suspended', - abortController: suspendedAbortController, - }; - const activeTimeout: InstanceAiRunTimeoutDetails = { - reason: 'idle_timeout', - surface: 'active-run', - timeoutMs: 600_000, - elapsedMs: 650_000, - idleMs: 606_000, - }; - const suspendedTimeout: InstanceAiRunTimeoutDetails = { - reason: 'max_lifetime', - surface: 'suspended-run', - timeoutMs: 1_800_000, - elapsedMs: 1_801_000, - idleMs: 900_000, - }; - - runState.sweepTimedOut.mockReturnValue({ - activeThreadIds: ['thread-active'], - suspendedThreadIds: ['thread-suspended'], - confirmationRequestIds: ['request-1'], - activeTimeouts: { 'thread-active': activeTimeout }, - suspendedTimeouts: { 'thread-suspended': suspendedTimeout }, - }); - runState.cancelActiveRun.mockReturnValue({ - runId: 'run-active', - abortController: activeAbortController, - }); - runState.cancelSuspendedRun.mockReturnValue(suspended); - runState.getPendingConfirmation.mockReturnValue({ threadId: 'thread-confirmation' }); - runState.getActiveRunId.mockReturnValue('run-confirmation'); - backgroundTasks.timeoutTimedOutTasks.mockResolvedValue([ - { - threadId: 'thread-bg', - taskId: 'task-1', - role: 'workflow-builder', - timeoutReason: 'idle_timeout', - }, - ]); - - await service.sweepTimedOutWork(123_456); - - expect(runState.sweepTimedOut).toHaveBeenCalledWith(policy, 123_456); - expect(runState.cancelActiveRun).toHaveBeenCalledWith('thread-active'); - expect(runState.cancelSuspendedRun).toHaveBeenCalledWith('thread-suspended'); - expect(activeAbortController.signal.aborted).toBe(true); - expect(suspendedAbortController.signal.aborted).toBe(true); - expect(finalizeCancelledSuspendedRun).toHaveBeenCalledWith( - suspended, - INSTANCE_AI_RUN_TIMEOUT_REASON, - ); - expect(runState.rejectPendingConfirmation).toHaveBeenCalledWith('request-1'); - expect(onPendingConfirmationRejected).toHaveBeenCalledWith('request-1'); - expect(backgroundTasks.timeoutTimedOutTasks).toHaveBeenCalledWith( - policy, - 123_456, - expect.objectContaining({ shouldSkipTask: expect.any(Function) }), - ); - const timeoutOptions = backgroundTasks.timeoutTimedOutTasks.mock.calls[0]?.[2]; - runState.hasPendingConfirmationForThread.mockReturnValueOnce(true); - expect( - timeoutOptions?.shouldSkipTask?.({ - threadId: 'thread-bg', - taskId: 'task-1', - role: 'workflow-builder', - }), - ).toBe(true); - expect(runState.hasPendingConfirmationForThread).toHaveBeenCalledWith('thread-bg'); - expect(eventBus.publish).toHaveBeenCalledWith( - 'thread-active', - expect.objectContaining({ responseId: 'run-timeout:run-active' }), - ); - expect(eventBus.publish).toHaveBeenCalledWith( - 'thread-confirmation', - expect.objectContaining({ responseId: 'run-timeout:run-confirmation' }), - ); - expect(service.consumeRunTimeout('run-active')).toEqual({ - timedOut: true, - details: activeTimeout, - }); - expect(service.consumeRunTimeout('run-suspended')).toEqual({ - timedOut: true, - details: suspendedTimeout, - }); - expect(service.hasTimedOutActiveRunThread('thread-active')).toBe(true); - }); - - it('lets suspended-run finalization own the timeout notice', () => { - const { service, runState, eventBus, finalizeCancelledSuspendedRun } = createLivenessService(); - const abortController = new AbortController(); - const suspended = { - runId: 'run-suspended', - threadId: 'thread-suspended', - abortController, - }; - runState.cancelSuspendedRun.mockReturnValue(suspended); - - service.cancelTimedOutSuspendedRun('thread-suspended'); - - expect(eventBus.publish).not.toHaveBeenCalled(); - expect(finalizeCancelledSuspendedRun).toHaveBeenCalledWith( - suspended, - INSTANCE_AI_RUN_TIMEOUT_REASON, - ); - expect(abortController.signal.aborted).toBe(true); - }); - - it('deduplicates timeout notices per run', () => { - const { service, eventBus } = createLivenessService(); - - service.publishRunTimeoutNotice('thread-1', 'run-1'); - service.publishRunTimeoutNotice('thread-1', 'run-1'); - - expect(eventBus.publish).toHaveBeenCalledTimes(1); - expect(eventBus.publish).toHaveBeenCalledWith( - 'thread-1', - expect.objectContaining({ responseId: 'run-timeout:run-1' }), - ); - }); - - it('publishes a notice per run, and evicts the oldest dedupe entry past the cap', () => { - const { service, eventBus } = createLivenessService(); - - service.publishRunTimeoutNotice('thread-1', 'run-a'); - service.publishRunTimeoutNotice('thread-1', 'run-b'); - expect(eventBus.publish).toHaveBeenCalledTimes(2); - - // Push `run-a` out of the capped FIFO, after which its notice can repeat — - // the accepted trade-off for a bounded guard. - for (let i = 0; i < InstanceAiLivenessService.NOTICE_DEDUPE_CACHE_SIZE; i++) { - service.publishRunTimeoutNotice('thread-1', `filler-${i}`); - } - eventBus.publish.mockClear(); - service.publishRunTimeoutNotice('thread-1', 'run-a'); - - expect(eventBus.publish).toHaveBeenCalledTimes(1); - }); - - it('keeps sweeping run state when background task timeout handling fails', async () => { - const { service, backgroundTasks, logger } = createLivenessService(); - backgroundTasks.timeoutTimedOutTasks.mockRejectedValue(new Error('failed')); - - await service.sweepTimedOutWork(); - - expect(logger.warn).toHaveBeenCalledWith('Failed to sweep timed-out background tasks', { - error: 'failed', - }); - }); -}); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts index d2b1df266ec..d5c4dec6b3a 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts @@ -32,14 +32,11 @@ import { setupSandboxWorkspace, withSandboxLifecycleTrace, type InstanceAiContext, - type ManagedBackgroundTask, } from '@n8n/instance-ai'; import { InstanceAiSandboxService, - type InstanceAiSandboxBackgroundTasks, type InstanceAiSandboxProxy, - type InstanceAiSandboxRunState, type InstanceAiSandboxServiceOptions, type InstanceAiSandboxSettings, } from '../sandbox'; @@ -49,8 +46,6 @@ const fakeUser = { id: 'user-1' } as User; type Overrides = { resolveTracingConfig?: InstanceAiSandboxServiceOptions['resolveTracingConfig']; config?: Partial; - runState?: Partial; - backgroundTasks?: Partial; settingsService?: Partial; aiService?: Partial; }; @@ -58,15 +53,6 @@ type Overrides = { function createSandboxService(overrides: Overrides = {}) { const logger = { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }; const errorReporter = { error: vi.fn(), warn: vi.fn() } as unknown as ErrorReporter; - const runState: InstanceAiSandboxRunState = { - getActiveRunId: vi.fn(() => undefined), - hasSuspendedRun: vi.fn(() => false), - ...overrides.runState, - }; - const backgroundTasks: InstanceAiSandboxBackgroundTasks = { - getRunningTasks: vi.fn(() => [] as ManagedBackgroundTask[]), - ...overrides.backgroundTasks, - }; const settingsService: InstanceAiSandboxSettings = { resolveDaytonaConfig: vi.fn(async () => ({ apiKey: 'test-daytona-key' })), resolveN8nSandboxConfig: vi.fn(async () => ({})), @@ -81,14 +67,12 @@ function createSandboxService(overrides: Overrides = {}) { config: overrides.config as InstanceAiConfig, logger, errorReporter, - runState, - backgroundTasks, settingsService, aiService, resolveTracingConfig: overrides.resolveTracingConfig, }; const service = new InstanceAiSandboxService(options); - return { service, logger, errorReporter, runState, backgroundTasks, settingsService, aiService }; + return { service, logger, errorReporter, settingsService, aiService }; } describe('InstanceAiSandboxService', () => { @@ -867,37 +851,6 @@ describe('InstanceAiSandboxService', () => { } }); - it('keeps an in-use sandbox alive when the expiry timer fires', async () => { - vi.useFakeTimers(); - try { - const getRunningTasks = vi.fn(() => [{ taskId: 'task-1' }] as ManagedBackgroundTask[]); - const { service } = createSandboxService({ - config: { sandboxEnabled: true, sandboxProvider: 'daytona', builderSandboxTtlMs: 1000 }, - backgroundTasks: { getRunningTasks }, - }); - const sandbox = { id: 'sandbox-1' }; - const workspace = { init: vi.fn(async () => {}), destroy: vi.fn(async () => {}) }; - (createSandbox as Mock).mockResolvedValue(sandbox); - (createWorkspace as Mock).mockReturnValue(workspace); - (setupSandboxWorkspace as Mock).mockResolvedValue(undefined); - - await service.getOrCreateWorkspace('thread-1', fakeUser, {} as InstanceAiContext); - - vi.advanceTimersByTime(1000); - - // In-use sandboxes are touched (re-scheduled) rather than dropped. - const reused = await service.getOrCreateWorkspace( - 'thread-1', - fakeUser, - {} as InstanceAiContext, - ); - expect(reused).toBeDefined(); - expect(createSandbox).toHaveBeenCalledTimes(1); - } finally { - vi.useRealTimers(); - } - }); - it('clears scheduled expiry timers on stopSandboxExpiryTimers', async () => { vi.useFakeTimers(); const clearTimeoutSpy = vi.spyOn(global, 'clearTimeout'); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts index 26b1cc5ed8b..7985039a514 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts @@ -1,9 +1,3 @@ -const terminalOutcomeStorageMock = { - getUndelivered: vi.fn(), - markDelivered: vi.fn(), - upsert: vi.fn(), -}; - // Manual mock — must be declared before any import that touches the mocked module. vi.mock('@n8n/instance-ai', () => ({ orchestratorAgentId: (runId: string) => `orchestrator-${runId}`, @@ -92,16 +86,10 @@ vi.mock('@n8n/instance-ai', () => ({ }; } }, - TerminalOutcomeStorage: class { - getUndelivered = terminalOutcomeStorageMock.getUndelivered; - markDelivered = terminalOutcomeStorageMock.markDelivered; - upsert = terminalOutcomeStorageMock.upsert; - }, })); import type { Mock } from 'vitest'; import type { InstanceAiEvent } from '@n8n/api-types'; -import type { ManagedBackgroundTask, TerminalOutcome } from '@n8n/instance-ai'; import { InstanceAiTerminalOutcomeService, @@ -118,28 +106,9 @@ type Deps = { telemetry: { track: Mock }; errorReporter: { report: Mock }; logger: { warn: Mock; debug: Mock; error: Mock }; - runState: { getRunIdsForMessageGroup: Mock; cancelThread: Mock }; - suspendedThreads: { dropPendingConfirmationsForThread: Mock }; - tracing: { finalizeRunTracing: Mock; buildMessageTraceMetadata: Mock }; - publishRunFinish: Mock; + runState: { getRunIdsForMessageGroup: Mock }; }; -function makeTerminalOutcome(overrides: Partial = {}): TerminalOutcome { - return { - id: 'group-1:task-1:completed', - threadId: 'thread-a', - runId: 'run-1', - messageGroupId: 'group-1', - correlationId: 'message-1', - taskId: 'task-1', - agentId: 'agent-builder', - status: 'completed', - userFacingMessage: 'The background workflow-builder task finished.', - createdAt: '2026-05-01T00:00:00.000Z', - ...overrides, - }; -} - function createService(): { service: InstanceAiTerminalOutcomeService; deps: Deps; @@ -159,35 +128,15 @@ function createService(): { logger: { warn: vi.fn(), debug: vi.fn(), error: vi.fn() }, runState: { getRunIdsForMessageGroup: vi.fn(() => ['run-1']), - cancelThread: vi.fn(), }, - suspendedThreads: { dropPendingConfirmationsForThread: vi.fn(async () => {}) }, - tracing: { - finalizeRunTracing: vi.fn(async () => {}), - buildMessageTraceMetadata: vi.fn(() => ({ completion_source: 'orchestrator' })), - }, - publishRunFinish: vi.fn( - (_threadId: string, runId: string, status: 'completed' | 'cancelled' | 'errored') => { - events.push({ - type: 'run-finish', - runId, - agentId: 'agent-001', - payload: { status: status === 'errored' ? 'error' : status }, - } as InstanceAiEvent); - }, - ), }; const options = { eventBus: deps.eventBus, - agentMemory: {}, telemetry: deps.telemetry, errorReporter: deps.errorReporter, logger: deps.logger, runState: deps.runState, - suspendedThreads: deps.suspendedThreads, - tracing: deps.tracing, - publishRunFinish: deps.publishRunFinish, } as unknown as InstanceAiTerminalOutcomeServiceOptions; return { service: new InstanceAiTerminalOutcomeService(options), deps }; @@ -195,230 +144,17 @@ function createService(): { beforeEach(() => { vi.clearAllMocks(); - terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([]); - terminalOutcomeStorageMock.markDelivered.mockResolvedValue(undefined); - terminalOutcomeStorageMock.upsert.mockResolvedValue(undefined); -}); - -describe('InstanceAiTerminalOutcomeService — terminal outcome replay', () => { - it('publishes recovered background outcomes and marks them delivered', async () => { - const outcome = makeTerminalOutcome(); - terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]); - const { service, deps } = createService(); - - await service.replayUndeliveredTerminalOutcomes('thread-a'); - - expect(deps.eventBus.publish).toHaveBeenCalledWith('thread-a', { - type: 'text-block', - runId: outcome.runId, - agentId: 'orchestrator-run-1', - responseId: `background-outcome:${outcome.id}`, - payload: { text: outcome.userFacingMessage }, - }); - expect(terminalOutcomeStorageMock.markDelivered).toHaveBeenCalledWith( - 'thread-a', - outcome.id, - expect.any(String), - ); - expect(deps.telemetry.track).toHaveBeenCalledWith( - 'instance_ai_terminal_response_decision', - expect.objectContaining({ source: 'terminal_outcome_replay', action: 'replay_event' }), - ); - }); - - it('deduplicates replay by response id only', async () => { - const outcome = makeTerminalOutcome({ id: 'group-1:task-2:completed' }); - terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]); - const { service, deps } = createService(); - // Same text under a different response id must not suppress the replay. - deps.eventBus.events.push({ - type: 'text-block', - runId: outcome.runId, - agentId: 'orchestrator-run-1', - responseId: 'background-outcome:different-id', - payload: { text: outcome.userFacingMessage }, - } as InstanceAiEvent); - - await service.replayUndeliveredTerminalOutcomes('thread-a'); - - expect(deps.eventBus.publish).toHaveBeenCalledTimes(1); - expect(deps.eventBus.publish).toHaveBeenCalledWith( - 'thread-a', - expect.objectContaining({ responseId: `background-outcome:${outcome.id}` }), - ); - - // A second replay finds the exact response id already emitted and skips it. - await service.replayUndeliveredTerminalOutcomes('thread-a'); - - expect(deps.eventBus.publish).toHaveBeenCalledTimes(1); - expect(deps.telemetry.track).toHaveBeenCalledWith( - 'instance_ai_terminal_response_decision', - expect.objectContaining({ source: 'terminal_outcome_replay', action: 'already-emitted' }), - ); - }); - - it('leaves the outcome undelivered when publishing the line fails', async () => { - const outcome = makeTerminalOutcome(); - terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]); - const { service, deps } = createService(); - deps.eventBus.publish.mockImplementation(() => { - throw new Error('bus unavailable'); - }); - - await service.replayUndeliveredTerminalOutcomes('thread-a'); - - expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled(); - expect(deps.logger.warn).toHaveBeenCalledWith( - 'Failed to replay Instance AI terminal outcome', - expect.objectContaining({ threadId: 'thread-a', runId: outcome.runId }), - ); - }); - - it('leaves the outcome undelivered when the drain drops the published line', async () => { - const outcome = makeTerminalOutcome(); - terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]); - const { service, deps } = createService(); - // The publish enqueues without throwing, but the line never reaches the - // log — the shape of a dropped drain batch. - deps.eventBus.publish.mockImplementation(() => {}); - - await service.replayUndeliveredTerminalOutcomes('thread-a'); - - expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled(); - expect(deps.telemetry.track).not.toHaveBeenCalledWith( - 'instance_ai_terminal_response_decision', - expect.anything(), - ); - }); - - it('checks persisted outcomes on repeated replay calls', async () => { - const { service } = createService(); - - await service.replayUndeliveredTerminalOutcomes('thread-a'); - await service.replayUndeliveredTerminalOutcomes('thread-a'); - - expect(terminalOutcomeStorageMock.getUndelivered).toHaveBeenCalledTimes(2); - }); -}); - -describe('InstanceAiTerminalOutcomeService — background outcome recording', () => { - function makeTask(overrides: Partial = {}): ManagedBackgroundTask { - return { - taskId: 'task-1', - threadId: 'thread-a', - runId: 'run-1', - role: 'workflow-builder', - agentId: 'agent-builder', - status: 'completed', - result: 'done', - startedAt: 0, - lastActivityAt: 0, - abortController: new AbortController(), - corrections: [], - messageGroupId: 'group-1', - ...overrides, - } as ManagedBackgroundTask; - } - - it('persists, publishes, and marks the outcome delivered on success', async () => { - const { service, deps } = createService(); - - await service.recordBackgroundTerminalOutcome(makeTask()); - - expect(terminalOutcomeStorageMock.upsert).toHaveBeenCalledTimes(1); - expect(deps.eventBus.publish).toHaveBeenCalledWith( - 'thread-a', - expect.objectContaining({ - type: 'text-block', - payload: { text: 'The background workflow-builder task finished.' }, - }), - ); - expect(terminalOutcomeStorageMock.markDelivered).toHaveBeenCalledTimes(1); - }); - - it('keeps the outcome pending and replays it later when persistence fails', async () => { - terminalOutcomeStorageMock.upsert.mockRejectedValueOnce(new Error('db down')); - const { service, deps } = createService(); - - await service.recordBackgroundTerminalOutcome(makeTask()); - - expect(deps.telemetry.track).toHaveBeenCalledWith( - 'instance_ai_terminal_outcome_persistence_failure', - expect.objectContaining({ phase: 'metadata' }), - ); - expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled(); - - // The pending outcome is recovered on the next replay. - await service.replayUndeliveredTerminalOutcomes('thread-a'); - expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled(); - }); - - it('does not mark the outcome delivered when the drain drops the line', async () => { - const { service, deps } = createService(); - deps.eventBus.publish.mockImplementation(() => {}); - - await service.recordBackgroundTerminalOutcome(makeTask()); - - expect(terminalOutcomeStorageMock.upsert).toHaveBeenCalledTimes(1); - expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled(); - expect(deps.telemetry.track).toHaveBeenCalledWith( - 'instance_ai_terminal_outcome_persistence_failure', - expect.objectContaining({ phase: 'event' }), - ); - expect(deps.telemetry.track).not.toHaveBeenCalledWith( - 'instance_ai_terminal_response_decision', - expect.anything(), - ); - }); -}); - -describe('InstanceAiTerminalOutcomeService — durable-log outcome lines', () => { - it('publishes the outcome line as a persisted text-block, not a trailing delta', async () => { - // A trailing delta would race the coalescer's idle flush on an immediate - // page reload; a text-block is persisted before it is emitted live. - const { deps } = createService(); - const service = new InstanceAiTerminalOutcomeService({ - eventBus: deps.eventBus, - agentMemory: {}, - telemetry: deps.telemetry, - logger: deps.logger, - runState: deps.runState, - suspendedThreads: deps.suspendedThreads, - tracing: deps.tracing, - publishRunFinish: deps.publishRunFinish, - } as never); - - await service.recordBackgroundTerminalOutcome({ - taskId: 'task-dl', - threadId: 'thread-dl', - runId: 'run-dl', - role: 'workflow-builder', - agentId: 'agent-builder', - status: 'cancelled', - result: undefined, - startedAt: 0, - lastActivityAt: 0, - abortController: new AbortController(), - corrections: [], - } as never); - - const line = deps.eventBus.events.find( - (event) => event.type === 'text-block' || event.type === 'text-delta', - ); - expect(line?.type).toBe('text-block'); - }); }); describe('InstanceAiTerminalOutcomeService — terminal response guard wiring', () => { - it('publishes fallback output before run-finish on a silent completed run', async () => { + it('publishes fallback output on a silent completed run', async () => { const { service, deps } = createService(); await service.evaluateTerminalResponse('thread-a', 'run-1', 'completed', { messageGroupId: 'group-1', }); - deps.publishRunFinish('thread-a', 'run-1', 'completed'); - expect(deps.eventBus.events.map((event) => event.type)).toEqual(['text-delta', 'run-finish']); + expect(deps.eventBus.events.map((event) => event.type)).toEqual(['text-delta']); }); it('reports a silent completed run so the stall is not lost', async () => { @@ -465,16 +201,15 @@ describe('InstanceAiTerminalOutcomeService — terminal response guard wiring', expect(deps.eventBus.events).toEqual([]); }); - it('publishes fallback error before run-finish on a silent failed run', async () => { + it('publishes fallback error on a silent failed run', async () => { const { service, deps } = createService(); await service.evaluateTerminalResponse('thread-a', 'run-1', 'errored', { messageGroupId: 'group-1', errorMessage: 'Safe user-facing error', }); - deps.publishRunFinish('thread-a', 'run-1', 'errored'); - expect(deps.eventBus.events.map((event) => event.type)).toEqual(['error', 'run-finish']); + expect(deps.eventBus.events.map((event) => event.type)).toEqual(['error']); }); it('forwards a structured error code onto the emitted error event', async () => { @@ -490,40 +225,15 @@ describe('InstanceAiTerminalOutcomeService — terminal response guard wiring', expect(errorEvent?.payload).toMatchObject({ code: 'quota_exhausted' }); }); - it('clears malformed confirmation suspension and finishes the run after the guard error', async () => { + it('publishes the guard error for a malformed confirmation', async () => { const { service, deps } = createService(); - const abortController = new AbortController(); const decision = await service.evaluateWaitingResponse('thread-a', 'run-1', undefined, { messageGroupId: 'group-1', }); + expect(decision?.reason).toBe('confirmation-invalid'); - - const finalization = await service.finishInvalidConfirmationRun({ - threadId: 'thread-a', - runId: 'run-1', - promptVersion: 'progressive@1', - abortController, - }); - - expect(finalization.status).toBe('error'); - expect(finalization.reason).toBe('invalid_confirmation_payload'); - expect(deps.publishRunFinish).toHaveBeenCalledWith( - 'thread-a', - 'run-1', - 'errored', - expect.any(String), - 'progressive@1', - ); - expect(deps.runState.cancelThread).toHaveBeenCalledWith('thread-a'); - expect(deps.suspendedThreads.dropPendingConfirmationsForThread).toHaveBeenCalledWith( - 'thread-a', - ); - expect(abortController.signal.aborted).toBe(true); - expect(deps.eventBus.events.at(-1)).toMatchObject({ - type: 'run-finish', - payload: { status: 'error' }, - }); + expect(deps.eventBus.events.at(-1)).toMatchObject({ type: 'error' }); }); it('reads events across the message group when a group id is provided', async () => { diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-test.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-test.controller.test.ts index e3151ed8609..220a0ccb50a 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-test.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-test.controller.test.ts @@ -11,32 +11,21 @@ vi.mock('../eval/execution.service', () => ({ EvalExecutionService: vi.fn(), })); -import type { ProjectRepository, UserRepository, WorkflowRepository } from '@n8n/db'; +import type { WorkflowRepository } from '@n8n/db'; import type { Request, Response } from 'express'; import { mock } from 'vitest-mock-extended'; import { ForbiddenError } from '@n8n/errors'; -import type { InstanceAiMemoryService } from '../instance-ai-memory.service'; import { InstanceAiTestController } from '../instance-ai-test.controller'; import type { InstanceAiService } from '../instance-ai.service'; -import type { InstanceAiThreadRepository } from '../repositories/instance-ai-thread.repository'; +import type { AgentExecutionThreadRepository } from '../../agents/repositories/agent-execution-thread.repository'; describe('InstanceAiTestController', () => { const instanceAiService = mock(); - const threadRepo = mock(); + const threadRepo = mock(); const workflowRepo = mock(); - const userRepo = mock(); - const memoryService = mock(); - const projectRepo = mock(); - const controller = new InstanceAiTestController( - instanceAiService, - threadRepo, - workflowRepo, - userRepo, - memoryService, - projectRepo, - ); + const controller = new InstanceAiTestController(instanceAiService, threadRepo, workflowRepo); const originalEnv = process.env; @@ -109,28 +98,29 @@ describe('InstanceAiTestController', () => { }); describe('getIdleState', () => { - it('should report idle when there is no running work', () => { - instanceAiService.hasRunningWorkForTest.mockReturnValue(false); + it('should report idle when no Assistant thread is running', async () => { + threadRepo.find.mockResolvedValue([{ id: 't1' }] as never); + instanceAiService.getLiveRun.mockResolvedValue({ status: 'suspended', runIds: [] }); - const result = controller.getIdleState(); + const result = await controller.getIdleState(); - expect(instanceAiService.hasRunningWorkForTest).toHaveBeenCalled(); + expect(instanceAiService.getLiveRun).toHaveBeenCalledWith('t1'); expect(result).toEqual({ idle: true }); }); - it('should report non-idle when there is running work', () => { - instanceAiService.hasRunningWorkForTest.mockReturnValue(true); + it('should report non-idle when an Assistant thread is running', async () => { + threadRepo.find.mockResolvedValue([{ id: 't1' }] as never); + instanceAiService.getLiveRun.mockResolvedValue({ status: 'running', runIds: [] }); - const result = controller.getIdleState(); + const result = await controller.getIdleState(); - expect(instanceAiService.hasRunningWorkForTest).toHaveBeenCalled(); expect(result).toEqual({ idle: false }); }); - it('should throw ForbiddenError when trace replay is not enabled', () => { + it('should throw ForbiddenError when trace replay is not enabled', async () => { delete process.env.E2E_TESTS; - expect(() => controller.getIdleState()).toThrow(ForbiddenError); + await expect(controller.getIdleState()).rejects.toThrow(ForbiddenError); }); }); @@ -154,66 +144,17 @@ describe('InstanceAiTestController', () => { }); }); - describe('background timeout simulation', () => { - it('should create an owned thread and start a stuck background task', async () => { - const user = { id: 'user-1' } as never; - const simulation = { - threadId: 'thread-1', - runId: 'run-1', - messageGroupId: 'mg-1', - taskId: 'task-1', - agentId: 'agent-1', - timeoutAt: 123, - }; - userRepo.findOneByOrFail.mockResolvedValue(user); - projectRepo.getPersonalProjectForUserOrFail.mockResolvedValue({ - id: 'personal-project-id', - } as never); - instanceAiService.startStuckBackgroundTaskForTest.mockResolvedValue(simulation); - - const result = await controller.startBackgroundTimeoutSimulation({ - userId: 'user-1', - threadId: 'thread-1', - }); - - expect(memoryService.ensureThread).toHaveBeenCalledWith( - 'user-1', - 'thread-1', - 'personal-project-id', - { source: 'playwright', origin: 'internal' }, - ); - expect(instanceAiService.startStuckBackgroundTaskForTest).toHaveBeenCalledWith( - user, - 'thread-1', - ); - expect(result).toEqual(simulation); - }); - - it('should run the liveness sweep at a simulated time', async () => { - const result = await controller.runLivenessSweep({ now: 123 }); - - expect(instanceAiService.runLivenessSweepForTest).toHaveBeenCalledWith(123); - expect(result).toEqual({ ok: true }); - }); - }); - describe('reset', () => { it('should clear per-thread state and delete threads + workflows', async () => { threadRepo.find.mockResolvedValue([{ id: 't1' }, { id: 't2' }] as never); workflowRepo.find.mockResolvedValue([{ id: 'w1' }, { id: 'w2' }, { id: 'w3' }] as never); - const deleteExecute = vi.fn().mockResolvedValue(undefined); - const deleteQb = { execute: deleteExecute }; - const queryBuilder = { delete: vi.fn().mockReturnValue(deleteQb) }; - threadRepo.createQueryBuilder.mockReturnValue(queryBuilder as never); const result = await controller.reset(); - expect(instanceAiService.cancelAllBackgroundTasks).toHaveBeenCalled(); expect(instanceAiService.clearTraceContextsForTest).toHaveBeenCalled(); expect(instanceAiService.clearThreadState).toHaveBeenCalledWith('t1'); expect(instanceAiService.clearThreadState).toHaveBeenCalledWith('t2'); - expect(queryBuilder.delete).toHaveBeenCalled(); - expect(deleteExecute).toHaveBeenCalled(); + expect(threadRepo.delete).toHaveBeenCalledWith({ agentId: 'n8n-assistant' }); expect(workflowRepo.delete).toHaveBeenCalledWith('w1'); expect(workflowRepo.delete).toHaveBeenCalledWith('w2'); expect(workflowRepo.delete).toHaveBeenCalledWith('w3'); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-tracing.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-tracing.service.test.ts index c2581c9eda7..ca7ebf3822c 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-tracing.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-tracing.service.test.ts @@ -21,7 +21,6 @@ import { type InstanceAiTracingAiService, type InstanceAiTracingEventLog, type InstanceAiTracingEventReader, - type InstanceAiTracingRunState, } from '../tracing'; type FakeTraceRun = { @@ -53,7 +52,6 @@ function createService( overrides: { logger?: Partial; eventReader?: Partial; - runState?: Partial; eventLog?: Partial; aiService?: Partial; } = {}, @@ -63,10 +61,6 @@ function createService( getEventsForRun: vi.fn(async () => []), ...overrides.eventReader, }; - const runState: InstanceAiTracingRunState = { - attachTracing: vi.fn(), - ...overrides.runState, - }; const eventLog: InstanceAiTracingEventLog = { findLangsmithAnchor: vi.fn(async () => undefined), ...overrides.eventLog, @@ -80,12 +74,11 @@ function createService( const service = new InstanceAiTracingService({ logger, eventReader, - runState, eventLog, aiService, }); - return { service, logger, eventReader, runState, eventLog, aiService }; + return { service, logger, eventReader, eventLog, aiService }; } describe('InstanceAiTracingService', () => { @@ -120,7 +113,7 @@ describe('InstanceAiTracingService', () => { it('degrades instead of throwing when the events read fails', async () => { // Most callers run inside a run's terminal catch block: a throw here - // would skip run-finish and hang the client until the liveness sweep. + // would skip run-finish and hang the client. const { service, logger } = createService({ eventReader: { getEventsForRun: vi.fn(async () => { @@ -132,7 +125,6 @@ describe('InstanceAiTracingService', () => { await expect( service.buildMessageTraceMetadata('thread-1', 'run-1', { status: 'cancelled', - cancellationReason: 'timeout', }), ).resolves.toEqual({ completion_source: 'orchestrator', @@ -140,7 +132,7 @@ describe('InstanceAiTracingService', () => { // it can't be read as a genuinely empty run. first_visible_state: 'empty', first_visible_state_unavailable: true, - cancellation_type: 'idle_timeout', + cancellation_type: 'explicit', }); expect(logger.warn).toHaveBeenCalledWith( 'Failed to read run events for Instance AI trace metadata', @@ -171,7 +163,7 @@ describe('InstanceAiTracingService', () => { describe('resume trace registration', () => { it('keeps a resume trace detached until the checkpoint claim succeeds', async () => { - const { service, runState } = createService(); + const { service } = createService(); const baseTracing = makeTraceContext({ rootRun: { id: 'root-base', traceId: 'trace-base' }, }); @@ -195,13 +187,11 @@ describe('InstanceAiTracingService', () => { expect(result).toBe(resumeTracing); expect(service.getTraceContext('run-1')).toBeUndefined(); - expect(runState.attachTracing).not.toHaveBeenCalled(); service.registerTraceContext('run-1', 'thread-a', resumeTracing, 'group-1'); expect(service.getTraceContext('run-1')).toBe(resumeTracing); expect(service.getMessageGroupId('run-1')).toBe('group-1'); - expect(runState.attachTracing).toHaveBeenCalledWith('thread-a', resumeTracing); }); }); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts index a7d53d5098f..a425f6f2195 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts @@ -44,7 +44,6 @@ import type { InstanceAiEvalRestoreThreadRequest, InstanceAiEvalThreadMemoryResponse, InstanceAiSendMessageRequest, - InstanceAiCorrectTaskRequest, InstanceAiConfirmRequest, InstanceAiEnsureThreadRequest, InstanceAiThreadMessagesQuery, @@ -1526,41 +1525,6 @@ describe('InstanceAiController', () => { }); }); - describe('cancelTask', () => { - it('should require instanceAi:message scope', () => { - expect(scopeOf('cancelTask')).toEqual({ scope: 'instanceAi:message', globalOnly: true }); - }); - - it('should cancel the background task', async () => { - memoryService.checkThreadOwnership.mockResolvedValue('owned'); - - const result = await controller.cancelTask(req, res, THREAD_ID, 'task-1'); - - expect(result).toEqual({ ok: true }); - expect(instanceAiService.routeCancelBackgroundTask).toHaveBeenCalledWith(THREAD_ID, 'task-1'); - }); - }); - - describe('correctTask', () => { - it('should require instanceAi:message scope', () => { - expect(scopeOf('correctTask')).toEqual({ scope: 'instanceAi:message', globalOnly: true }); - }); - - it('should send correction to the task', async () => { - memoryService.checkThreadOwnership.mockResolvedValue('owned'); - const payload = mock({ message: 'fix this' }); - - const result = await controller.correctTask(req, res, THREAD_ID, 'task-1', payload); - - expect(result).toEqual({ ok: true }); - expect(instanceAiService.routeCorrectionToTask).toHaveBeenCalledWith( - THREAD_ID, - 'task-1', - 'fix this', - ); - }); - }); - describe('preference card routes', () => { it('should require instanceAi:message scope', () => { expect(scopeOf('undoPreference')).toEqual({ @@ -2290,7 +2254,6 @@ describe('InstanceAiController', () => { const result = await controller.getThreadMessages(req, res, THREAD_ID, query); expect(result).toMatchObject({ nextEventId: 42 }); - expect(instanceAiService.replayUndeliveredTerminalOutcomes).toHaveBeenCalledWith(THREAD_ID); expect(memoryService.getRichMessages).toHaveBeenCalledWith(USER_ID, THREAD_ID, { limit: 50, page: 0, diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts index 7d52cef7a0f..29aa705742a 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts @@ -6054,209 +6054,6 @@ describe('InstanceAiService — deterministic workflow setup follow-up', () => { }); }); -type TaskControlInternals = { - instanceSettings: { isMultiMain: boolean }; - publisher: { publishCommand: Mock }; - backgroundTasks: { getTaskSnapshots: Mock }; - logger: { error: Mock }; - runState: { hasLiveRun: Mock }; - eventLog: { flush: Mock }; - interruptedRunSweeper: { cancelUnfinishedRuns: Mock }; - sendCorrectionToTask: Mock; - cancelBackgroundTask: Mock; - cancelRun: Mock; - clearThreadState: Mock; - routeCorrectionToTask: InstanceAiService['routeCorrectionToTask']; - routeCancelBackgroundTask: InstanceAiService['routeCancelBackgroundTask']; - routeCancelRun: InstanceAiService['routeCancelRun']; - routeClearThreadState: InstanceAiService['routeClearThreadState']; - handleRelayTaskControl: InstanceAiService['handleRelayTaskControl']; -}; - -function buildTaskControlService(isMultiMain: boolean): TaskControlInternals { - const service = Object.create(InstanceAiService.prototype) as unknown as TaskControlInternals; - service.instanceSettings = { isMultiMain }; - service.publisher = { publishCommand: vi.fn().mockResolvedValue(undefined) }; - service.backgroundTasks = { getTaskSnapshots: vi.fn(() => []) }; - service.logger = { error: vi.fn() }; - // Inert cancel-time zombie fallback — its behavior is covered by the - // dedicated 'routeCancelRun zombie fallback' suite; these tests are about - // routing. - service.runState = { hasLiveRun: vi.fn(() => false) }; - service.eventLog = { flush: vi.fn(async () => {}) }; - service.interruptedRunSweeper = { cancelUnfinishedRuns: vi.fn(async () => 0) }; - service.sendCorrectionToTask = vi.fn(() => 'queued'); - service.cancelBackgroundTask = vi.fn(); - service.cancelRun = vi.fn(); - service.clearThreadState = vi.fn(async () => {}); - return service; -} - -describe('InstanceAiService — cross-main task-control routing', () => { - describe('routeCorrectionToTask', () => { - it('broadcasts when the task is not local and multi-main', async () => { - const service = buildTaskControlService(true); - service.sendCorrectionToTask.mockReturnValue('task-not-found'); - - await service.routeCorrectionToTask('thread-a', 'task-1', 'try again'); - - expect(service.sendCorrectionToTask).toHaveBeenCalledWith('thread-a', 'task-1', 'try again'); - expect(service.publisher.publishCommand).toHaveBeenCalledWith({ - command: 'relay-instance-ai-task-control', - payload: { - threadId: 'thread-a', - taskId: 'task-1', - action: 'correct', - correction: 'try again', - }, - }); - }); - - it('does not broadcast when the correction was applied locally', async () => { - const service = buildTaskControlService(true); - service.sendCorrectionToTask.mockReturnValue('queued'); - - await service.routeCorrectionToTask('thread-a', 'task-1', 'try again'); - - expect(service.publisher.publishCommand).not.toHaveBeenCalled(); - }); - - it('does not broadcast in single-main even on a local miss', async () => { - const service = buildTaskControlService(false); - service.sendCorrectionToTask.mockReturnValue('task-not-found'); - - await service.routeCorrectionToTask('thread-a', 'task-1', 'try again'); - - expect(service.publisher.publishCommand).not.toHaveBeenCalled(); - }); - }); - - describe('routeCancelBackgroundTask', () => { - it('broadcasts when the task is not local and multi-main', async () => { - const service = buildTaskControlService(true); - service.backgroundTasks.getTaskSnapshots.mockReturnValue([]); - - await service.routeCancelBackgroundTask('thread-a', 'task-1'); - - expect(service.cancelBackgroundTask).toHaveBeenCalledWith('thread-a', 'task-1'); - expect(service.publisher.publishCommand).toHaveBeenCalledWith({ - command: 'relay-instance-ai-task-control', - payload: { threadId: 'thread-a', taskId: 'task-1', action: 'cancel-task' }, - }); - }); - - it('does not broadcast when the task is local', async () => { - const service = buildTaskControlService(true); - service.backgroundTasks.getTaskSnapshots.mockReturnValue([{ taskId: 'task-1' }]); - - await service.routeCancelBackgroundTask('thread-a', 'task-1'); - - expect(service.cancelBackgroundTask).toHaveBeenCalledWith('thread-a', 'task-1'); - expect(service.publisher.publishCommand).not.toHaveBeenCalled(); - }); - }); - - describe('routeCancelRun / routeClearThreadState', () => { - it('routeCancelRun cancels locally and always fans out in multi-main', async () => { - const service = buildTaskControlService(true); - - await service.routeCancelRun('thread-a'); - - expect(service.cancelRun).toHaveBeenCalledWith('thread-a'); - expect(service.publisher.publishCommand).toHaveBeenCalledWith({ - command: 'relay-instance-ai-task-control', - payload: { threadId: 'thread-a', action: 'cancel-thread' }, - }); - }); - - it('routeCancelRun does not fan out in single-main', async () => { - const service = buildTaskControlService(false); - - await service.routeCancelRun('thread-a'); - - expect(service.cancelRun).toHaveBeenCalledWith('thread-a'); - expect(service.publisher.publishCommand).not.toHaveBeenCalled(); - }); - - it('routeClearThreadState clears locally and fans out in multi-main', async () => { - const service = buildTaskControlService(true); - - await service.routeClearThreadState('thread-a', 'owner-a'); - - expect(service.clearThreadState).toHaveBeenCalledWith('thread-a', 'owner-a'); - expect(service.publisher.publishCommand).toHaveBeenCalledWith({ - command: 'relay-instance-ai-task-control', - payload: { threadId: 'thread-a', action: 'clear-thread', userId: 'owner-a' }, - }); - }); - }); - - describe('handleRelayTaskControl', () => { - it('applies a relayed correction via the local method and never re-broadcasts', async () => { - const service = buildTaskControlService(true); - - await service.handleRelayTaskControl({ - threadId: 'thread-a', - taskId: 'task-1', - action: 'correct', - correction: 'fix it', - }); - - expect(service.sendCorrectionToTask).toHaveBeenCalledWith('thread-a', 'task-1', 'fix it'); - expect(service.publisher.publishCommand).not.toHaveBeenCalled(); - }); - - it('ignores a correction relay missing its correction text', async () => { - const service = buildTaskControlService(true); - - await service.handleRelayTaskControl({ - threadId: 'thread-a', - taskId: 'task-1', - action: 'correct', - }); - - expect(service.sendCorrectionToTask).not.toHaveBeenCalled(); - }); - - it.each(['owner-a', undefined])( - 'routes task controls with owner %s to the local methods', - async (userId) => { - const service = buildTaskControlService(true); - - await service.handleRelayTaskControl({ - threadId: 'thread-a', - taskId: 'task-1', - action: 'cancel-task', - }); - await service.handleRelayTaskControl({ threadId: 'thread-a', action: 'cancel-thread' }); - await service.handleRelayTaskControl({ - threadId: 'thread-a', - action: 'clear-thread', - userId, - }); - - expect(service.cancelBackgroundTask).toHaveBeenCalledWith('thread-a', 'task-1'); - expect(service.cancelRun).toHaveBeenCalledWith('thread-a'); - expect(service.clearThreadState).toHaveBeenCalledWith('thread-a', userId); - expect(service.publisher.publishCommand).not.toHaveBeenCalled(); - }, - ); - - it('swallows and logs errors from a local action (no unhandled rejection on the sibling main)', async () => { - const service = buildTaskControlService(true); - service.clearThreadState.mockRejectedValue(new Error('db exploded')); - - await expect( - service.handleRelayTaskControl({ threadId: 'thread-a', action: 'clear-thread' }), - ).resolves.toBeUndefined(); - expect(service.logger.error).toHaveBeenCalledWith( - 'Failed to apply relayed Instance AI task-control', - expect.objectContaining({ threadId: 'thread-a', action: 'clear-thread' }), - ); - }); - }); -}); - describe('InstanceAiService — clearThreadState agent-builder cleanup', () => { type Internals = { threadPushRef: Map; diff --git a/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts b/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts deleted file mode 100644 index d8390fc98fd..00000000000 --- a/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import type { User } from '@n8n/db'; -import { RunDebugBuffer, RunStateRegistry } from '@n8n/instance-ai'; - -import { InstanceAiService } from '../instance-ai.service'; - -type RunDebugGatingInternals = { - instanceAiConfig: { runDebugEnabled: boolean }; - aiConfig: { modelStreamIdleTimeoutMs: number; modelStreamFirstOutputTimeoutMs: number }; - runDebugBuffer: RunDebugBuffer; - runState: RunStateRegistry; - buildOrchestratorAgentStreamOptions: ( - user: User, - threadId: string, - runId: string, - signal: AbortSignal, - ) => Record; - buildOrchestratorResumeAgentOptions: ( - user: User, - threadId: string, - runId: string, - agentRunId: string, - toolCallId: string, - signal: AbortSignal, - ) => Record; - getRunDebug: (runId: string) => ReturnType; -}; - -function createRunDebugGatingService(runDebugEnabled: boolean): RunDebugGatingInternals { - const service = Object.create(InstanceAiService.prototype) as RunDebugGatingInternals; - service.instanceAiConfig = { runDebugEnabled }; - service.aiConfig = { modelStreamIdleTimeoutMs: 90_000, modelStreamFirstOutputTimeoutMs: 180_000 }; - service.runDebugBuffer = new RunDebugBuffer(); - service.runState = new RunStateRegistry((user: User) => user.id); - return service; -} - -describe('InstanceAiService run debug gating', () => { - const user = { id: 'user-1' } as User; - const threadId = 'thread-1'; - const runId = 'run-1'; - const signal = new AbortController().signal; - - it('does not attach step hooks or create run records when run debug is disabled', () => { - const service = createRunDebugGatingService(false); - - const streamOptions = service.buildOrchestratorAgentStreamOptions( - user, - threadId, - runId, - signal, - ); - const resumeOptions = service.buildOrchestratorResumeAgentOptions( - user, - threadId, - runId, - 'agent-run-1', - 'tool-call-1', - signal, - ); - - expect(streamOptions.onStepStart).toBeUndefined(); - expect(streamOptions.onStepEnd).toBeUndefined(); - expect(streamOptions.onStepFinish).toBeUndefined(); - expect(resumeOptions.onStepStart).toBeUndefined(); - expect(resumeOptions.onStepEnd).toBeUndefined(); - expect(resumeOptions.onStepFinish).toBeUndefined(); - expect(service.getRunDebug(runId)).toBeUndefined(); - // Both terminal paths opt into raw-usage recovery so stopped/errored runs bill. - expect(streamOptions.recoverUsageOnAbort).toBe(true); - expect(resumeOptions.recoverUsageOnAbort).toBe(true); - expect(streamOptions.modelStreamIdleTimeoutMs).toBe(90_000); - expect(resumeOptions.modelStreamIdleTimeoutMs).toBe(90_000); - expect(streamOptions.modelStreamFirstOutputTimeoutMs).toBe(180_000); - expect(resumeOptions.modelStreamFirstOutputTimeoutMs).toBe(180_000); - // Both paths must carry the run's signal, or a stop cannot reach the agent - // loop and its sub-agents (AGENT-453). - expect(streamOptions.abortSignal).toBe(signal); - expect(resumeOptions.abortSignal).toBe(signal); - // Both paths must carry the request-level Anthropic cache directive; without it - // on resume, HITL turns reprocess the whole conversation uncached (INS-759). - const cacheDirective = { anthropic: { cacheControl: { type: 'ephemeral' } } }; - expect(streamOptions.providerOptions).toEqual(cacheDirective); - expect(resumeOptions.providerOptions).toEqual(cacheDirective); - }); - - it('attaches step hooks and creates run records when run debug is enabled', () => { - const service = createRunDebugGatingService(true); - - const streamOptions = service.buildOrchestratorAgentStreamOptions( - user, - threadId, - runId, - signal, - ); - const resumeOptions = service.buildOrchestratorResumeAgentOptions( - user, - threadId, - runId, - 'agent-run-1', - 'tool-call-1', - signal, - ); - - expect(typeof streamOptions.onStepStart).toBe('function'); - expect(typeof streamOptions.onStepEnd).toBe('function'); - expect(typeof streamOptions.onStepFinish).toBe('function'); - expect(typeof resumeOptions.onStepStart).toBe('function'); - expect(typeof resumeOptions.onStepEnd).toBe('function'); - expect(typeof resumeOptions.onStepFinish).toBe('function'); - expect(service.getRunDebug(runId)).toEqual( - expect.objectContaining({ - runId, - threadId, - steps: [], - workflowCode: [], - }), - ); - }); -}); diff --git a/packages/cli/src/modules/instance-ai/__tests__/run-trace-metadata.test.ts b/packages/cli/src/modules/instance-ai/__tests__/run-trace-metadata.test.ts index 64be51450d7..a50548b7359 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/run-trace-metadata.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/run-trace-metadata.test.ts @@ -96,27 +96,4 @@ describe('buildInstanceAiRunTraceMetadata', () => { first_tool_name: 'workflows', }); }); - - it('records timeout cancellation type and idle tail without extra fields', () => { - const metadata = buildInstanceAiRunTraceMetadata([], { - status: 'cancelled', - cancellationReason: 'timeout', - runTimeout: { - timedOut: true, - details: { - reason: 'idle_timeout', - surface: 'active-run', - timeoutMs: 600_000, - elapsedMs: 650_200, - idleMs: 606_400.4, - }, - }, - }); - - expect(metadata).toEqual({ - first_visible_state: 'empty', - cancellation_type: 'idle_timeout', - idle_tail_ms: 606_400, - }); - }); }); diff --git a/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts b/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts index f863b493e0c..efd306289ce 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts @@ -1,38 +1,22 @@ import type { InstanceAiErrorEvent, InstanceAiEvent } from '@n8n/api-types'; import type { Logger } from '@n8n/backend-common'; -import type { User } from '@n8n/db'; import { InstanceAiTerminalResponseGuard, orchestratorAgentId, - TerminalOutcomeStorage, - type InstanceAiTraceContext, - type ManagedBackgroundTask, - type PatchableThreadMemory, type RunStateRegistry, - type TerminalOutcome, type TerminalResponseDecision, type TerminalResponseStatus, type WorkSummary, } from '@n8n/instance-ai'; -import { getErrorMessage } from '@n8n/utils/errors/get-error-message'; - import { OperationalError } from 'n8n-workflow'; import type { Telemetry } from '@/telemetry'; import type { InProcessEventBus } from './event-bus/in-process-event-bus'; import type { InstanceAiErrorReporterService } from './instance-ai-error-reporter.service'; -import type { - InstanceAiTracingService, - MessageTraceFinalization, -} from './tracing/instance-ai-tracing.service'; type InstanceAiErrorCode = NonNullable; -function getBackgroundOutcomeResponseId(outcome: TerminalOutcome): string { - return `background-outcome:${outcome.id}`; -} - // The slice of each collaborator the terminal-outcome coordinator actually // uses. Anchored to the concrete types via `Pick` so the signatures stay in // sync with the source. @@ -50,68 +34,28 @@ export type InstanceAiTerminalOutcomeTelemetry = Pick; export type InstanceAiTerminalOutcomeErrorReporter = Pick; -export type InstanceAiTerminalOutcomeRunState = Pick< - RunStateRegistry, - 'getRunIdsForMessageGroup' | 'cancelThread' ->; - -export interface InstanceAiTerminalOutcomeSuspendedThreads { - dropPendingConfirmationsForThread(threadId: string): Promise; -} - -export type InstanceAiTerminalOutcomeTracing = Pick< - InstanceAiTracingService, - 'finalizeRunTracing' | 'buildMessageTraceMetadata' ->; +export type InstanceAiTerminalOutcomeRunState = Pick; export interface InstanceAiTerminalOutcomeServiceOptions { eventBus: InstanceAiTerminalOutcomeEventBus; - agentMemory: PatchableThreadMemory; telemetry: InstanceAiTerminalOutcomeTelemetry; errorReporter: InstanceAiTerminalOutcomeErrorReporter; logger: Logger; runState: InstanceAiTerminalOutcomeRunState; - suspendedThreads: InstanceAiTerminalOutcomeSuspendedThreads; - tracing: InstanceAiTerminalOutcomeTracing; - /** - * Publishes the run-finish event plus the success heartbeat. Owned by the - * run loop (`InstanceAiService`), which also emits it on the foreground path. - */ - publishRunFinish: ( - threadId: string, - runId: string, - status: 'completed' | 'cancelled' | 'errored', - reason?: string, - promptVersion?: string, - ) => void; } /** - * Owns the terminal-response guard and the durable replay of background task - * outcomes for Instance AI conversations. + * Owns the terminal-response guard for Instance AI conversations. * - * Two responsibilities live here: - * - * 1. **Terminal-response guard.** Whenever a run reaches a terminal state - * (completed / cancelled / errored) or starts waiting on a confirmation, - * it consults {@link InstanceAiTerminalResponseGuard} against the run's - * emitted events and publishes a fallback line when the agent went silent, - * suppresses duplicate completions, and flags malformed confirmations. - * - * 2. **Terminal-outcome durability.** Background tasks finish out of band from - * the foreground run, so their user-facing summary is persisted to - * {@link TerminalOutcomeStorage} and published as a durable text-block, then - * replayed on reconnect so a closed SSE stream never drops the result. + * Whenever a run reaches a terminal state (completed / cancelled / errored) or + * starts waiting on a confirmation, it consults + * {@link InstanceAiTerminalResponseGuard} against the run's emitted events and + * publishes a fallback line when the agent went silent, suppresses duplicate + * completions, and flags malformed confirmations. */ export class InstanceAiTerminalOutcomeService { - private readonly pendingTerminalOutcomes = new Map(); - - private terminalOutcomeStorage?: TerminalOutcomeStorage; - private readonly eventBus: InstanceAiTerminalOutcomeEventBus; - private readonly agentMemory: PatchableThreadMemory; - private readonly telemetry: InstanceAiTerminalOutcomeTelemetry; private readonly errorReporter: InstanceAiTerminalOutcomeErrorReporter; @@ -120,22 +64,12 @@ export class InstanceAiTerminalOutcomeService { private readonly runState: InstanceAiTerminalOutcomeRunState; - private readonly suspendedThreads: InstanceAiTerminalOutcomeSuspendedThreads; - - private readonly tracing: InstanceAiTerminalOutcomeTracing; - - private readonly publishRunFinish: InstanceAiTerminalOutcomeServiceOptions['publishRunFinish']; - constructor(options: InstanceAiTerminalOutcomeServiceOptions) { this.eventBus = options.eventBus; - this.agentMemory = options.agentMemory; this.telemetry = options.telemetry; this.errorReporter = options.errorReporter; this.logger = options.logger; this.runState = options.runState; - this.suspendedThreads = options.suspendedThreads; - this.tracing = options.tracing; - this.publishRunFinish = options.publishRunFinish; } async evaluateTerminalResponse( @@ -257,237 +191,4 @@ export class InstanceAiTerminalOutcomeService { this.eventBus.publish(threadId, decision.event); } } - - private createTerminalOutcomeStorage(): TerminalOutcomeStorage { - this.terminalOutcomeStorage ??= new TerminalOutcomeStorage(this.agentMemory); - return this.terminalOutcomeStorage; - } - - async finishInvalidConfirmationRun(args: { - threadId: string; - runId: string; - promptVersion?: string; - abortController: AbortController; - tracing?: InstanceAiTraceContext; - }): Promise { - this.runState.cancelThread(args.threadId); - void this.suspendedThreads.dropPendingConfirmationsForThread(args.threadId); - args.abortController.abort(); - await this.tracing.finalizeRunTracing(args.runId, args.tracing, { - status: 'error', - reason: 'invalid_confirmation_payload', - }); - this.publishRunFinish( - args.threadId, - args.runId, - 'errored', - 'I need your input to continue, but I could not display the prompt. Please try again.', - args.promptVersion, - ); - return { - status: 'error', - reason: 'invalid_confirmation_payload', - metadata: await this.tracing.buildMessageTraceMetadata(args.threadId, args.runId, { - status: 'error', - }), - }; - } - - private buildBackgroundTerminalOutcome(task: ManagedBackgroundTask): TerminalOutcome { - const status = - task.status === 'failed' ? 'failed' : task.status === 'cancelled' ? 'cancelled' : 'completed'; - const userFacingMessage = - status === 'completed' - ? `The background ${task.role} task finished.` - : status === 'cancelled' - ? `The background ${task.role} task was cancelled.` - : `The background ${task.role} task failed before I could complete that part.`; - - return { - id: `${task.messageGroupId ?? task.runId}:${task.taskId}:${status}`, - threadId: task.threadId, - runId: task.runId, - messageGroupId: task.messageGroupId, - correlationId: task.messageGroupId, - taskId: task.taskId, - agentId: task.agentId, - status, - userFacingMessage, - createdAt: new Date().toISOString(), - }; - } - - async replayUndeliveredTerminalOutcomes(threadId: string): Promise { - const storage = this.createTerminalOutcomeStorage(); - const noOutcomes: TerminalOutcome[] = []; - const persistedOutcomes = await storage.getUndelivered(threadId).catch((error) => { - this.logger.warn('Failed to load undelivered Instance AI terminal outcomes', { - threadId, - error: getErrorMessage(error), - }); - return noOutcomes; - }); - const inMemoryOutcomes = [...this.pendingTerminalOutcomes.values()].filter( - (outcome) => outcome.threadId === threadId, - ); - const outcomes = new Map(); - for (const outcome of [...persistedOutcomes, ...inMemoryOutcomes]) { - outcomes.set(outcome.id, outcome); - } - const persistedOutcomeIds = new Set(persistedOutcomes.map((outcome) => outcome.id)); - - for (const outcome of outcomes.values()) { - const responseId = getBackgroundOutcomeResponseId(outcome); - let delivery: 'published' | 'already-emitted' | 'dropped' = 'dropped'; - try { - delivery = await this.publishTerminalOutcomeLine(outcome, responseId); - } catch (error) { - this.logger.warn('Failed to replay Instance AI terminal outcome', { - threadId, - runId: outcome.runId, - taskId: outcome.taskId, - error: getErrorMessage(error), - }); - } - // Left undelivered on purpose: the next replay retries it. - if (delivery === 'dropped') continue; - - const action = delivery === 'published' ? 'replay_event' : 'already-emitted'; - - if (persistedOutcomeIds.has(outcome.id)) { - await storage - .markDelivered(threadId, outcome.id, new Date().toISOString()) - .catch((error) => { - this.logger.warn('Failed to mark Instance AI terminal outcome as delivered', { - threadId, - runId: outcome.runId, - taskId: outcome.taskId, - error: getErrorMessage(error), - }); - }); - } - this.pendingTerminalOutcomes.delete(outcome.id); - this.telemetry.track('instance_ai_terminal_response_decision', { - thread_id: threadId, - run_id: outcome.runId, - message_group_id: outcome.messageGroupId, - task_id: outcome.taskId, - source: 'terminal_outcome_replay', - status: outcome.status, - action, - visibility_source: 'background-outcome', - }); - } - } - - /** - * Publish the outcome line as a durable text-block and read it back. - * `publish` only enqueues — the drain persists asynchronously and settles - * flush waiters even when it had to drop a batch — so only the read-back - * makes the line trustworthy as a delivery record. 'dropped' means it never - * reached the log; the caller must leave the outcome undelivered so a later - * replay retries it. - */ - private async publishTerminalOutcomeLine( - outcome: TerminalOutcome, - responseId: string, - ): Promise<'published' | 'already-emitted' | 'dropped'> { - const isOutcomeLine = (event: InstanceAiEvent) => event.responseId === responseId; - const alreadyPublished = ( - await this.eventBus.getEventsForRun(outcome.threadId, outcome.runId) - ).some(isOutcomeLine); - if (alreadyPublished) return 'already-emitted'; - - this.eventBus.publish(outcome.threadId, { - type: 'text-block', - runId: outcome.runId, - agentId: orchestratorAgentId(outcome.runId), - responseId, - payload: { text: outcome.userFacingMessage }, - }); - // The adapter's read settles the thread's drain before querying, so the - // block is either in the log by now or was dropped. - const durable = (await this.eventBus.getEventsForRun(outcome.threadId, outcome.runId)).some( - isOutcomeLine, - ); - return durable ? 'published' : 'dropped'; - } - - async recordBackgroundTerminalOutcome(task: ManagedBackgroundTask): Promise { - const outcome = this.buildBackgroundTerminalOutcome(task); - let persisted = false; - try { - await this.createTerminalOutcomeStorage().upsert(task.threadId, outcome); - persisted = true; - } catch (error) { - this.pendingTerminalOutcomes.set(outcome.id, outcome); - this.logger.warn('Failed to persist Instance AI terminal outcome', { - threadId: task.threadId, - runId: task.runId, - taskId: task.taskId, - error: getErrorMessage(error), - }); - this.telemetry.track('instance_ai_terminal_outcome_persistence_failure', { - thread_id: task.threadId, - run_id: task.runId, - task_id: task.taskId, - status: outcome.status, - phase: 'metadata', - }); - } - - const responseId = getBackgroundOutcomeResponseId(outcome); - let delivery: 'published' | 'already-emitted' | 'dropped' = 'dropped'; - try { - delivery = await this.publishTerminalOutcomeLine(outcome, responseId); - } catch (error) { - this.logger.warn('Failed to publish Instance AI terminal outcome line', { - threadId: task.threadId, - runId: task.runId, - taskId: task.taskId, - error: getErrorMessage(error), - }); - } - if (delivery === 'dropped') { - // Leave the outcome undelivered — the metadata row (or the pending-map - // entry when the upsert failed too) makes the next replay retry it. - this.telemetry.track('instance_ai_terminal_outcome_persistence_failure', { - thread_id: task.threadId, - run_id: task.runId, - task_id: task.taskId, - status: outcome.status, - phase: 'event', - }); - return; - } - - this.telemetry.track('instance_ai_terminal_response_decision', { - thread_id: task.threadId, - run_id: task.runId, - message_group_id: task.messageGroupId, - task_id: task.taskId, - source: 'background_outcome', - status: outcome.status, - action: delivery === 'published' ? 'emit' : 'already-emitted', - visibility_source: 'background-outcome', - }); - - if (!persisted) return; - - try { - await this.createTerminalOutcomeStorage().markDelivered( - task.threadId, - outcome.id, - new Date().toISOString(), - ); - this.pendingTerminalOutcomes.delete(outcome.id); - } catch (error) { - this.logger.warn('Failed to mark Instance AI terminal outcome as delivered', { - threadId: task.threadId, - runId: task.runId, - taskId: task.taskId, - error: getErrorMessage(error), - }); - } - } } diff --git a/packages/cli/src/modules/instance-ai/instance-ai-test.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai-test.controller.ts index a8e78f1e2d8..d05434d022f 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai-test.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai-test.controller.ts @@ -1,11 +1,9 @@ -import { ProjectRepository, UserRepository, WorkflowRepository } from '@n8n/db'; -import { Body, Delete, Get, Param, Post, RestController } from '@n8n/decorators'; +import { WorkflowRepository } from '@n8n/db'; +import { Delete, Get, Param, Post, RestController } from '@n8n/decorators'; import type { Request, Response } from 'express'; -import { v4 as uuidv4 } from 'uuid'; import { ForbiddenError } from '@n8n/errors'; -import { InstanceAiMemoryService } from './instance-ai-memory.service'; import { InstanceAiService } from './instance-ai.service'; import { AgentExecutionThreadRepository } from '../agents/repositories/agent-execution-thread.repository'; import { ASSISTANT_AGENT_ID } from './assistant-turn-options'; @@ -20,9 +18,6 @@ export class InstanceAiTestController { private readonly instanceAiService: InstanceAiService, private readonly threadRepo: AgentExecutionThreadRepository, private readonly workflowRepo: WorkflowRepository, - private readonly userRepo: UserRepository, - private readonly memoryService: InstanceAiMemoryService, - private readonly projectRepo: ProjectRepository, ) {} @Post('/test/tool-trace', { skipAuth: true }) @@ -44,30 +39,18 @@ export class InstanceAiTestController { } @Get('/test/idle', { skipAuth: true }) - getIdleState() { + async getIdleState() { this.assertTraceReplayEnabled(); - return { idle: !this.instanceAiService.hasRunningWorkForTest() }; - } - - @Post('/test/background-timeout/start', { skipAuth: true }) - async startBackgroundTimeoutSimulation(@Body payload: { userId: string; threadId?: string }) { - this.assertTraceReplayEnabled(); - const threadId = payload.threadId ?? uuidv4(); - const user = await this.userRepo.findOneByOrFail({ id: payload.userId }); - const personalProject = await this.projectRepo.getPersonalProjectForUserOrFail(user.id); - - await this.memoryService.ensureThread(user.id, threadId, personalProject.id, { - source: 'playwright', - origin: 'internal', + const threads = await this.threadRepo.find({ + where: { agentId: ASSISTANT_AGENT_ID }, + select: ['id'], }); - return await this.instanceAiService.startStuckBackgroundTaskForTest(user, threadId); - } - - @Post('/test/liveness-sweep', { skipAuth: true }) - async runLivenessSweep(@Body payload: { now?: number } = {}) { - this.assertTraceReplayEnabled(); - await this.instanceAiService.runLivenessSweepForTest(payload.now); - return { ok: true }; + for (const { id } of threads) { + if ((await this.instanceAiService.getLiveRun(id)).status === 'running') { + return { idle: false }; + } + } + return { idle: true }; } @Delete('/test/tool-trace/:slug', { skipAuth: true }) @@ -86,14 +69,13 @@ export class InstanceAiTestController { * test's recorded responses (observed: a follow-up test's recording referencing * the previous test's workflow name). * - * This endpoint cancels background tasks, clears per-thread in-memory state, - * and deletes all thread + workflow rows. + * This endpoint clears per-thread in-memory state and deletes all thread + + * workflow rows. */ @Post('/test/reset', { skipAuth: true }) async reset() { this.assertTraceReplayEnabled(); - this.instanceAiService.cancelAllBackgroundTasks(); this.instanceAiService.clearTraceContextsForTest(); const threads = await this.threadRepo.find({ diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index 12ce86afd84..ef955bfe3da 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -11,7 +11,6 @@ import { InstanceAiSendMessageRequest, InstanceAiEventsQuery, instanceAiGatewayKeySchema, - InstanceAiCorrectTaskRequest, InstanceAiEnsureThreadRequest, InstanceAiPersistPendingAgentRequest, InstanceAiThreadMessagesQuery, @@ -154,11 +153,6 @@ export class InstanceAiController { } } - private requireRunDebugEnabled(): void { - if (!this.instanceAiService.isRunDebugEnabled()) { - throw new NotFoundError('Run debug is not enabled'); - } - } // Each BrotliCompress stream allocates ~8.6 MB of native memory for its // dictionary, and the compression middleware retains streams via closures on // the response object for the lifetime of the HTTP keep-alive connection. @@ -339,12 +333,7 @@ export class InstanceAiController { req.once('close', cleanup); res.once('finish', cleanup); - // 2. Re-publish any terminal outcomes that never reached the client. - if (ownership === 'owned') { - await this.instanceAiService.replayUndeliveredTerminalOutcomes(threadId); - } - - // 3. Set SSE headers. + // 2. Set SSE headers. // Disable response compression — SSE streams small chunks where compression // overhead exceeds the benefit, and each Brotli compressor retains ~8.6 MB // of native memory for the lifetime of the connection. @@ -355,7 +344,7 @@ export class InstanceAiController { res.setHeader('X-Accel-Buffering', 'no'); res.flushHeaders(); - // 4. Determine replay cursor + // 3. Determine replay cursor // Last-Event-ID header (browser auto-reconnect) takes precedence over query param. // Both are validated as non-negative integers; invalid values fall back to 0. const headerValue = req.headers['last-event-id']; @@ -363,7 +352,7 @@ export class InstanceAiController { const cursor = Number.isFinite(parsedHeader) && parsedHeader >= 0 ? parsedHeader : (query.lastEventId ?? 0); - // 5. Collect live message groups. + // 4. Collect live message groups. // Multiple groups can be active simultaneously when a background task // from an older turn outlives its original turn. const liveRun = await this.instanceAiService.getLiveRun(threadId); @@ -410,7 +399,7 @@ export class InstanceAiController { ); }; - // 6. Replay missed events from the durable log — survives restarts and is + // 5. Replay missed events from the durable log — survives restarts and is // valid on any main (the table is in the shared DB). The reads are // async, so live events can land mid-bootstrap: buffer them across // every await (the @@ -553,7 +542,7 @@ export class InstanceAiController { bootstrapping = false; - // 7. Keep-alive + // 6. Keep-alive keepAlive = setInterval(() => { res.write(': ping\n\n'); res.flush?.(); @@ -636,35 +625,6 @@ export class InstanceAiController { return { ok: true }; } - @Post('/chat/:threadId/tasks/:taskId/cancel') - @GlobalScope('instanceAi:message') - async cancelTask( - req: AuthenticatedRequest, - _res: Response, - @Param('threadId') threadId: string, - @Param('taskId') taskId: string, - ) { - this.requireInstanceAiEnabled(); - await this.assertThreadAccess(req.user.id, threadId); - await this.instanceAiService.routeCancelBackgroundTask(threadId, taskId); - return { ok: true }; - } - - @Post('/chat/:threadId/tasks/:taskId/correct') - @GlobalScope('instanceAi:message') - async correctTask( - req: AuthenticatedRequest, - _res: Response, - @Param('threadId') threadId: string, - @Param('taskId') taskId: string, - @Body payload: InstanceAiCorrectTaskRequest, - ) { - this.requireInstanceAiEnabled(); - await this.assertThreadAccess(req.user.id, threadId); - await this.instanceAiService.routeCorrectionToTask(threadId, taskId, payload.message); - return { ok: true }; - } - // ── Preference card (the save_user_preference result in the chat) ──────── // // The thread check is the ownership boundary. The row check is inside @@ -1033,7 +993,6 @@ export class InstanceAiController { ) { this.requireInstanceAiEnabled(); await this.assertThreadAccess(req.user.id, threadId); - await this.instanceAiService.replayUndeliveredTerminalOutcomes(threadId); // ?raw=true returns the old format for the thread inspector if (query.raw === 'true') { @@ -1100,35 +1059,6 @@ export class InstanceAiController { return await this.instanceAiService.getThreadStatus(threadId); } - @Get('/debug/runs/:runId') - @GlobalScope('instanceAi:message') - async getRunDebug(req: AuthenticatedRequest, _res: Response, @Param('runId') runId: string) { - this.requireInstanceAiEnabled(); - this.requireRunDebugEnabled(); - const record = this.instanceAiService.getRunDebug(runId); - if (!record) { - throw new NotFoundError('Run debug record not found'); - } - await this.assertThreadAccess(req.user.id, record.threadId); - return record; - } - - @Get('/debug/threads/:threadId/runs') - @GlobalScope('instanceAi:message') - async listThreadDebugRuns( - req: AuthenticatedRequest, - _res: Response, - @Param('threadId') threadId: string, - ) { - this.requireInstanceAiEnabled(); - this.requireRunDebugEnabled(); - await this.assertThreadAccess(req.user.id, threadId); - return { - threadId, - runs: this.instanceAiService.listThreadDebugRuns(threadId), - }; - } - // ── Evaluation endpoints ────────────────────────────────────────────────── // Runs for minutes; the eval client (N8nClient) disables undici's 300s timeout for it. diff --git a/packages/cli/src/modules/instance-ai/instance-ai.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.service.ts index f8cb2433a6d..05a21c748c9 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.service.ts @@ -48,7 +48,6 @@ import { type InstanceAiConfig, } from '@n8n/config'; import { UserRepository, type User } from '@n8n/db'; -import { OnPubSubEvent } from '@n8n/decorators'; import { Container, Service } from '@n8n/di'; import { CONCISE_PROMPT_VERSION, @@ -69,17 +68,13 @@ import { threadProvenanceMetadata, createInternalOperationTraceContext, emitAgentSnapshotTraceEvent, - createInstanceAiLivenessPolicyConfig, - InstanceAiLivenessPolicy, McpClientManager, createDomainAccessTracker, - BackgroundTaskManager, MemoryTaskRegistry, classifyAttachments, buildAttachmentManifest, getDateTimeSection, isParseableAttachment, - enrichMessageWithBackgroundTasks, isQuotaExhaustedError, PlannedTaskCoordinator, PlannedTaskStorage, @@ -88,7 +83,6 @@ import { RunStateRegistry, shutdownProductTelemetryProviders, tokenUsageToBuilderUsageItems, - RunDebugBuffer, truncateToTitle, generateTitleForRun, patchThread, @@ -102,7 +96,6 @@ import { saveAgentBuilderTarget, type DomainAccessTracker, type InstanceAiContext, - type ManagedBackgroundTask, type McpServerConfig, type ModelConfig, type AgentSnapshotArtifact, @@ -113,7 +106,6 @@ import { type PlannedTaskService, type PlannedWorkflowVerification, type ServiceProxyConfig, - type SuspendedRunState, type WorkflowBuildOutcome, type ProjectSummary, type WorkflowLoopWorkItemRecord, @@ -123,7 +115,6 @@ import { type WorkSummary, deriveInstanceContextReach, type RunTokenUsage, - type RunDebugRecord, WorkflowTaskCoordinator, WorkflowLoopStorage, ThreadTaskStorage, @@ -137,7 +128,7 @@ import { getErrorMessage } from '@n8n/utils/errors/get-error-message'; import { isRecord } from '@n8n/utils/is-record'; import { lazyImport } from '@n8n/utils/lazy-import'; import { setSchemaBaseDirs } from '@n8n/workflow-sdk'; -import { ErrorReporter, InstanceSettings } from 'n8n-core'; +import { ErrorReporter } from 'n8n-core'; import { OperationalError, UnexpectedError, UserError } from 'n8n-workflow'; import { nanoid } from 'nanoid'; @@ -148,8 +139,6 @@ import { InstanceAiAgentContextAdapterService } from '@/modules/agents/instance- import { modelStreamStallOptions } from '@/modules/agents/model-stream-stall-options'; import { userHasScopes } from '@/permissions.ee/check-access'; import { Push } from '@/push'; -import { Publisher } from '@/scaling/pubsub/publisher.service'; -import type { PubSubCommandMap } from '@/scaling/pubsub/pubsub.event-map'; import { AI_PREFERENCES_CLEARED_BLOCK, AiPreferenceService, @@ -186,7 +175,6 @@ import { import { InstanceAiGatewayService } from './instance-ai-gateway.service'; import { InstanceAiMemoryService } from './instance-ai-memory.service'; import { InstanceAiModelService } from './instance-ai-model.service'; -import { InstanceAiRunProbe } from './instance-ai-run-probe'; import { InstanceAiSettingsService } from './instance-ai-settings.service'; import { InstanceAiTemporaryWorkflowService } from './instance-ai-temporary-workflow.service'; import { InstanceAiTerminalOutcomeService } from './instance-ai-terminal-outcome.service'; @@ -213,7 +201,6 @@ import { } from './internal-messages'; import { loadOnboardingSkill } from './onboarding'; import { ONBOARDING_OPENING } from './onboarding-opening'; -import { INSTANCE_AI_RUN_TIMEOUT_REASON, InstanceAiLivenessService } from './liveness'; import { InstanceAiMcpRegistryService } from './mcp'; import { runMetricsModelLabel } from './observability'; import { @@ -520,14 +507,6 @@ function buildSuspensionTraceOutputs(runId: string, suspension: SuspensionInfo | }; } -/** - * Upper bound on how long `shutdown()` will wait for in-flight executeRun / - * processResumedStream promises to drain after their abortControllers fire. - * Sized well below n8n's `gracefulShutdownTimeoutInS` (30s default) so a - * stuck agent can't burn the whole budget here. - */ -const INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS = 5 * 1000; - function isTextMessagePart(part: unknown): part is { type: 'text'; text: string } { return ( typeof part === 'object' && @@ -539,8 +518,20 @@ function isTextMessagePart(part: unknown): part is { type: 'text'; text: string ); } +/** Planned tasks the scheduler may start in one tick. */ const MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD = 5; +/** The finished build task a workflow-verification follow-up reports on. */ +type WorkflowVerificationSourceTask = { + taskId: string; + role: string; + status: 'completed'; + result?: string; + error?: string; + plannedTaskId?: string; + workItemId?: string; +}; + /** * Circuit breaker for machine-started follow-up runs (verification, synthesize, * replan, …). A follow-up that dies before the agent can settle its trigger @@ -613,9 +604,7 @@ export class InstanceAiService { private readonly formBaseUrl: string; - private readonly runState = new RunStateRegistry((user) => user.id); - - private readonly backgroundTasks: BackgroundTaskManager; + private readonly runState = new RunStateRegistry(); private readonly memoryTaskRegistry = new MemoryTaskRegistry(); @@ -669,12 +658,6 @@ export class InstanceAiService { private readonly terminalOutcome: InstanceAiTerminalOutcomeService; - private readonly liveness: InstanceAiLivenessService>; - - /** Owns DB persistence of suspended runs + orphan-confirmation restoration. */ - - private readonly runDebugBuffer = new RunDebugBuffer(); - /** Default IANA timezone for the instance (from GENERIC_TIMEZONE env var). */ private readonly defaultTimeZone: string; @@ -684,27 +667,9 @@ export class InstanceAiService { private readonly taskProjector: WorkflowVerificationTaskProjector; - /** - * In-flight `executeRun` / `processResumedStream` promises. Tracked so - * `shutdown()` can drain them before n8n closes the DB connection — the - * SDK's abort-driven `cleanupRun` and `executeRun`'s `finally` block - * both write to the DB during teardown, and racing the connection close - * surfaces as `DriverAlreadyReleasedError` for callers. - */ - private readonly inFlightExecutions = new Set>(); - - /** - * Run IDs whose post-stream terminal handling should be skipped when their - * abort fires. Populated by `shutdown()` for runs that were sitting on an - * inline HITL confirmation and for suspended runs, and drained by - * `shouldPreserveHitlOnShutdown(runId)`. - */ - private readonly preserveHitlOnShutdown = new Set(); - constructor( logger: Logger, globalConfig: GlobalConfig, - private readonly instanceSettings: InstanceSettings, private readonly adapterService: InstanceAiAdapterService, private readonly eventBus: InProcessEventBus, private readonly eventLog: DurableEventLog, @@ -727,10 +692,8 @@ export class InstanceAiService { ssrfProtectionService: SsrfProtectionService, private readonly eventService: EventService, private readonly evalCredentialAllowlists: EvalThreadCredentialAllowlistService, - runProbe: InstanceAiRunProbe, private readonly modelService: InstanceAiModelService, private readonly creditService: InstanceAiCreditService, - private readonly publisher: Publisher, private readonly instanceAiErrorReporter: InstanceAiErrorReporterService, private readonly push: Push, private readonly conversationHistoryService: InstanceAiConversationHistoryService, @@ -739,7 +702,6 @@ export class InstanceAiService { private readonly aiUsageService: AiUsageService, ) { this.logger = logger.scoped('instance-ai'); - runProbe.registerActiveRunCountProvider(() => this.runState.activeRunCount()); this.workflowObligations = new WorkflowVerificationObligationService( this.agentMemory, (threadId) => this.runState.isSetupPanelEnabled(threadId), @@ -752,26 +714,6 @@ export class InstanceAiService { ); this.instanceAiConfig = globalConfig.instanceAi; this.aiConfig = globalConfig.ai; - this.backgroundTasks = new BackgroundTaskManager( - MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD, - this.instanceAiConfig.maxConcurrentSubAgents, - ); - const livenessPolicyConfig = createInstanceAiLivenessPolicyConfig({ - confirmationTimeoutMs: this.instanceAiConfig.confirmationTimeout, - }); - this.liveness = new InstanceAiLivenessService>({ - policy: new InstanceAiLivenessPolicy(livenessPolicyConfig), - backgroundTaskIdleTimeoutMs: livenessPolicyConfig.backgroundTaskIdleTimeoutMs, - runState: this.runState, - backgroundTasks: this.backgroundTasks, - eventBus: this.eventBus, - logger: this.logger, - // Suspended turns live in Agents checkpoints now. Their TTL owns expiry. - finalizeCancelledSuspendedRun: () => {}, - onPendingConfirmationRejected: (requestId) => { - void requestId; - }, - }); this.tracing = new InstanceAiTracingService({ logger: this.logger, // `first_visible_state` has to see the run's streamed text, which lives @@ -779,7 +721,6 @@ export class InstanceAiService { eventReader: { getEventsForRun: async (threadId, runId) => await this.readRunEvents(threadId, [runId]), }, - runState: this.runState, eventLog: this.eventLogRepository, aiService: this.aiService, }); @@ -787,8 +728,6 @@ export class InstanceAiService { config: this.instanceAiConfig, logger: this.logger, errorReporter: this.errorReporter, - runState: this.runState, - backgroundTasks: this.backgroundTasks, settingsService: this.settingsService, aiService: this.aiService, resolveTracingConfig: async (threadId, userId) => { @@ -807,19 +746,10 @@ export class InstanceAiService { getEventsForRun: async (threadId, runId) => await this.readRunEvents(threadId, [runId]), getEventsForRuns: async (threadId, runIds) => await this.readRunEvents(threadId, runIds), }, - agentMemory: this.agentMemory, telemetry: this.telemetry, errorReporter: this.instanceAiErrorReporter, logger: this.logger, runState: this.runState, - // Pending cards are suspended Agents checkpoints; nothing separate to drop. - suspendedThreads: { dropPendingConfirmationsForThread: async () => {} }, - tracing: this.tracing, - publishRunFinish: (threadId, runId, status, reason, promptVersion) => { - this.publishRunFinish(threadId, runId, status, reason, undefined, undefined, { - promptVersion, - }); - }, }); this.defaultTimeZone = globalConfig.generic.timezone; const restEndpoint = globalConfig.endpoints.rest; @@ -842,7 +772,6 @@ export class InstanceAiService { }); }); }); - this.liveness.start(); } private async createProxyRunConfig(user: Pick): Promise<{ @@ -1164,10 +1093,6 @@ export class InstanceAiService { }); } - isRunDebugEnabled(): boolean { - return this.instanceAiConfig.runDebugEnabled; - } - /** What observational memory holds for a thread: the live observations and the * compaction cursor. An eval asserts on these rows instead of parsing the * rendered system prompt. Refuses a thread the caller does not own, so the @@ -1197,21 +1122,6 @@ export class InstanceAiService { }; } - getRunDebug(runId: string) { - return this.runDebugBuffer.get(runId); - } - - listThreadDebugRuns(threadId: string) { - return this.runDebugBuffer.listByThread(threadId).map((record: RunDebugRecord) => ({ - runId: record.runId, - threadId: record.threadId, - startedAt: record.startedAt, - stepCount: record.steps.length, - workflowCodeCount: record.workflowCode.length, - label: record.label, - })); - } - clearTraceContextsForTest(): void { this.tracing.clearTraceContextsForTest(); } @@ -1246,7 +1156,6 @@ export class InstanceAiService { ) { throw new BadRequestError(`Unknown Instance AI prompt version "${promptVersion}"`); } - this.liveness.clearThreadState(threadId); const { runId } = await this.enqueueAssistantTurn(user, threadId, message, { runId: `run_${nanoid()}`, messageGroupId: `mg_${nanoid()}`, @@ -1270,109 +1179,6 @@ export class InstanceAiService { void this.cancelAwaitingApprovalPlan(threadId); } - /** Send a correction message to a running background task. */ - sendCorrectionToTask( - threadId: string, - taskId: string, - correction: string, - ): 'queued' | 'task-completed' | 'task-not-found' { - return this.backgroundTasks.queueCorrection(threadId, taskId, correction); - } - - /** Cancel a single background task by ID. */ - cancelBackgroundTask(threadId: string, taskId: string): void { - const task = this.backgroundTasks.cancelTask(threadId, taskId); - if (!task) return; - - void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled'); - this.eventBus.publish(threadId, { - type: 'agent-completed', - runId: task.runId, - agentId: task.agentId, - payload: { role: task.role, result: '', status: 'cancelled' }, - }); - - void this.terminalOutcome.recordBackgroundTerminalOutcome(task); - - const user = this.runState.getThreadUser(threadId); - if (user) { - void this.handlePlannedTaskSettlement(user, task, 'cancelled'); - } - } - - // ── Cross-main task-control routing ─────────────────────────────────────── - // User actions (correct/cancel/clear) can land on a different main than the - // one running the task/run. Each `route*` method applies the action locally - // and, when the target isn't local (or is thread-wide), broadcasts so the - // owning main applies it. Broadcast + local-gate — no shared ownership store. - // `applyTaskControlLocally` is the single action → local-method mapping, - // shared by the route entry points and the `@OnPubSubEvent` relay handler - // (which never re-broadcasts, so there's no loop). These wrappers are the - // controller entry points; internal callers keep using the local methods - // directly so they don't trigger cross-main broadcasts. - - private broadcastTaskControl(payload: PubSubCommandMap['relay-instance-ai-task-control']): void { - if (!this.instanceSettings.isMultiMain) return; - void this.publisher - .publishCommand({ command: 'relay-instance-ai-task-control', payload }) - .catch((error: unknown) => - this.logger.error('Failed to relay Instance AI task-control to sibling mains', { - threadId: payload.threadId, - action: payload.action, - error, - }), - ); - } - - /** Apply a task-control action to this main's local slice of the thread. - * Returns whether the action's target was found locally: task-scoped - * actions report a local hit to gate re-broadcast, thread-wide actions - * always report a miss so they fan out to every main. */ - private async applyTaskControlLocally({ - threadId, - taskId, - action, - correction, - userId, - }: PubSubCommandMap['relay-instance-ai-task-control']): Promise { - switch (action) { - case 'correct': - // A relay without its correction text is malformed: nothing to apply. - if (!taskId || correction === undefined) return true; - return this.sendCorrectionToTask(threadId, taskId, correction) !== 'task-not-found'; - case 'cancel-task': { - if (!taskId) return true; - const isLocal = this.backgroundTasks - .getTaskSnapshots(threadId) - .some((task) => task.taskId === taskId); - this.cancelBackgroundTask(threadId, taskId); - return isLocal; - } - case 'cancel-thread': - // A thread's run + tasks can be spread across mains, so always fan out. - this.cancelRun(threadId); - return false; - case 'clear-thread': - await this.clearThreadState(threadId, userId); - return false; - } - } - - private async routeTaskControl( - payload: PubSubCommandMap['relay-instance-ai-task-control'], - ): Promise { - const foundLocally = await this.applyTaskControlLocally(payload); - if (!foundLocally) this.broadcastTaskControl(payload); - } - - async routeCorrectionToTask(threadId: string, taskId: string, correction: string): Promise { - await this.routeTaskControl({ threadId, taskId, action: 'correct', correction }); - } - - async routeCancelBackgroundTask(threadId: string, taskId: string): Promise { - await this.routeTaskControl({ threadId, taskId, action: 'cancel-task' }); - } - /** Stop the running turn, or cancel the suspended one. Works from any main. */ async routeCancelRun(user: User, threadId: string): Promise { const thread = await this.systemAgents.getThread(ASSISTANT_AGENT_ID, user, threadId); @@ -1390,163 +1196,9 @@ export class InstanceAiService { } } + /** Thread deletion clears this main's in-memory state only. */ async routeClearThreadState(threadId: string, userId?: string): Promise { - await this.routeTaskControl({ - threadId, - action: 'clear-thread', - ...(userId ? { userId } : {}), - }); - } - - /** Apply a task-control action relayed from another main to this main's local - * slice of the thread. Never re-broadcasts. Not self-sent, so this never - * fires on the originating main. */ - @OnPubSubEvent('relay-instance-ai-task-control', { instanceType: 'main' }) - async handleRelayTaskControl( - payload: PubSubCommandMap['relay-instance-ai-task-control'], - ): Promise { - // Guard the whole handler: it runs as a fire-and-forget pubsub listener, so - // a throw (e.g. from the async clearThreadState) would surface as an - // unhandled rejection on this sibling main instead of being contained. - try { - await this.applyTaskControlLocally(payload); - } catch (error) { - this.logger.error('Failed to apply relayed Instance AI task-control', { - threadId: payload.threadId, - taskId: payload.taskId, - action: payload.action, - error, - }); - } - } - - /** Cancel all background tasks across all threads. Test-only. */ - cancelAllBackgroundTasks(): number { - const cancelled = this.backgroundTasks.cancelAll(); - for (const task of cancelled) { - void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled'); - } - return cancelled.length; - } - - async startStuckBackgroundTaskForTest( - user: User, - threadId: string, - ): Promise<{ - threadId: string; - runId: string; - messageGroupId: string; - taskId: string; - agentId: string; - timeoutAt: number; - }> { - const messageId = `msg_${nanoid()}`; - const messageText = 'I started a background workflow-builder task.'; - const { runId, messageGroupId } = this.runState.startRun({ threadId, user }); - if (!messageGroupId) { - throw new UnexpectedError('Failed to create message group for timeout simulation'); - } - const taskId = `task_${nanoid()}`; - const agentId = `agent_${nanoid()}`; - - this.eventBus.publish(threadId, { - type: 'run-start', - runId, - agentId: orchestratorAgentId(runId), - userId: user.id, - payload: { messageId, messageGroupId }, - }); - this.eventBus.publish(threadId, { - type: 'text-delta', - runId, - agentId: orchestratorAgentId(runId), - responseId: `test-background-start:${runId}`, - payload: { text: messageText }, - }); - this.eventBus.publish(threadId, { - type: 'agent-spawned', - runId, - agentId, - payload: { - parentId: orchestratorAgentId(runId), - role: 'workflow-builder', - tools: [], - taskId, - kind: 'builder', - title: 'Building workflow', - subtitle: 'Timeout simulation', - goal: 'Simulate a stuck background task timeout', - }, - }); - - // Persist the assistant message so the UI can render it after navigation. - await this.agentMemory.saveMessages({ - threadId, - resourceId: user.id, - messages: [ - { - id: messageId, - createdAt: new Date(), - type: 'llm', - role: 'assistant', - content: [{ type: 'text', text: messageText }], - }, - ], - }); - - const outcome = this.backgroundTasks.spawn({ - taskId, - threadId, - runId, - role: 'workflow-builder', - agentId, - messageGroupId, - run: async (signal) => - await new Promise((resolve) => { - signal.addEventListener('abort', () => resolve('aborted'), { once: true }); - }), - onFailed: (task) => { - this.eventBus.publish(threadId, { - type: 'agent-completed', - runId, - agentId, - payload: { - role: task.role, - result: '', - error: task.error ?? 'Unknown error', - }, - }); - }, - onSettled: async (task) => { - await this.terminalOutcome.recordBackgroundTerminalOutcome(task); - }, - }); - - if (outcome.status !== 'started') { - throw new UnexpectedError('Failed to start stuck background task simulation'); - } - - this.runState.clearActiveRun(threadId); - this.eventBus.publish(threadId, { - type: 'run-finish', - runId, - agentId: orchestratorAgentId(runId), - userId: user.id, - payload: { status: 'completed' }, - }); - - return { - threadId, - runId, - messageGroupId, - taskId, - agentId, - timeoutAt: outcome.task.lastActivityAt + this.liveness.backgroundTaskIdleTimeoutMs + 1, - }; - } - - async runLivenessSweepForTest(now?: number): Promise { - await this.liveness.sweepTimedOutWork(now); + await this.clearThreadState(threadId, userId); } // ── Gateway lifecycle (delegated to LocalGatewayRegistry) ─────────────── @@ -1561,17 +1213,6 @@ export class InstanceAiService { return this.tracing.getTraceEvents(slug); } - hasRunningWorkForTest(): boolean { - const threadIds = new Set(this.tracing.getTrackedThreadIds()); - - for (const threadId of threadIds) { - if (this.runState.getActiveRunId(threadId)) return true; - if (this.backgroundTasks.getRunningTasks(threadId).length > 0) return true; - } - - return false; - } - activateTraceSlug(slug: string): void { this.tracing.activateTraceSlug(slug); } @@ -1585,31 +1226,7 @@ export class InstanceAiService { * Must be called when a thread is deleted so the maps don't leak. */ async clearThreadState(threadId: string, userId?: string): Promise { - this.liveness.clearThreadState(threadId); - - // Clear run-state registry entries (active/suspended runs, confirmations, - // user, time zone, and message-group mappings). - const { active, suspended } = this.runState.clearThread(threadId); - if (active) { - active.abortController.abort(); - await this.tracing.finalizeRunTracing(active.runId, active.tracing, { - status: 'cancelled', - reason: 'thread_cleared', - }); - } - if (suspended) { - suspended.abortController.abort(); - await this.tracing.finalizeRunTracing(suspended.runId, suspended.tracing, { - status: 'cancelled', - reason: 'thread_cleared', - }); - } - - // Cancel background tasks belonging to this thread - for (const task of this.backgroundTasks.cancelThread(threadId)) { - task.abortController.abort(); - await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled'); - } + this.runState.clearThread(threadId); await this.tracing.finalizeRemainingMessageTraceRoots(threadId, { status: 'cancelled', reason: 'thread_cleared', @@ -1646,79 +1263,6 @@ export class InstanceAiService { } async shutdown(): Promise { - this.liveness.shutdown(); - - const { activeRuns, suspendedRuns, pendingThreadIds } = this.runState.shutdown(); - const threadsWithPendingHitl = new Set(pendingThreadIds); - for (const run of activeRuns) { - // Runs holding an inline HITL confirmation (`create-tasks`, - // sub-agent `ask-user`) sit in `activeRuns` because the orchestrator - // is alive — it's just awaiting the in-process Promise. Their - // `instance_ai_pending_confirmations` row survives the restart and - // `handleOrphanedConfirmation` will issue the user-visible - // `restart_lost_confirmation` UserError + `run-finish` when (if) - // the user clicks confirm. If we publish run-finish here, the fold - // would render the plan/ask card as cancelled before the user has a - // chance to see it on reload. - if (threadsWithPendingHitl.has(run.threadId)) { - await this.tracing.finalizeRunTracing(run.runId, run.tracing, { - status: 'cancelled', - reason: 'service_shutdown', - }); - // Record the policy *before* the abort fires so the run's catch - // handler (which runs synchronously off the abort) sees the - // flag. The catch path consults `shouldPreserveHitlOnShutdown` - // and skips the terminal-fallback / run-finish writes that - // would otherwise overwrite the plan/ask card. - this.preserveHitlOnShutdown.add(run.runId); - run.abortController.abort(); - continue; - } - - // Truly mid-stream run: the durable run-finish is all history needs — - // the fold derives the terminal tree from the log after restart. - this.publishRunFinish( - run.threadId, - run.runId, - 'cancelled', - 'service_shutdown', - undefined, - undefined, - { promptVersion: run.promptVersion }, - ); - await this.tracing.finalizeRunTracing(run.runId, run.tracing, { - status: 'cancelled', - reason: 'service_shutdown', - }); - run.abortController.abort(); - } - for (const run of suspendedRuns) { - // Suspended runs are recoverable from the checkpoint store + pending - // confirmation index, so leave the run-finish unpublished and the - // snapshot untouched. We only need to abort the in-process stream; - // the DB rows are intentionally preserved across restart. The flag - // keeps the card publish alive if the abort lands before the card - // reaches the log. - await this.tracing.finalizeRunTracing(run.runId, run.tracing, { - status: 'cancelled', - reason: 'service_shutdown', - }); - this.preserveHitlOnShutdown.add(run.runId); - run.abortController.abort(); - } - for (const task of this.backgroundTasks.cancelAll()) { - task.abortController.abort(); - await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled'); - } - - // Drain the now-aborted executeRun / processResumedStream promises - // before returning. Each one's finally block does DB work - // (`schedulePlannedTasks`, `dropPendingConfirmationsForThread`) and - // the SDK's abort-driven `cleanupRun` issues `checkpointStore.delete`, - // all of which would race the connection close in `exitSuccessFully` - // and surface as `DriverAlreadyReleasedError` otherwise. Bounded so - // a hung agent can't block n8n's own graceful-shutdown deadline. - await this.drainInFlightExecutions(INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS); this.instanceAiErrorReporter.endAllRuns(); const threadsWithTraces = new Set(this.tracing.getTrackedThreadIds()); @@ -1739,6 +1283,7 @@ export class InstanceAiService { // teardown paths. this.domainAccessTrackersByThread.clear(); + this.runState.clear(); this.tracing.clear(); // Flush in-flight drains + open coalesce buffers so the tail of every @@ -1755,23 +1300,6 @@ export class InstanceAiService { this.logger.debug('Instance AI service shut down'); } - private async drainInFlightExecutions(timeoutMs: number): Promise { - if (this.inFlightExecutions.size === 0) return; - - const drain = Promise.allSettled([...this.inFlightExecutions]); - const timeout = new Promise<'timeout'>((resolve) => { - setTimeout(() => resolve('timeout'), timeoutMs).unref(); - }); - - const outcome = await Promise.race([drain.then(() => 'drained' as const), timeout]); - if (outcome === 'timeout') { - this.logger.warn('Timed out waiting for in-flight Instance AI runs to drain', { - timeoutMs, - stillInFlight: this.inFlightExecutions.size, - }); - } - } - /** * One prune pass: expire stale checkpoints, hard-delete tombstones past the * GC horizon, drop expired pending confirmations, and delete expired @@ -1989,10 +1517,7 @@ export class InstanceAiService { private buildWorkflowVerificationFollowUpMessage(input: { obligation: WorkflowVerificationObligation; outcome?: WorkflowBuildOutcome; - sourceTask?: Pick< - ManagedBackgroundTask, - 'taskId' | 'role' | 'status' | 'result' | 'error' | 'plannedTaskId' | 'workItemId' - >; + sourceTask?: WorkflowVerificationSourceTask; }): string { const payload = { obligation: input.obligation, @@ -2011,15 +1536,6 @@ export class InstanceAiService { return { memory, taskStorage, plannedTaskService }; } - /** - * Replays any undelivered background-task outcomes for a thread so a - * reconnecting client never misses a result that completed while its stream - * was closed. Delegates to {@link InstanceAiTerminalOutcomeService}. - */ - async replayUndeliveredTerminalOutcomes(threadId: string): Promise { - await this.terminalOutcome.replayUndeliveredTerminalOutcomes(threadId); - } - private async syncPlannedTasksToUi(threadId: string, graph: PlannedTaskGraph): Promise { const { taskStorage } = await this.createPlannedTaskState(); const tasks = await this.taskProjector.projectPlannedTaskList(threadId, graph); @@ -2258,12 +1774,6 @@ export class InstanceAiService { }); this.domainAccessTrackersByThread.set(threadId, domainTracker); context.domainAccessTracker = domainTracker; - if (this.isRunDebugEnabled()) { - context.recordWorkflowCodeSnapshot = (snapshot) => { - this.runDebugBuffer.ensure(runId, threadId); - this.runDebugBuffer.recordWorkflowCode(runId, snapshot); - }; - } browserMcpServer?.setDomainGate({ tracker: domainTracker, @@ -2441,14 +1951,9 @@ export class InstanceAiService { runtimeSkills, runtimeSkillCatalog: allRuntimeSkills, oauth2CallbackUrl: this.oauth2CallbackUrl, - cancelBackgroundTask: async (taskId) => this.cancelBackgroundTask(threadId, taskId), - touchRun: () => this.runState.touchActiveRun(threadId), - touchBackgroundTask: (taskId) => this.backgroundTasks.touchTask(threadId, taskId), plannedTaskService, schedulePlannedTasks: async () => await this.schedulePlannedTasks(user, threadId), iterationLog, - sendCorrectionToTask: (taskId, correction) => - this.sendCorrectionToTask(threadId, taskId, correction), workflowTaskService: workflowTasks, workspace: runtimeWorkspace, workspaceRoot, @@ -2678,43 +2183,6 @@ export class InstanceAiService { } } - private async handlePlannedTaskSettlement( - user: User, - task: ManagedBackgroundTask, - status: 'succeeded' | 'failed' | 'cancelled', - { reschedule = true }: { reschedule?: boolean } = {}, - ): Promise { - if (!task.plannedTaskId) return; - - const { plannedTaskService } = await this.createPlannedTaskState(); - let graph: PlannedTaskGraph | null = null; - - if (status === 'succeeded') { - graph = await plannedTaskService.markSucceeded(task.threadId, task.plannedTaskId, { - result: task.result, - outcome: task.outcome, - }); - } else if (status === 'failed') { - graph = await plannedTaskService.markFailed(task.threadId, task.plannedTaskId, { - error: task.error, - }); - } else { - graph = await plannedTaskService.markCancelled(task.threadId, task.plannedTaskId, { - error: task.error, - }); - } - - if (graph) { - await this.syncPlannedTasksToUi(task.threadId, graph); - } - - // Callers that tore the whole thread down (user stop, run timeout, - // shutdown) opt out: the scheduler must not start a follow-up run. The - // graph stays intact so the next user message can pick it back up. - if (!reschedule) return; - await this.schedulePlannedTasks(user, task.threadId); - } - /** * Setup panel v2 ground truth at run start: re-analyze the workflows this * thread announced and tell the agent what is open and what the user @@ -3055,7 +2523,8 @@ export class InstanceAiService { private createPlannedTaskRunGate(): PlannedTaskRunGate { return { - hasLiveRun: (threadId) => this.runState.hasLiveRun(threadId), + // The Agents runtime queues follow-ups behind a live turn. + hasLiveRun: () => false, }; } @@ -3159,10 +2628,7 @@ export class InstanceAiService { private toWorkflowVerificationSourceTask( verification: PlannedWorkflowVerification, - ): Pick< - ManagedBackgroundTask, - 'taskId' | 'role' | 'status' | 'result' | 'error' | 'plannedTaskId' | 'workItemId' - > { + ): WorkflowVerificationSourceTask { return { taskId: verification.task.backgroundTaskId ?? verification.task.id, role: 'workflow-builder', @@ -3228,11 +2694,7 @@ export class InstanceAiService { await this.syncPlannedTasksToUi(threadId, graph); - const availableSlots = Math.max( - 0, - MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD - - this.backgroundTasks.getRunningTasks(threadId).length, - ); + const availableSlots = MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD; const pendingWorkflowVerification = await this.workflowObligations.findPendingPlannedWorkflowVerification(threadId, graph); const action = await plannedTaskService.tick(threadId, { @@ -3717,7 +3179,7 @@ export class InstanceAiService { attachmentManifest = buildAttachmentManifest(classifiedAttachments); } const turnHadFileAttachments = nonStructuredAttachments.length > 0; - const enrichedMessage = await this.buildMessageWithRunningTasks(threadId, message); + const enrichedMessage = message; const messageBody = !message && hasParseableAttachment ? `The user attached file(s) without a message. Inspect the first parseable attachment with parse-file and provide a concise summary.\n\n${attachmentManifest}` @@ -4063,7 +3525,7 @@ export class InstanceAiService { } if (status === 'suspended') { - this.emitRunMetrics(threadId, 'suspended', { + this.emitRunMetrics('suspended', { modelId, workSummary: result.workSummary, usage: result.usage, @@ -4281,37 +3743,16 @@ export class InstanceAiService { const graph = await plannedTaskService.getGraph(threadId); const task = graph?.tasks.find((t) => t.id === checkpointTaskId); if (task && task.status === 'running') { - // If the orchestrator spawned a detached sub-agent inside this - // checkpoint's turn (builder, research, data-table) and - // that child is still running, leave the checkpoint running. The - // child's settlement path re-emits `orchestrate-checkpoint` so the - // orchestrator re-enters the same checkpoint context and can then - // call `complete-checkpoint`. - const inflightChildren = this.backgroundTasks.getRunningTasksByParentCheckpoint( + this.logger.warn('Checkpoint run ended without reporting completion — marking failed', { threadId, checkpointTaskId, - ); - if (inflightChildren.length > 0) { - this.logger.debug( - 'Checkpoint run ended with in-flight child tasks — deferring finalization', - { - threadId, - checkpointTaskId, - inflightTaskIds: inflightChildren.map((t) => t.taskId), - }, - ); - } else { - this.logger.warn('Checkpoint run ended without reporting completion — marking failed', { - threadId, - checkpointTaskId, - }); - await plannedTaskService.markCheckpointFailed(threadId, checkpointTaskId, { - error: 'Checkpoint run ended without reporting completion', - }); - const nextGraph = await plannedTaskService.getGraph(threadId); - if (nextGraph) { - await this.syncPlannedTasksToUi(threadId, nextGraph); - } + }); + await plannedTaskService.markCheckpointFailed(threadId, checkpointTaskId, { + error: 'Checkpoint run ended without reporting completion', + }); + const nextGraph = await plannedTaskService.getGraph(threadId); + if (nextGraph) { + await this.syncPlannedTasksToUi(threadId, nextGraph); } } } catch (error) { @@ -4815,17 +4256,6 @@ export class InstanceAiService { } } - private async buildMessageWithRunningTasks(threadId: string, message: string): Promise { - return await enrichMessageWithBackgroundTasks( - message, - this.backgroundTasks.getRunningTasks(threadId), - { - formatTask: async (task: ManagedBackgroundTask) => - `[Running task — ${task.role}]: taskId=${task.taskId}`, - }, - ); - } - private trackConfirmationRequest( userId: string, threadId: string, @@ -4953,7 +4383,7 @@ export class InstanceAiService { status: effectiveStatus, ...(userId ? { user_id: userId } : {}), }); - this.emitBrowserCredentialSetupOutcomes(threadId, runId, status, reason); + this.emitBrowserCredentialSetupOutcomes(threadId, runId, status); if (status === 'errored') { this.telemetry.track('Builder generation errored', { thread_id: threadId, @@ -5037,15 +4467,14 @@ export class InstanceAiService { * Emit one terminal telemetry event per browser-assisted credential setup * attempt of the finished run (NODE-5511). Consumes the pending record so * every attempt yields exactly one success or failure event. When the flow - * itself never failed, the run's own termination (user stop, timeout, - * stream error) is reported as the error code so aborts aren't counted as + * itself never failed, the run's own termination (user stop or stream + * error) is reported as the error code so aborts aren't counted as * flow failures. */ private emitBrowserCredentialSetupOutcomes( threadId: string, runId: string, runStatus: 'completed' | 'cancelled' | 'errored', - runFinishReason?: string, ): void { const browserSetup = this.pendingBrowserCredentialSetups.get(runId); if (!browserSetup) return; @@ -5055,9 +4484,7 @@ export class InstanceAiService { ? 'not_attempted' : runStatus === 'errored' ? 'run_errored' - : runFinishReason === INSTANCE_AI_RUN_TIMEOUT_REASON - ? 'run_timed_out' - : 'run_cancelled'; + : 'run_cancelled'; for (const attempt of browserSetup.attempts) { this.telemetry.track('Instance AI Browser Use credential setup completed', { user_id: browserSetup.userId, @@ -5112,7 +4539,7 @@ export class InstanceAiService { contextReach: options?.contextReach, }, ); - this.emitRunMetrics(threadId, status, options); + this.emitRunMetrics(status, options); if (status === 'completed' && options?.userId && options?.modelId) { void this.refineTitleIfNeeded(threadId, options.userId, options.modelId); } @@ -5181,16 +4608,12 @@ export class InstanceAiService { /** Emit a typed event consumed by the Prometheus Instance AI metrics collector. */ private emitRunMetrics( - threadId: string, status: 'completed' | 'cancelled' | 'errored' | 'suspended', options?: { modelId?: ModelConfig; workSummary?: WorkSummary; usage?: RunTokenUsage }, ): void { - const startedAt = this.runState.getActiveRun(threadId)?.startedAt; this.eventService.emit('instance-ai-run-finished', { status: status === 'errored' ? 'error' : status, - // Duration is reported once, by the run's terminal event. - durationMs: - status !== 'suspended' && startedAt !== undefined ? Date.now() - startedAt : undefined, + durationMs: undefined, model: runMetricsModelLabel(options?.modelId), toolCalls: options?.workSummary?.totalToolCalls ?? 0, toolErrors: options?.workSummary?.totalToolErrors ?? 0, diff --git a/packages/cli/src/modules/instance-ai/liveness/index.ts b/packages/cli/src/modules/instance-ai/liveness/index.ts deleted file mode 100644 index f8b8911214c..00000000000 --- a/packages/cli/src/modules/instance-ai/liveness/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export { - INSTANCE_AI_RUN_TIMEOUT_REASON, - InstanceAiLivenessService, - type InstanceAiLivenessBackgroundTasks, - type InstanceAiConsumedRunTimeout, - type InstanceAiLivenessEventBus, - type InstanceAiLivenessPendingConfirmation, - type InstanceAiLivenessRunState, - type InstanceAiLivenessServiceOptions, - type InstanceAiLivenessSweepResult, - type InstanceAiLivenessTimedOutActiveRun, - type InstanceAiLivenessTimedOutSuspendedRun, - type InstanceAiLivenessTimedOutTask, -} from './instance-ai-liveness.service'; diff --git a/packages/cli/src/modules/instance-ai/liveness/instance-ai-liveness.service.ts b/packages/cli/src/modules/instance-ai/liveness/instance-ai-liveness.service.ts deleted file mode 100644 index 0f6549f210a..00000000000 --- a/packages/cli/src/modules/instance-ai/liveness/instance-ai-liveness.service.ts +++ /dev/null @@ -1,274 +0,0 @@ -import type { InstanceAiEvent } from '@n8n/api-types'; -import type { Logger } from '@n8n/backend-common'; -import { Time } from '@n8n/constants'; -import { - orchestratorAgentId, - type InstanceAiLivenessPolicy, - type InstanceAiLivenessTimeoutReason, -} from '@n8n/instance-ai'; -import { getErrorMessage } from '@n8n/utils/errors/get-error-message'; - -import type { InstanceAiRunTimeoutDetails } from '../run-timeout-details'; - -export const INSTANCE_AI_RUN_TIMEOUT_REASON = 'timeout'; - -const RUN_TIMEOUT_MESSAGE = - 'The run stopped making progress, so I cancelled it. You can retry or adjust the request.'; - -export type InstanceAiLivenessTimedOutActiveRun = { - runId: string; - abortController: AbortController; -}; - -export type InstanceAiLivenessTimedOutSuspendedRun = { - runId: string; - threadId: string; - abortController: AbortController; -}; - -export type InstanceAiLivenessTimedOutTask = { - threadId: string; - taskId: string; - role: string; - timeoutReason?: InstanceAiLivenessTimeoutReason; -}; - -export type InstanceAiLivenessPendingConfirmation = { - threadId: string; -}; - -export type InstanceAiLivenessSweepResult = { - activeThreadIds: string[]; - suspendedThreadIds: string[]; - confirmationRequestIds: string[]; - activeTimeouts?: Record; - suspendedTimeouts?: Record; - confirmationTimeouts?: Record; -}; - -export type InstanceAiConsumedRunTimeout = { - timedOut: boolean; - details?: InstanceAiRunTimeoutDetails; -}; - -export type InstanceAiLivenessRunState< - TSuspendedRun extends InstanceAiLivenessTimedOutSuspendedRun, -> = { - sweepTimedOut: (policy: InstanceAiLivenessPolicy, now?: number) => InstanceAiLivenessSweepResult; - cancelActiveRun: (threadId: string) => InstanceAiLivenessTimedOutActiveRun | undefined; - cancelSuspendedRun: (threadId: string) => TSuspendedRun | undefined; - getActiveRunId: (threadId: string) => string | undefined; - getPendingConfirmation: (requestId: string) => InstanceAiLivenessPendingConfirmation | undefined; - hasPendingConfirmationForThread: (threadId: string) => boolean; - rejectPendingConfirmation: (requestId: string) => boolean; -}; - -export type InstanceAiLivenessBackgroundTasks = { - timeoutTimedOutTasks: ( - policy: InstanceAiLivenessPolicy, - now?: number, - options?: { - shouldSkipTask?: (task: InstanceAiLivenessTimedOutTask) => boolean; - }, - ) => Promise; -}; - -export type InstanceAiLivenessEventBus = { - publish: (threadId: string, event: InstanceAiEvent) => void; -}; - -export type InstanceAiLivenessServiceOptions< - TSuspendedRun extends InstanceAiLivenessTimedOutSuspendedRun, -> = { - policy: InstanceAiLivenessPolicy; - backgroundTaskIdleTimeoutMs: number; - runState: InstanceAiLivenessRunState; - backgroundTasks: InstanceAiLivenessBackgroundTasks; - eventBus: InstanceAiLivenessEventBus; - finalizeCancelledSuspendedRun: (suspended: TSuspendedRun, reason: string) => void; - onPendingConfirmationRejected?: (requestId: string) => void; - logger: Logger; -}; - -export class InstanceAiLivenessService< - TSuspendedRun extends InstanceAiLivenessTimedOutSuspendedRun, -> { - private timeoutInterval?: NodeJS.Timeout; - - private readonly timedOutRunIds = new Map(); - - private readonly timedOutActiveRunThreads = new Set(); - - /** - * Runs whose timeout notice has already been published, so a run cancelled - * by the sweep and then finalised by its own abort handler only gets the - * notice once. Capped FIFO (see {@link NOTICE_DEDUPE_CACHE_SIZE}): dedupe - * only has to hold across a single run's cancellation window, and a run - * lives on one main for one process lifetime. - */ - private readonly noticedRunIds = new Set(); - - /** Max retained run ids in {@link noticedRunIds}; oldest evicted first. */ - static readonly NOTICE_DEDUPE_CACHE_SIZE = 1000; - - constructor(private readonly options: InstanceAiLivenessServiceOptions) {} - - get backgroundTaskIdleTimeoutMs(): number { - return this.options.backgroundTaskIdleTimeoutMs; - } - - start(): void { - if (!this.options.policy.hasEnabledTimeouts()) return; - - this.timeoutInterval = setInterval(() => { - void this.sweepTimedOutWork(); - }, Time.minutes.toMilliseconds); - } - - shutdown(): void { - if (this.timeoutInterval) { - clearInterval(this.timeoutInterval); - this.timeoutInterval = undefined; - } - this.timedOutRunIds.clear(); - this.timedOutActiveRunThreads.clear(); - this.noticedRunIds.clear(); - } - - clearThreadState(threadId: string): void { - this.timedOutActiveRunThreads.delete(threadId); - } - - markRunTimedOut(runId: string, details?: InstanceAiRunTimeoutDetails): void { - this.timedOutRunIds.set(runId, details); - } - - consumeRunTimedOut(runId: string): boolean { - const timedOut = this.timedOutRunIds.has(runId); - if (timedOut) this.timedOutRunIds.delete(runId); - return timedOut; - } - - consumeRunTimeout(runId: string): InstanceAiConsumedRunTimeout { - const timedOut = this.timedOutRunIds.has(runId); - if (!timedOut) return { timedOut: false }; - - const details = this.timedOutRunIds.get(runId); - this.timedOutRunIds.delete(runId); - return details ? { timedOut: true, details } : { timedOut: true }; - } - - hasTimedOutActiveRunThread(threadId: string): boolean { - return this.timedOutActiveRunThreads.has(threadId); - } - - async sweepTimedOutWork(now = Date.now()): Promise { - const { - activeThreadIds, - suspendedThreadIds, - confirmationRequestIds, - activeTimeouts, - suspendedTimeouts, - confirmationTimeouts, - } = this.options.runState.sweepTimedOut(this.options.policy, now); - - for (const threadId of activeThreadIds) { - this.options.logger.debug('Cancelling timed-out active run', { threadId }); - this.cancelTimedOutActiveRun(threadId, activeTimeouts?.[threadId]); - } - - for (const threadId of suspendedThreadIds) { - this.options.logger.debug('Auto-rejecting timed-out suspended run', { threadId }); - this.cancelTimedOutSuspendedRun(threadId, suspendedTimeouts?.[threadId]); - } - - for (const reqId of confirmationRequestIds) { - this.options.logger.debug('Auto-rejecting timed-out sub-agent confirmation', { - requestId: reqId, - }); - const pending = this.options.runState.getPendingConfirmation(reqId); - if (pending) { - const runId = this.options.runState.getActiveRunId(pending.threadId); - if (runId) { - this.markRunTimedOut(runId, confirmationTimeouts?.[reqId]); - this.publishRunTimeoutNotice(pending.threadId, runId); - } - } - this.options.runState.rejectPendingConfirmation(reqId); - this.options.onPendingConfirmationRejected?.(reqId); - } - - try { - const timedOutTasks = await this.options.backgroundTasks.timeoutTimedOutTasks( - this.options.policy, - now, - { - shouldSkipTask: (task) => - this.options.runState.hasPendingConfirmationForThread(task.threadId), - }, - ); - for (const task of timedOutTasks) { - this.options.logger.debug('Timed out background task', { - threadId: task.threadId, - taskId: task.taskId, - role: task.role, - timeoutReason: task.timeoutReason, - }); - } - } catch (error) { - this.options.logger.warn('Failed to sweep timed-out background tasks', { - error: getErrorMessage(error), - }); - } - } - - cancelTimedOutActiveRun(threadId: string, details?: InstanceAiRunTimeoutDetails): void { - const active = this.options.runState.cancelActiveRun(threadId); - if (!active) return; - - this.markRunTimedOut(active.runId, details); - this.timedOutActiveRunThreads.add(threadId); - this.publishRunTimeoutNotice(threadId, active.runId); - active.abortController.abort(); - } - - cancelTimedOutSuspendedRun(threadId: string, details?: InstanceAiRunTimeoutDetails): void { - const suspended = this.options.runState.cancelSuspendedRun(threadId); - if (!suspended) return; - - this.markRunTimedOut(suspended.runId, details); - suspended.abortController.abort(); - this.options.finalizeCancelledSuspendedRun(suspended, INSTANCE_AI_RUN_TIMEOUT_REASON); - } - - publishRunTimeoutNotice(threadId: string, runId: string): void { - // Deduped locally rather than by scanning the run's events: the notice is - // a delta, and under the durable log deltas are ephemeral (never stored, - // persisted only once their segment coalesces), so a read-back cannot see - // a notice published moments earlier. - if (this.noticedRunIds.has(runId)) return; - - // Recorded only after the publish lands: marking first would suppress the - // notice permanently if publish threw. - this.options.eventBus.publish(threadId, { - type: 'text-delta', - runId, - agentId: orchestratorAgentId(runId), - responseId: `run-timeout:${runId}`, - payload: { text: RUN_TIMEOUT_MESSAGE }, - }); - this.rememberNoticedRunId(runId); - } - - /** - * Record a notified run id, evicting the oldest once the cap is reached. A - * Set keeps insertion order, so the first value is the oldest entry. - */ - private rememberNoticedRunId(runId: string): void { - this.noticedRunIds.add(runId); - if (this.noticedRunIds.size > InstanceAiLivenessService.NOTICE_DEDUPE_CACHE_SIZE) { - const oldest = this.noticedRunIds.values().next().value; - if (oldest !== undefined) this.noticedRunIds.delete(oldest); - } - } -} diff --git a/packages/cli/src/modules/instance-ai/run-timeout-details.ts b/packages/cli/src/modules/instance-ai/run-timeout-details.ts deleted file mode 100644 index 14738c59f69..00000000000 --- a/packages/cli/src/modules/instance-ai/run-timeout-details.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { InstanceAiLivenessSurface, InstanceAiLivenessTimeoutReason } from '@n8n/instance-ai'; - -export type InstanceAiRunTimeoutDetails = { - reason: InstanceAiLivenessTimeoutReason; - surface: InstanceAiLivenessSurface; - timeoutMs: number; - elapsedMs: number; - idleMs: number; -}; diff --git a/packages/cli/src/modules/instance-ai/run-trace-metadata.ts b/packages/cli/src/modules/instance-ai/run-trace-metadata.ts index 87a0bf14a5d..e360766249c 100644 --- a/packages/cli/src/modules/instance-ai/run-trace-metadata.ts +++ b/packages/cli/src/modules/instance-ai/run-trace-metadata.ts @@ -1,9 +1,4 @@ import type { InstanceAiEvent } from '@n8n/api-types'; -import type { InstanceAiLivenessTimeoutReason } from '@n8n/instance-ai'; - -import type { InstanceAiRunTimeoutDetails } from './run-timeout-details'; - -const RUN_TIMEOUT_REASON = 'timeout'; export type InstanceAiFirstVisibleState = | 'assistant_text' @@ -12,17 +7,9 @@ export type InstanceAiFirstVisibleState = | 'task_card' | 'empty'; -export type InstanceAiCancellationType = 'explicit' | InstanceAiLivenessTimeoutReason; - -export type InstanceAiRunTimeoutTraceContext = { - timedOut: boolean; - details?: InstanceAiRunTimeoutDetails; -}; - export type InstanceAiRunTraceMetadataOptions = { status: 'completed' | 'cancelled' | 'error'; cancellationReason?: string; - runTimeout?: InstanceAiRunTimeoutTraceContext; }; type FirstVisibleSummary = { @@ -80,18 +67,6 @@ function getFirstVisibleSummary(events: InstanceAiEvent[]): FirstVisibleSummary return withFirstToolName(sawToolCall ? 'tool_call' : 'empty', firstToolName); } -function getCancellationType( - options: InstanceAiRunTraceMetadataOptions, -): InstanceAiCancellationType | undefined { - if (options.status !== 'cancelled') return undefined; - - if (options.runTimeout?.timedOut || options.cancellationReason === RUN_TIMEOUT_REASON) { - return options.runTimeout?.details?.reason ?? 'idle_timeout'; - } - - return 'explicit'; -} - export function buildInstanceAiRunTraceMetadata( events: InstanceAiEvent[], options: InstanceAiRunTraceMetadataOptions, @@ -105,13 +80,8 @@ export function buildInstanceAiRunTraceMetadata( metadata.first_tool_name = firstVisible.firstToolName; } - const cancellationType = getCancellationType(options); - if (cancellationType) { - metadata.cancellation_type = cancellationType; - } - - if (options.runTimeout?.details?.idleMs !== undefined) { - metadata.idle_tail_ms = Math.round(options.runTimeout.details.idleMs); + if (options.status === 'cancelled') { + metadata.cancellation_type = 'explicit'; } return metadata; diff --git a/packages/cli/src/modules/instance-ai/sandbox/index.ts b/packages/cli/src/modules/instance-ai/sandbox/index.ts index 1f7b13775a9..2e5740ad605 100644 --- a/packages/cli/src/modules/instance-ai/sandbox/index.ts +++ b/packages/cli/src/modules/instance-ai/sandbox/index.ts @@ -1,8 +1,6 @@ export { InstanceAiSandboxService, - type InstanceAiSandboxBackgroundTasks, type InstanceAiSandboxProxy, - type InstanceAiSandboxRunState, type InstanceAiSandboxServiceOptions, type InstanceAiSandboxSettings, type RuntimeSandboxEntry, diff --git a/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts b/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts index 68c6d493b3c..dcb8d3ae843 100644 --- a/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts +++ b/packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts @@ -12,7 +12,6 @@ import { type ServiceProxyConfig, type InstanceAiContext, type Logger, - type ManagedBackgroundTask, type SandboxConfig, } from '@n8n/instance-ai'; import type { ErrorReporter } from 'n8n-core'; @@ -130,17 +129,6 @@ function withThreadScopedSandboxIdentity(config: SandboxConfig, threadId: string }; } -/** Thread-run state the sandbox lifecycle consults to know when a workspace is still in use. */ -export type InstanceAiSandboxRunState = { - getActiveRunId: (threadId: string) => string | undefined; - hasSuspendedRun: (threadId: string) => boolean; -}; - -/** Background-task view the sandbox lifecycle consults to know when a workspace is still in use. */ -export type InstanceAiSandboxBackgroundTasks = { - getRunningTasks: (threadId: string) => ManagedBackgroundTask[]; -}; - /** Settings collaborator that resolves provider credentials from admin config. */ export type InstanceAiSandboxSettings = { resolveDaytonaConfig: () => Promise<{ apiUrl?: string; apiKey?: string }>; @@ -164,8 +152,6 @@ export type InstanceAiSandboxServiceOptions = { config: InstanceAiConfig; logger: Logger; errorReporter: ErrorReporter; - runState: InstanceAiSandboxRunState; - backgroundTasks: InstanceAiSandboxBackgroundTasks; settingsService: InstanceAiSandboxSettings; aiService: InstanceAiSandboxProxy; resolveTracingConfig?: ( @@ -359,7 +345,7 @@ export class InstanceAiSandboxService { if (existing) { if ( existing.configFingerprint !== cacheState.fingerprint || - (this.isSandboxEntryExpired(existing) && !this.isSandboxInUse(threadId)) + this.isSandboxEntryExpired(existing) ) { this.evictSandboxEntry( threadId, @@ -653,14 +639,6 @@ export class InstanceAiSandboxService { this.scheduleSandboxExpiry(threadId, entry); } - private isSandboxInUse(threadId: string): boolean { - return Boolean( - this.options.runState.getActiveRunId(threadId) || - this.options.runState.hasSuspendedRun(threadId) || - this.options.backgroundTasks.getRunningTasks(threadId).length > 0, - ); - } - private scheduleSandboxExpiry(threadId: string, entry: RuntimeSandboxEntry): void { if (this.sandboxTtlMs <= 0) return; if (entry.cleanupTimer) clearTimeout(entry.cleanupTimer); @@ -673,10 +651,6 @@ export class InstanceAiSandboxService { entry.cleanupTimer = setTimeout(() => { const current = this.sandboxes.get(threadId); if (current !== entry) return; - if (this.isSandboxInUse(threadId)) { - this.touchSandboxEntry(threadId, entry); - return; - } this.evictSandboxEntry(threadId, entry, 'idle', true); }, delay); entry.cleanupTimer.unref(); diff --git a/packages/cli/src/modules/instance-ai/tracing/index.ts b/packages/cli/src/modules/instance-ai/tracing/index.ts index 817e345a5b8..5f4f82efd7d 100644 --- a/packages/cli/src/modules/instance-ai/tracing/index.ts +++ b/packages/cli/src/modules/instance-ai/tracing/index.ts @@ -2,7 +2,6 @@ export { InstanceAiTracingService, type InstanceAiTracingAiService, type InstanceAiTracingEventReader, - type InstanceAiTracingRunState, type InstanceAiTracingEventLog, type InstanceAiTracingServiceOptions, type MessageTraceFinalization, diff --git a/packages/cli/src/modules/instance-ai/tracing/instance-ai-tracing.service.ts b/packages/cli/src/modules/instance-ai/tracing/instance-ai-tracing.service.ts index 67b8c92a058..c71bf46f74f 100644 --- a/packages/cli/src/modules/instance-ai/tracing/instance-ai-tracing.service.ts +++ b/packages/cli/src/modules/instance-ai/tracing/instance-ai-tracing.service.ts @@ -8,9 +8,7 @@ import { submitLangsmithUserFeedback, type BrowserExtensionTraceContext, type InstanceAiTraceContext, - type ManagedBackgroundTask, type ModelConfig, - type RunStateRegistry, type ServiceProxyConfig, } from '@n8n/instance-ai'; import { getErrorMessage } from '@n8n/utils/errors/get-error-message'; @@ -63,8 +61,6 @@ export type InstanceAiTracingEventReader = { getEventsForRun: (threadId: string, runId: string) => Promise; }; -export type InstanceAiTracingRunState = Pick, 'attachTracing'>; - export type InstanceAiTracingEventLog = Pick; export type InstanceAiTracingAiService = Pick; @@ -72,7 +68,6 @@ export type InstanceAiTracingAiService = Pick; - error?: string; - metadata?: Record; - }, - ): Promise { - if (!traceContext) return; - - try { - if ( - traceContext.actorRun.id !== traceContext.rootRun.id && - traceContext.actorRun.endTime === undefined - ) { - await traceContext.finishRun(traceContext.actorRun, { - outputs: { - status: options.status, - ...options.outputs, - }, - metadata: { - final_status: options.status, - ...options.metadata, - }, - ...(options.error ? { error: options.error } : {}), - }); - } - await traceContext.finishRun(traceContext.rootRun, { - outputs: { - status: options.status, - ...options.outputs, - }, - metadata: { - final_status: options.status, - ...options.metadata, - }, - ...(options.error ? { error: options.error } : {}), - }); - } catch (error) { - this.logger.warn('Failed to finalize Instance AI detached trace run', { - taskId, - traceRunId: traceContext.rootRun.id, - error: getErrorMessage(error), - }); - } finally { - releaseTraceClient(traceContext.rootRun.traceId); - } - } - async finalizeRunTracing( runId: string, tracing: InstanceAiTraceContext | undefined, @@ -461,26 +400,6 @@ export class InstanceAiTracingService { } } - async finalizeBackgroundTaskTracing( - task: ManagedBackgroundTask, - status: 'completed' | 'failed' | 'cancelled', - ): Promise { - await this.finalizeDetachedTraceRun(task.taskId, task.traceContext, { - status, - outputs: { - taskId: task.taskId, - agentId: task.agentId, - role: task.role, - ...(task.result ? { result: task.result } : {}), - }, - ...(status === 'failed' && task.error ? { error: task.error } : {}), - metadata: { - ...(task.plannedTaskId ? { planned_task_id: task.plannedTaskId } : {}), - ...(task.workItemId ? { work_item_id: task.workItemId } : {}), - }, - }); - } - async submitLangsmithFeedback( user: User, threadId: string, diff --git a/packages/cli/src/modules/instance-ai/workflow-verification-task-projector.ts b/packages/cli/src/modules/instance-ai/workflow-verification-task-projector.ts index 09d09eb658a..5b144c99a5a 100644 --- a/packages/cli/src/modules/instance-ai/workflow-verification-task-projector.ts +++ b/packages/cli/src/modules/instance-ai/workflow-verification-task-projector.ts @@ -5,7 +5,6 @@ import { orchestratorAgentId, ThreadTaskStorage, WorkflowLoopStorage, - type ManagedBackgroundTask, type PlannedTaskGraph, type PlannedTaskRecord, type WorkflowBuildOutcome, @@ -280,61 +279,6 @@ export class WorkflowVerificationTaskProjector { return verifyRow(task.id, obligation); } - /** Sync a direct builder's checklist rows from its background-task lifecycle. */ - async syncFromBackgroundTask(task: ManagedBackgroundTask): Promise { - if (task.plannedTaskId || task.parentCheckpointId || task.role !== 'workflow-builder') return; - - const items = await this.directItemsFromTask(task); - if (!items) return; - await this.upsertItems(task.threadId, task.runId, items); - } - - private async directItemsFromTask(task: ManagedBackgroundTask): Promise { - const build = { id: task.taskId, description: BUILD_DESCRIPTION }; - const verify = { id: verifyRowId(task.taskId), description: VERIFY_DESCRIPTION }; - - switch (task.status) { - case 'running': - return [ - { ...build, detail: DETAIL.building, status: 'in_progress' }, - { ...verify, detail: DETAIL.waitingForBuild, status: 'todo' }, - ]; - case 'failed': - return [ - { ...build, detail: task.error ?? DETAIL.blocked, status: 'failed' }, - { ...verify, detail: DETAIL.buildIncomplete, status: 'cancelled' }, - ]; - case 'cancelled': - return [ - { ...build, detail: task.error ?? DETAIL.cancelled, status: 'cancelled' }, - { ...verify, detail: DETAIL.buildIncomplete, status: 'cancelled' }, - ]; - case 'completed': - return await this.directItemsFromCompletedTask(task, build, verify); - } - } - - private async directItemsFromCompletedTask( - task: ManagedBackgroundTask, - build: { id: string; description: string }, - verify: { id: string; description: string }, - ): Promise { - const outcome = parseWorkflowBuildOutcome(task.outcome); - const workItemId = task.workItemId ?? outcome?.workItemId; - if (!workItemId) { - return [ - { ...build, detail: DETAIL.submitted, status: 'done' }, - { ...verify, detail: DETAIL.noWorkflow, status: 'cancelled' }, - ]; - } - - const obligation = await this.obligations.getObligation(task.threadId, workItemId, { - source: 'direct', - }); - if (!obligation) return undefined; - return directTaskItems(task.taskId, outcome, obligation); - } - /** Re-derive every direct builder's checklist rows from workflow-loop storage. */ async syncFromWorkflowLoop(threadId: string, runId: string): Promise { try { @@ -382,22 +326,6 @@ export class WorkflowVerificationTaskProjector { } } - private async upsertItems(threadId: string, runId: string, items: TaskItem[]): Promise { - try { - const taskStorage = new ThreadTaskStorage(this.agentMemory); - const existing = (await taskStorage.get(threadId)) ?? { tasks: [] }; - const { tasks, changed } = upsertTaskItemsInOrder(existing.tasks, items); - if (!changed) return; - await this.saveAndPublish(taskStorage, threadId, runId, { tasks }); - } catch (error) { - this.logger.warn('Failed to update direct workflow builder task checklist items', { - threadId, - taskIds: items.map((item) => item.id), - error: getErrorMessage(error), - }); - } - } - private async saveAndPublish( taskStorage: ThreadTaskStorage, threadId: string, diff --git a/packages/cli/src/scaling/constants.ts b/packages/cli/src/scaling/constants.ts index 4bbbde84df5..c3560220167 100644 --- a/packages/cli/src/scaling/constants.ts +++ b/packages/cli/src/scaling/constants.ts @@ -54,7 +54,6 @@ export const IMMEDIATE_COMMANDS = new Set([ 'cancel-agent-background-job', 'wake-agent-background-job', 'relay-instance-ai-event', - 'relay-instance-ai-task-control', 'agent-chat-subscription-changed', 'agent-chat-integration-changed', // Correlated request/response pairs: the subscriber debounces by command name, diff --git a/packages/cli/src/scaling/pubsub/pubsub.event-map.ts b/packages/cli/src/scaling/pubsub/pubsub.event-map.ts index f677737fa07..7fd977b9098 100644 --- a/packages/cli/src/scaling/pubsub/pubsub.event-map.ts +++ b/packages/cli/src/scaling/pubsub/pubsub.event-map.ts @@ -238,20 +238,6 @@ export type PubSubCommandMap = { storedEvent: { id?: number; event: InstanceAiEvent }; }; - /** - * Relay an Instance AI task-control action (correction / cancel / clear) to - * sibling mains. The action may target a background task or run held - * in another main's in-memory state. Broadcast + local-gate: every main applies - * it only to its own local slice of the thread. - */ - 'relay-instance-ai-task-control': { - threadId: string; - userId?: string; - taskId?: string; - action: 'correct' | 'cancel-task' | 'cancel-thread' | 'clear-thread'; - correction?: string; - }; - /** * Relay human message events between main instances. * Used for cross-client synchronization when a user sends a message diff --git a/packages/cli/src/scaling/pubsub/pubsub.types.ts b/packages/cli/src/scaling/pubsub/pubsub.types.ts index 73d8e395c94..631ebee8d9d 100644 --- a/packages/cli/src/scaling/pubsub/pubsub.types.ts +++ b/packages/cli/src/scaling/pubsub/pubsub.types.ts @@ -70,7 +70,6 @@ export namespace PubSub { export type WakeAgentBackgroundJob = ToCommand<'wake-agent-background-job'>; export type RelayChatStreamEvent = ToCommand<'relay-chat-stream-event'>; export type RelayInstanceAiEvent = ToCommand<'relay-instance-ai-event'>; - export type RelayInstanceAiTaskControl = ToCommand<'relay-instance-ai-task-control'>; export type RelayChatHumanMessage = ToCommand<'relay-chat-human-message'>; export type RelayChatMessageEdit = ToCommand<'relay-chat-message-edit'>; export type ClearTestWebhooks = ToCommand<'clear-test-webhooks'>; @@ -122,7 +121,6 @@ export namespace PubSub { | Commands.WakeAgentBackgroundJob | Commands.RelayChatStreamEvent | Commands.RelayInstanceAiEvent - | Commands.RelayInstanceAiTaskControl | Commands.RelayChatHumanMessage | Commands.RelayChatMessageEdit | Commands.ClearTestWebhooks