github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-10-17 13:39:44 +02:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity Graph

refactor(core): Remove the Assistant run lifecycle leftovers (no-changelog)

Browse Source 
The Agents runtime owns turn liveness, cancel and suspensions, so remove:
background tasks and their task-control relay, the liveness service and
policy, the resumable stream executor and stream runner, the run debug
buffer and endpoints, the shutdown drain, terminal outcome replay, and the
in-process discovery evals. RunStateRegistry keeps per-thread settings only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jaakko Husso 2026-10-06 02:13:21 +03:00
parent 19728e2428
commit fa11244c70
No known key found for this signature in database
93 changed files with 381 additions and 12918 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/WORKFLOWS.md vendored
View File

@ -201,8 +201,7 @@ suite. Test cases are pulled at run time from the LangTracer suite
keep `--source disk` for authoring). To
re-run after pushing a fix, dispatch `ci-instance-ai-evals.yml` with the PR
number (optionally `tier: full` for broader coverage) — results post back to
the PR. The lighter `test-evals-discovery.yml` still runs on every push as part
of `ci-pull-requests.yml`.
the PR.
**`ci-instance-ai-evals.yml` is the PR gate; `test-evals-instance-ai.yml` is
the lab bench.** The gate deliberately exposes only PR re-runs. Anything that

20
.github/workflows/ci-pull-requests.yml vendored
View File

@ -176,7 +176,6 @@ jobs:
packages/@n8n/instance-ai/evaluations/**
packages/cli/src/modules/instance-ai/**
packages/core/src/execution-engine/eval-mock-helpers.ts
.github/workflows/test-evals-discovery.yml
db:
packages/cli/src/databases/**
packages/cli/src/modules/*/database/**
@ -510,25 +509,6 @@ jobs:
ref: ${{ needs.install-and-build.outputs.commit_sha }}
secrets: inherit
# In-process discovery eval — asserts the orchestrator reaches for browser/computer-use
# tools at OAuth/screenshot moments. Lightweight (no Docker), runs in parallel with the
# heavy workflow eval. Non-blocking initially; promote to required after stability.
instance-ai-discovery-evals:
name: Instance AI Discovery Evals
needs: [install-and-build, format-check]
if: >-
!cancelled() &&
needs.install-and-build.result == 'success' &&
needs.format-check.result == 'success' &&
needs.install-and-build.outputs.instance_ai_workflow_eval == 'true' &&
github.repository == 'n8n-io/n8n' &&
github.event_name != 'merge_group' &&
(github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
uses: ./.github/workflows/test-evals-discovery.yml
with:
branch: ${{ needs.install-and-build.outputs.commit_sha }}
secrets: inherit
# This job is required by GitHub branch protection rules.
# PRs cannot be merged unless this job passes.
required-checks:

125
.github/workflows/test-evals-discovery.yml vendored
View File

@ -1,125 +0,0 @@
name: 'Test: Instance AI Discovery Evals'
run-name: "Test: Instance AI Discovery Evals (branch=${{ inputs.branch || 'master' }}, filter=${{ inputs.filter || 'all' }}, trials=${{ inputs.trials }})"
on:
workflow_call:
inputs:
branch:
description: 'GitHub branch to test'
required: false
type: string
default: 'master'
filter:
description: 'Filter scenarios by id (e.g. "slack-oauth")'
required: false
type: string
default: ''
trials:
description: 'Trials per scenario'
required: false
type: number
default: 3
workflow_dispatch:
inputs:
branch:
description: 'GitHub branch to test'
required: false
default: 'master'
filter:
description: 'Filter scenarios by id (e.g. "slack-oauth")'
required: false
default: ''
trials:
description: 'Trials per scenario'
required: false
default: '3'
jobs:
run-discovery-evals:
name: 'Run Discovery Evals'
runs-on: blacksmith-4vcpu-ubuntu-2204
timeout-minutes: 15
env:
NODE_OPTIONS: '--max-old-space-size=7168'
permissions:
contents: read
pull-requests: write
steps:
- name: Checkout
uses: useblacksmith/checkout@bcec731f1eb1367240608d1c889a75b96db6ec53 # v1.5.0
with:
ref: ${{ inputs.branch || github.ref }}
fetch-depth: 1
- name: Setup Environment
uses: ./.github/actions/setup-nodejs
- name: Export Node Types
run: |
./packages/cli/bin/n8n export:nodes --output ./packages/@n8n/ai-workflow-builder.ee/evaluations/nodes.json
- name: Run Discovery Evals
id: eval
working-directory: packages/@n8n/instance-ai
env:
ANTHROPIC_API_KEY: ${{ secrets.EVALS_ANTHROPIC_KEY }}
N8N_ENABLED_MODULES: agents,instance-ai
LANGSMITH_TRACING: 'true'
LANGSMITH_ENDPOINT: ${{ secrets.EVALS_LANGSMITH_ENDPOINT }}
LANGSMITH_API_KEY: ${{ secrets.EVALS_LANGSMITH_API_KEY }}
LANGSMITH_REVISION_ID: ${{ github.sha }}
LANGSMITH_BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
FILTER: ${{ inputs.filter }}
TRIALS: ${{ inputs.trials || 3 }}
run: |
set -o pipefail
EVAL_ARGS=(--trials "$TRIALS" --fail-on-zero-pass)
if [ -n "$FILTER" ]; then
EVAL_ARGS+=(--filter "$FILTER")
fi
pnpm eval:discovery "${EVAL_ARGS[@]}" 2>&1 | tee discovery-eval-output.txt
- name: Post eval results to PR
if: ${{ always() && github.event.pull_request.number && hashFiles('packages/@n8n/instance-ai/discovery-eval-output.txt') != '' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
EVAL_OUTCOME: ${{ steps.eval.outcome }}
HEAD_REF: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref_name }}
COMMIT_SHA: ${{ github.sha }}
run: |
if [ "$EVAL_OUTCOME" = "success" ]; then
STATUS_ICON="✅"
else
STATUS_ICON="❌"
fi
{
echo "### Instance AI Discovery Eval ${STATUS_ICON}"
echo ""
echo "Branch: \`${HEAD_REF}\` · Commit: \`${COMMIT_SHA}\`"
echo ""
echo "<details><summary>Eval output</summary>"
echo ""
echo '```'
cat packages/@n8n/instance-ai/discovery-eval-output.txt
echo '```'
echo ""
echo "</details>"
} > /tmp/discovery-comment.md
COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
--jq '.[] | select(.body | startswith("### Instance AI Discovery Eval")) | .id' | tail -1)
if [ -n "$COMMENT_ID" ]; then
gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" -X PATCH -F body=@/tmp/discovery-comment.md
else
gh pr comment "${{ github.event.pull_request.number }}" --body-file /tmp/discovery-comment.md
fi
- name: Upload Results
if: ${{ always() }}
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: instance-ai-discovery-eval-results
path: packages/@n8n/instance-ai/discovery-eval-output.txt
retention-days: 14

1
packages/@n8n/decorators/src/pubsub/pubsub-metadata.ts
View File

@ -34,7 +34,6 @@ export type PubSubEventName =
| 'resume-agent-workflow-tool'
| 'relay-chat-stream-event'
| 'relay-instance-ai-event'
| 'relay-instance-ai-task-control'
| 'relay-chat-human-message'
| 'relay-chat-message-edit'
| 'reload-sso-provisioning-configuration'

20
packages/@n8n/instance-ai/evaluations/README.md
View File

@ -9,7 +9,6 @@ Six harnesses live here:
- **`eval:instance-ai`** — end-to-end build + mocked execution + LLM verification (drives a running n8n instance)
- **`eval:agents`** — standalone Agent build cases from the LangTracer `agents` suite (author new ones in `data/agents/`)
- **`eval:subagent`** — legacy command name for the workflow-build compatibility corpus; it drives the live orchestrator/skill build path, scored by binary checks
- **`eval:discovery`** — orchestrator in-process, scored against required or forbidden tool/dispatch events (no n8n server)
- **`eval:pairwise`** — live orchestrator workflow builds, scored by an LLM judge panel against do/don't lists. Intended for head-to-head comparison with `ai-workflow-builder.ee` on the same dataset
- **`eval:computer-use`** — grades the computer-use agent (file / OAuth / doc-reading tasks) against fixtures; see [`computer-use/README.md`](computer-use/README.md)
@ -20,7 +19,6 @@ Sections:
- [Running e2e + workflow-build evals](#running-evals)
- [Regression detection](#regression-detection)
- [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows)
- [Running discovery evals](#discovery-evals)
- [Running pairwise evals](#pairwise-evals)
- [How the e2e harness works](#how-the-e2e-harness-works)
- [How the workflow-build harness works](#how-the-workflow-build-harness-works)
@ -546,24 +544,6 @@ lanes flat-out at `lanes * 4` (each lane's per-lane build cap) starves it,
surfacing as verifier/MCP timeouts. Treat `lanes * 4` as an aggressive upper
bound for keys with rate-limit headroom, not the starting point.
## Discovery evals
Discovery evals run the orchestrator in-process and assert first-hop tool or
background-agent routing from captured `tool-call`, `tool-result`, `tool-error`, and
`agent-spawned` events. Use them when a regression is about which path the
agent chooses, not whether a generated workflow executes.
To inspect runtime skill loading, run a focused verbose pass:
```bash
pnpm eval:discovery --filter data-table-skill-loading --trials 3 --verbose --fail-on-zero-pass
```
Verbose output lists each trial's completed tool calls with argument previews.
For data-table routing, look for `load_skill(skillId="data-table-manager")`
and `data-tables(action="list")`, and verify there are no planning,
workflow-builder, or spawned-agent entries in the spawned-agent section.
## Pairwise evals
Pairwise evals score a built workflow against the dataset's `dos` / `donts`

134
packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts
View File

@ -1,134 +0,0 @@
/**
* Smoke + shape tests for the discovery scenario loader.
*
* Loads every JSON in evaluations/data/discovery/ off disk, parses it, and
* asserts the shape required by `runExpectedToolsInvokedCheck`. Catches
* malformed scenario files at unit-test time before they reach the eval
* runner.
*/
import { discoveryTestCaseSchema, loadDiscoveryTestCasesWithFiles } from '../data/discovery';
import { runExpectedToolsInvokedCheck } from '../discovery/expected-tools-invoked';
describe('loadDiscoveryTestCasesWithFiles', () => {
const cases = loadDiscoveryTestCasesWithFiles();
it('finds at least the v1 scenario set', () => {
const slugs = cases.map((c) => c.fileSlug).sort();
expect(slugs).toEqual(
expect.arrayContaining([
'slack-oauth-credential-setup',
'google-oauth-credential-setup',
'screenshot-dashboard',
'http-node-config-no-browser',
'oauth-with-computer-use-disabled',
'workflow-builder-no-credential-ask',
'folder-scoped-workflow-list',
]),
);
});
it.each([
'slack-oauth-credential-setup',
'google-oauth-credential-setup',
'screenshot-dashboard',
'http-node-config-no-browser',
'oauth-with-computer-use-disabled',
'workflow-builder-no-credential-ask',
'folder-scoped-workflow-list',
])('%s parses with a valid expectedToolInvocations rule', (slug) => {
const entry = cases.find((c) => c.fileSlug === slug);
expect(entry).toBeDefined();
const { testCase } = entry!;
expect(testCase.id).toBe(slug);
expect(testCase.userMessage.length).toBeGreaterThan(0);
// Calling the check with an empty outcome must not throw on rule validation;
// it should simply report pass/fail. This pins the JSON shape.
expect(() =>
runExpectedToolsInvokedCheck(testCase, {
workflowIds: [],
executionIds: [],
dataTableIds: [],
artifactRefs: [],
finalText: '',
toolCalls: [],
agentActivities: [],
}),
).not.toThrow();
});
it('keeps positive and negative scenarios in the set (so we test both directions)', () => {
const positive = cases.filter((c) => Boolean(c.testCase.expectedToolInvocations.anyOf?.length));
const negative = cases.filter((c) =>
Boolean(c.testCase.expectedToolInvocations.noneOf?.length),
);
expect(positive.length).toBeGreaterThan(0);
expect(negative.length).toBeGreaterThan(0);
});
});
describe('discoveryTestCaseSchema', () => {
const valid = {
id: 'my-scenario',
userMessage: 'do the thing',
expectedToolInvocations: { anyOf: ['build-workflow'] },
};
it('accepts a minimal valid case', () => {
expect(discoveryTestCaseSchema.safeParse(valid).success).toBe(true);
});
it('rejects a typo-d key instead of passing vacuously', () => {
const typo = { ...valid, expectedToolInvocation: { anyOf: ['x'] } };
expect(discoveryTestCaseSchema.safeParse(typo).success).toBe(false);
});
it('rejects empty expectations — a case must assert something', () => {
const empty = { ...valid, expectedToolInvocations: {} };
expect(discoveryTestCaseSchema.safeParse(empty).success).toBe(false);
});
it('rejects an empty expectation list — dead config must fail at load time', () => {
const emptyList = { ...valid, expectedToolInvocations: { allOfToolCalls: [] } };
expect(discoveryTestCaseSchema.safeParse(emptyList).success).toBe(false);
});
it.each([
[
'connected with tool categories',
{ status: 'connected', toolCategories: ['screenshot'] },
true,
],
['connected without tool categories', { status: 'connected' }, false],
['disconnected', { status: 'disconnected' }, true],
['disabledByUser', { status: 'disabledByUser' }, true],
['unavailable', { status: 'unavailable' }, true],
['an unknown status', { status: 'on-fire' }, false],
])('validates each computerUse channel strictly: %s', (_name, channel, ok) => {
const withChannel = {
...valid,
instanceState: {
computerUse: { localComputer: channel, browser: { status: 'unavailable' } },
},
};
expect(discoveryTestCaseSchema.safeParse(withChannel).success).toBe(ok);
});
it('requires both channels — a half-filled state is dead config', () => {
const halfFilled = {
...valid,
instanceState: { computerUse: { localComputer: { status: 'disconnected' } } },
};
expect(discoveryTestCaseSchema.safeParse(halfFilled).success).toBe(false);
});
it('rejects the retired localGateway key instead of passing vacuously', () => {
const retired = { ...valid, instanceState: { localGateway: { status: 'disconnected' } } };
expect(discoveryTestCaseSchema.safeParse(retired).success).toBe(false);
});
});

14
packages/@n8n/instance-ai/evaluations/data/discovery/config-evals-skill-loading.json
View File

@ -1,14 +0,0 @@
{
"id": "config-evals-skill-loading",
"userMessage": "Set up an evaluation on my existing workflow that scores the AI Agent's answers against a test dataset. Use a config-based evaluation — do not add any evaluation nodes to the canvas.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["config-evals"]
}
],
"noneOf": ["evals", "spawn_sub_agent:eval-setup", "eval-setup-with-agent"]
},
"rationale": "Regression coverage for runtime skill loading of the config-evals skill. Config-based evaluation work must load the config-evals skill, and must never wire on-canvas evaluation nodes via the node-based evals tool or the eval-setup sub-agent."
}

18
packages/@n8n/instance-ai/evaluations/data/discovery/data-table-natural-list-skill-loading.json
View File

@ -1,18 +0,0 @@
{
"id": "data-table-natural-list-skill-loading",
"userMessage": "What data tables do I have?",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["data-table-manager"]
},
{
"toolName": "data-tables",
"argsContainAny": ["list"]
}
],
"noneOf": ["create-tasks", "build-workflow", "spawn_sub_agent:workflow-builder"]
},
"rationale": "Regression coverage for natural standalone data-table requests. A plain request to list existing Data Tables should load the data-table-manager skill and use the direct data-tables list action, without routing through planning, workflow-building, or background agents."
}

18
packages/@n8n/instance-ai/evaluations/data/discovery/data-table-skill-loading.json
View File

@ -1,18 +0,0 @@
{
"id": "data-table-skill-loading",
"userMessage": "Use the data-table-manager skill to list my n8n Data Tables and summarize what tables are available. This is standalone data-table work; do not build or modify a workflow.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["data-table-manager"]
},
{
"toolName": "data-tables",
"argsContainAny": ["list"]
}
],
"noneOf": ["create-tasks", "build-workflow", "spawn_sub_agent:workflow-builder"]
},
"rationale": "Regression coverage for runtime skill loading. Standalone data-table work must load the data-table-manager skill and call data-tables directly, without routing through planning, workflow-building, or task creation."
}

22
packages/@n8n/instance-ai/evaluations/data/discovery/data-table-workflow-skill-loading.json
View File

@ -1,22 +0,0 @@
{
"id": "data-table-workflow-skill-loading",
"userMessage": "Create me a workflow that implements an n8n form to capture responses into a data table. The form should be a decklist submission form for a MTG tournament, asking for player name and a deck list as text fields they'll fill.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["data-table-manager"]
},
{
"toolName": "load_skill",
"argsContainAny": ["workflow-builder"]
},
{
"toolName": "build-workflow"
}
],
"noneOf": ["create-tasks"]
},
"rationale": "Regression coverage for single-workflow build prompts that depend on Data Tables. The orchestrator should load both the data-table-manager skill and the workflow-builder skill, include workflow-local table requirements in the build, and avoid task planning or delegation.",
"timeoutMs": 180000
}

16
packages/@n8n/instance-ai/evaluations/data/discovery/folder-scoped-workflow-list.json
View File

@ -1,16 +0,0 @@
{
"id": "folder-scoped-workflow-list",
"userMessage": "Which workflows are in the Clients/Acme folder? Just list them, do not change anything.",
"instanceState": { "folderExploration": true },
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "workflows",
"args": { "action": "list" },
"argsContainAny": ["folderPath"]
}
],
"noneOf": ["build-workflow", "create-tasks", "spawn_sub_agent:workflow-builder"]
},
"rationale": "When the user names a FOLDER, the agent must address it with folderPath on workflows(list) rather than guessing a name filter or dumping the inventory. Folder membership is stored, not encoded in workflow names; a query prefix returns exactly the wrong set. Guards the discoverability of the folder-exploration capability."
}

24
packages/@n8n/instance-ai/evaluations/data/discovery/google-oauth-credential-setup.json
View File

@ -1,24 +0,0 @@
{
"id": "google-oauth-credential-setup",
"userMessage": "I need to set up a Google OAuth credential so I can read my calendar. Walk me through the Google Cloud Console so I can find my Client ID and Client Secret to enter into n8n.",
"instanceState": {
"computerUse": {
"localComputer": { "status": "disconnected" },
"browser": { "status": "connected", "toolCategories": ["browser"] }
}
},
"expectedToolInvocations": {
"anyOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["credential-setup-with-computer-use"]
},
{ "toolName": "browser_connect" },
{ "toolName": "browser_navigate" },
{ "toolName": "browser_snapshot" },
{ "toolName": "browser_screenshot" }
]
},
"rationale": "Google OAuth involves multiple console screens; the orchestrator must reach for the Computer Use credential skill or browser_* tools rather than asking the user to copy values into chat.",
"timeoutMs": 120000
}

14
packages/@n8n/instance-ai/evaluations/data/discovery/http-node-config-no-browser.json
View File

@ -1,14 +0,0 @@
{
"id": "http-node-config-no-browser",
"userMessage": "How do I set the request timeout on my HTTP Request node?",
"instanceState": {
"computerUse": {
"localComputer": { "status": "disconnected" },
"browser": { "status": "connected", "toolCategories": ["browser"] }
}
},
"expectedToolInvocations": {
"noneOf": ["browser_navigate", "browser_snapshot", "browser_screenshot"]
},
"rationale": "Negative control. A pure node-config question must NOT trigger Computer Use browser tooling. Guards against an over-eager regression that would bias every credential-adjacent question into browser mode."
}

183
packages/@n8n/instance-ai/evaluations/data/discovery/index.ts
View File

@ -1,183 +0,0 @@
import { readFileSync, readdirSync } from 'fs';
import { basename, join } from 'path';
import { z } from 'zod';
import type { ComputerUseChannelState, ComputerUseState } from '../../../src/types';
import type { DiscoveryMcpState } from '../../discovery/stub-mcp-registry';
import type { DiscoveryConfirmations, DiscoveryTestCase } from '../../discovery/types';
const forbiddenToolCallSchema = z
.object({
toolName: z.string().min(1),
args: z
.record(z.string(), z.unknown())
.refine((pattern) => Object.keys(pattern).length > 0, { message: 'args must not be empty' })
.optional(),
argsContainAny: z.array(z.string().min(1)).min(1).optional(),
declined: z.boolean().optional(),
})
.strict();
/** Mirrors `ComputerUseState` (src/types.ts) — the annotation makes tsc flag
* these schemas when the source union drifts. */
const computerUseChannelStateSchema: z.ZodType<ComputerUseChannelState> = z.discriminatedUnion(
'status',
[
z.object({ status: z.literal('unavailable') }).strict(),
z.object({ status: z.literal('disconnected') }).strict(),
z.object({ status: z.literal('disabledByUser') }).strict(),
z.object({ status: z.literal('connected'), toolCategories: z.array(z.string()) }).strict(),
],
);
const computerUseStateSchema: z.ZodType<ComputerUseState> = z
.object({
localComputer: computerUseChannelStateSchema,
browser: computerUseChannelStateSchema,
})
.strict();
const mcpStateSchema: z.ZodType<DiscoveryMcpState> = z
.object({
registry: z.array(z.string().min(1)).optional(),
connected: z
.array(
z
.object({ slug: z.string().min(1), tools: z.array(z.string().min(1)).optional() })
.strict(),
)
.optional(),
})
.strict();
const confirmationDecisionSchema = z.enum(['approve', 'deny']);
/** An empty map is dead config: the default (approve everything) already applies. */
const confirmationsSchema: z.ZodType<DiscoveryConfirmations> = z
.record(
z.string().min(1),
z.union([
confirmationDecisionSchema,
z
.object({
decision: confirmationDecisionSchema,
resumeWith: z
.record(z.string(), z.unknown())
.refine((fields) => Object.keys(fields).length > 0, {
message: 'resumeWith must not be empty',
})
.refine((fields) => !('approved' in fields), {
message: 'resumeWith must not set `approved` — use `decision` instead',
})
.optional(),
})
.strict(),
]),
)
.refine((entries) => Object.keys(entries).length > 0, {
message: 'confirmations must not be empty',
});
/** Strict authoring schema for discovery cases — a typo'd key or an empty
* expectation must fail at load time, not pass vacuously at run time (the
* workflow-case loader has had this guarantee for a while; discovery cases
* were a blind JSON.parse cast until TRUST-261's cleanup). */
export const discoveryTestCaseSchema = z
.object({
id: z.string().min(1),
userMessage: z.string().min(1),
instanceState: z
.object({
computerUse: computerUseStateSchema.optional(),
mcp: mcpStateSchema.optional(),
folderExploration: z.boolean().optional(),
})
.strict()
.optional(),
confirmations: confirmationsSchema.optional(),
expectedToolInvocations: z
.object({
// min(1) on every list: an empty expectation array is dead config that
// would otherwise pass here and only surface as a run-time failure.
anyOf: z.array(z.string().min(1)).min(1).optional(),
noneOf: z.array(z.string().min(1)).min(1).optional(),
anyOfToolCalls: z.array(forbiddenToolCallSchema).min(1).optional(),
allOfToolCalls: z.array(forbiddenToolCallSchema).min(1).optional(),
noneOfToolCalls: z.array(forbiddenToolCallSchema).min(1).optional(),
})
.strict()
.refine((expectations) => Object.values(expectations).some((v) => v !== undefined), {
message: 'expectedToolInvocations needs at least one expectation key',
}),
rationale: z.string().optional(),
maxSteps: z.number().int().positive().optional(),
timeoutMs: z.number().int().positive().optional(),
})
.strict();
export interface DiscoveryTestCaseWithFile {
testCase: DiscoveryTestCase;
/** Filename without extension, e.g. "slack-oauth-credential-setup" */
fileSlug: string;
}
function parseTestCaseFile(filePath: string): DiscoveryTestCase {
const content = readFileSync(filePath, 'utf-8');
let raw: unknown;
try {
raw = JSON.parse(content);
} catch (error) {
throw new Error(
`Failed to parse discovery test case ${filePath}: ${error instanceof Error ? error.message : String(error)}`,
);
}
const parsed = discoveryTestCaseSchema.safeParse(raw);
if (!parsed.success) {
const issues = parsed.error.issues
.map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`)
.join('; ');
throw new Error(`Invalid discovery test case ${filePath}: ${issues}`);
}
return parsed.data;
}
function parseSubstringList(value: string | undefined): string[] {
if (!value) return [];
return value
.split(',')
.map((s) => s.trim().toLowerCase())
.filter((s) => s.length > 0);
}
function getJsonFiles(filter?: string, exclude?: string): string[] {
const dir = __dirname;
let files = readdirSync(dir).filter((f) => f.endsWith('.json'));
const includeTokens = parseSubstringList(filter);
if (includeTokens.length > 0) {
files = files.filter((f) => {
const lower = f.toLowerCase();
return includeTokens.some((t) => lower.includes(t));
});
}
const excludeTokens = parseSubstringList(exclude);
if (excludeTokens.length > 0) {
files = files.filter((f) => {
const lower = f.toLowerCase();
return !excludeTokens.some((t) => lower.includes(t));
});
}
return files.map((f) => join(dir, f));
}
export function loadDiscoveryTestCasesWithFiles(
filter?: string,
exclude?: string,
): DiscoveryTestCaseWithFile[] {
return getJsonFiles(filter, exclude).map((f) => ({
testCase: parseTestCaseFile(f),
fileSlug: basename(f, '.json'),
}));
}

25
packages/@n8n/instance-ai/evaluations/data/discovery/intent-ambiguous-alert-response-clarifies.json
View File

@ -1,25 +0,0 @@
{
"id": "intent-ambiguous-alert-response-clarifies",
"userMessage": "I need to ingest a cybersecurity alert, analyze it, and take actions depending on the finding.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["intent-recognition"]
},
{
"toolName": "ask-user",
"argsContainAny": [
"fixed rules",
"predefined rules",
"human judgment",
"AI judgment",
"case-by-case",
"decide which action"
]
}
]
},
"rationale": "The request does not say whether fixed rules or judgment choose each action. The assistant must clarify this anchor-defining choice before it selects an artifact.",
"timeoutMs": 120000
}

22
packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-creative-publisher.json
View File

@ -1,22 +0,0 @@
{
"id": "intent-explicit-agent-creative-publisher",
"userMessage": "Build me an agent that decides what faceless YouTube videos to make and drafts each script.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["agent-builder"]
},
{
"toolName": "build-agent"
}
],
"noneOfToolCalls": [
{
"toolName": "build-workflow"
}
]
},
"rationale": "An explicit Agent request with open-ended topic selection and creative work must use the Agent build path. This trims publishing integrations from the sourced failure so the case tests routing only.",
"timeoutMs": 120000
}

22
packages/@n8n/instance-ai/evaluations/data/discovery/intent-explicit-agent-fixed-duty.json
View File

@ -1,22 +0,0 @@
{
"id": "intent-explicit-agent-fixed-duty",
"userMessage": "Build me an agent that fetches new GitHub stars each night and sends me a digest.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["agent-builder"]
},
{
"toolName": "build-agent"
}
],
"noneOfToolCalls": [
{
"toolName": "build-workflow"
}
]
},
"rationale": "The user explicitly selected an Agent artifact. A fixed scheduled duty can be an Agent task, so the assistant must not override that selection with a workflow.",
"timeoutMs": 120000
}

26
packages/@n8n/instance-ai/evaluations/data/discovery/intent-fixed-scheduled-market-summary.json
View File

@ -1,26 +0,0 @@
{
"id": "intent-fixed-scheduled-market-summary",
"userMessage": "Every weekday at 8am, fetch the latest closing prices for the S&P 500 and Nasdaq, format them into a two-line summary, and email it to me.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["workflow-builder"]
},
{
"toolName": "build-workflow"
}
],
"noneOfToolCalls": [
{
"toolName": "build-agent"
},
{
"toolName": "load_skill",
"argsContainAny": ["agent-builder"]
}
]
},
"rationale": "This scheduled report has fixed inputs, formatting, and delivery. It guards against routing every recurring role-like task to an Agent.",
"timeoutMs": 120000
}

26
packages/@n8n/instance-ai/evaluations/data/discovery/intent-role-shaped-scheduled-analyst.json
View File

@ -1,26 +0,0 @@
{
"id": "intent-role-shaped-scheduled-analyst",
"userMessage": "Set up a daily market analyst. It should research market changes, decide which developments matter, and send me a briefing each morning.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["intent-recognition"]
},
{
"toolName": "load_skill",
"argsContainAny": ["agent-builder"]
},
{
"toolName": "build-agent"
}
],
"noneOfToolCalls": [
{
"toolName": "build-workflow"
}
]
},
"rationale": "A persistent delegated role owns an open-ended research and judgment duty. The schedule is an Agent task and does not justify a workflow shell.",
"timeoutMs": 120000
}

18
packages/@n8n/instance-ai/evaluations/data/discovery/mcp-broken-connection-reconnect.json
View File

@ -1,18 +0,0 @@
{
"id": "mcp-broken-connection-reconnect",
"userMessage": "Search my Notion for the Q3 roadmap page and tell me what's in it.",
"instanceState": {
"mcp": {
"registry": ["notion", "linear"],
"connected": [{ "slug": "notion", "tools": [] }]
}
},
"expectedToolInvocations": {
"allOfToolCalls": [
{ "toolName": "mcp-servers", "args": { "action": "connect", "serverSlugs": ["notion"] } }
],
"noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "search" } }]
},
"rationale": "A connected server serving no tools is a broken connection: the agent must reconnect the slug it already has, not fall back to a registry search that filters connected servers out and dead-ends.",
"maxSteps": 12
}

18
packages/@n8n/instance-ai/evaluations/data/discovery/mcp-connect-unconnected-service.json
View File

@ -1,18 +0,0 @@
{
"id": "mcp-connect-unconnected-service",
"userMessage": "Pull up the notes from my last product review in Notion and give me a short summary.",
"instanceState": {
"mcp": { "registry": ["notion", "linear"], "connected": [] }
},
"expectedToolInvocations": {
"allOfToolCalls": [
{ "toolName": "mcp-servers", "args": { "action": "search" } },
{ "toolName": "mcp-servers", "args": { "action": "connect", "serverSlugs": ["notion"] } }
],
"noneOfToolCalls": [
{ "toolName": "mcp-servers", "args": { "action": "connect", "serverSlugs": ["linear"] } }
]
},
"rationale": "Nothing is connected: the agent must find Notion in the registry and offer it, not declare it unreachable or recite manual setup steps. Linear is a decoy.",
"maxSteps": 12
}

17
packages/@n8n/instance-ai/evaluations/data/discovery/mcp-declined-tool-call-no-reconnect.json
View File

@ -1,17 +0,0 @@
{
"id": "mcp-declined-tool-call-no-reconnect",
"userMessage": "Find the 'Launch checklist' page in my Notion and summarise what's still open.",
"instanceState": {
"mcp": {
"registry": ["notion", "linear"],
"connected": [{ "slug": "notion", "tools": ["notion-search"] }]
}
},
"confirmations": { "mcp_notion_notion-search": "deny" },
"expectedToolInvocations": {
"allOfToolCalls": [{ "toolName": "mcp_notion_notion-search", "declined": true }],
"noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "connect" } }]
},
"rationale": "A refused tool call is the user withholding permission, not a broken connection: the agent must report back rather than reconnect a server that is already connected.",
"maxSteps": 12
}

13
packages/@n8n/instance-ai/evaluations/data/discovery/mcp-no-registry-match.json
View File

@ -1,13 +0,0 @@
{
"id": "mcp-no-registry-match",
"userMessage": "Pull up my latest deals from HubSpot and give me a short summary.",
"instanceState": {
"mcp": { "registry": ["notion", "linear"], "connected": [] }
},
"expectedToolInvocations": {
"allOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "search" } }],
"noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "connect" } }]
},
"rationale": "HubSpot is not in the registry: the agent must search before declaring it unavailable, and must not offer an invented slug or a near-miss server.",
"maxSteps": 12
}

17
packages/@n8n/instance-ai/evaluations/data/discovery/mcp-not-offered-for-workflow-build.json
View File

@ -1,17 +0,0 @@
{
"id": "mcp-not-offered-for-workflow-build",
"userMessage": "Build me a workflow that posts a standup reminder to our Slack channel every weekday at 9am.",
"instanceState": {
"mcp": { "registry": ["slack", "notion"], "connected": [] }
},
"expectedToolInvocations": {
"allOfToolCalls": [{ "toolName": "build-workflow" }],
"noneOfToolCalls": [
{ "toolName": "mcp-servers", "args": { "action": "search" } },
{ "toolName": "mcp-servers", "args": { "action": "connect" } }
]
},
"rationale": "A workflow talks to a service through its node and credential, not an MCP connection. Slack is in the registry to make the wrong move available. `connected` is allowed — only offering a connection is the regression.",
"maxSteps": 24,
"timeoutMs": 120000
}

16
packages/@n8n/instance-ai/evaluations/data/discovery/mcp-uses-connected-server-tools.json
View File

@ -1,16 +0,0 @@
{
"id": "mcp-uses-connected-server-tools",
"userMessage": "Pull up the notes from my last product review in Notion and give me a short summary.",
"instanceState": {
"mcp": { "registry": ["notion", "linear"], "connected": [{ "slug": "notion" }] }
},
"expectedToolInvocations": {
"anyOfToolCalls": [
{ "toolName": "mcp_notion_notion-search" },
{ "toolName": "mcp_notion_notion-fetch" }
],
"noneOfToolCalls": [{ "toolName": "mcp-servers", "args": { "action": "connect" } }]
},
"rationale": "Notion is already connected: the agent must use its tools instead of re-offering a connection the user already has.",
"maxSteps": 12
}

14
packages/@n8n/instance-ai/evaluations/data/discovery/oauth-with-computer-use-disabled.json
View File

@ -1,14 +0,0 @@
{
"id": "oauth-with-computer-use-disabled",
"userMessage": "I need to set up a Slack OAuth credential. Walk me through the Slack admin so I can find my Client ID and Client Secret to enter into n8n.",
"instanceState": {
"computerUse": {
"localComputer": { "status": "unavailable" },
"browser": { "status": "unavailable" }
}
},
"expectedToolInvocations": {
"noneOf": ["browser_connect", "browser_navigate", "browser_snapshot", "browser_screenshot"]
},
"rationale": "When Computer Use is disabled globally the orchestrator must not pretend any browser tool exists. It should explain the standard credential setup path instead. Pairs with the OAuth-positive scenarios to prove discovery is gated on gateway state, not blindly triggered by the user message."
}

14
packages/@n8n/instance-ai/evaluations/data/discovery/screenshot-dashboard.json
View File

@ -1,14 +0,0 @@
{
"id": "screenshot-dashboard",
"userMessage": "Take a screenshot of the Stripe dashboard I have open in my browser so I can attach it to this workflow.",
"instanceState": {
"computerUse": {
"localComputer": { "status": "disconnected" },
"browser": { "status": "connected", "toolCategories": ["browser"] }
}
},
"expectedToolInvocations": {
"anyOf": ["browser_screenshot", "browser_snapshot", "browser_navigate"]
},
"rationale": "Direct browser-tool discovery — the user names a browser action verbatim. The orchestrator must reach for any browser tool (snapshot/screenshot/navigate are all valid first moves to inspect the page) rather than falling back to research."
}

24
packages/@n8n/instance-ai/evaluations/data/discovery/slack-oauth-credential-setup.json
View File

@ -1,24 +0,0 @@
{
"id": "slack-oauth-credential-setup",
"userMessage": "I need to set up a Slack OAuth credential. Walk me through the Slack admin so I can find my Client ID and Client Secret to enter into n8n.",
"instanceState": {
"computerUse": {
"localComputer": { "status": "disconnected" },
"browser": { "status": "connected", "toolCategories": ["browser"] }
}
},
"expectedToolInvocations": {
"anyOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["credential-setup-with-computer-use"]
},
{ "toolName": "browser_connect" },
{ "toolName": "browser_navigate" },
{ "toolName": "browser_snapshot" },
{ "toolName": "browser_screenshot" }
]
},
"rationale": "OAuth credential setup is the canonical browser-discovery moment. The orchestrator must reach for the Computer Use credential skill or browser_* tools directly; falling through to research/ask-user without browser involvement is the regression.",
"timeoutMs": 120000
}

19
packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-agent-builder-leak.json
View File

@ -1,19 +0,0 @@
{
"id": "workflow-builder-no-agent-builder-leak",
"userMessage": "Build a workflow that checks an RSS feed every hour and posts new items to a Slack channel.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["workflow-builder"]
}
],
"noneOfToolCalls": [
{
"toolName": "build-agent"
}
]
},
"rationale": "Regression gate for the agent-builder path into Instance AI. A plain workflow-building prompt must route through the workflow-builder skill and must never invoke the build-agent orchestration tool. Meaningful in both module modes; the strongest signal is a run with N8N_ENABLED_MODULES including 'agents', where build-agent is registered (it is always-loaded, not deferred) yet must still not be selected.",
"timeoutMs": 120000
}

29
packages/@n8n/instance-ai/evaluations/data/discovery/workflow-builder-no-credential-ask.json
View File

@ -1,29 +0,0 @@
{
"id": "workflow-builder-no-credential-ask",
"userMessage": "Build a workflow that lets customers request a meeting time in natural language, parses the preferred time with OpenAI, creates a Google Calendar event, and sends a Gmail confirmation email.",
"expectedToolInvocations": {
"allOfToolCalls": [
{
"toolName": "load_skill",
"argsContainAny": ["workflow-builder"]
},
{
"toolName": "build-workflow"
}
],
"noneOf": ["create-tasks"],
"noneOfToolCalls": [
{
"toolName": "ask-user",
"argsContainAny": ["credential", "credentials", "account", "auth", "api key", "token"]
},
{
"toolName": "ask-user",
"argsContainAny": ["timezone", "time zone"]
}
]
},
"rationale": "Regression coverage for INS-204. A clear single-workflow build should route through the workflow-builder skill directly, may use credentials(action=\"list\") for discovery, and must not ask the user which credential/account to use when the builder can auto-select or mock unresolved credentials. It should also use contextual timezone data rather than asking for it.",
"maxSteps": 24,
"timeoutMs": 150000
}

269
packages/@n8n/instance-ai/evaluations/discovery/__tests__/confirmation-policy.test.ts
View File

@ -1,269 +0,0 @@
import type { SuspensionInfo } from '../../../src/utils/stream-helpers';
import { discoveryTestCaseSchema } from '../../data/discovery';
import {
buildConfirmationPolicy,
resolveConfirmation,
unmatchedConfirmations,
} from '../confirmation-policy';
import { credentialAutoSetupResponder } from '../credential-approval';
import { createMcpConnectResponder, createStubMcpRegistry } from '../stub-mcp-registry';
import type { DiscoveryTestCase } from '../types';
function scenario(overrides: Partial<DiscoveryTestCase> = {}): DiscoveryTestCase {
return {
id: 'test',
userMessage: 'do the thing',
expectedToolInvocations: { anyOf: ['mcp-servers'] },
...overrides,
};
}
function suspension(
toolName: string,
suspendPayload: Record<string, unknown> = {},
): SuspensionInfo {
return { toolCallId: 'call-1', requestId: 'req-1', toolName, suspendPayload };
}
const connectPayload = {
mcpConnectRequest: { servers: [{ serverSlug: 'notion' }, { serverSlug: 'linear' }] },
};
const credentialPayload = {
credentialRequests: [{ credentialType: 'slackOAuth2Api' }],
};
function mcpResponders() {
const registry = createStubMcpRegistry({ registry: ['notion', 'linear'], connected: [] });
return { registry, responders: [createMcpConnectResponder(registry)] };
}
describe('buildConfirmationPolicy', () => {
it('is empty when the scenario declares nothing', () => {
expect(buildConfirmationPolicy(scenario()).size).toBe(0);
});
it('normalizes a bare decision into an answer', () => {
const policy = buildConfirmationPolicy(scenario({ confirmations: { 'mcp-servers': 'deny' } }));
expect(policy.get('mcp-servers')).toEqual({ decision: 'deny' });
});
it('keeps an answer object as declared', () => {
const answer = { decision: 'approve' as const, resumeWith: { connectedSlugs: ['notion'] } };
const policy = buildConfirmationPolicy(scenario({ confirmations: { 'mcp-servers': answer } }));
expect(policy.get('mcp-servers')).toEqual(answer);
});
});
describe('resolveConfirmation', () => {
const empty = buildConfirmationPolicy(scenario());
it('approves a suspension no policy entry covers', () => {
expect(resolveConfirmation(suspension('some-tool'), empty)).toEqual({ approved: true });
});
it('approves when the suspension is unknown', () => {
expect(resolveConfirmation(undefined, empty)).toEqual({ approved: true });
});
it('denies when the scenario says so', async () => {
const policy = buildConfirmationPolicy(scenario({ confirmations: { 'mcp-servers': 'deny' } }));
const { responders, registry } = mcpResponders();
expect(
resolveConfirmation(suspension('mcp-servers', connectPayload), policy, responders),
).toEqual({ approved: false });
await expect(registry.service.listConnections()).resolves.toEqual([]);
});
it('denies an mcp tool call by its prefixed name', () => {
const policy = buildConfirmationPolicy(
scenario({ confirmations: { 'mcp_notion_notion-search': 'deny' } }),
);
const approval = { type: 'approval', toolName: 'mcp_notion_notion-search', args: {} };
expect(resolveConfirmation(suspension('mcp_notion_notion-search', approval), policy)).toEqual({
approved: false,
});
expect(resolveConfirmation(suspension('mcp_linear_list_issues', approval), policy)).toEqual({
approved: true,
});
});
it('falls back to the tool name the approval gate echoes into the payload', () => {
const policy = buildConfirmationPolicy(
scenario({ confirmations: { 'mcp_notion_notion-search': 'deny' } }),
);
const gated: SuspensionInfo = {
toolCallId: 'call-1',
requestId: 'req-1',
suspendPayload: { type: 'approval', toolName: 'mcp_notion_notion-search', args: {} },
};
expect(resolveConfirmation(gated, policy)).toEqual({ approved: false });
});
it('takes the first responder that recognises the payload', () => {
const { responders } = mcpResponders();
const all = [credentialAutoSetupResponder, ...responders];
expect(resolveConfirmation(suspension('mcp-servers', connectPayload), empty, all)).toEqual({
approved: true,
connectedSlugs: ['notion', 'linear'],
});
expect(resolveConfirmation(suspension('credentials', credentialPayload), empty, all)).toEqual({
approved: true,
autoSetup: { credentialType: 'slackOAuth2Api' },
});
});
it('lets the scenario override responder fields without losing their side effects', async () => {
const { registry, responders } = mcpResponders();
const policy = buildConfirmationPolicy(
scenario({
confirmations: {
'mcp-servers': { decision: 'approve', resumeWith: { connectedSlugs: ['notion'] } },
},
}),
);
expect(
resolveConfirmation(suspension('mcp-servers', connectPayload), policy, responders),
).toEqual({ approved: true, connectedSlugs: ['notion'] });
await expect(registry.service.listConnections()).resolves.toEqual([
{ slug: 'notion' },
{ slug: 'linear' },
]);
});
it('keeps a deny denied even when resumeWith claims approval', () => {
const policy = buildConfirmationPolicy(
scenario({
confirmations: { 'mcp-servers': { decision: 'deny', resumeWith: { approved: true } } },
}),
);
const { responders } = mcpResponders();
expect(
resolveConfirmation(suspension('mcp-servers', connectPayload), policy, responders),
).toEqual({ approved: false });
});
it('keeps an approve approved even when resumeWith claims refusal', () => {
const policy = buildConfirmationPolicy(
scenario({
confirmations: { 'ask-user': { decision: 'approve', resumeWith: { approved: false } } },
}),
);
expect(resolveConfirmation(suspension('ask-user'), policy)).toEqual({ approved: true });
});
it('carries resumeWith when no responder recognises the payload', () => {
const policy = buildConfirmationPolicy(
scenario({
confirmations: { 'ask-user': { decision: 'approve', resumeWith: { answer: 'yes' } } },
}),
);
expect(resolveConfirmation(suspension('ask-user'), policy)).toEqual({
approved: true,
answer: 'yes',
});
});
});
describe('unmatchedConfirmations', () => {
const policy = buildConfirmationPolicy(
scenario({
confirmations: {
'mcp_notion_notion-search': 'deny',
credentials: { decision: 'approve', resumeWith: { autoSetup: { credentialType: 'x' } } },
},
}),
);
it('reports every behaviour-changing answer no suspension asked for', () => {
expect(unmatchedConfirmations(policy, [])).toEqual(['mcp_notion_notion-search', 'credentials']);
});
it('reports nothing once each declared tool has suspended', () => {
const asked = [suspension('mcp_notion_notion-search'), suspension('credentials')];
expect(unmatchedConfirmations(policy, asked)).toEqual([]);
});
it('exempts a bare approve, which only asks for the default', () => {
const approveOnly = buildConfirmationPolicy(
scenario({ confirmations: { 'ask-user': 'approve' } }),
);
expect(unmatchedConfirmations(approveOnly, [])).toEqual([]);
});
it('matches the tool name echoed into the suspend payload', () => {
const gated: SuspensionInfo = {
toolCallId: 'call-1',
requestId: 'req-1',
suspendPayload: { type: 'approval', toolName: 'mcp_notion_notion-search' },
};
expect(unmatchedConfirmations(policy, [gated, suspension('credentials')])).toEqual([]);
});
it('ignores suspensions the scenario never declared', () => {
expect(
unmatchedConfirmations(buildConfirmationPolicy(scenario()), [suspension('nodes')]),
).toEqual([]);
});
});
describe('credentialAutoSetupResponder', () => {
it('requests browser auto-setup for the first credential', () => {
expect(credentialAutoSetupResponder(credentialPayload)).toEqual({
autoSetup: { credentialType: 'slackOAuth2Api' },
});
});
it.each([
['a payload with no credential requests', {}],
['an empty request list', { credentialRequests: [] }],
['a request with no credential type', { credentialRequests: [{}] }],
])('passes on %s', (_label, payload) => {
expect(credentialAutoSetupResponder(payload)).toBeUndefined();
});
});
describe('createMcpConnectResponder', () => {
it('marks the offered slugs live on the registry', async () => {
const { registry, responders } = mcpResponders();
expect(responders[0](connectPayload)).toEqual({ connectedSlugs: ['notion', 'linear'] });
await expect(registry.service.listConnections()).resolves.toEqual([
{ slug: 'notion' },
{ slug: 'linear' },
]);
});
it('passes on a payload carrying no connect request', () => {
const { responders } = mcpResponders();
expect(responders[0]({ credentialRequests: [] })).toBeUndefined();
});
});
describe('discoveryTestCaseSchema', () => {
const base = { id: 'x', userMessage: 'y', expectedToolInvocations: { anyOf: ['a'] } };
it.each([
['a bare decision', { 'mcp-servers': 'deny' }],
['an answer object', { 'mcp-servers': { decision: 'approve' } }],
['an answer with resume data', { credentials: { decision: 'approve', resumeWith: { a: 1 } } }],
])('accepts %s', (_label, confirmations) => {
expect(discoveryTestCaseSchema.safeParse({ ...base, confirmations }).success).toBe(true);
});
it.each([
['an unknown decision', { confirmations: { 'mcp-servers': 'maybe' } }],
['an empty map', { confirmations: {} }],
['a missing decision', { confirmations: { 'mcp-servers': { resumeWith: { a: 1 } } } }],
[
'an empty resumeWith',
{ confirmations: { 'mcp-servers': { decision: 'approve', resumeWith: {} } } },
],
['a misspelled key', { confirmation: { 'mcp-servers': 'deny' } }],
[
'a resumeWith setting the reserved `approved` field',
{ confirmations: { 'mcp-servers': { decision: 'deny', resumeWith: { approved: true } } } },
],
])('rejects %s', (_label, invalid) => {
expect(discoveryTestCaseSchema.safeParse({ ...base, ...invalid }).success).toBe(false);
});
});

696
packages/@n8n/instance-ai/evaluations/discovery/__tests__/expected-tools-invoked.test.ts
View File

@ -1,696 +0,0 @@
import type { AgentActivity, CapturedToolCall, EventOutcome } from '../../types';
import { evaluateDiscoveryTrial, runExpectedToolsInvokedCheck } from '../expected-tools-invoked';
import type { DiscoveryTestCase, DiscoveryTrialFacts } from '../types';
type ToolCallInput = Pick<CapturedToolCall, 'toolName'> &
Partial<Pick<CapturedToolCall, 'args' | 'result'>>;
function makeToolCall(tc: ToolCallInput, i: number): CapturedToolCall {
return {
toolCallId: `call-${i}`,
toolName: tc.toolName,
args: tc.args ?? {},
...(tc.result === undefined ? {} : { result: tc.result }),
durationMs: 0,
};
}
function makeOutcome(opts: {
toolCalls?: ToolCallInput[];
agents?: Array<Pick<AgentActivity, 'role' | 'tools'> & { toolCalls?: ToolCallInput[] }>;
}): EventOutcome {
return {
workflowIds: [],
executionIds: [],
dataTableIds: [],
artifactRefs: [],
finalText: '',
toolCalls: (opts.toolCalls ?? []).map(makeToolCall),
agentActivities: (opts.agents ?? []).map((a, i) => ({
agentId: `agent-${i}`,
role: a.role,
tools: a.tools,
toolCalls: (a.toolCalls ?? []).map(makeToolCall),
textContent: '',
reasoning: '',
status: 'completed',
})),
};
}
const slackOauthScenario: DiscoveryTestCase = {
id: 'test',
userMessage: 'Help me set up Slack credentials',
expectedToolInvocations: {
anyOf: ['load_skill', 'browser_navigate'],
},
};
describe('runExpectedToolsInvokedCheck', () => {
describe('anyOf — positive cases', () => {
it('passes when the expected top-level tool was invoked', () => {
const result = runExpectedToolsInvokedCheck(
slackOauthScenario,
makeOutcome({ toolCalls: [{ toolName: 'load_skill' }] }),
);
expect(result.pass).toBe(true);
expect(result.invokedTools).toContain('load_skill');
});
it('passes when a named sub-agent was spawned (matched via spawn_sub_agent: prefix)', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: '',
expectedToolInvocations: { anyOf: ['spawn_sub_agent:researcher'] },
},
makeOutcome({
agents: [{ role: 'researcher', tools: ['credentials'] }],
}),
);
expect(result.pass).toBe(true);
expect(result.spawnedAgents).toContain('spawn_sub_agent:researcher');
});
it('passes when the spawned sub-agent had the expected tool attached (sub-agent tools list)', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: '',
expectedToolInvocations: { anyOf: ['credentials'] },
},
makeOutcome({
agents: [{ role: 'researcher', tools: ['credentials'] }],
}),
);
expect(result.pass).toBe(true);
expect(result.invokedTools).toContain('credentials');
});
});
describe('anyOf — negative cases', () => {
it('fails when none of the expected tools or sub-agents appear', () => {
const result = runExpectedToolsInvokedCheck(
slackOauthScenario,
makeOutcome({ toolCalls: [{ toolName: 'research' }] }),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('Expected at least one of');
expect(result.comment).toContain('browser_navigate');
});
it('fails when the orchestrator only ran research with no browser dispatch', () => {
const result = runExpectedToolsInvokedCheck(
slackOauthScenario,
makeOutcome({
toolCalls: [{ toolName: 'research' }, { toolName: 'ask-user' }],
}),
);
expect(result.pass).toBe(false);
});
});
describe('noneOf — negative scenarios (over-eager invocation guard)', () => {
const httpNodeConfigScenario: DiscoveryTestCase = {
id: 'test',
userMessage: 'How do I set the timeout on my HTTP node?',
expectedToolInvocations: {
noneOf: ['browser_navigate', 'spawn_sub_agent:credential-helper'],
},
};
it('passes when the forbidden tool was not invoked', () => {
const result = runExpectedToolsInvokedCheck(
httpNodeConfigScenario,
makeOutcome({ toolCalls: [{ toolName: 'nodes' }] }),
);
expect(result.pass).toBe(true);
});
it('fails when the forbidden tool was invoked', () => {
const result = runExpectedToolsInvokedCheck(
httpNodeConfigScenario,
makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('Expected none of');
expect(result.comment).toContain('browser_navigate');
});
it('fails when the forbidden sub-agent was spawned', () => {
const result = runExpectedToolsInvokedCheck(
httpNodeConfigScenario,
makeOutcome({
agents: [{ role: 'credential-helper', tools: ['browser_navigate'] }],
}),
);
expect(result.pass).toBe(false);
});
});
describe('combined anyOf + noneOf', () => {
const combinedScenario: DiscoveryTestCase = {
id: 'test',
userMessage: '',
expectedToolInvocations: {
anyOf: ['browser_navigate'],
noneOf: ['delegate'],
},
};
it('passes when anyOf matches and noneOf is not violated', () => {
const result = runExpectedToolsInvokedCheck(
combinedScenario,
makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }),
);
expect(result.pass).toBe(true);
});
it('fails when noneOf is violated even if anyOf matched', () => {
const result = runExpectedToolsInvokedCheck(
combinedScenario,
makeOutcome({
toolCalls: [{ toolName: 'browser_navigate' }, { toolName: 'delegate' }],
}),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('Expected none of');
});
});
describe('noneOfToolCalls — actual tool-call guard', () => {
const planningScenario: DiscoveryTestCase = {
id: 'test',
userMessage: 'Build a Gmail and Calendar workflow',
expectedToolInvocations: {
anyOf: ['create-tasks'],
noneOfToolCalls: [{ toolName: 'ask-user', argsContainAny: ['credential'] }],
},
};
it('passes when ask-user is available to a spawned agent but is not called', () => {
const result = runExpectedToolsInvokedCheck(
planningScenario,
makeOutcome({
toolCalls: [{ toolName: 'create-tasks' }],
agents: [{ role: 'delegate', tools: ['credentials', 'ask-user'] }],
}),
);
expect(result.pass).toBe(true);
expect(result.invokedTools).toContain('ask-user');
});
it('fails when the forbidden tool call happens with matching args', () => {
const result = runExpectedToolsInvokedCheck(
planningScenario,
makeOutcome({
toolCalls: [
{ toolName: 'create-tasks' },
{
toolName: 'ask-user',
args: { question: 'Which Google Calendar credential should I use?' },
},
],
agents: [{ role: 'delegate', tools: ['credentials', 'ask-user'] }],
}),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('Expected no actual tool call matching');
expect(result.comment).toContain('credential');
});
it('passes when the same tool is called for unrelated args', () => {
const result = runExpectedToolsInvokedCheck(
planningScenario,
makeOutcome({
toolCalls: [
{ toolName: 'create-tasks' },
{ toolName: 'ask-user', args: { question: 'Which failure branch should run?' } },
],
agents: [{ role: 'delegate', tools: ['credentials', 'ask-user'] }],
}),
);
expect(result.pass).toBe(true);
});
});
describe('anyOfToolCalls — actual tool-call alternatives', () => {
const credentialSetupScenario: DiscoveryTestCase = {
id: 'test',
userMessage: 'Help me set up Slack credentials',
expectedToolInvocations: {
anyOfToolCalls: [
{ toolName: 'load_skill', argsContainAny: ['credential-setup-with-computer-use'] },
{ toolName: 'browser_navigate' },
],
},
};
it('passes when one expected actual tool call happened with matching args', () => {
const result = runExpectedToolsInvokedCheck(
credentialSetupScenario,
makeOutcome({
toolCalls: [
{
toolName: 'load_skill',
args: { skillId: 'credential-setup-with-computer-use' },
},
],
}),
);
expect(result.pass).toBe(true);
});
it('passes when a browser fallback tool call happened', () => {
const result = runExpectedToolsInvokedCheck(
credentialSetupScenario,
makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] }),
);
expect(result.pass).toBe(true);
});
it('fails when only an unrelated skill was loaded', () => {
const result = runExpectedToolsInvokedCheck(
credentialSetupScenario,
makeOutcome({ toolCalls: [{ toolName: 'load_skill', args: { skillId: 'other-skill' } }] }),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('credential-setup-with-computer-use');
});
});
describe('allOfToolCalls — actual tool-call requirements', () => {
const dataTableScenario: DiscoveryTestCase = {
id: 'test',
userMessage: 'List my n8n Data Tables.',
expectedToolInvocations: {
allOfToolCalls: [
{ toolName: 'load_skill', argsContainAny: ['data-table-manager'] },
{ toolName: 'data-tables', argsContainAny: ['list'] },
],
},
};
it('passes when every expected actual tool call happened with matching args', () => {
const result = runExpectedToolsInvokedCheck(
dataTableScenario,
makeOutcome({
toolCalls: [
{ toolName: 'load_skill', args: { skillId: 'data-table-manager' } },
{ toolName: 'data-tables', args: { action: 'list' } },
],
}),
);
expect(result.pass).toBe(true);
});
it('fails when a tool is only available to a spawned agent but was not called', () => {
const result = runExpectedToolsInvokedCheck(
dataTableScenario,
makeOutcome({
toolCalls: [{ toolName: 'load_skill', args: { skillId: 'data-table-manager' } }],
agents: [{ role: 'workflow-builder', tools: ['data-tables'] }],
}),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('Expected actual tool call matching');
expect(result.comment).toContain('data-tables');
});
it('fails when the tool call args do not match the expectation', () => {
const result = runExpectedToolsInvokedCheck(
dataTableScenario,
makeOutcome({
toolCalls: [
{ toolName: 'load_skill', args: { skillId: 'data-table-manager' } },
{ toolName: 'data-tables', args: { action: 'schema' } },
],
}),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('list');
});
});
describe('args — structured argument matching', () => {
const connectNotion: DiscoveryTestCase = {
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
anyOfToolCalls: [
{ toolName: 'mcp-servers', args: { action: 'connect', serverSlugs: ['notion'] } },
],
},
};
it('passes on a deep-partial match, ignoring unlisted keys', () => {
const result = runExpectedToolsInvokedCheck(
connectNotion,
makeOutcome({
toolCalls: [
{
toolName: 'mcp-servers',
args: { action: 'connect', serverSlugs: ['notion'], reason: 'unlocks search' },
},
],
}),
);
expect(result.pass).toBe(true);
});
it('matches an array as a subset, not by length or order', () => {
const result = runExpectedToolsInvokedCheck(
connectNotion,
makeOutcome({
toolCalls: [
{
toolName: 'mcp-servers',
args: { action: 'connect', serverSlugs: ['linear', 'notion'] },
},
],
}),
);
expect(result.pass).toBe(true);
});
it('fails when a listed key differs, even though a substring match would pass', () => {
const result = runExpectedToolsInvokedCheck(
connectNotion,
makeOutcome({
toolCalls: [
{
toolName: 'mcp-servers',
args: {
action: 'connect',
serverSlugs: ['linear'],
reason: 'Linear covers this instead of Notion',
},
},
],
}),
);
expect(result.pass).toBe(false);
});
it('fails when the action differs', () => {
const result = runExpectedToolsInvokedCheck(
connectNotion,
makeOutcome({
toolCalls: [{ toolName: 'mcp-servers', args: { action: 'search', queries: ['notion'] } }],
}),
);
expect(result.pass).toBe(false);
});
});
describe('declined tool calls', () => {
const declined = {
declined: true,
message: 'Tool "mcp_notion_notion-search" was not approved',
};
it('does not count a declined call as a violation', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
noneOfToolCalls: [{ toolName: 'mcp_notion_notion-search' }],
},
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }],
}),
);
expect(result.pass).toBe(true);
});
it('does not count a declined call as satisfying a positive expectation', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
anyOfToolCalls: [{ toolName: 'mcp_notion_notion-search' }],
},
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }],
}),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('declined');
});
it('still counts a call that was approved and ran', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
anyOfToolCalls: [{ toolName: 'mcp_notion_notion-search' }],
},
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: { ok: true } }],
}),
);
expect(result.pass).toBe(true);
});
it('matches a refusal when the expectation asks for one', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
allOfToolCalls: [{ toolName: 'mcp_notion_notion-search', declined: true }],
},
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }],
}),
);
expect(result.pass).toBe(true);
});
it('does not match a call that ran when the expectation asks for a refusal', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
allOfToolCalls: [{ toolName: 'mcp_notion_notion-search', declined: true }],
},
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: { ok: true } }],
}),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('a declined result');
});
it('does not count a sub-agent call the user refused as invoked', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: { noneOf: ['mcp_notion_notion-search'] },
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }],
agents: [
{
role: 'researcher',
tools: [],
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }],
},
],
}),
);
expect(result.pass).toBe(true);
expect(result.invokedTools).not.toContain('mcp_notion_notion-search');
});
it('counts a sub-agent call that ran as invoked', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: { anyOf: ['mcp_notion_notion-search'] },
},
makeOutcome({
agents: [
{
role: 'researcher',
tools: [],
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: { ok: true } }],
},
],
}),
);
expect(result.pass).toBe(true);
});
it('forbids a refusal when noneOfToolCalls asks for one', () => {
const result = runExpectedToolsInvokedCheck(
{
id: 'test',
userMessage: 'Search my Notion.',
expectedToolInvocations: {
noneOfToolCalls: [{ toolName: 'mcp_notion_notion-search', declined: true }],
},
},
makeOutcome({
toolCalls: [{ toolName: 'mcp_notion_notion-search', result: declined }],
}),
);
expect(result.pass).toBe(false);
});
});
describe('rule validation', () => {
it('throws when neither anyOf nor noneOf is provided', () => {
expect(() =>
runExpectedToolsInvokedCheck(
{ id: 'x', userMessage: '', expectedToolInvocations: {} },
makeOutcome({}),
),
).toThrow(/anyOf.*noneOf/);
});
it('throws when both anyOf and noneOf are empty', () => {
expect(() =>
runExpectedToolsInvokedCheck(
{ id: 'x', userMessage: '', expectedToolInvocations: { anyOf: [], noneOf: [] } },
makeOutcome({}),
),
).toThrow();
});
});
});
describe('evaluateDiscoveryTrial', () => {
const trial = (overrides: Partial<DiscoveryTrialFacts> = {}): DiscoveryTrialFacts => ({
streamStatus: 'completed',
timeoutMs: 60_000,
unmatchedConfirmations: [],
...overrides,
});
const negativeOnly: DiscoveryTestCase = {
id: 'test',
userMessage: 'Set up a Slack credential.',
expectedToolInvocations: { noneOf: ['browser_navigate'] },
};
const positiveOnly: DiscoveryTestCase = {
id: 'test',
userMessage: 'Screenshot my dashboard.',
expectedToolInvocations: { anyOf: ['browser_navigate'] },
};
const satisfied = makeOutcome({ toolCalls: [{ toolName: 'browser_navigate' }] });
it('passes a satisfied expectation on a completed run', () => {
expect(evaluateDiscoveryTrial(negativeOnly, makeOutcome({}), trial()).pass).toBe(true);
expect(evaluateDiscoveryTrial(positiveOnly, satisfied, trial()).pass).toBe(true);
});
it.each([
['negative-only', negativeOnly, makeOutcome({})],
['positive-only', positiveOnly, satisfied],
])('fails a satisfied %s expectation when the run stopped at its step cap', (_l, s, outcome) => {
const result = evaluateDiscoveryTrial(s, outcome, trial({ streamStatus: 'step-exhausted' }));
expect(result.pass).toBe(false);
expect(result.comment).toContain('step cap');
});
it.each(['errored', 'suspended'] as const)(
'fails a satisfied expectation when the run %s',
(streamStatus) => {
const result = evaluateDiscoveryTrial(positiveOnly, satisfied, trial({ streamStatus }));
expect(result.pass).toBe(false);
expect(result.comment).toContain('Run did not complete');
},
);
it('fails a satisfied expectation when the run exceeded its budget', () => {
const result = evaluateDiscoveryTrial(
positiveOnly,
satisfied,
trial({ streamStatus: 'timed-out', timeoutMs: 150_000 }),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('exceeded its 150000ms budget');
});
it('reports the run error alongside the invalid trial', () => {
const result = evaluateDiscoveryTrial(
negativeOnly,
makeOutcome({}),
trial({ streamStatus: 'errored', runError: 'overloaded_error' }),
);
expect(result.comment).toContain('errored: overloaded_error');
});
it('keeps the expectation diagnostic when an invalid trial also failed its expectation', () => {
const result = evaluateDiscoveryTrial(
positiveOnly,
makeOutcome({ toolCalls: [{ toolName: 'nodes' }] }),
trial({ streamStatus: 'timed-out' }),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('budget and was abandoned');
expect(result.comment).toContain('Expected at least one of');
});
it('fails when a declared confirmation answer was never asked for', () => {
const result = evaluateDiscoveryTrial(
negativeOnly,
makeOutcome({}),
trial({ unmatchedConfirmations: ['mcp_notion_notion-search'] }),
);
expect(result.pass).toBe(false);
expect(result.comment).toContain('mcp_notion_notion-search');
expect(result.comment).toContain('never ran');
});
});

71
packages/@n8n/instance-ai/evaluations/discovery/__tests__/stream-status.test.ts
View File

@ -1,71 +0,0 @@
import type { ExecuteResumableStreamResult } from '../../../src/runtime/resumable-stream-executor';
import { resolveStreamStatus } from '../stream-status';
function streamResult(
overrides: Partial<ExecuteResumableStreamResult> = {},
): ExecuteResumableStreamResult {
return {
status: 'completed',
agentRunId: 'run-1',
workSummary: {
toolCalls: [],
totalToolCalls: 0,
totalToolErrors: 0,
askedClarifyingQuestion: false,
},
...overrides,
};
}
describe('resolveStreamStatus', () => {
it('reports a clean finish as completed', () => {
expect(resolveStreamStatus(streamResult({ finishReason: 'stop' }), false)).toBe('completed');
});
it('reports a run that hit its step cap as step-exhausted', () => {
expect(resolveStreamStatus(streamResult({ finishReason: 'max-iterations' }), false)).toBe(
'step-exhausted',
);
});
it.each(['length', 'content-filter', 'error', 'other', 'tool-calls'] as const)(
'reports a run cut short by %s as errored',
(finishReason) => {
expect(resolveStreamStatus(streamResult({ finishReason }), false)).toBe('errored');
},
);
it('never reads a missing finish chunk as a clean finish', () => {
expect(resolveStreamStatus(streamResult(), false)).toBe('errored');
});
it('prefers timed-out over the step cap', () => {
expect(resolveStreamStatus(streamResult({ finishReason: 'max-iterations' }), true)).toBe(
'timed-out',
);
});
it('prefers errored over the step cap', () => {
expect(
resolveStreamStatus(
streamResult({ status: 'errored', finishReason: 'max-iterations' }),
false,
),
).toBe('errored');
});
it('reports the budget sentinel as timed-out', () => {
expect(resolveStreamStatus('timed-out', false)).toBe('timed-out');
});
it.each([streamResult({ status: 'suspended' }), streamResult({ finishReason: 'paused' })])(
'reports a suspended or paused run as suspended',
(result) => {
expect(resolveStreamStatus(result, false)).toBe('suspended');
},
);
it('never reads a cancelled run as a clean finish', () => {
expect(resolveStreamStatus(streamResult({ status: 'cancelled' }), false)).toBe('timed-out');
});
});

283
packages/@n8n/instance-ai/evaluations/discovery/cli.ts
View File

@ -1,283 +0,0 @@
#!/usr/bin/env node
// ---------------------------------------------------------------------------
// CLI for browser/computer-use discovery evaluation.
//
// Usage:
// pnpm eval:discovery # run all scenarios, 3 trials each
// pnpm eval:discovery --filter slack-oauth --verbose
// pnpm eval:discovery --trials 5
//
// Loads scenarios from evaluations/data/discovery/, runs each scenario × N
// trials via the in-process runner, reports per-scenario pass-rates, exits
// non-zero on any scenario below threshold, or on any scenario with zero passes
// when --fail-on-zero-pass is set.
// ---------------------------------------------------------------------------
import { runDiscoveryScenario, type DiscoveryRunResult } from './runner';
import type { DiscoveryTestCase } from './types';
import { loadDiscoveryTestCasesWithFiles } from '../data/discovery';
// ---------------------------------------------------------------------------
// Args
// ---------------------------------------------------------------------------
interface CliArgs {
filter?: string;
verbose: boolean;
trials: number;
passThreshold: number;
timeoutMs: number;
/** Optional iteration cap; unset uses the SDK default. */
maxSteps?: number;
modelId: string;
concurrency: number;
nodesJsonPath?: string;
failOnZeroPass: boolean;
}
const DEFAULT_MODEL = process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6';
function previewArgs(args: Record<string, unknown>): string {
const entries = Object.entries(args).slice(0, 3);
if (entries.length === 0) return '';
return entries
.map(([k, v]) => {
let preview: string;
if (typeof v === 'string') {
preview = v.length > 40 ? `"${v.slice(0, 40)}…"` : `"${v}"`;
} else if (typeof v === 'number' || typeof v === 'boolean' || v === null) {
preview = String(v);
} else if (Array.isArray(v)) {
preview = `[${String(v.length)}]`;
} else {
preview = '{…}';
}
return `${k}=${preview}`;
})
.join(', ');
}
function requirePositiveInt(raw: string | undefined, flag: string): number {
const n = Number(raw);
if (!Number.isFinite(n) || n < 1 || !Number.isInteger(n)) {
console.error(`Invalid value for ${flag}: ${String(raw)} (expected a positive integer)`);
process.exit(1);
}
return n;
}
function requireNumberInRange(
raw: string | undefined,
flag: string,
lo: number,
hi: number,
): number {
const n = Number(raw);
if (!Number.isFinite(n) || n < lo || n > hi) {
console.error(`Invalid value for ${flag}: ${String(raw)} (expected number in [${lo}, ${hi}])`);
process.exit(1);
}
return n;
}
function parseArgs(argv: string[]): CliArgs {
const args: CliArgs = {
verbose: false,
trials: 3,
passThreshold: 2 / 3,
timeoutMs: 60_000,
modelId: DEFAULT_MODEL,
concurrency: 3,
failOnZeroPass: false,
};
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
switch (arg) {
case '--verbose':
case '-v':
args.verbose = true;
break;
case '--filter':
args.filter = argv[++i];
break;
case '--trials':
args.trials = requirePositiveInt(argv[++i], '--trials');
break;
case '--pass-threshold':
args.passThreshold = requireNumberInRange(argv[++i], '--pass-threshold', 0, 1);
break;
case '--timeout':
args.timeoutMs = requirePositiveInt(argv[++i], '--timeout');
break;
case '--max-steps':
args.maxSteps = requirePositiveInt(argv[++i], '--max-steps');
break;
case '--model':
args.modelId = argv[++i];
break;
case '--concurrency':
args.concurrency = requirePositiveInt(argv[++i], '--concurrency');
break;
case '--nodes-json':
args.nodesJsonPath = argv[++i];
break;
case '--fail-on-zero-pass':
args.failOnZeroPass = true;
break;
default:
break;
}
}
return args;
}
// ---------------------------------------------------------------------------
// Local mode
// ---------------------------------------------------------------------------
interface ScenarioAggregate {
scenario: DiscoveryTestCase;
results: DiscoveryRunResult[];
passCount: number;
passRate: number;
}
async function runLocalMode(args: CliArgs): Promise<void> {
const cases = loadDiscoveryTestCasesWithFiles(args.filter);
if (cases.length === 0) {
console.log('No discovery scenarios found.');
return;
}
console.log(
`Running ${String(cases.length)} discovery scenario(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
);
const aggregates: ScenarioAggregate[] = [];
// Per scenario, run trials in parallel up to `concurrency`. Across scenarios, run sequentially
// — keeps the load profile predictable and per-scenario reports atomic.
for (const { testCase, fileSlug } of cases) {
process.stdout.write(`▸ ${fileSlug} ... `);
const trialResults: DiscoveryRunResult[] = [];
for (let i = 0; i < args.trials; i += args.concurrency) {
const batchSize = Math.min(args.concurrency, args.trials - i);
const batch = await Promise.all(
Array.from(
{ length: batchSize },
async () =>
await runDiscoveryScenario({
scenario: testCase,
modelId: args.modelId,
maxSteps: args.maxSteps,
timeoutMs: args.timeoutMs,
...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
}),
),
);
trialResults.push(...batch);
}
const passCount = trialResults.filter((r) => r.check.pass).length;
const passRate = passCount / trialResults.length;
const status = passRate >= args.passThreshold ? '✓' : '✗';
console.log(
`${status} ${String(passCount)}/${String(trialResults.length)} passed (${(passRate * 100).toFixed(0)}%)`,
);
if (args.verbose) {
for (const [idx, r] of trialResults.entries()) {
const icon = r.check.pass ? ' ✓' : ' ✗';
const trial = `trial ${String(idx + 1)}`;
console.log(`${icon} ${trial} (${(r.durationMs / 1000).toFixed(1)}s) — ${r.check.comment}`);
if (r.runError) console.log(` run error: ${r.runError}`);
if (r.outcome.toolCalls.length > 0) {
console.log(' tool calls:');
for (const tc of r.outcome.toolCalls) {
const argsPreview = previewArgs(tc.args);
console.log(` • ${tc.toolName}(${argsPreview})`);
}
}
if (r.outcome.agentActivities.length > 0) {
console.log(' spawned sub-agents:');
for (const a of r.outcome.agentActivities) {
console.log(` • role=${a.role}, tools=[${a.tools.join(', ')}]`);
}
}
}
}
aggregates.push({ scenario: testCase, results: trialResults, passCount, passRate });
}
printSummary(aggregates, args);
const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
const zeroPassScenarios = aggregates.filter((a) => a.passCount === 0);
const shouldFail = args.failOnZeroPass
? zeroPassScenarios.length > 0
: failingScenarios.length > 0;
if (shouldFail) {
process.exitCode = 1;
}
}
function printSummary(aggregates: ScenarioAggregate[], args: CliArgs): void {
console.log('\n=== Summary ===');
const totalTrials = aggregates.reduce((sum, a) => sum + a.results.length, 0);
const totalPasses = aggregates.reduce((sum, a) => sum + a.passCount, 0);
const passingScenarios = aggregates.filter((a) => a.passRate >= args.passThreshold).length;
const totalDurationMs = aggregates
.flatMap((a) => a.results)
.reduce((sum, r) => sum + r.durationMs, 0);
console.log(
`Scenarios: ${String(passingScenarios)}/${String(aggregates.length)} above threshold (${(args.passThreshold * 100).toFixed(0)}%)`,
);
console.log(
`Trials: ${String(totalPasses)}/${String(totalTrials)} passed (${((totalPasses / totalTrials) * 100).toFixed(0)}%)`,
);
console.log(`Total time: ${(totalDurationMs / 1000).toFixed(1)}s`);
const failingScenarios = aggregates.filter((a) => a.passRate < args.passThreshold);
if (failingScenarios.length > 0) {
console.log('\nFailing scenarios:');
for (const a of failingScenarios) {
console.log(` ✗ ${a.scenario.id} (${(a.passRate * 100).toFixed(0)}%)`);
const firstFailure = a.results.find((r) => !r.check.pass);
if (firstFailure) {
console.log(` ${firstFailure.check.comment}`);
}
}
}
}
// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------
async function main(): Promise<void> {
const args = parseArgs(process.argv.slice(2));
if (!process.env.ANTHROPIC_API_KEY) {
console.error(
'Error: ANTHROPIC_API_KEY is required to run discovery evaluations (the runner calls Anthropic in-process).',
);
process.exit(1);
}
await runLocalMode(args);
}
main()
.then(() => {
process.stdout.write('', () => process.exit(process.exitCode ?? 0));
})
.catch((error) => {
console.error('Fatal error:', error);
process.exit(1);
});

70
packages/@n8n/instance-ai/evaluations/discovery/confirmation-policy.ts
View File

@ -1,70 +0,0 @@
// ---------------------------------------------------------------------------
// How the harness answers confirmation / HITL suspensions.
//
// Production waits for the user; the harness answers from the scenario, keyed on
// the suspending tool's name and defaulting to approve, so a scenario only
// declares the answers it wants to differ. A bare approval resumes
// `{ approved: true }`, which is the whole contract for an `approval` card — an
// MCP tool call included.
//
// Cards that need the user to *supply* something (a credential choice, a
// connect card's slugs, a Q&A wizard's answers) are filled in by an
// ApprovalResponder living with that domain — `credential-approval.ts`,
// `stub-mcp-registry.ts`. This module stays payload-agnostic: to cover a new
// card kind, write a responder next to its domain and register it in the
// runner. A scenario can always bypass the lot with `resumeWith`.
// ---------------------------------------------------------------------------
import { isRecord } from '@n8n/utils/is-record';
import type { ConfirmationAnswer, DiscoveryTestCase } from './types';
import type { SuspensionInfo } from '../../src/utils/stream-helpers';
export type ConfirmationPolicy = Map<string, ConfirmationAnswer>;
// Supplies what an approving user would have filled in for one kind of card.
export type ApprovalResponder = (
payload: Record<string, unknown>,
) => Record<string, unknown> | undefined;
export function buildConfirmationPolicy(scenario: DiscoveryTestCase): ConfirmationPolicy {
const policy: ConfirmationPolicy = new Map();
for (const [toolName, answer] of Object.entries(scenario.confirmations ?? {})) {
policy.set(toolName, typeof answer === 'string' ? { decision: answer } : answer);
}
return policy;
}
export function resolveConfirmation(
suspension: SuspensionInfo | undefined,
policy: ConfirmationPolicy,
responders: ApprovalResponder[] = [],
): Record<string, unknown> {
const answer = policy.get(suspendedToolName(suspension));
if (answer?.decision === 'deny') return { ...answer.resumeWith, approved: false };
const payload = suspension?.suspendPayload ?? {};
for (const responder of responders) {
const responderAnswer = responder(payload);
if (responderAnswer) return { ...responderAnswer, ...answer?.resumeWith, approved: true };
}
return { ...answer?.resumeWith, approved: true };
}
export function unmatchedConfirmations(
policy: ConfirmationPolicy,
suspensions: Iterable<SuspensionInfo>,
): string[] {
const asked = new Set([...suspensions].map(suspendedToolName));
return [...policy]
.filter(
([tool, answer]) => !asked.has(tool) && (answer.decision === 'deny' || answer.resumeWith),
)
.map(([tool]) => tool);
}
function suspendedToolName(suspension: SuspensionInfo | undefined): string {
if (suspension?.toolName) return suspension.toolName;
const payload = suspension?.suspendPayload;
return isRecord(payload) && typeof payload.toolName === 'string' ? payload.toolName : '';
}

19
packages/@n8n/instance-ai/evaluations/discovery/credential-approval.ts
View File

@ -1,19 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { ApprovalResponder } from './confirmation-policy';
/** Approving a credential card means "Auto-setup with browser". A bare approval
* takes the "user picked an existing credential" branch, so the orchestrator never
* sees `needsBrowserSetup=true` — the flag that wires it to the Computer Use
* credential setup skill. The card offers one credential at a time once browser
* setup is in play, so the first request is the one `autoSetup` names. */
export const credentialAutoSetupResponder: ApprovalResponder = (payload) => {
const requests = payload.credentialRequests;
if (!Array.isArray(requests)) return undefined;
const first: unknown = requests[0];
if (!isRecord(first) || typeof first.credentialType !== 'string' || !first.credentialType) {
return undefined;
}
return { autoSetup: { credentialType: first.credentialType } };
};

250
packages/@n8n/instance-ai/evaluations/discovery/expected-tools-invoked.ts
View File

@ -1,250 +0,0 @@
// ---------------------------------------------------------------------------
// Discovery check — assert the orchestrator reached for the expected tool(s).
//
// Reads the captured event outcome (`toolCalls` + `agentActivities`) and
// compares against a `DiscoveryTestCase.expectedToolInvocations` rule.
//
// "Invoked" means either:
// - a top-level `tool-call` event with that tool name, OR
// - an `agent-spawned` event whose payload `tools` array contains that name
// (the sub-agent had access — even if it has not yet called it), OR
// - the rule names `spawn_sub_agent:<role>` and a sub-agent with that role
// was spawned.
//
// The asymmetry (sub-agent existence counts as discovery) lets dispatch checks
// assert that a specialized background agent was reached even before it emits
// its own tool calls.
// ---------------------------------------------------------------------------
import { isRecord } from '@n8n/utils/is-record';
import type { EventOutcome } from '../types';
import type {
DiscoveryCheckResult,
DiscoveryTestCase,
DiscoveryTrialFacts,
ExpectedToolInvocations,
ForbiddenToolCall,
} from './types';
const SPAWN_PREFIX = 'spawn_sub_agent:';
function collectInvokedTools(outcome: EventOutcome): string[] {
const tools = new Set<string>();
for (const tc of outcome.toolCalls) {
if (tc.toolName && !wasDeclined(tc)) tools.add(tc.toolName);
}
for (const agent of outcome.agentActivities) {
for (const t of agent.tools) tools.add(t);
for (const tc of agent.toolCalls) {
if (tc.toolName && !wasDeclined(tc)) tools.add(tc.toolName);
}
}
return [...tools];
}
function collectSpawnedAgents(outcome: EventOutcome): string[] {
return outcome.agentActivities
.filter((a) => a.role.length > 0)
.map((a) => `${SPAWN_PREFIX}${a.role}`);
}
function matches(name: string, invokedTools: string[], spawnedAgents: string[]): boolean {
if (name.startsWith(SPAWN_PREFIX)) {
return spawnedAgents.includes(name);
}
return invokedTools.includes(name);
}
function validateRule(rule: ExpectedToolInvocations): void {
const hasAnyOf = Array.isArray(rule.anyOf) && rule.anyOf.length > 0;
const hasNoneOf = Array.isArray(rule.noneOf) && rule.noneOf.length > 0;
const hasAnyOfToolCalls = Array.isArray(rule.anyOfToolCalls) && rule.anyOfToolCalls.length > 0;
const hasAllOfToolCalls = Array.isArray(rule.allOfToolCalls) && rule.allOfToolCalls.length > 0;
const hasNoneOfToolCalls = Array.isArray(rule.noneOfToolCalls) && rule.noneOfToolCalls.length > 0;
if (!hasAnyOf && !hasNoneOf && !hasAnyOfToolCalls && !hasAllOfToolCalls && !hasNoneOfToolCalls) {
throw new Error(
'expectedToolInvocations must specify a non-empty `anyOf`, `noneOf`, `anyOfToolCalls`, `allOfToolCalls`, or `noneOfToolCalls` list',
);
}
}
function wasDeclined(toolCall: EventOutcome['toolCalls'][number]): boolean {
return isRecord(toolCall.result) && toolCall.result.declined === true;
}
function toolCallMatchesExpectation(
toolCall: EventOutcome['toolCalls'][number],
expectation: ForbiddenToolCall,
): boolean {
if (wasDeclined(toolCall) !== (expectation.declined ?? false)) return false;
if (toolCall.toolName !== expectation.toolName) return false;
if (expectation.args && !matchesArgPattern(expectation.args, toolCall.args)) return false;
const argsContainAny = expectation.argsContainAny ?? [];
if (argsContainAny.length === 0) return true;
const argsText = JSON.stringify(toolCall.args).toLowerCase();
return argsContainAny.some((term) => argsText.includes(term.toLowerCase()));
}
function matchesArgPattern(pattern: unknown, actual: unknown): boolean {
if (Array.isArray(pattern)) {
return (
Array.isArray(actual) &&
pattern.every((item) => actual.some((candidate) => matchesArgPattern(item, candidate)))
);
}
if (isRecord(pattern)) {
return (
isRecord(actual) &&
Object.entries(pattern).every(([key, value]) => matchesArgPattern(value, actual[key]))
);
}
return pattern === actual;
}
function describeActualToolCalls(outcome: EventOutcome): string {
return (
outcome.toolCalls
.map((tc) => (wasDeclined(tc) ? `${tc.toolName} (declined)` : tc.toolName))
.join(', ') || '∅'
);
}
function formatToolCallExpectation(expectation: ForbiddenToolCall): string {
const clauses: string[] = [];
if (expectation.declined) clauses.push('a declined result');
if (expectation.args) clauses.push(`args matching ${JSON.stringify(expectation.args)}`);
if (expectation.argsContainAny && expectation.argsContainAny.length > 0) {
clauses.push(`args containing one of [${expectation.argsContainAny.join(', ')}]`);
}
return clauses.length > 0
? `${expectation.toolName} with ${clauses.join(' and ')}`
: expectation.toolName;
}
export function runExpectedToolsInvokedCheck(
scenario: DiscoveryTestCase,
outcome: EventOutcome,
): DiscoveryCheckResult {
validateRule(scenario.expectedToolInvocations);
const invokedTools = collectInvokedTools(outcome);
const spawnedAgents = collectSpawnedAgents(outcome);
const { anyOf, noneOf, anyOfToolCalls, allOfToolCalls, noneOfToolCalls } =
scenario.expectedToolInvocations;
if (anyOf && anyOf.length > 0) {
const matched = anyOf.find((name) => matches(name, invokedTools, spawnedAgents));
if (!matched) {
return {
pass: false,
comment: `Expected at least one of [${anyOf.join(', ')}] to be invoked. Invoked: [${invokedTools.join(', ') || '∅'}]; spawned: [${spawnedAgents.join(', ') || '∅'}].`,
invokedTools,
spawnedAgents,
};
}
}
if (noneOf && noneOf.length > 0) {
const violated = noneOf.find((name) => matches(name, invokedTools, spawnedAgents));
if (violated) {
return {
pass: false,
comment: `Expected none of [${noneOf.join(', ')}] to be invoked, but [${violated}] was reached.`,
invokedTools,
spawnedAgents,
};
}
}
if (anyOfToolCalls && anyOfToolCalls.length > 0) {
const matched = anyOfToolCalls.find((expectation) =>
outcome.toolCalls.some((toolCall) => toolCallMatchesExpectation(toolCall, expectation)),
);
if (!matched) {
const actualToolCalls = describeActualToolCalls(outcome);
return {
pass: false,
comment: `Expected at least one actual tool call matching [${anyOfToolCalls.map(formatToolCallExpectation).join(', ')}]. Actual tool calls: [${actualToolCalls}].`,
invokedTools,
spawnedAgents,
};
}
}
if (allOfToolCalls && allOfToolCalls.length > 0) {
for (const expectation of allOfToolCalls) {
const matched = outcome.toolCalls.find((toolCall) =>
toolCallMatchesExpectation(toolCall, expectation),
);
if (!matched) {
const actualToolCalls = describeActualToolCalls(outcome);
return {
pass: false,
comment: `Expected actual tool call matching [${formatToolCallExpectation(expectation)}]. Actual tool calls: [${actualToolCalls}].`,
invokedTools,
spawnedAgents,
};
}
}
}
if (noneOfToolCalls && noneOfToolCalls.length > 0) {
for (const expectation of noneOfToolCalls) {
const violated = outcome.toolCalls.find((toolCall) =>
toolCallMatchesExpectation(toolCall, expectation),
);
if (violated) {
return {
pass: false,
comment: `Expected no actual tool call matching [${formatToolCallExpectation(expectation)}], but saw ${violated.toolName} with args ${JSON.stringify(violated.args)}.`,
invokedTools,
spawnedAgents,
};
}
}
}
return {
pass: true,
comment: 'Discovery expectation satisfied.',
invokedTools,
spawnedAgents,
};
}
/**
* Only a run that finished on its own terms can settle an expectation, so anything else
* fails whichever way the expectation points — a truncated run that satisfied a positive
* expectation still means the agent never got to finish what it was doing.
*/
export function evaluateDiscoveryTrial(
scenario: DiscoveryTestCase,
outcome: EventOutcome,
trial: DiscoveryTrialFacts,
): DiscoveryCheckResult {
const check = runExpectedToolsInvokedCheck(scenario, outcome);
const invalid = invalidTrialReason(trial);
if (!invalid) return check;
return { ...check, pass: false, comment: check.pass ? invalid : `${invalid} ${check.comment}` };
}
function invalidTrialReason(trial: DiscoveryTrialFacts): string | undefined {
switch (trial.streamStatus) {
case 'timed-out':
return `Run exceeded its ${String(trial.timeoutMs)}ms budget and was abandoned.`;
case 'step-exhausted':
return 'Run stopped at its step cap instead of finishing.';
case 'errored':
case 'suspended':
return `Run did not complete (${trial.runError ? `${trial.streamStatus}: ${trial.runError}` : trial.streamStatus}).`;
case 'completed':
return trial.unmatchedConfirmations.length > 0
? `Scenario declared confirmation answers for [${trial.unmatchedConfirmations.join(', ')}] that no suspension asked for, so those decisions never ran.`
: undefined;
}
}

359
packages/@n8n/instance-ai/evaluations/discovery/runner.ts
View File

@ -1,359 +0,0 @@
// ---------------------------------------------------------------------------
// In-process discovery runner.
//
// Drives the orchestrator with a scenario's userMessage + instanceState,
// captures InstanceAi events into the CapturedEvent[] shape that
// extractOutcomeFromEvents consumes, then runs the discovery check. No
// Docker, no n8n server — the orchestrator runs in-process against
// stubbed services.
//
// What's tested: the orchestrator's first dispatch decision. Tools are NOT
// stubbed — when the orchestrator loads a runtime skill or reaches for a
// Computer Use browser tool, the tool-call event fires before any downstream
// failure, so the discovery check still sees the dispatch intent. The wall-clock
// timeout bounds the trial, not the run: at the budget the trial stops and fails,
// and the abandoned stream is left to unwind on its own. Scenarios or --max-steps
// can additionally opt into an iteration cap.
// ---------------------------------------------------------------------------
import type { InstanceAiEvent, TaskList } from '@n8n/api-types';
import { nanoid } from 'nanoid';
import {
buildConfirmationPolicy,
resolveConfirmation,
unmatchedConfirmations,
type ApprovalResponder,
} from './confirmation-policy';
import { credentialAutoSetupResponder } from './credential-approval';
import { evaluateDiscoveryTrial } from './expected-tools-invoked';
import { resolveStreamStatus } from './stream-status';
import { createStubLocalMcpServer } from './stub-local-mcp';
import {
createMcpConnectResponder,
createStubMcpRegistry,
createStubMcpToolRegistry,
StubMcpClientManager,
stubMcpServerConfigs,
type StubMcpRegistry,
} from './stub-mcp-registry';
import type { DiscoveryCheckResult, DiscoveryStreamStatus, DiscoveryTestCase } from './types';
import { createInstanceAgent } from '../../src/agent/instance-agent';
import type { InstanceAiEventBus } from '../../src/event-bus';
import type { Logger } from '../../src/logger';
import {
executeResumableStream,
normalizeStreamSource,
} from '../../src/runtime/resumable-stream-executor';
import { loadInstanceAiRuntimeSkillSource } from '../../src/skills/runtime-skills';
import type {
BuilderTurnStream,
InstanceAiContext,
InstanceAiBuilderDelegate,
ComputerUseState,
ModelConfig,
OrchestrationContext,
TaskStorage,
} from '../../src/types';
import { isAgentFeatureEnabled } from '../../src/utils/agent-feature-enabled';
import { asResumable, type SuspensionInfo } from '../../src/utils/stream-helpers';
import { createInMemoryEventBus, wrapEventBusWithObserver } from '../harness/in-memory-event-bus';
import { createStubServices, defaultNodesJsonPath } from '../harness/stub-services';
import { createStubWorkspace, stubWorkspaceRoot } from '../harness/stub-workspace';
import { extractOutcomeFromEvents } from '../outcome/event-parser';
import type { CapturedEvent, EventOutcome } from '../types';
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
export interface DiscoveryRunOptions {
scenario: DiscoveryTestCase;
modelId: ModelConfig;
/** Defaults to `defaultNodesJsonPath()`. */
nodesJsonPath?: string;
/** Hard cap on agent steps. Unset leaves the SDK's own 30-iteration ceiling in place. */
maxSteps?: number;
/** Per-trial timeout in ms. */
timeoutMs?: number;
}
export interface DiscoveryRunResult {
scenario: DiscoveryTestCase;
check: DiscoveryCheckResult;
events: CapturedEvent[];
outcome: EventOutcome;
durationMs: number;
/** Final agent status — useful for diagnosing why noop / unexpected loops happened. */
streamStatus: DiscoveryStreamStatus;
/** Populated when the run errored before reaching the check. */
runError?: string;
}
export async function runDiscoveryScenario(
options: DiscoveryRunOptions,
): Promise<DiscoveryRunResult> {
const started = Date.now();
// Unset by default: the orchestrator legitimately explores past any small fixed cap
// (data-table-workflow needs >8 iterations). Unset still lands on the SDK's own
// 30-iteration ceiling, which reports `step-exhausted`.
const maxSteps = options.scenario?.maxSteps ?? options.maxSteps;
const timeoutMs = options.scenario?.timeoutMs ?? options.timeoutMs ?? 60_000;
const nodesJsonPath = options.nodesJsonPath ?? defaultNodesJsonPath();
const events: CapturedEvent[] = [];
let streamStatus: DiscoveryRunResult['streamStatus'] = 'completed';
let runError: string | undefined;
const confirmationPolicy = buildConfirmationPolicy(options.scenario);
const suspensions = new Map<string, SuspensionInfo>();
const abortController = new AbortController();
let mcpManager: StubMcpClientManager | undefined;
let timeoutHandle: NodeJS.Timeout | undefined;
const budgetExpired = new Promise<'timed-out'>((resolve) => {
timeoutHandle = setTimeout(() => {
abortController.abort();
resolve('timed-out');
}, timeoutMs);
});
try {
const services = await createStubServices({ nodesJsonPath });
const mcpState = options.scenario.instanceState?.mcp;
const mcpRegistry = mcpState ? createStubMcpRegistry(mcpState) : undefined;
const context: InstanceAiContext = {
...applyInstanceState(services.context, options.scenario, mcpRegistry),
...(isAgentFeatureEnabled() ? { builderDelegate: createStubBuilderDelegate() } : {}),
workspace: createStubWorkspace(),
workspaceRoot: stubWorkspaceRoot,
};
mcpManager = new StubMcpClientManager(createStubMcpToolRegistry(mcpState ?? {}));
const threadId = 'discovery-thread-' + nanoid(6);
const runId = 'discovery-run-' + nanoid(6);
const approvalResponders: ApprovalResponder[] = [
credentialAutoSetupResponder,
...(mcpRegistry ? [createMcpConnectResponder(mcpRegistry)] : []),
];
const eventBus = wrapEventBusWithObserver(createInMemoryEventBus(), (event) => {
events.push(toCapturedEvent(event));
});
// `OrchestrationContext` is required for the orchestrator to receive tools like
// `create-tasks` and runtime skills. Discovery scenarios measure first-step
// tool-call decisions, not background execution.
const orchestrationContext = createStubOrchestrationContext({
context,
modelId: options.modelId,
eventBus,
threadId,
runId,
abortSignal: abortController.signal,
});
const { agent } = await createInstanceAgent({
modelId: options.modelId,
context,
orchestrationContext,
mcpServers: stubMcpServerConfigs(mcpState ?? {}),
mcpManager,
// No memory: discovery measures stateless first-step tool dispatch.
memoryConfig: {},
thinkingEnabled: false,
});
const streamSource = normalizeStreamSource(
await agent.stream(options.scenario.userMessage, {
maxIterations: maxSteps,
abortSignal: abortController.signal,
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' as const } },
},
}),
);
const run = executeResumableStream({
agent: asResumable(agent),
stream: streamSource,
context: {
threadId,
runId,
agentId: 'n8n-instance-agent',
eventBus,
signal: abortController.signal,
logger: silentLogger(),
},
control: {
mode: 'auto',
onSuspension: (suspension) => suspensions.set(suspension.requestId, suspension),
waitForConfirmation: async (requestId: string): Promise<Record<string, unknown>> =>
await Promise.resolve(
resolveConfirmation(suspensions.get(requestId), confirmationPolicy, approvalResponders),
),
},
});
void run.catch(() => {});
const result = await Promise.race([run, budgetExpired]);
streamStatus = resolveStreamStatus(result, abortController.signal.aborted);
} catch (error) {
runError = error instanceof Error ? error.message : String(error);
streamStatus = abortController.signal.aborted ? 'timed-out' : 'errored';
} finally {
clearTimeout(timeoutHandle);
await mcpManager?.disconnect();
}
const observedEvents = [...events];
const outcome = extractOutcomeFromEvents(observedEvents);
const check = evaluateDiscoveryTrial(options.scenario, outcome, {
streamStatus,
timeoutMs,
...(runError ? { runError } : {}),
unmatchedConfirmations: unmatchedConfirmations(confirmationPolicy, suspensions.values()),
});
return {
scenario: options.scenario,
check,
// An abandoned stream can still publish, so hand back what the verdict saw.
events: [...events],
outcome,
durationMs: Date.now() - started,
streamStatus,
...(runError ? { runError } : {}),
};
}
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
function applyInstanceState(
base: InstanceAiContext,
scenario: DiscoveryTestCase,
mcpRegistry: StubMcpRegistry | undefined,
): InstanceAiContext {
const state = scenario.instanceState;
if (!state) return base;
const computerUse: ComputerUseState | undefined = state.computerUse;
// Both channels can serve tools, so the stub server gets the union.
const liveToolCategories = computerUse
? Object.values(computerUse).flatMap((channel) =>
channel.status === 'connected' ? channel.toolCategories : [],
)
: [];
const localMcpServer =
liveToolCategories.length > 0
? createStubLocalMcpServer({
capabilities: liveToolCategories.filter(
(c): c is 'browser' | 'filesystem' | 'shell' =>
c === 'browser' || c === 'filesystem' || c === 'shell',
),
})
: base.localMcpServer;
return {
...base,
...(computerUse ? { computerUseState: computerUse } : {}),
...(localMcpServer ? { localMcpServer } : {}),
...(mcpRegistry ? { mcpService: mcpRegistry.service } : {}),
...(state.folderExploration !== undefined
? { folderExplorationEnabled: state.folderExploration }
: {}),
};
}
function silentLogger(): Logger {
return { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} };
}
function completedBuilderTurn(): BuilderTurnStream {
return {
fullStream: (async function* () {
await Promise.resolve();
yield {
type: 'tool-call',
toolCallId: 'discovery-write',
toolName: 'write_config',
input: {},
};
yield { type: 'tool-result', toolCallId: 'discovery-write', output: { ok: true } };
})(),
text: Promise.resolve('Agent configured for discovery evaluation.'),
};
}
function createStubBuilderDelegate(): InstanceAiBuilderDelegate {
return {
createAgent: async (name) =>
await Promise.resolve({
agentId: 'discovery-agent',
projectId: 'discovery-project',
name,
}),
streamBuild: async () => await Promise.resolve(completedBuilderTurn()),
resumeBuild: async () => await Promise.resolve(completedBuilderTurn()),
findOpenSuspensions: async () => await Promise.resolve([]),
cancelOpenSuspension: async () => await Promise.resolve(),
resolveAgentName: async () => await Promise.resolve(undefined),
};
}
interface StubOrchestrationContextOptions {
context: InstanceAiContext;
modelId: ModelConfig;
eventBus: InstanceAiEventBus;
threadId: string;
runId: string;
abortSignal: AbortSignal;
}
function createStubOrchestrationContext(
opts: StubOrchestrationContextOptions,
): OrchestrationContext {
const taskStorage: TaskStorage = {
// eslint-disable-next-line @typescript-eslint/require-await
get: async (): Promise<TaskList | null> => null,
save: async (): Promise<void> => {},
};
return {
threadId: opts.threadId,
runId: opts.runId,
userId: opts.context.userId,
orchestratorAgentId: 'n8n-instance-agent',
modelId: opts.modelId,
eventBus: opts.eventBus,
logger: silentLogger(),
runtimeSkills: loadInstanceAiRuntimeSkillSource(),
abortSignal: opts.abortSignal,
taskStorage,
// Surface the localMcpServer so Computer Use browser tools are available to the
// orchestrator.
...(opts.context.localMcpServer ? { localMcpServer: opts.context.localMcpServer } : {}),
// Registers the `workspace_*` file tools for build-workflow
...(opts.context.workspace ? { workspace: opts.context.workspace } : {}),
...(opts.context.workspaceRoot ? { workspaceRoot: opts.context.workspaceRoot } : {}),
// Used for the orchestrator's untrusted-content doctrine and other domain references.
// Provide the same context the orchestrator sees.
domainContext: opts.context,
};
}
function toCapturedEvent(event: InstanceAiEvent): CapturedEvent {
return {
timestamp: Date.now(),
type: event.type,
// `extractOutcomeFromEvents` reads `data.payload.toolName` etc. — our
// InstanceAiEvent already has that shape, so we pass it through directly.
data: event,
};
}

38
packages/@n8n/instance-ai/evaluations/discovery/stream-status.ts
View File

@ -1,38 +0,0 @@
import type { FinishReason } from '@n8n/agents';
import type { DiscoveryStreamStatus } from './types';
import type { ExecuteResumableStreamResult } from '../../src/runtime/resumable-stream-executor';
export function resolveStreamStatus(
result: ExecuteResumableStreamResult | 'timed-out',
aborted: boolean,
): DiscoveryStreamStatus {
if (result === 'timed-out' || aborted) return 'timed-out';
if (result.status === 'errored') return 'errored';
if (result.status === 'suspended') return 'suspended';
if (result.status === 'cancelled') return 'timed-out';
if (result.finishReason === undefined) return 'errored';
return statusForFinishReason(result.finishReason);
}
function statusForFinishReason(finishReason: FinishReason): DiscoveryStreamStatus {
switch (finishReason) {
case 'stop':
return 'completed';
case 'max-iterations':
return 'step-exhausted';
case 'paused':
return 'suspended';
case 'tool-calls':
case 'length':
case 'content-filter':
case 'error':
case 'guardrail':
case 'other':
return 'errored';
default: {
const unhandled: never = finishReason;
throw new Error(`Unhandled finish reason: ${String(unhandled)}`);
}
}
}

88
packages/@n8n/instance-ai/evaluations/discovery/stub-local-mcp.ts
View File

@ -1,88 +0,0 @@
// ---------------------------------------------------------------------------
// Stub LocalMcpServer for discovery evals.
//
// A case declares its connected channels through `instanceState.computerUse`, and
// the runner plugs in this stub so the agent holds the matching tools without
// spinning up the real computer-use daemon or the Chrome extension. The stub
// advertises tools by name and category. It never receives actual `callTool` invocations during
// discovery scenarios — the orchestrator's first dispatch decision is what we
// measure, not downstream tool execution.
// ---------------------------------------------------------------------------
import type { McpTool, McpToolCallRequest, McpToolCallResult } from '@n8n/api-types';
import type { LocalMcpServer } from '../../src/types';
const STUB_BROWSER_TOOL_NAMES = [
'browser_connect',
'browser_tab_open',
'browser_navigate',
'browser_snapshot',
'browser_content',
'browser_click',
'browser_type',
'browser_screenshot',
'browser_capture_secret',
'browser_create_credential',
];
const STUB_FILESYSTEM_TOOL_NAMES = ['fs_read_file', 'fs_search_files', 'fs_list_dir'];
const STUB_SHELL_TOOL_NAMES = ['shell_run'];
function makeStubTool(name: string, category: string): McpTool {
return {
name,
description: `Stub ${category} tool — no-op for discovery evals.`,
inputSchema: { type: 'object', properties: {} },
annotations: { category },
};
}
export interface CreateStubLocalMcpServerOptions {
/** Capabilities to advertise. Each capability publishes a small set of stub tool names. */
capabilities: ReadonlyArray<'browser' | 'filesystem' | 'shell'>;
}
export function createStubLocalMcpServer(options: CreateStubLocalMcpServerOptions): LocalMcpServer {
const tools: McpTool[] = [];
if (options.capabilities.includes('browser')) {
for (const name of STUB_BROWSER_TOOL_NAMES) tools.push(makeStubTool(name, 'browser'));
}
if (options.capabilities.includes('filesystem')) {
for (const name of STUB_FILESYSTEM_TOOL_NAMES) tools.push(makeStubTool(name, 'filesystem'));
}
if (options.capabilities.includes('shell')) {
for (const name of STUB_SHELL_TOOL_NAMES) tools.push(makeStubTool(name, 'shell'));
}
return {
getAvailableTools: () => tools,
getToolsByCategory: (category: string) =>
tools.filter((t) => {
const annotations = t.annotations;
return (
annotations !== undefined &&
typeof annotations === 'object' &&
(annotations as Record<string, unknown>).category === category
);
}),
// eslint-disable-next-line @typescript-eslint/require-await
callTool: async (req: McpToolCallRequest): Promise<McpToolCallResult> => {
// Return a normal tool-error result rather than throwing. native agent logs an
// error stack trace for thrown tool errors; an `isError: true` result is
// treated as expected and surfaced as a `tool-error` event without spam.
// The discovery check still records the underlying `tool-call` event, so
// dispatch is captured either way.
return {
content: [
{
type: 'text' as const,
text: `Stub: ${req.name} would have executed in production. Discovery evals measure dispatch, not execution.`,
},
],
isError: true,
};
},
};
}

201
packages/@n8n/instance-ai/evaluations/discovery/stub-mcp-registry.ts
View File

@ -1,201 +0,0 @@
// ---------------------------------------------------------------------------
// Stub MCP registry + connections for discovery evals. Fakes the registry
// service; `stub-local-mcp.ts` fakes the computer-use gateway.
//
// ---------------------------------------------------------------------------
import { sanitizeToolName, wrapToolForApproval, type BuiltTool } from '@n8n/agents';
import { DEFAULT_INSTANCE_AI_PERMISSIONS, type InstanceAiPermissions } from '@n8n/api-types';
import { isRecord } from '@n8n/utils/is-record';
import type { ApprovalResponder } from './confirmation-policy';
import type { Logger } from '../../src/logger';
import { McpClientManager } from '../../src/mcp/mcp-client-manager';
import { createToolRegistry } from '../../src/tool-registry';
import type {
InstanceAiMcpService,
InstanceAiToolRegistry,
McpRegistryConnectServerSummary,
McpRegistryServerSummary,
McpServerConfig,
} from '../../src/types';
/** What production derives for cli's shared e2e fixtures (`registry/mock-servers.ts`).
* Search below drops credential options and real relevance scoring. */
const CATALOGUE: McpRegistryConnectServerSummary[] = [
{
slug: 'notion',
title: 'Notion',
description: 'Connect to the Notion MCP Server',
usesCredentials: [
{ credentialType: 'notionMcpOAuth2Api', name: 'OAuth2', value: 'oAuth2' },
],
tools: ['notion-search', 'notion-fetch', 'notion-create-pages'],
},
{
slug: 'linear',
title: 'Linear',
description: 'Connect to the Linear MCP Server',
usesCredentials: [
{ credentialType: 'linearMcpOAuth2Api', name: 'OAuth2', value: 'oAuth2' },
],
tools: ['list_issues', 'get_issue', 'save_issue'],
},
{
slug: 'slack',
title: 'Slack',
description: 'Connect to the Slack MCP Server',
usesCredentials: [
{ credentialType: 'slackMcpOAuth2Api', name: 'OAuth2', value: 'oAuth2' },
],
tools: [],
},
];
export interface StubMcpConnection {
slug: string;
/** Defaults to the catalogue's tools; empty models a broken connection. */
tools?: string[];
}
export interface DiscoveryMcpState {
registry?: string[];
connected?: StubMcpConnection[];
}
export interface StubMcpRegistry {
service: InstanceAiMcpService;
markConnected: (slugs: string[]) => void;
}
function catalogueServer(slug: string): McpRegistryConnectServerSummary {
const server = CATALOGUE.find((entry) => entry.slug === slug);
if (!server) {
const known = CATALOGUE.map((entry) => entry.slug).join(', ');
throw new Error(`Unknown MCP registry slug "${slug}" — expected one of ${known}.`);
}
return server;
}
export function createStubMcpRegistry(state: DiscoveryMcpState): StubMcpRegistry {
const servers = (state.registry ?? []).map(catalogueServer);
const connected = new Set((state.connected ?? []).map((entry) => entry.slug));
const matches = (server: McpRegistryServerSummary, query: string): boolean =>
`${server.slug} ${server.title} ${server.description}`
.toLowerCase()
.includes(query.trim().toLowerCase());
return {
markConnected: (slugs) => slugs.forEach((slug) => connected.add(slug)),
service: {
// Connected servers are filtered out, as in the adapter.
search: async (queries) =>
await Promise.resolve(
servers.filter(
(server) =>
!connected.has(server.slug) && queries.some((query) => matches(server, query)),
).map(({ slug, title, description, tools }) => ({ slug, title, description, tools })),
),
getServers: async (slugs) =>
await Promise.resolve(servers.filter((server) => slugs.includes(server.slug))),
listConnections: async () => await Promise.resolve([...connected].map((slug) => ({ slug }))),
},
};
}
/** Approving a connect card claims the offered slugs on resume *and* makes them
* live here — `resumeConnect` counts a slug only when both hold. */
export function createMcpConnectResponder(registry: StubMcpRegistry): ApprovalResponder {
return (payload) => {
const request = payload.mcpConnectRequest;
if (!isRecord(request) || !Array.isArray(request.servers)) return undefined;
const slugs = request.servers
.map((server) => (isRecord(server) ? server.serverSlug : undefined))
.filter((slug): slug is string => typeof slug === 'string' && slug.length > 0);
if (slugs.length === 0) return undefined;
registry.markConnected(slugs);
return { connectedSlugs: slugs };
};
}
/** `listConnectedMcpServices` pairs a tool with its slug through this name. */
function serverName(slug: string): string {
return `mcp_${slug}`;
}
export function stubMcpServerConfigs(state: DiscoveryMcpState): McpServerConfig[] {
return (state.connected ?? []).map(({ slug }) => ({
name: serverName(slug),
url: `https://${slug}.mcp.invalid/mcp`,
metadata: { serverSlug: slug },
}));
}
/** Mirrors `mcp-tool-resolver`'s output, down to the server-prefixed name. */
function stubMcpTool(rawName: string, slug: string): BuiltTool {
return {
name: sanitizeToolName(`${serverName(slug)}_${rawName}`),
description: `${slug} MCP tool.`,
inputSchema: { type: 'object', properties: {} },
mcpTool: true,
mcpServerName: serverName(slug),
mcpToolName: rawName,
// An empty or apologetic result reads to the agent as an expired connection
// and sends it to `connect` — a stub artefact that looks like a regression.
handler: async () =>
await Promise.resolve({
ok: true,
items: [{ id: `${slug}-eval-stub-1`, title: 'Eval harness result' }],
}),
};
}
export function createStubMcpToolRegistry(state: DiscoveryMcpState): InstanceAiToolRegistry {
const registry = createToolRegistry();
for (const { slug, tools } of state.connected ?? []) {
for (const rawName of tools ?? catalogueServer(slug).tools) {
const tool = stubMcpTool(rawName, slug);
registry.set(tool.name, tool);
}
}
return registry;
}
/** Production wraps MCP tools in the same gate whenever the caller asks for
* approval; the real path goes through `McpConnection.listTools`, which this stub
* bypasses. */
function withApprovalGate(tools: InstanceAiToolRegistry): InstanceAiToolRegistry {
const gated = createToolRegistry();
for (const [name, tool] of tools) {
gated.set(name, wrapToolForApproval(tool, { requireApproval: true }));
}
return gated;
}
/** `CreateInstanceAgentOptions.mcpManager` is the concrete class, so this
* subclasses rather than structurally fakes it. */
export class StubMcpClientManager extends McpClientManager {
constructor(private readonly stubTools: InstanceAiToolRegistry) {
super();
}
override async getRegularTools(
_configs: McpServerConfig[],
_logger?: Logger,
{
mcpRead,
mcpWrite,
}: Pick<InstanceAiPermissions, 'mcpRead' | 'mcpWrite'> = DEFAULT_INSTANCE_AI_PERMISSIONS,
) {
const requireApproval = mcpRead !== 'always_allow' || mcpWrite !== 'always_allow';
const tools = requireApproval ? withApprovalGate(this.stubTools) : this.stubTools;
return await Promise.resolve({ tools, connectionFailures: [] });
}
override async disconnect() {
await Promise.resolve();
}
}

119
packages/@n8n/instance-ai/evaluations/discovery/types.ts
View File

@ -1,119 +0,0 @@
// ---------------------------------------------------------------------------
// Tool-discovery scenario types — guards browser/computer-use discoverability.
//
// Discovery scenarios run the orchestrator against a user message that should
// (or should not) cause a specific tool / sub-agent to be reached for, and
// assert the captured events match the expectation. Workflow-build scenarios
// in evaluations/data/workflows/ assert the *output workflow*; discovery
// scenarios assert the *agent's tool/dispatch behavior*.
// ---------------------------------------------------------------------------
import type { DiscoveryMcpState } from './stub-mcp-registry';
import type { ComputerUseState } from '../../src/types';
/**
* Pass condition for tool invocations.
*
* - `anyOf` — pass if at least one of the listed tool names was invoked
* (top-level orchestrator call, or via a spawned sub-agent's tool list).
* - `noneOf` — pass only if NONE of the listed tool names was invoked.
* Used for negative scenarios that guard against over-eager invocation.
* - `anyOfToolCalls` — pass if at least one listed actual tool call happened.
* Unlike `anyOf`, this can check serialized tool args.
* - `allOfToolCalls` — pass only if EVERY listed actual tool call happened.
* Unlike `anyOf`, this checks completed/errored tool calls only; a spawned
* sub-agent having the tool available does not count as a match.
* - `noneOfToolCalls` — pass only if NONE of the listed tool calls happened.
* Unlike `noneOf`, this checks actual tool calls only; a spawned sub-agent
* having the tool available does not count as a violation.
*
* Both forms accept a sub-agent role prefix `spawn_sub_agent:<role>` to match
* an `agent-spawned` event whose role equals `<role>`.
*/
export interface ForbiddenToolCall {
toolName: string;
/** Deep-partial match: listed keys must match, extra keys ignored, pattern
* arrays matched as subsets. Prefer this for schema'd fields — a substring
* can't tell a value from the same word in a free-text sibling. */
args?: Record<string, unknown>;
/** Case-insensitive substrings over the serialized args, for free-text
* fields. Combines with `args`; both must hold on the same call. */
argsContainAny?: string[];
/** Match a call the user refused (an approval-gated tool resuming with
* `{ declined: true }`) instead of one that ran. Defaults to false, so an
* expectation never matches a refusal by accident. */
declined?: boolean;
}
export interface ExpectedToolInvocations {
anyOf?: string[];
noneOf?: string[];
anyOfToolCalls?: ForbiddenToolCall[];
allOfToolCalls?: ForbiddenToolCall[];
noneOfToolCalls?: ForbiddenToolCall[];
}
/**
* Optional instance configuration overrides for a discovery scenario.
* Mirrors a subset of `SystemPromptOptions` so a scenario can pin the gateway
* status (e.g. `disabledGlobally` to test the "explain how to enable" branch).
*/
export interface DiscoveryInstanceState {
computerUse?: ComputerUseState;
mcp?: DiscoveryMcpState;
/** Turn the folder-exploration capability on for the run (default off). */
folderExploration?: boolean;
}
export type ConfirmationDecision = 'approve' | 'deny';
export interface ConfirmationAnswer {
decision: ConfirmationDecision;
resumeWith?: Record<string, unknown>;
}
// Keyed by the suspending tool's name
export type DiscoveryConfirmations = Record<string, ConfirmationDecision | ConfirmationAnswer>;
export interface DiscoveryTestCase {
/** Unique scenario identifier — also used as the scenario filename (without .json). */
id: string;
/** The user message sent to the orchestrator. */
userMessage: string;
/** Optional instance state overrides applied when constructing the agent. */
instanceState?: DiscoveryInstanceState;
/** Answers for the tools listed; every other suspension is approved. */
confirmations?: DiscoveryConfirmations;
/** Pass condition. At least one expectation key is required. */
expectedToolInvocations: ExpectedToolInvocations;
/** Free-form note explaining what regression this scenario protects against. */
rationale?: string;
/** Override the runner step cap when orchestrator-inline work needs more iterations. */
maxSteps?: number;
/** Override the runner timeout when a scenario needs more time. */
timeoutMs?: number;
}
export type DiscoveryStreamStatus =
| 'completed'
| 'errored'
| 'timed-out'
| 'suspended'
| 'step-exhausted';
export interface DiscoveryTrialFacts {
streamStatus: DiscoveryStreamStatus;
timeoutMs: number;
runError?: string;
unmatchedConfirmations: string[];
}
export interface DiscoveryCheckResult {
pass: boolean;
/** Human-readable reason — included in failure reports. */
comment: string;
/** Tool names actually invoked during the run (top-level + via sub-agents). */
invokedTools: string[];
/** `spawn_sub_agent:<role>` markers for every spawned sub-agent. */
spawnedAgents: string[];
}

1
packages/@n8n/instance-ai/package.json
View File

@ -23,7 +23,6 @@
"eval:pairwise:compare": "tsx evaluations/cli/compare-pairwise.ts",
"eval:subagent": "tsx evaluations/subagent/cli.ts",
"eval:computer-use": "tsx evaluations/computer-use/cli.ts",
"eval:discovery": "tsx evaluations/discovery/cli.ts",
"prompts:print": "tsx scripts/print-prompts.ts"
},
"main": "dist/index.js",

6
packages/@n8n/instance-ai/skills/planned-task-runtime/SKILL.md
View File

@ -45,12 +45,6 @@ If a task appears stuck, tell the user and stop; do not try to detect completion
yourself. Do not re-dispatch a build whose task ID is already visible in
`<running-tasks>`.
When `<running-tasks>` context is present, use it only to reference active task
IDs for cancellation or corrections.
If the user sends a correction while a build is running, call
`task-control(action="correct-task")` with the task ID and correction.
## Synthesize follow-up
When `<planned-task-follow-up type="synthesize">` is present, all planned tasks

50
packages/@n8n/instance-ai/src/__tests__/index.test.ts
View File

@ -66,7 +66,6 @@ vi.mock('../storage', () => ({
ThreadTaskStorage: class ThreadTaskStorage {},
PlannedTaskStorage: class PlannedTaskStorage {},
getThread: () => ({ id: 'thread-1' }),
TerminalOutcomeStorage: class TerminalOutcomeStorage {},
patchThread: () => ({ id: 'thread-1' }),
WorkflowLoopStorage: class WorkflowLoopStorage {},
}));
@ -108,40 +107,10 @@ vi.mock('../workspace/sandbox-setup', () => ({
setupSandboxWorkspace: () => undefined,
}));
vi.mock('../workspace/snapshot-manager', () => ({ SnapshotManager: class SnapshotManager {} }));
vi.mock('../runtime/background-task-manager', () => ({
BackgroundTaskManager: class BackgroundTaskManager {},
enrichMessageWithRunningTasks: () => ({ content: 'message' }),
}));
vi.mock('../runtime/run-state-registry', () => ({ RunStateRegistry: class RunStateRegistry {} }));
vi.mock('../runtime/terminal-response-guard', () => ({
InstanceAiTerminalResponseGuard: class InstanceAiTerminalResponseGuard {},
}));
vi.mock('../runtime/resumable-stream-executor', () => ({
executeResumableStream: () => ({ status: 'done' }),
}));
vi.mock('../runtime/stream-runner', () => ({
resumeAgentRun: () => ({ status: 'resumed' }),
streamAgentRun: () => ({ status: 'streamed' }),
}));
vi.mock('../runtime/liveness-policy', () => {
const INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG = {
confirmationTimeoutMs: 1,
backgroundTaskIdleTimeoutMs: 1,
backgroundTaskMaxLifetimeMs: 1,
activeRunIdleTimeoutMs: 0,
activeRunMaxLifetimeMs: 1,
};
class InstanceAiLivenessPolicy {
hasEnabledTimeouts() {
return true;
}
}
return {
INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
createInstanceAiLivenessPolicyConfig: () => INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
InstanceAiLivenessPolicy,
};
});
vi.mock('../workflow-loop', () => ({
workflowBuildOutcomeSchema: { safeParse: () => ({ success: true }) },
attemptRecordSchema: { safeParse: () => ({ success: false }) },
@ -188,7 +157,6 @@ describe('@n8n/instance-ai public entrypoint', () => {
expect(entrypoint.getParseableAttachmentMimeTypes).toEqual(expect.any(Function));
expect(entrypoint.mapAgentChunkToEvent).toEqual(expect.any(Function));
expect(entrypoint.wrapUntrustedData).toEqual(expect.any(Function));
expect(entrypoint.BackgroundTaskManager).toEqual(expect.any(Function));
expect(entrypoint.IdRemapper).toEqual(expect.any(Function));
expect(entrypoint.McpClientManager).toEqual(expect.any(Function));
expect(entrypoint.UnsupportedAttachmentError).toEqual(expect.any(Function));
@ -242,9 +210,6 @@ describe('@n8n/instance-ai public entrypoint', () => {
expect(construct(entrypoint.ThreadTaskStorage)).toBeInstanceOf(entrypoint.ThreadTaskStorage);
expect(construct(entrypoint.PlannedTaskStorage)).toBeInstanceOf(entrypoint.PlannedTaskStorage);
expect(call(entrypoint.getThread)).toEqual({ id: 'thread-1' });
expect(construct(entrypoint.TerminalOutcomeStorage)).toBeInstanceOf(
entrypoint.TerminalOutcomeStorage,
);
expect(call(entrypoint.patchThread)).toEqual({ id: 'thread-1' });
expect(construct(entrypoint.WorkflowLoopStorage)).toBeInstanceOf(
entrypoint.WorkflowLoopStorage,
@ -277,25 +242,10 @@ describe('@n8n/instance-ai public entrypoint', () => {
expect(call(entrypoint.createWorkspace)).toEqual({ type: 'workspace' });
expect(construct(entrypoint.SnapshotManager)).toBeInstanceOf(entrypoint.SnapshotManager);
expect(construct(entrypoint.BackgroundTaskManager)).toBeInstanceOf(
entrypoint.BackgroundTaskManager,
);
expect(call(entrypoint.enrichMessageWithRunningTasks)).toEqual({ content: 'message' });
expect(call(entrypoint.enrichMessageWithBackgroundTasks)).toEqual({ content: 'message' });
expect(construct(entrypoint.RunStateRegistry)).toBeInstanceOf(entrypoint.RunStateRegistry);
expect(construct(entrypoint.InstanceAiTerminalResponseGuard)).toBeInstanceOf(
entrypoint.InstanceAiTerminalResponseGuard,
);
expect(call(entrypoint.executeResumableStream)).toEqual({ status: 'done' });
expect(call(entrypoint.resumeAgentRun)).toEqual({ status: 'resumed' });
expect(call(entrypoint.streamAgentRun)).toEqual({ status: 'streamed' });
expect(call(entrypoint.createInstanceAiLivenessPolicyConfig)).toEqual(
entrypoint.INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
);
expect(construct(entrypoint.InstanceAiLivenessPolicy)).toBeInstanceOf(
entrypoint.InstanceAiLivenessPolicy,
);
expect(construct(entrypoint.WorkflowTaskCoordinator)).toBeInstanceOf(
entrypoint.WorkflowTaskCoordinator,
);

282
packages/@n8n/instance-ai/src/debug/__tests__/run-debug-buffer.test.ts
View File

@ -1,282 +0,0 @@
import type { GenerateTextStepEndEvent, GenerateTextStepStartEvent } from 'ai';
import { describe, expect, it } from 'vitest';
import { mock } from 'vitest-mock-extended';
import { z } from 'zod';
import type { Logger } from '../../logger';
import {
RunDebugBuffer,
buildRunDebugLabel,
createRunDebugStepHooks,
sanitizeStepFinish,
sanitizeStepStart,
} from '../run-debug-buffer';
import { sanitizeDebugSnapshotValue } from '../sanitize-debug-snapshot';
function makeStepEndEvent(text: string): GenerateTextStepEndEvent {
return {
stepNumber: 0,
text,
toolCalls: [],
finishReason: 'tool-calls',
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
response: {
modelId: 'test-model',
timestamp: new Date('2026-01-01T00:00:00.000Z'),
messages: [{ role: 'assistant', content: text }],
},
} as unknown as GenerateTextStepEndEvent;
}
function makeStepStartEvent(): GenerateTextStepStartEvent {
return {
stepNumber: 0,
system: 'You are helpful',
messages: [{ role: 'user', content: 'hello' }],
tools: { search: { description: 'search', inputSchema: { type: 'object' } } },
toolChoice: 'auto',
activeTools: ['search'],
abortSignal: new AbortController().signal,
} as unknown as GenerateTextStepStartEvent;
}
describe('RunDebugBuffer', () => {
it('pairs step start and finish by run-scoped step index', () => {
const buffer = new RunDebugBuffer();
buffer.ensure('run-1', 'thread-1');
buffer.recordStepStart('run-1', 0, makeStepStartEvent());
buffer.recordStepFinish('run-1', 0, makeStepEndEvent('done'));
const record = buffer.get('run-1');
expect(record?.steps).toHaveLength(1);
expect(record?.steps[0]?.stepNumber).toBe(0);
expect(record?.steps[0]?.input?.messages).toBeDefined();
expect(record?.steps[0]?.output?.text).toBe('done');
});
it('records multiple orchestrator iterations even when SDK stepNumber stays 0', () => {
const buffer = new RunDebugBuffer();
buffer.ensure('run-1', 'thread-1');
const hooks = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' });
for (const label of ['first', 'second', 'third']) {
hooks.onStepStart(makeStepStartEvent());
hooks.onStepEnd(makeStepEndEvent(label));
}
const record = buffer.get('run-1');
expect(record?.steps).toHaveLength(3);
expect(record?.steps.map((step) => step.stepNumber)).toEqual([0, 1, 2]);
expect(record?.steps.map((step) => step.output?.text)).toEqual(['first', 'second', 'third']);
});
it('continues step numbering after resume hooks are recreated', () => {
const buffer = new RunDebugBuffer();
buffer.ensure('run-1', 'thread-1');
const firstPass = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' });
firstPass.onStepStart(makeStepStartEvent());
firstPass.onStepEnd(makeStepEndEvent('before suspend'));
const resumePass = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' });
resumePass.onStepStart(makeStepStartEvent());
resumePass.onStepEnd(makeStepEndEvent('after resume'));
const record = buffer.get('run-1');
expect(record?.steps).toHaveLength(2);
expect(record?.steps[1]?.stepNumber).toBe(1);
expect(record?.steps[1]?.output?.text).toBe('after resume');
});
it('keeps full tool definitions and strips abortSignal from step start', () => {
const sanitized = sanitizeStepStart(
{
stepNumber: 1,
system: 'system prompt',
messages: [{ role: 'user', content: 'hello' }],
tools: {
search: { description: 'search', inputSchema: { type: 'object', properties: { q: {} } } },
},
toolChoice: 'auto',
activeTools: ['search'],
abortSignal: new AbortController().signal,
} as unknown as GenerateTextStepStartEvent,
4,
);
expect(sanitized.stepNumber).toBe(4);
expect(sanitized.sdkStepNumber).toBe(1);
expect(sanitized.activeTools).toEqual(['search']);
expect(sanitized.tools).toEqual({
search: { description: 'search', inputSchema: { type: 'object', properties: { q: {} } } },
});
expect(sanitized).not.toHaveProperty('abortSignal');
});
it('drops step-start data that repeats other captured fields', () => {
const stepTools = [{ type: 'function', name: 'search', inputSchema: { type: 'object' } }];
const sanitized = sanitizeStepStart(
{
stepNumber: 0,
instructions: 'system prompt',
messages: [{ role: 'user', content: 'hello' }],
tools: { search: { description: 'search', inputSchema: z.object({ q: z.string() }) } },
stepTools,
promptMessages: [{ role: 'system', content: 'system prompt' }],
steps: [{ stepNumber: 0, text: 'earlier step' }],
modelId: 'claude',
} as unknown as GenerateTextStepStartEvent,
0,
);
expect(sanitized.stepTools).toEqual(stepTools);
expect(sanitized.modelId).toBe('claude');
for (const key of ['tools', 'promptMessages', 'steps']) {
expect(sanitized).not.toHaveProperty(key);
}
});
it('drops empty step-start containers and keeps populated ones', () => {
const sanitized = sanitizeStepStart(
{
stepNumber: 0,
runtimeContext: {},
toolsContext: { search: { region: 'eu' } },
activeTools: [],
} as unknown as GenerateTextStepStartEvent,
0,
);
expect(sanitized).not.toHaveProperty('runtimeContext');
expect(sanitized).not.toHaveProperty('activeTools');
expect(sanitized.toolsContext).toEqual({ search: { region: 'eu' } });
});
it('keeps tool descriptions without Zod internals when stepTools is missing', () => {
const sanitized = sanitizeStepStart(
{
stepNumber: 0,
tools: { search: { description: 'search', inputSchema: z.object({ q: z.string() }) } },
} as unknown as GenerateTextStepStartEvent,
0,
);
expect(sanitized.tools).toEqual({ search: { description: 'search' } });
});
it('does not truncate long captured strings', () => {
const longSystem = 'x'.repeat(5_000);
const sanitized = sanitizeDebugSnapshotValue(longSystem);
expect(sanitized).toHaveLength(5_000);
});
it('captures full tool call inputs and outputs on step finish', () => {
const sanitized = sanitizeStepFinish(
{
stepNumber: 0,
text: '',
toolCalls: [
{
toolCallId: 'tc-1',
toolName: 'build-workflow',
input: { code: 'full workflow code payload' },
},
],
toolResults: [
{
toolCallId: 'tc-1',
toolName: 'build-workflow',
output: { success: true, workflowId: 'wf-1' },
},
],
finishReason: 'tool-calls',
usage: { inputTokens: 1, outputTokens: 2, totalTokens: 3 },
response: {
modelId: 'test-model',
timestamp: new Date('2026-01-01T00:00:00.000Z'),
messages: [{ role: 'assistant', content: '' }],
body: { secret: 'raw-provider-body' },
},
} as unknown as GenerateTextStepEndEvent,
0,
);
expect(sanitized.toolCalls).toEqual([
{
toolCallId: 'tc-1',
toolName: 'build-workflow',
input: { code: 'full workflow code payload' },
},
]);
expect(sanitized.toolResults).toEqual([
{
toolCallId: 'tc-1',
toolName: 'build-workflow',
output: { success: true, workflowId: 'wf-1' },
},
]);
expect(sanitized.response).not.toHaveProperty('body');
});
it('evicts oldest run when cap is exceeded', () => {
const logger = mock<Logger>();
const buffer = new RunDebugBuffer(logger);
for (let index = 0; index < 51; index++) {
const runId = `run-${index}`;
buffer.ensure(runId, 'thread-1');
buffer.recordStepStart(runId, 0, {
stepNumber: 0,
messages: [],
} as unknown as GenerateTextStepStartEvent);
}
expect(buffer.get('run-0')).toBeUndefined();
expect(buffer.get('run-50')).toBeDefined();
expect(logger.warn).toHaveBeenCalled();
});
it('createRunDebugStepHooks writes into the buffer', () => {
const buffer = new RunDebugBuffer();
buffer.ensure('run-1', 'thread-1');
const hooks = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' });
hooks.onStepStart({
stepNumber: 2,
messages: [{ role: 'user', content: 'ping' }],
} as unknown as GenerateTextStepStartEvent);
expect(buffer.get('run-1')?.steps[0]?.stepNumber).toBe(0);
expect(buffer.get('run-1')?.steps[0]?.input?.sdkStepNumber).toBe(2);
});
it('exposes onStepEnd and keeps onStepFinish as a compatibility alias', () => {
const buffer = new RunDebugBuffer();
const hooks = createRunDebugStepHooks(buffer, { runId: 'run-1', threadId: 'thread-1' });
expect(typeof hooks.onStepEnd).toBe('function');
expect(hooks.onStepFinish).toBe(hooks.onStepEnd);
});
it('stores a run label on first ensure', () => {
const buffer = new RunDebugBuffer();
buffer.ensure('run-1', 'thread-1', 'build a weather workflow');
expect(buffer.get('run-1')?.label).toBe('build a weather workflow');
buffer.ensure('run-1', 'thread-1', 'ignored');
expect(buffer.get('run-1')?.label).toBe('build a weather workflow');
});
});
describe('buildRunDebugLabel', () => {
it('uses the user message when no resume reason is provided', () => {
expect(buildRunDebugLabel({ message: 'build a simple workflow' })).toBe(
'build a simple workflow',
);
});
it('maps resume reasons to action labels', () => {
expect(buildRunDebugLabel({ resumeReason: 'approval' })).toBe('Resume · approval');
expect(buildRunDebugLabel({ resumeReason: 'replan' })).toBe('Follow-up · replan');
});
});

49
packages/@n8n/instance-ai/src/debug/__tests__/sanitize-debug-snapshot.test.ts
View File

@ -1,49 +0,0 @@
import {
sanitizeDebugSnapshotRecord,
sanitizeDebugSnapshotValue,
} from '../sanitize-debug-snapshot';
describe('sanitizeDebugSnapshotValue', () => {
it('redacts secrets and omits runtime-only payloads', () => {
expect(
sanitizeDebugSnapshotRecord({
apiKey: 'plain-value',
message: 'token sk-abcdefghijklmnop',
abortSignal: new AbortController().signal,
response: { body: 'raw provider response', status: 500 },
}),
).toEqual({
apiKey: '[redacted]',
message: '[REDACTED]',
response: { status: 500 },
});
});
it('scrubs error messages', () => {
expect(sanitizeDebugSnapshotValue(new Error('failed with sk-abcdefghijklmnop'))).toEqual({
name: 'Error',
message: 'failed with [REDACTED]',
});
});
it('handles circular values', () => {
const value: Record<string, unknown> = {};
value.self = value;
expect(sanitizeDebugSnapshotValue(value)).toEqual({ self: '[Circular]' });
});
it('serializes a shared reference in each place it appears', () => {
const shared = { anthropic: { eagerInputStreaming: false } };
expect(
sanitizeDebugSnapshotValue({
first: { providerOptions: shared },
second: [{ providerOptions: shared }],
}),
).toEqual({
first: { providerOptions: shared },
second: [{ providerOptions: shared }],
});
});
});

306
packages/@n8n/instance-ai/src/debug/run-debug-buffer.ts
View File

@ -1,306 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import { scrubSecretsInText } from '@n8n/utils/scrub-secrets';
import type { GenerateTextStepEndEvent, GenerateTextStepStartEvent } from 'ai';
import type { Logger } from '../logger';
import { sanitizeDebugSnapshotRecord, sanitizeDebugSnapshotValue } from './sanitize-debug-snapshot';
const MAX_RUNS = 50;
const MAX_STEPS_PER_RUN = 200;
const MAX_WORKFLOW_SNAPSHOTS_PER_RUN = 100;
export interface WorkflowCodeSnapshotInput {
code: string;
source: 'full-code' | 'patch';
patches?: unknown;
workflowId?: string;
toolCallId?: string;
success: boolean;
errors?: string[];
capturedAt: number;
}
export type SanitizedStepStart = {
stepNumber: number;
sdkStepNumber?: number;
} & Record<string, unknown>;
export type SanitizedStepFinish = {
stepNumber: number;
sdkStepNumber?: number;
} & Record<string, unknown>;
export type WorkflowCodeSnapshot = WorkflowCodeSnapshotInput;
export interface RunDebugStep {
stepNumber: number;
input?: SanitizedStepStart;
output?: SanitizedStepFinish;
}
export interface RunDebugRecord {
threadId: string;
runId: string;
startedAt: number;
label?: string;
/** Next run-scoped step index; survives step-cap eviction. */
nextStepIndex: number;
steps: RunDebugStep[];
workflowCode: WorkflowCodeSnapshot[];
}
export interface RunDebugStepHookOptions {
runId: string;
threadId: string;
}
/**
* Step-start keys that repeat data captured elsewhere:
* - `tools` holds live Zod schemas; `stepTools` has the JSON Schema the model received.
* - `promptMessages` repeats `instructions` and `messages`.
* - `steps` repeats earlier steps, which the buffer records on their own.
*/
const DUPLICATE_STEP_START_KEYS = new Set(['tools', 'promptMessages', 'steps']);
function isEmptyContainer(value: unknown): boolean {
if (Array.isArray(value)) return value.length === 0;
return isRecord(value) && Object.keys(value).length === 0;
}
function captureStepStartPayload(event: GenerateTextStepStartEvent): Record<string, unknown> {
const hasStepTools = 'stepTools' in event && Array.isArray(event.stepTools);
const payload: Record<string, unknown> = {};
for (const [key, value] of Object.entries(event)) {
if (key === 'tools' && !hasStepTools) {
payload.tools = summarizeToolSet(value);
continue;
}
if (DUPLICATE_STEP_START_KEYS.has(key) || isEmptyContainer(value)) continue;
payload[key] = value;
}
return sanitizeDebugSnapshotRecord(payload);
}
/**
* Keeps tool descriptions and plain JSON schemas when the SDK does not report
* `stepTools`. Zod schemas are dropped: serialized, they are only method stubs.
*/
function summarizeToolSet(tools: unknown): unknown {
if (!isRecord(tools)) return undefined;
const summary: Record<string, { description?: string; inputSchema?: unknown }> = {};
for (const [name, tool] of Object.entries(tools)) {
if (!isRecord(tool)) continue;
summary[name] = {
...(typeof tool.description === 'string' ? { description: tool.description } : {}),
...(tool.inputSchema !== undefined && !isZodSchema(tool.inputSchema)
? { inputSchema: tool.inputSchema }
: {}),
};
}
return summary;
}
/** Matches Zod 3 (`_def`) and Zod 4 (`_zod`) schemas without depending on one Zod copy. */
function isZodSchema(value: unknown): boolean {
return isRecord(value) && ('_def' in value || '_zod' in value);
}
function captureStepFinishPayload(event: GenerateTextStepEndEvent): Record<string, unknown> {
return sanitizeDebugSnapshotRecord(event);
}
export function sanitizeStepStart(
event: GenerateTextStepStartEvent,
stepNumber: number,
): SanitizedStepStart {
return {
...captureStepStartPayload(event),
stepNumber,
sdkStepNumber: event.stepNumber,
};
}
export function sanitizeStepFinish(
event: GenerateTextStepEndEvent,
stepNumber: number,
): SanitizedStepFinish {
return {
...captureStepFinishPayload(event),
stepNumber,
sdkStepNumber: event.stepNumber,
};
}
export function createRunDebugStepHooks(
buffer: RunDebugBuffer,
options: RunDebugStepHookOptions,
): {
onStepStart: (event: GenerateTextStepStartEvent) => void;
onStepEnd: (event: GenerateTextStepEndEvent) => void;
/** @deprecated Use `onStepEnd` instead. */
onStepFinish: (event: GenerateTextStepEndEvent) => void;
} {
// The agent runtime calls streamText/generateText once per loop iteration. The AI SDK
// resets stepNumber to 0 on each call, so we allocate a run-scoped sequence instead.
let stepIndex = buffer.getNextStepIndex(options.runId);
const onStepEnd = (event: GenerateTextStepEndEvent) => {
buffer.recordStepFinish(options.runId, stepIndex, event);
stepIndex++;
};
return {
onStepStart: (event) => {
buffer.recordStepStart(options.runId, stepIndex, event);
},
onStepEnd,
onStepFinish: onStepEnd,
};
}
export function buildRunDebugLabel(options: {
message?: string;
resumeReason?: string;
}): string {
const resumeLabels: Record<string, string> = {
approval: 'Resume · approval',
background_task_completed: 'Follow-up · background task completed',
workflow_verification: 'Follow-up · workflow verification',
workflow_setup: 'Follow-up · workflow setup',
planned_checkpoint: 'Follow-up · planned checkpoint',
replan: 'Follow-up · replan',
synthesize: 'Follow-up · synthesize',
};
if (options.resumeReason && resumeLabels[options.resumeReason]) {
return resumeLabels[options.resumeReason];
}
const trimmed = options.message?.trim();
if (trimmed) {
return trimmed.length > 80 ? `${trimmed.slice(0, 80)}…` : trimmed;
}
return 'Orchestrator run';
}
export class RunDebugBuffer {
private readonly records = new Map<string, RunDebugRecord>();
constructor(private readonly logger?: Logger) {}
ensure(runId: string, threadId: string, label?: string): void {
if (this.records.has(runId)) return;
this.evictOldestRunIfNeeded();
this.records.set(runId, {
threadId,
runId,
startedAt: Date.now(),
label: label ? scrubSecretsInText(label.trim()) : undefined,
nextStepIndex: 0,
steps: [],
workflowCode: [],
});
}
getNextStepIndex(runId: string): number {
return this.records.get(runId)?.nextStepIndex ?? 0;
}
recordStepStart(runId: string, stepIndex: number, event: GenerateTextStepStartEvent): void {
const record = this.records.get(runId);
if (!record) return;
const existing = record.steps.find((step) => step.stepNumber === stepIndex);
if (existing) {
existing.input = sanitizeStepStart(event, stepIndex);
return;
}
this.evictOldestStepIfNeeded(record);
record.steps.push({
stepNumber: stepIndex,
input: sanitizeStepStart(event, stepIndex),
});
record.steps.sort((a, b) => a.stepNumber - b.stepNumber);
}
recordStepFinish(runId: string, stepIndex: number, event: GenerateTextStepEndEvent): void {
const record = this.records.get(runId);
if (!record) return;
const existing = record.steps.find((step) => step.stepNumber === stepIndex);
if (existing) {
existing.output = sanitizeStepFinish(event, stepIndex);
record.nextStepIndex = stepIndex + 1;
return;
}
this.evictOldestStepIfNeeded(record);
record.steps.push({
stepNumber: stepIndex,
output: sanitizeStepFinish(event, stepIndex),
});
record.steps.sort((a, b) => a.stepNumber - b.stepNumber);
record.nextStepIndex = stepIndex + 1;
}
recordWorkflowCode(runId: string, snapshot: WorkflowCodeSnapshotInput): void {
const record = this.records.get(runId);
if (!record) return;
if (record.workflowCode.length >= MAX_WORKFLOW_SNAPSHOTS_PER_RUN) {
record.workflowCode.shift();
this.logger?.warn('Evicted oldest workflow code snapshot from run debug buffer', {
runId,
maxSnapshots: MAX_WORKFLOW_SNAPSHOTS_PER_RUN,
});
}
record.workflowCode.push({
...snapshot,
code: scrubSecretsInText(snapshot.code),
patches: sanitizeDebugSnapshotValue(snapshot.patches),
errors: snapshot.errors?.map((error) => scrubSecretsInText(error)),
});
}
get(runId: string): RunDebugRecord | undefined {
const record = this.records.get(runId);
if (!record) return undefined;
return structuredClone(record);
}
listByThread(threadId: string): RunDebugRecord[] {
return [...this.records.values()]
.filter((record) => record.threadId === threadId)
.sort((a, b) => a.startedAt - b.startedAt)
.map((record) => structuredClone(record));
}
private evictOldestRunIfNeeded(): void {
if (this.records.size < MAX_RUNS) return;
const oldest = [...this.records.values()].sort((a, b) => a.startedAt - b.startedAt)[0];
if (!oldest) return;
this.records.delete(oldest.runId);
this.logger?.warn('Evicted oldest run from debug buffer', {
runId: oldest.runId,
threadId: oldest.threadId,
maxRuns: MAX_RUNS,
});
}
private evictOldestStepIfNeeded(record: RunDebugRecord): void {
if (record.steps.length < MAX_STEPS_PER_RUN) return;
const removed = record.steps.shift();
this.logger?.warn('Evicted oldest step from run debug buffer', {
runId: record.runId,
stepNumber: removed?.stepNumber,
maxSteps: MAX_STEPS_PER_RUN,
});
}
}

119
packages/@n8n/instance-ai/src/debug/sanitize-debug-snapshot.ts
View File

@ -1,119 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import { isSensitiveKey } from '@n8n/utils/redaction/sensitive-key';
import { scrubSecretsInText } from '@n8n/utils/scrub-secrets';
const OMIT_KEYS = new Set(['abortSignal']);
function shouldOmitKey(key: string, parentKey?: string): boolean {
if (OMIT_KEYS.has(key)) {
return true;
}
// Raw provider HTTP payloads are not useful in the debug buffer.
if (key === 'body' && parentKey === 'response') {
return true;
}
return false;
}
function redactSensitiveKey(key: string, value: unknown, seen: WeakSet<object>): unknown {
if (isSensitiveKey(key) && typeof value === 'string') {
return '[redacted]';
}
return sanitizeDebugSnapshotValue(value, key, seen);
}
/**
* Full-fidelity JSON-safe snapshot for the in-memory run debug buffer.
* Unlike trace sanitization, this does not truncate strings, arrays, or object keys.
*/
export function sanitizeDebugSnapshotValue(
value: unknown,
keyHint?: string,
seen?: WeakSet<object>,
): unknown {
const ancestors = seen ?? new WeakSet<object>();
if (value === undefined || value === null) {
return value;
}
if (typeof value === 'string') {
if (keyHint && isSensitiveKey(keyHint)) {
return '[redacted]';
}
return scrubSecretsInText(value);
}
if (typeof value === 'number' || typeof value === 'boolean') {
return value;
}
if (typeof value === 'bigint') {
return value.toString();
}
if (typeof value === 'function') {
return `[function ${value.name || 'anonymous'}]`;
}
if (value instanceof AbortSignal) {
return '[AbortSignal]';
}
if (value instanceof Date) {
return value.toISOString();
}
if (value instanceof Error) {
return {
name: value.name,
message: scrubSecretsInText(value.message),
};
}
if (value instanceof Uint8Array) {
return `[binary ${value.byteLength} bytes]`;
}
// Only objects on the current path count as circular. A shared reference that
// appears in several sibling branches is serialized in each of them.
if (Array.isArray(value)) {
if (ancestors.has(value)) {
return '[Circular]';
}
ancestors.add(value);
const sanitized = value.map((entry) => sanitizeDebugSnapshotValue(entry, keyHint, ancestors));
ancestors.delete(value);
return sanitized;
}
if (isRecord(value)) {
if (ancestors.has(value)) {
return '[Circular]';
}
ancestors.add(value);
const sanitized: Record<string, unknown> = {};
for (const [key, entryValue] of Object.entries(value)) {
if (shouldOmitKey(key, keyHint)) {
continue;
}
sanitized[key] = redactSensitiveKey(key, entryValue, ancestors);
}
ancestors.delete(value);
return sanitized;
}
if (typeof value === 'symbol') {
return value.toString();
}
return '[unsupported value]';
}
export function sanitizeDebugSnapshotRecord(value: unknown): Record<string, unknown> {
const sanitized = sanitizeDebugSnapshotValue(value);
return isRecord(sanitized) ? sanitized : { value: sanitized };
}

93
packages/@n8n/instance-ai/src/index.ts
View File

@ -13,12 +13,8 @@ import type * as ValidateAttachmentsMod from './parsers/validate-attachments';
import type * as PlannedTaskPermissionsMod from './planned-tasks/planned-task-permissions';
import type * as PlannedTaskServiceMod from './planned-tasks/planned-task-service';
import type * as PromptProfilesMod from './prompts/prompt-profiles';
import type * as BackgroundTaskManagerMod from './runtime/background-task-manager';
import type * as InstanceContextStateMod from './runtime/instance-context-state';
import type * as LivenessPolicyMod from './runtime/liveness-policy';
import type * as ResumableStreamExecutorMod from './runtime/resumable-stream-executor';
import type * as RunStateRegistryMod from './runtime/run-state-registry';
import type * as StreamRunnerMod from './runtime/stream-runner';
import type * as TerminalResponseGuardMod from './runtime/terminal-response-guard';
import type * as MaterializeRuntimeSkillsMod from './skills/materialize-runtime-skills';
import type * as RuntimeSkillsMod from './skills/runtime-skills';
@ -170,21 +166,9 @@ const loadSnapshotManager = lazyModule(
const loadRunStateRegistry = lazyModule(
() => require('./runtime/run-state-registry') as typeof RunStateRegistryMod,
);
const loadBackgroundTaskManager = lazyModule(
() => require('./runtime/background-task-manager') as typeof BackgroundTaskManagerMod,
);
const loadTerminalResponseGuard = lazyModule(
() => require('./runtime/terminal-response-guard') as typeof TerminalResponseGuardMod,
);
const loadResumableStreamExecutor = lazyModule(
() => require('./runtime/resumable-stream-executor') as typeof ResumableStreamExecutorMod,
);
const loadStreamRunner = lazyModule(
() => require('./runtime/stream-runner') as typeof StreamRunnerMod,
);
const loadLivenessPolicy = lazyModule(
() => require('./runtime/liveness-policy') as typeof LivenessPolicyMod,
);
const loadWorkflowLoop = lazyModule(() => require('./workflow-loop') as typeof WorkflowLoopMod);
const loadWorkflowLoopRuntime = lazyModule(
() => require('./workflow-loop/runtime') as typeof WorkflowLoopRuntimeMod,
@ -405,10 +389,6 @@ export const PlannedTaskStorage: typeof StorageMod.PlannedTaskStorage = lazyClas
() => loadStorage().PlannedTaskStorage,
);
export const getThread: typeof StorageMod.getThread = lazyFunction(() => loadStorage().getThread);
export type TerminalOutcomeStorage = StorageMod.TerminalOutcomeStorage;
export const TerminalOutcomeStorage: typeof StorageMod.TerminalOutcomeStorage = lazyClass(
() => loadStorage().TerminalOutcomeStorage,
);
export const patchThread: typeof StorageMod.patchThread = lazyFunction(
() => loadStorage().patchThread,
);
@ -422,7 +402,6 @@ export type {
IterationLog,
PatchableThreadMemory,
ThreadPatch,
TerminalOutcome,
WorkflowSetupRoutingClaim,
WorkflowLoopWorkItemRecord,
} from './storage';
@ -488,10 +467,6 @@ defineLazyExport(
'N8N_WORKSPACE_DIR_ENV',
() => loadMaterializeRuntimeSkills().N8N_WORKSPACE_DIR_ENV,
);
defineLazyExport(
'INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG',
() => loadLivenessPolicy().INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
);
defineLazyExport('workflowBuildOutcomeSchema', () => loadWorkflowLoop().workflowBuildOutcomeSchema);
defineLazyExport(
'workflowVerificationEvidenceSchema',
@ -538,20 +513,8 @@ export const SnapshotManager: typeof SnapshotManagerMod.SnapshotManager = lazyCl
() => loadSnapshotManager().SnapshotManager,
);
export type { InstanceAiEventBus, StoredEvent } from './event-bus';
export type BackgroundTaskManager = BackgroundTaskManagerMod.BackgroundTaskManager;
export const BackgroundTaskManager: typeof BackgroundTaskManagerMod.BackgroundTaskManager =
lazyClass(() => loadBackgroundTaskManager().BackgroundTaskManager);
export const enrichMessageWithRunningTasks: typeof BackgroundTaskManagerMod.enrichMessageWithRunningTasks =
lazyFunction(() => loadBackgroundTaskManager().enrichMessageWithRunningTasks);
export const enrichMessageWithBackgroundTasks: typeof BackgroundTaskManagerMod.enrichMessageWithRunningTasks =
enrichMessageWithRunningTasks;
export type {
BackgroundTaskStatus,
ManagedBackgroundTask,
SpawnManagedBackgroundTaskOptions,
} from './runtime/background-task-manager';
export { MemoryTaskRegistry } from './runtime/memory-task-registry';
export type RunStateRegistry<TUser = unknown> = RunStateRegistryMod.RunStateRegistry<TUser>;
export type RunStateRegistry = RunStateRegistryMod.RunStateRegistry;
export const RunStateRegistry: typeof RunStateRegistryMod.RunStateRegistry = lazyClass(
() => loadRunStateRegistry().RunStateRegistry,
);
@ -562,21 +525,7 @@ export {
formatWorkflowSetupStateNote,
observeWorkflowSetupStates,
} from './tools/workflows/setup-panel-state';
export type { RunDebugRecord } from './debug/run-debug-buffer';
export {
RunDebugBuffer,
buildRunDebugLabel,
createRunDebugStepHooks,
} from './debug/run-debug-buffer';
export type {
ActiveRunState,
BackgroundTaskStatusSnapshot,
ConfirmationData,
PendingConfirmation,
RunStateTimeoutDetails,
StartedRunState,
SuspendedRunState,
} from './runtime/run-state-registry';
export type { ConfirmationData } from './runtime/run-state-registry';
export type InstanceAiTerminalResponseGuard =
TerminalResponseGuardMod.InstanceAiTerminalResponseGuard;
export const InstanceAiTerminalResponseGuard: typeof TerminalResponseGuardMod.InstanceAiTerminalResponseGuard =
@ -586,18 +535,6 @@ export type {
TerminalResponseStatus,
TerminalVisibilitySource,
} from './runtime/terminal-response-guard';
export const executeResumableStream: typeof ResumableStreamExecutorMod.executeResumableStream =
lazyFunction(() => loadResumableStreamExecutor().executeResumableStream);
export type {
AutoResumeControl,
ExecuteResumableStreamOptions,
ExecuteResumableStreamResult,
ManualSuspensionControl,
ResumableStreamContext,
ResumableStreamControl,
ResumableStreamSource,
TraceStatus,
} from './runtime/resumable-stream-executor';
export type { WorkSummary, ToolCallSummary } from './stream/work-summary-accumulator';
export type {
AgentChunkPublisherOptions,
@ -617,30 +554,6 @@ export const mergeInstanceContextReach: typeof InstanceContextReachMod.mergeInst
export type { RunTokenUsage, BuilderUsageItem } from './stream/usage-accumulator';
export const tokenUsageToBuilderUsageItems: typeof UsageAccumulatorMod.tokenUsageToBuilderUsageItems =
lazyFunction(() => loadUsageAccumulator().tokenUsageToBuilderUsageItems);
export const resumeAgentRun: typeof StreamRunnerMod.resumeAgentRun = lazyFunction(
() => loadStreamRunner().resumeAgentRun,
);
export const streamAgentRun: typeof StreamRunnerMod.streamAgentRun = lazyFunction(
() => loadStreamRunner().streamAgentRun,
);
export const createInstanceAiLivenessPolicyConfig: typeof LivenessPolicyMod.createInstanceAiLivenessPolicyConfig =
lazyFunction(() => loadLivenessPolicy().createInstanceAiLivenessPolicyConfig);
export declare const INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG: typeof LivenessPolicyMod.INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG;
export type InstanceAiLivenessPolicy = LivenessPolicyMod.InstanceAiLivenessPolicy;
export const InstanceAiLivenessPolicy: typeof LivenessPolicyMod.InstanceAiLivenessPolicy =
lazyClass(() => loadLivenessPolicy().InstanceAiLivenessPolicy);
export type {
InstanceAiLivenessDecision,
InstanceAiLivenessInput,
InstanceAiLivenessPolicyConfig,
InstanceAiLivenessSurface,
InstanceAiLivenessTimeoutReason,
} from './runtime/liveness-policy';
export type {
StreamableAgent,
StreamRunOptions,
StreamRunResult,
} from './runtime/stream-runner';
export type WorkflowTaskCoordinator = WorkflowLoopMod.WorkflowTaskCoordinator;
export const WorkflowTaskCoordinator: typeof WorkflowLoopMod.WorkflowTaskCoordinator = lazyClass(
() => loadWorkflowLoop().WorkflowTaskCoordinator,
@ -728,7 +641,7 @@ export type {
PlannedTaskSchedulerAction,
PlannedTaskService,
OrchestrationContext,
BackgroundTaskResult,
TraceStatus,
InstanceAiToolTraceOptions,
InstanceAiTraceContext,
InstanceAiTraceRun,

852
packages/@n8n/instance-ai/src/runtime/__tests__/background-task-manager.test.ts
View File

@ -1,852 +0,0 @@
import type { BackgroundTaskResult } from '../../types';
import { BackgroundTaskManager, enrichMessageWithRunningTasks } from '../background-task-manager';
import type {
ManagedBackgroundTask,
SpawnManagedBackgroundTaskOptions,
} from '../background-task-manager';
import { InstanceAiLivenessPolicy } from '../liveness-policy';
const day = 24 * 60 * 60_000;
function makeSpawnOptions(
overrides: Partial<SpawnManagedBackgroundTaskOptions> = {},
): SpawnManagedBackgroundTaskOptions {
return {
taskId: 'task-1',
threadId: 'thread-1',
runId: 'run-1',
role: 'builder',
agentId: 'agent-1',
run: vi.fn().mockResolvedValue('done'),
...overrides,
};
}
describe('BackgroundTaskManager', () => {
let manager: BackgroundTaskManager;
beforeEach(() => {
manager = new BackgroundTaskManager(3);
});
describe('liveness timeouts', () => {
const policy = new InstanceAiLivenessPolicy({
confirmationTimeoutMs: day,
backgroundTaskIdleTimeoutMs: 10_000,
backgroundTaskMaxLifetimeMs: 30_000,
activeRunIdleTimeoutMs: 10_000,
activeRunMaxLifetimeMs: 30_000,
});
it('fails and settles idle running tasks', async () => {
const onFailed = vi.fn((_task: ManagedBackgroundTask) => undefined);
const onSettled = vi.fn((_task: ManagedBackgroundTask) => undefined);
let signal: AbortSignal | undefined;
manager.spawn(
makeSpawnOptions({
run: async (abortSignal) => {
signal = abortSignal;
await new Promise(() => {});
return 'never';
},
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' },
onFailed,
onSettled,
}),
);
const task = manager.getRunningTasks('thread-1')[0];
task.startedAt = 0;
task.lastActivityAt = 0;
const timedOut = await manager.timeoutTimedOutTasks(policy, 10_000);
expect(timedOut).toHaveLength(1);
expect(signal?.aborted).toBe(true);
expect(onFailed).toHaveBeenCalledWith(
expect.objectContaining({
status: 'failed',
timeoutReason: 'idle_timeout',
}),
);
expect(onFailed.mock.calls[0]?.[0].error).toEqual(expect.stringContaining('timed out'));
expect(onSettled).toHaveBeenCalledWith(
expect.objectContaining({ status: 'failed', timeoutReason: 'idle_timeout' }),
);
expect(manager.getRunningTasks('thread-1')).toHaveLength(0);
const next = manager.spawn(
makeSpawnOptions({
taskId: 'task-2',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' },
}),
);
expect(next.status).toBe('started');
});
it('keeps touched tasks alive until they exceed the idle timeout', async () => {
manager.spawn(
makeSpawnOptions({
run: async () => await new Promise(() => {}),
}),
);
const task = manager.getRunningTasks('thread-1')[0];
task.startedAt = 0;
manager.touchTask('thread-1', 'task-1', 9_000);
const timedOut = await manager.timeoutTimedOutTasks(policy, 10_000);
expect(timedOut).toEqual([]);
expect(manager.getRunningTasks('thread-1')).toHaveLength(1);
});
it('skips timeout checks for tasks currently waiting on HITL', async () => {
manager.spawn(
makeSpawnOptions({
run: async () => await new Promise(() => {}),
}),
);
const task = manager.getRunningTasks('thread-1')[0];
task.startedAt = 0;
task.lastActivityAt = 0;
const timedOut = await manager.timeoutTimedOutTasks(policy, 30_000, {
shouldSkipTask: (candidate) => candidate.threadId === 'thread-1',
});
expect(timedOut).toEqual([]);
expect(manager.getRunningTasks('thread-1')).toHaveLength(1);
});
});
describe('spawn', () => {
it('spawns a task and tracks it as running', () => {
const result = manager.spawn(makeSpawnOptions());
expect(result.status).toBe('started');
expect(manager.getRunningTasks('thread-1')).toHaveLength(1);
expect(manager.getRunningTasks('thread-1')[0].taskId).toBe('task-1');
});
it('rejects spawn when concurrent limit is reached', () => {
const onLimitReached = vi.fn();
const createTraceContext = vi.fn();
manager.spawn(
makeSpawnOptions({ taskId: 't1', run: async () => await new Promise(() => {}) }),
);
manager.spawn(
makeSpawnOptions({ taskId: 't2', run: async () => await new Promise(() => {}) }),
);
manager.spawn(
makeSpawnOptions({ taskId: 't3', run: async () => await new Promise(() => {}) }),
);
const result = manager.spawn(
makeSpawnOptions({ taskId: 't4', onLimitReached, createTraceContext }),
);
expect(result.status).toBe('limit-reached');
expect(onLimitReached).toHaveBeenCalledWith(expect.stringContaining('limit of 3'));
expect(createTraceContext).not.toHaveBeenCalled();
});
describe('instance-wide ceiling', () => {
const forever = async () => await new Promise<string>(() => {});
it('rejects a spawn once the total across threads is reached', () => {
// Per-thread 5 so it can't fire; the total of 2 is what we're exercising.
const capped = new BackgroundTaskManager(5, 2);
const onLimitReached = vi.fn();
capped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1', run: forever }));
capped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2', run: forever }));
const result = capped.spawn(
makeSpawnOptions({ taskId: 't3', threadId: 'thread-3', onLimitReached }),
);
expect(result.status).toBe('limit-reached');
expect(onLimitReached).toHaveBeenCalledWith(expect.stringContaining('this n8n instance'));
});
it('is unlimited when the ceiling is -1', () => {
const uncapped = new BackgroundTaskManager(5, -1);
uncapped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1', run: forever }));
uncapped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2', run: forever }));
const result = uncapped.spawn(makeSpawnOptions({ taskId: 't3', threadId: 'thread-3' }));
expect(result.status).toBe('started');
});
it('frees a slot when a task settles', async () => {
const capped = new BackgroundTaskManager(5, 1);
capped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1' }));
await vi.waitFor(() => expect(capped.runningTaskCount()).toBe(0));
const result = capped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2' }));
expect(result.status).toBe('started');
});
it('counts running tasks across every thread', () => {
const capped = new BackgroundTaskManager(5, 10);
capped.spawn(makeSpawnOptions({ taskId: 't1', threadId: 'thread-1', run: forever }));
capped.spawn(makeSpawnOptions({ taskId: 't2', threadId: 'thread-2', run: forever }));
expect(capped.runningTaskCount()).toBe(2);
});
});
it('creates lazy trace context only after a task is accepted', async () => {
const traceContext = { projectName: 'instance-ai' } as never;
const createTraceContext = vi.fn().mockResolvedValue(traceContext);
const run = vi.fn().mockResolvedValue('done');
manager.spawn(makeSpawnOptions({ createTraceContext, run }));
await flushPromises();
expect(createTraceContext).toHaveBeenCalledTimes(1);
expect(run).toHaveBeenCalledWith(
expect.any(AbortSignal),
expect.any(Function),
expect.any(Function),
{ traceContext },
);
});
it('calls onCompleted and onSettled when run resolves with string', async () => {
const onCompleted = vi.fn();
const onSettled = vi.fn();
const { promise, resolve } = createDeferred<string>();
manager.spawn(
makeSpawnOptions({
run: async () => await promise,
onCompleted,
onSettled,
}),
);
resolve('result-text');
await flushPromises();
expect(onCompleted).toHaveBeenCalledWith(
expect.objectContaining({ status: 'completed', result: 'result-text' }),
);
expect(onSettled).toHaveBeenCalled();
});
it('calls onCompleted with structured result', async () => {
const onCompleted = vi.fn();
const { promise, resolve } = createDeferred<string | BackgroundTaskResult>();
manager.spawn(
makeSpawnOptions({
run: async () => await promise,
onCompleted,
}),
);
resolve({ text: 'summary', outcome: { workflowId: 'wf-1' } });
await flushPromises();
expect(onCompleted).toHaveBeenCalledWith(
expect.objectContaining({
status: 'completed',
result: 'summary',
outcome: { workflowId: 'wf-1' },
}),
);
});
it('calls onFailed and onSettled when run rejects', async () => {
const onFailed = vi.fn();
const onSettled = vi.fn();
const { promise, reject } = createDeferred<string | BackgroundTaskResult>();
manager.spawn(
makeSpawnOptions({
run: async () => await promise,
onFailed,
onSettled,
}),
);
reject(new Error('boom'));
await flushPromises();
expect(onFailed).toHaveBeenCalledWith(
expect.objectContaining({ status: 'failed', error: 'boom' }),
);
expect(onSettled).toHaveBeenCalled();
});
it('does not call onFailed when aborted', async () => {
const onFailed = vi.fn();
const { promise, reject } = createDeferred<string | BackgroundTaskResult>();
manager.spawn(
makeSpawnOptions({
run: async () => await promise,
onFailed,
}),
);
manager.cancelTask('thread-1', 'task-1');
reject(new Error('aborted'));
await flushPromises();
expect(onFailed).not.toHaveBeenCalled();
});
it('does not call onSettled when aborted', async () => {
const onSettled = vi.fn();
const { promise, reject } = createDeferred<string | BackgroundTaskResult>();
manager.spawn(
makeSpawnOptions({
run: async () => await promise,
onSettled,
}),
);
manager.cancelTask('thread-1', 'task-1');
reject(new Error('aborted'));
await flushPromises();
expect(onSettled).not.toHaveBeenCalled();
});
it('removes task from map after settlement', async () => {
const { promise, resolve } = createDeferred<string>();
manager.spawn(makeSpawnOptions({ run: async () => await promise }));
expect(manager.getTaskSnapshots('thread-1')).toHaveLength(1);
resolve('done');
await flushPromises();
expect(manager.getTaskSnapshots('thread-1')).toHaveLength(0);
});
});
describe('single-flight dedupe', () => {
it('returns duplicate when plannedTaskId matches a running task', () => {
const first = manager.spawn(
makeSpawnOptions({
taskId: 'first',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' },
}),
);
expect(first.status).toBe('started');
const run = vi.fn(async (): Promise<string> => await new Promise(() => {}));
const second = manager.spawn(
makeSpawnOptions({
taskId: 'second',
run,
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-1' },
}),
);
expect(second.status).toBe('duplicate');
if (second.status === 'duplicate') {
expect(second.existing.taskId).toBe('first');
}
expect(run).not.toHaveBeenCalled();
expect(manager.getRunningTasks('thread-1')).toHaveLength(1);
});
it('does not create lazy trace context for duplicate spawns', () => {
manager.spawn(
makeSpawnOptions({
taskId: 'first',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-trace' },
}),
);
const createTraceContext = vi.fn();
const second = manager.spawn(
makeSpawnOptions({
taskId: 'second',
createTraceContext,
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-trace' },
}),
);
expect(second.status).toBe('duplicate');
expect(createTraceContext).not.toHaveBeenCalled();
});
it('allows a new spawn once the first planned-task settles', async () => {
const { promise, resolve } = createDeferred<string>();
manager.spawn(
makeSpawnOptions({
taskId: 'first',
run: async () => await promise,
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-2' },
}),
);
resolve('done');
await flushPromises();
const second = manager.spawn(
makeSpawnOptions({
taskId: 'second',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-2' },
}),
);
expect(second.status).toBe('started');
});
it('returns duplicate when workflowId + role matches a running task without plannedTaskId', () => {
manager.spawn(
makeSpawnOptions({
taskId: 'first',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', workflowId: 'wf-1' },
}),
);
const run = vi.fn(async (): Promise<string> => await new Promise(() => {}));
const second = manager.spawn(
makeSpawnOptions({
taskId: 'second',
run,
dedupeKey: { role: 'workflow-builder', workflowId: 'wf-1' },
}),
);
expect(second.status).toBe('duplicate');
expect(run).not.toHaveBeenCalled();
});
it('does not collapse two distinct plannedTaskIds that target the same workflowId', () => {
// A plan may include two work items for the same workflow — e.g.,
// initial build (planned-A) followed by a patch (planned-B). They are
// distinct planned tasks and must both run; collapsing them on
// workflowId would skip work the user approved.
const first = manager.spawn(
makeSpawnOptions({
taskId: 'task-A',
run: async () => await new Promise(() => {}),
dedupeKey: {
role: 'workflow-builder',
plannedTaskId: 'planned-A',
workflowId: 'wf-shared',
},
}),
);
expect(first.status).toBe('started');
const run = vi.fn(async (): Promise<string> => await new Promise(() => {}));
const second = manager.spawn(
makeSpawnOptions({
taskId: 'task-B',
run,
dedupeKey: {
role: 'workflow-builder',
plannedTaskId: 'planned-B',
workflowId: 'wf-shared',
},
}),
);
expect(second.status).toBe('started');
expect(run).toHaveBeenCalledTimes(1);
expect(manager.getRunningTasks('thread-1')).toHaveLength(2);
});
it('does not dedupe across roles for the same workflowId', () => {
manager.spawn(
makeSpawnOptions({
taskId: 'builder',
role: 'workflow-builder',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', workflowId: 'wf-1' },
}),
);
const other = manager.spawn(
makeSpawnOptions({
taskId: 'data-table-manager',
role: 'data-table-manager',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'data-table-manager', workflowId: 'wf-1' },
}),
);
expect(other.status).toBe('started');
expect(manager.getRunningTasks('thread-1')).toHaveLength(2);
});
it('limit-reached still fires even when dedupe would have passed', () => {
const filler = makeSpawnOptions({ run: async () => await new Promise(() => {}) });
manager.spawn({ ...filler, taskId: 't1' });
manager.spawn({ ...filler, taskId: 't2' });
manager.spawn({ ...filler, taskId: 't3' });
const onLimitReached = vi.fn();
const result = manager.spawn(
makeSpawnOptions({
taskId: 't4',
onLimitReached,
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-fresh' },
}),
);
expect(result.status).toBe('limit-reached');
expect(onLimitReached).toHaveBeenCalled();
});
it('cancelTask releases dedupe indices so a fresh spawn is allowed', () => {
manager.spawn(
makeSpawnOptions({
taskId: 'first',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-3' },
}),
);
manager.cancelTask('thread-1', 'first');
const second = manager.spawn(
makeSpawnOptions({
taskId: 'second',
run: async () => await new Promise(() => {}),
dedupeKey: { role: 'workflow-builder', plannedTaskId: 'planned-3' },
}),
);
expect(second.status).toBe('started');
});
});
describe('queueCorrection', () => {
it('queues correction for running task', () => {
manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) }));
expect(manager.queueCorrection('thread-1', 'task-1', 'fix this')).toBe('queued');
});
it('returns task-not-found for unknown task', () => {
expect(manager.queueCorrection('thread-1', 'unknown', 'fix')).toBe('task-not-found');
});
it('returns task-not-found for wrong thread', () => {
manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) }));
expect(manager.queueCorrection('thread-2', 'task-1', 'fix')).toBe('task-not-found');
});
it('drains corrections during run execution', async () => {
const drainedCorrections: string[][] = [];
const { promise, resolve } = createDeferred<string>();
manager.spawn(
makeSpawnOptions({
run: async (_signal, drain) => {
drainedCorrections.push(drain());
return await promise;
},
}),
);
// Queue correction before the run resolves
await flushPromises();
manager.queueCorrection('thread-1', 'task-1', 'correction-1');
resolve('done');
await flushPromises();
expect(drainedCorrections[0]).toEqual([]);
});
it('notifies waitForCorrection when a correction is queued', async () => {
let waitForCorrection: (() => Promise<void>) | undefined;
const { promise, resolve } = createDeferred<string>();
manager.spawn(
makeSpawnOptions({
run: async (_signal, _drain, waitFn) => {
waitForCorrection = waitFn;
return await promise;
},
}),
);
await flushPromises();
const correctionPromise = waitForCorrection!();
let resolved = false;
void correctionPromise.then(() => {
resolved = true;
});
await flushPromises();
expect(resolved).toBe(false);
manager.queueCorrection('thread-1', 'task-1', 'use openrouter node');
await flushPromises();
expect(resolved).toBe(true);
resolve('done');
await flushPromises();
});
it('resolves waitForCorrection immediately when corrections are already queued', async () => {
let waitForCorrection: (() => Promise<void>) | undefined;
const { promise, resolve } = createDeferred<string>();
manager.spawn(
makeSpawnOptions({
run: async (_signal, _drain, waitFn) => {
waitForCorrection = waitFn;
return await promise;
},
}),
);
await flushPromises();
manager.queueCorrection('thread-1', 'task-1', 'pending correction');
const correctionPromise = waitForCorrection!();
let resolved = false;
void correctionPromise.then(() => {
resolved = true;
});
await flushPromises();
expect(resolved).toBe(true);
resolve('done');
await flushPromises();
});
});
describe('cancelTask', () => {
it('cancels a running task and aborts its signal', () => {
manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) }));
const cancelled = manager.cancelTask('thread-1', 'task-1');
expect(cancelled).toBeDefined();
expect(cancelled!.status).toBe('cancelled');
expect(cancelled!.abortController.signal.aborted).toBe(true);
expect(manager.getRunningTasks('thread-1')).toHaveLength(0);
});
it('returns undefined for non-existent task', () => {
expect(manager.cancelTask('thread-1', 'unknown')).toBeUndefined();
});
it('returns undefined for wrong thread', () => {
manager.spawn(makeSpawnOptions({ run: async () => await new Promise(() => {}) }));
expect(manager.cancelTask('thread-2', 'task-1')).toBeUndefined();
});
});
describe('cancelThread', () => {
it('cancels all running tasks for a thread', () => {
manager.spawn(
makeSpawnOptions({ taskId: 't1', run: async () => await new Promise(() => {}) }),
);
manager.spawn(
makeSpawnOptions({ taskId: 't2', run: async () => await new Promise(() => {}) }),
);
const cancelled = manager.cancelThread('thread-1');
expect(cancelled).toHaveLength(2);
expect(manager.getRunningTasks('thread-1')).toHaveLength(0);
});
it('does not cancel tasks for other threads', () => {
manager.spawn(
makeSpawnOptions({
taskId: 't1',
threadId: 'thread-1',
run: async () => await new Promise(() => {}),
}),
);
manager.spawn(
makeSpawnOptions({
taskId: 't2',
threadId: 'thread-2',
run: async () => await new Promise(() => {}),
}),
);
manager.cancelThread('thread-1');
expect(manager.getRunningTasks('thread-2')).toHaveLength(1);
});
});
describe('cancelAll', () => {
it('cancels all tasks across all threads', () => {
manager.spawn(
makeSpawnOptions({
taskId: 't1',
threadId: 'thread-1',
run: async () => await new Promise(() => {}),
}),
);
manager.spawn(
makeSpawnOptions({
taskId: 't2',
threadId: 'thread-2',
run: async () => await new Promise(() => {}),
}),
);
const cancelled = manager.cancelAll();
expect(cancelled).toHaveLength(2);
expect(cancelled.every((t) => t.status === 'cancelled')).toBe(true);
expect(manager.getRunningTasks('thread-1')).toHaveLength(0);
expect(manager.getRunningTasks('thread-2')).toHaveLength(0);
});
});
describe('getTaskSnapshots / getRunningTasks', () => {
it('returns empty array for unknown thread', () => {
expect(manager.getTaskSnapshots('unknown')).toEqual([]);
expect(manager.getRunningTasks('unknown')).toEqual([]);
});
it('getRunningTasks excludes non-running tasks', async () => {
const { promise, resolve } = createDeferred<string>();
manager.spawn(makeSpawnOptions({ run: async () => await promise }));
resolve('done');
await flushPromises();
expect(manager.getRunningTasks('thread-1')).toHaveLength(0);
});
});
describe('getRunningTasksByParentCheckpoint', () => {
it('returns running tasks tagged with the given checkpoint id', () => {
manager.spawn(
makeSpawnOptions({
taskId: 'child-1',
run: async () => await new Promise(() => {}),
parentCheckpointId: 'cp-verify-1',
}),
);
manager.spawn(
makeSpawnOptions({
taskId: 'child-2',
run: async () => await new Promise(() => {}),
parentCheckpointId: 'cp-verify-1',
}),
);
manager.spawn(
makeSpawnOptions({
taskId: 'unrelated',
run: async () => await new Promise(() => {}),
}),
);
const children = manager.getRunningTasksByParentCheckpoint('thread-1', 'cp-verify-1');
expect(children.map((c) => c.taskId).sort()).toEqual(['child-1', 'child-2']);
});
it('excludes tasks tagged under a different checkpoint', () => {
manager.spawn(
makeSpawnOptions({
taskId: 'child-a',
run: async () => await new Promise(() => {}),
parentCheckpointId: 'cp-A',
}),
);
manager.spawn(
makeSpawnOptions({
taskId: 'child-b',
run: async () => await new Promise(() => {}),
parentCheckpointId: 'cp-B',
}),
);
const childrenA = manager.getRunningTasksByParentCheckpoint('thread-1', 'cp-A');
expect(childrenA.map((c) => c.taskId)).toEqual(['child-a']);
});
it('excludes tasks that have already settled', async () => {
const { promise, resolve } = createDeferred<string>();
manager.spawn(
makeSpawnOptions({
taskId: 'child-done',
parentCheckpointId: 'cp-verify-1',
run: async () => await promise,
}),
);
resolve('done');
await flushPromises();
expect(manager.getRunningTasksByParentCheckpoint('thread-1', 'cp-verify-1')).toHaveLength(0);
});
});
});
describe('enrichMessageWithRunningTasks', () => {
it('returns original message when no tasks', async () => {
const result = await enrichMessageWithRunningTasks('Hello', []);
expect(result).toBe('Hello');
});
it('prepends running-tasks block with default format', async () => {
const tasks = [{ taskId: 'task-1', role: 'builder' } as ManagedBackgroundTask];
const result = await enrichMessageWithRunningTasks('Hello', tasks);
expect(result).toContain('<running-tasks>');
expect(result).toContain('builder');
expect(result).toContain('task-1');
expect(result).toContain('Hello');
});
it('uses custom formatTask when provided', async () => {
const tasks = [{ taskId: 'task-1', role: 'builder' } as ManagedBackgroundTask];
const result = await enrichMessageWithRunningTasks('Hello', tasks, {
formatTask: (t) => `Custom: ${t.role}`,
});
expect(result).toContain('Custom: builder');
});
});
// --- Helpers ---
async function flushPromises(): Promise<void> {
await new Promise((resolve) => {
setTimeout(resolve, 0);
});
}
function createDeferred<T = unknown>(): {
promise: Promise<T>;
resolve: (value: T) => void;
reject: (reason: unknown) => void;
} {
let resolve!: (value: T) => void;
let reject!: (reason: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}

167
packages/@n8n/instance-ai/src/runtime/__tests__/liveness-policy.test.ts
View File

@ -1,167 +0,0 @@
import {
createInstanceAiLivenessPolicyConfig,
INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
InstanceAiLivenessPolicy,
} from '../liveness-policy';
const minute = 60_000;
const day = 24 * 60 * minute;
function createPolicy() {
return new InstanceAiLivenessPolicy(INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG);
}
describe('InstanceAiLivenessPolicy', () => {
it('keeps default liveness limits centralized and allows the existing confirmation override', () => {
expect(INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG).toEqual({
confirmationTimeoutMs: day,
backgroundTaskIdleTimeoutMs: 10 * minute,
backgroundTaskMaxLifetimeMs: 30 * minute,
activeRunIdleTimeoutMs: 0,
activeRunMaxLifetimeMs: day,
});
expect(createInstanceAiLivenessPolicyConfig({ confirmationTimeoutMs: 42_000 })).toEqual({
...INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
confirmationTimeoutMs: 42_000,
});
});
it('keeps active work alive while it is still reporting activity', () => {
const decision = createPolicy().evaluate({
surface: 'background-task',
startedAt: 0,
lastActivityAt: 9 * minute,
now: 10 * minute,
});
expect(decision).toEqual({ action: 'keep-alive' });
});
it('times out background work after the idle limit', () => {
const decision = createPolicy().evaluate({
surface: 'background-task',
startedAt: 0,
lastActivityAt: 0,
now: 10 * minute,
});
expect(decision).toMatchObject({
action: 'timeout',
reason: 'idle_timeout',
surface: 'background-task',
timeoutMs: 10 * minute,
});
});
it('times out active work at the max lifetime even with recent activity', () => {
const decision = createPolicy().evaluate({
surface: 'active-run',
startedAt: 0,
lastActivityAt: day + minute,
now: day + minute,
});
expect(decision).toMatchObject({
action: 'timeout',
reason: 'max_lifetime',
surface: 'active-run',
timeoutMs: day,
});
});
it('does not idle-timeout an active run that is waiting on another surface', () => {
const decision = createPolicy().evaluate({
surface: 'active-run',
startedAt: 0,
lastActivityAt: 0,
now: 6 * 60 * minute,
});
expect(decision).toEqual({ action: 'keep-alive' });
});
it('uses the one-day confirmation timeout for suspended runs and pending confirmations by default', () => {
const policy = createPolicy();
const shorterWorkTimeout = 30 * minute;
expect(
policy.evaluate({
surface: 'suspended-run',
startedAt: 0,
lastActivityAt: 0,
now: shorterWorkTimeout,
}),
).toEqual({ action: 'keep-alive' });
expect(
policy.evaluate({
surface: 'pending-confirmation',
startedAt: 0,
lastActivityAt: 0,
now: shorterWorkTimeout,
}),
).toEqual({ action: 'keep-alive' });
expect(
policy.evaluate({
surface: 'suspended-run',
startedAt: 0,
lastActivityAt: 0,
now: day,
}),
).toMatchObject({ action: 'timeout', reason: 'idle_timeout', timeoutMs: day });
expect(
policy.evaluate({
surface: 'pending-confirmation',
startedAt: 0,
lastActivityAt: 0,
now: day,
}),
).toMatchObject({ action: 'timeout', reason: 'idle_timeout', timeoutMs: day });
});
it('uses a configured confirmation timeout for suspended runs and pending confirmations', () => {
const policy = new InstanceAiLivenessPolicy(
createInstanceAiLivenessPolicyConfig({ confirmationTimeoutMs: 10 * minute }),
);
expect(
policy.evaluate({
surface: 'suspended-run',
startedAt: 0,
lastActivityAt: 0,
now: 10 * minute,
}),
).toMatchObject({ action: 'timeout', reason: 'idle_timeout' });
expect(
policy.evaluate({
surface: 'pending-confirmation',
startedAt: 0,
lastActivityAt: 0,
now: 10 * minute,
}),
).toMatchObject({ action: 'timeout', reason: 'idle_timeout' });
});
it('keeps a surface alive when its timeout is disabled', () => {
const policy = new InstanceAiLivenessPolicy({
confirmationTimeoutMs: 0,
backgroundTaskIdleTimeoutMs: 0,
backgroundTaskMaxLifetimeMs: 0,
activeRunIdleTimeoutMs: 0,
activeRunMaxLifetimeMs: 0,
});
expect(
policy.evaluate({
surface: 'pending-confirmation',
startedAt: 0,
lastActivityAt: 0,
now: 60 * minute,
}),
).toEqual({ action: 'keep-alive' });
});
});

772
packages/@n8n/instance-ai/src/runtime/__tests__/resumable-stream-executor.test.ts
View File

@ -1,772 +0,0 @@
import type { InstanceAiEvent } from '@n8n/api-types';
import type { SuspensionInfo } from '../../utils/stream-helpers';
import { executeResumableStream, normalizeStreamSource } from '../resumable-stream-executor';
function createEventBus() {
return {
publish: vi.fn(),
subscribe: vi.fn(),
getEventsAfter: vi.fn(),
getEventsForRun: vi.fn().mockReturnValue([]),
getEventsForRuns: vi.fn().mockReturnValue([]),
};
}
function createLogger() {
return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };
}
async function* fromChunks(chunks: unknown[]) {
for (const chunk of chunks) {
await Promise.resolve();
yield chunk;
}
}
function readableFromChunks(chunks: unknown[]) {
return new ReadableStream<unknown>({
start(controller) {
for (const chunk of chunks) {
controller.enqueue(chunk);
}
controller.close();
},
});
}
function textChunk(text: string) {
return { type: 'text-delta', delta: text };
}
function errorChunk(error: unknown) {
return { type: 'error', error };
}
function suspensionChunk(input: {
toolCallId: string;
toolName?: string;
suspendPayload?: Record<string, unknown>;
input?: Record<string, unknown>;
}) {
return {
type: 'tool-call-suspended',
toolCallId: input.toolCallId,
...(input.toolName ? { toolName: input.toolName } : {}),
...(input.input ? { input: input.input } : {}),
suspendPayload: input.suspendPayload ?? {},
};
}
function createDeferred<T>() {
let resolve!: (value: T | PromiseLike<T>) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((res, rej) => {
resolve = res;
reject = rej;
});
return { promise, resolve, reject };
}
interface PublishedEvent {
type: string;
responseId?: string;
payload?: {
requestId?: string;
toolCallId?: string;
text?: string;
};
}
describe('normalizeStreamSource', () => {
it('normalizes native @n8n/agents streams', async () => {
const source = normalizeStreamSource({
runId: 'agent-run-1',
stream: readableFromChunks([
textChunk('Hello'),
{
type: 'message',
message: {
role: 'assistant',
content: [{ type: 'text', text: ' world' }],
},
},
]),
getState: vi.fn(),
});
expect(source.runId).toBe('agent-run-1');
await expect(source.text).resolves.toBe('Hello');
});
});
describe('executeResumableStream', () => {
it('buffers the confirmation event in manual mode', async () => {
const eventBus = createEventBus();
const result = await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
textChunk('Working...'),
suspensionChunk({
toolCallId: 'tool-call-1',
toolName: 'ask-user',
suspendPayload: {
requestId: 'request-1',
message: 'Need approval',
},
}),
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
expect(result).toEqual(
expect.objectContaining({
status: 'suspended',
agentRunId: 'agent-run-1',
suspension: {
toolCallId: 'tool-call-1',
requestId: 'request-1',
toolName: 'ask-user',
suspendPayload: { requestId: 'request-1', message: 'Need approval' },
},
}),
);
expect(eventBus.publish).toHaveBeenCalledWith(
'thread-1',
expect.objectContaining({ type: 'text-delta', runId: 'run-1', agentId: 'agent-1' }),
);
expect(eventBus.publish).not.toHaveBeenCalledWith(
'thread-1',
expect.objectContaining({ type: 'confirmation-request' }),
);
expect(result.confirmationEvent?.type).toBe('confirmation-request');
expect(result.confirmationEvent?.payload.requestId).toBe('request-1');
});
it('returns errored status when stream contains an error chunk', async () => {
const error = new Error('Not Found');
const result = await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([textChunk('Working...'), errorChunk(error)]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
expect(result.status).toBe('errored');
expect(result.agentRunId).toBe('agent-run-1');
expect(result.error).toBe(error);
});
it('reports the terminal finish reason of a completed run', async () => {
const result = await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
textChunk('Working...'),
{ type: 'finish', finishReason: 'max-iterations' },
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
expect(result.status).toBe('completed');
expect(result.finishReason).toBe('max-iterations');
});
it('reports the finish reason of the resumed stream, not the suspended one', async () => {
const resume = vi.fn().mockResolvedValue({
runId: 'agent-run-2',
stream: readableFromChunks([
textChunk('Done.'),
{ type: 'finish', finishReason: 'max-iterations' },
]),
});
const result = await executeResumableStream({
agent: { resume },
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
suspensionChunk({
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: { requestId: 'request-1', message: 'Please confirm' },
}),
{ type: 'finish', finishReason: 'tool-calls' },
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'auto', waitForConfirmation: vi.fn().mockResolvedValue({ approved: true }) },
});
expect(result.status).toBe('completed');
expect(result.finishReason).toBe('max-iterations');
});
it('publishes only the quota error when a follow-on error chunk arrives', async () => {
const eventBus = createEventBus();
const quotaError = Object.assign(new Error('Have reached end of quota'), {
statusCode: 403,
errorCode: 'quota_exhausted',
});
const followOn = new Error('No output generated. Check the stream for errors.');
const result = await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([errorChunk(quotaError), errorChunk(followOn)]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
const errorEvents = eventBus.publish.mock.calls
.map(([, event]) => event as InstanceAiEvent)
.filter((event) => event.type === 'error');
expect(errorEvents).toHaveLength(1);
expect(errorEvents[0].payload).toMatchObject({ code: 'quota_exhausted' });
// The swallowed follow-on chunk must not overwrite result.error, or telemetry
// would log the generic "No output generated" while the user saw the quota error.
expect(result.error).toBe(quotaError);
});
it('captures terminal usage on an aborted run so cancelled runs are billed', async () => {
const controller = new AbortController();
// Yield a text chunk, abort mid-stream (as a user "stop" does), then let
// the agent emit its terminal finish chunk carrying the run's usage.
async function* abortingStream() {
await Promise.resolve();
yield textChunk('partial');
controller.abort();
yield {
type: 'finish',
finishReason: 'error',
usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
};
}
const result = await executeResumableStream({
agent: {},
stream: { runId: 'agent-run-1', fullStream: abortingStream() },
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
signal: controller.signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
expect(result.status).toBe('cancelled');
expect(result.usage).toMatchObject({
promptTokens: 10,
completionTokens: 5,
totalTokens: 15,
});
});
it('reports liveness activity for each consumed chunk', async () => {
const onActivity = vi.fn();
await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([textChunk('Working...'), textChunk('Done.')]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
signal: new AbortController().signal,
logger: createLogger(),
onActivity,
},
control: { mode: 'manual' },
});
expect(onActivity).toHaveBeenCalledTimes(2);
});
it('assigns stable response IDs from native start-step chunks', async () => {
const eventBus = createEventBus();
await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
{ type: 'start-step' },
textChunk('First'),
{
type: 'tool-call',
toolCallId: 'tool-call-1',
toolName: 'test-tool',
input: {},
},
textChunk(' step'),
{ type: 'finish-step' },
{ type: 'start-step' },
textChunk('Second step'),
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent);
const firstText = publishedEvents.find((event) => event.payload?.text === 'First');
const toolCall = publishedEvents.find((event) => event.type === 'tool-call');
const firstStepContinuation = publishedEvents.find((event) => event.payload?.text === ' step');
const secondText = publishedEvents.find((event) => event.payload?.text === 'Second step');
expect(firstText?.responseId).toBe('agent-run-1:step:1');
expect(toolCall?.responseId).toBe('agent-run-1:step:1');
expect(firstStepContinuation?.responseId).toBe('agent-run-1:step:1');
expect(secondText?.responseId).toBe('agent-run-1:step:2');
});
it('mints a synthetic per-segment responseId when the provider supplies none', async () => {
const eventBus = createEventBus();
// No start-step chunks at all; some providers never emit them. Deltas
// must still carry a responseId (segment identity keys the run reducer's
// block replace semantics), one per contiguous delta run.
await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
textChunk('First'),
textChunk(' segment'),
{
type: 'tool-call',
toolCallId: 'tool-call-1',
toolName: 'test-tool',
input: {},
},
textChunk('Second segment'),
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent);
// Match the segment's first delta by prefix; deltas publish per chunk.
const first = publishedEvents.find((event) => event.payload?.text?.startsWith('First'));
const toolCall = publishedEvents.find((event) => event.type === 'tool-call');
const second = publishedEvents.find((event) => event.payload?.text === 'Second segment');
expect(first?.responseId).toBeDefined();
// Sticky for the whole segment (the closing structural fact inherits it,
// mirroring provider-supplied step ids)...
expect(toolCall?.responseId).toBe(first?.responseId);
// ...and rolled after it, so two segments never share an id.
expect(second?.responseId).toBeDefined();
expect(second?.responseId).not.toBe(first?.responseId);
});
it('rolls the synthetic responseId at a finish-step boundary that maps to no event', async () => {
const eventBus = createEventBus();
// finish-step publishes nothing, but it still separates two segments; a
// sticky synthetic id across it would let the reducer's id-keyed replace
// pair blocks from different steps.
await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
textChunk('First step text'),
{ type: 'finish-step' },
textChunk('Second step text'),
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: { mode: 'manual' },
});
const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent);
// Match each segment's delta by prefix.
const first = publishedEvents.find((event) => event.payload?.text?.startsWith('First'));
const second = publishedEvents.find((event) => event.payload?.text?.startsWith('Second'));
expect(first?.responseId).toBeDefined();
expect(second?.responseId).toBeDefined();
expect(second?.responseId).not.toBe(first?.responseId);
});
it('stops consuming after a requested handoff while publishing the current tool result', async () => {
const eventBus = createEventBus();
let shouldStop = false;
eventBus.publish.mockImplementation((_: string, event: PublishedEvent) => {
if (event.type === 'tool-result') {
shouldStop = true;
}
});
const result = await executeResumableStream({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
{
type: 'tool-call',
toolCallId: 'tool-call-1',
toolName: 'create-tasks',
input: {},
},
{
type: 'tool-result',
toolCallId: 'tool-call-1',
output: { result: 'Plan approved. Started 1 task.', taskCount: 1 },
},
textChunk('This inline continuation should not be published.'),
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
stopSignal: () => (shouldStop ? { reason: 'planned-tasks-scheduled' } : undefined),
},
control: { mode: 'manual' },
});
const publishedEvents = eventBus.publish.mock.calls.map(([, event]) => event as PublishedEvent);
expect(result.status).toBe('completed');
expect(result.stopReason).toBe('planned-tasks-scheduled');
expect(publishedEvents.some((event) => event.type === 'tool-call')).toBe(true);
expect(publishedEvents.some((event) => event.type === 'tool-result')).toBe(true);
expect(
publishedEvents.some(
(event) => event.payload?.text === 'This inline continuation should not be published.',
),
).toBe(false);
});
it('auto-resumes suspended streams and passes drained corrections to resume data', async () => {
const eventBus = createEventBus();
const resume = vi.fn().mockResolvedValue({
runId: 'agent-run-2',
stream: readableFromChunks([textChunk('Done.')]),
});
const waitForConfirmation = vi.fn().mockResolvedValue({ approved: true });
let hasDrainedCorrection = false;
const result = await executeResumableStream({
agent: { resume },
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
suspensionChunk({
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: {
requestId: 'request-1',
message: 'Please confirm',
},
}),
]),
text: Promise.resolve('Initial text'),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: {
mode: 'auto',
waitForConfirmation,
drainCorrections: () => {
if (hasDrainedCorrection) {
return [];
}
hasDrainedCorrection = true;
return ['Prefer Slack instead of email'];
},
},
});
expect(waitForConfirmation).toHaveBeenCalledWith('request-1');
expect(resume).toHaveBeenCalledWith(
'stream',
expect.objectContaining({
__correctionOverride: true,
corrections: ['Prefer Slack instead of email'],
}),
{ runId: 'agent-run-1', toolCallId: 'tool-call-1' },
);
expect(result.status).toBe('completed');
expect(result.agentRunId).toBe('agent-run-2');
await expect(result.text ?? Promise.resolve('')).resolves.toBe('Done.');
expect(eventBus.publish).toHaveBeenCalledWith(
'thread-1',
expect.objectContaining({
type: 'text-delta',
// Correction lines are their own segments: each carries a unique
// synthetic responseId so the reducer never merges them.
responseId: expect.stringContaining(':correction:') as string,
payload: { text: '\n[USER CORRECTION]: Prefer Slack instead of email\n' },
}),
);
});
it('passes resume options from the control hook', async () => {
const resume = vi.fn().mockResolvedValue({
runId: 'agent-run-2',
stream: readableFromChunks([textChunk('Done.')]),
});
await executeResumableStream({
agent: { resume },
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
suspensionChunk({
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: { requestId: 'request-1' },
}),
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
signal: new AbortController().signal,
logger: createLogger(),
},
control: {
mode: 'auto',
waitForConfirmation: async () => await Promise.resolve({ approved: true }),
buildResumeOptions: ({ agentRunId, suspension }) => ({
runId: agentRunId,
toolCallId: suspension.toolCallId,
maxIterations: 5,
}),
},
});
expect(resume).toHaveBeenCalledWith(
'stream',
{ approved: true },
{ runId: 'agent-run-1', toolCallId: 'tool-call-1', maxIterations: 5 },
);
});
it('registers auto confirmations before the stream finishes draining', async () => {
const eventBus = createEventBus();
const finishGate = createDeferred<undefined>();
const approval = createDeferred<Record<string, unknown>>();
const waitStarted = createDeferred<undefined>();
const resume = vi.fn().mockResolvedValue({
runId: 'agent-run-2',
stream: readableFromChunks([textChunk('Done.')]),
});
const waitForConfirmation = vi.fn().mockImplementation(async () => {
waitStarted.resolve(undefined);
return await approval.promise;
});
const execution = executeResumableStream({
agent: { resume },
stream: {
runId: 'agent-run-1',
fullStream: (async function* () {
yield suspensionChunk({
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: {
requestId: 'request-1',
message: 'Please confirm',
},
});
await finishGate.promise;
yield { type: 'finish', finishReason: 'tool-calls' };
})(),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: {
mode: 'auto',
waitForConfirmation,
},
});
await waitStarted.promise;
expect(waitForConfirmation).toHaveBeenCalledWith('request-1');
expect(resume).not.toHaveBeenCalled();
const publishCalls = eventBus.publish.mock.calls as Array<[string, PublishedEvent]>;
const confirmationEvent = publishCalls.find(
([, event]) => event.type === 'confirmation-request',
);
expect(confirmationEvent?.[0]).toBe('thread-1');
expect(confirmationEvent?.[1].payload?.requestId).toBe('request-1');
approval.resolve({ approved: true });
finishGate.resolve(undefined);
await expect(execution).resolves.toEqual(
expect.objectContaining({
status: 'completed',
agentRunId: 'agent-run-2',
}),
);
});
it('surfaces only the first actionable suspension in a drain', async () => {
const eventBus = createEventBus();
const resume = vi.fn().mockResolvedValue({
runId: 'agent-run-2',
stream: readableFromChunks([textChunk('Done.')]),
});
const waitForConfirmation = vi.fn().mockResolvedValue({ approved: true });
const onSuspension = vi.fn((_: SuspensionInfo) => undefined);
await executeResumableStream({
agent: { resume },
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([
suspensionChunk({
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: {
requestId: 'request-1',
message: 'First confirmation',
},
}),
suspensionChunk({
toolCallId: 'tool-call-2',
toolName: 'pause-for-user',
suspendPayload: {
requestId: 'request-2',
message: 'Second confirmation',
},
}),
{ type: 'finish', finishReason: 'tool-calls' },
]),
},
context: {
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
eventBus,
signal: new AbortController().signal,
logger: createLogger(),
},
control: {
mode: 'auto',
waitForConfirmation,
onSuspension,
},
});
expect(onSuspension).toHaveBeenCalledTimes(1);
expect(onSuspension).toHaveBeenCalledWith({
requestId: 'request-1',
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: { requestId: 'request-1', message: 'First confirmation' },
});
expect(waitForConfirmation).toHaveBeenCalledTimes(1);
expect(waitForConfirmation).toHaveBeenCalledWith('request-1');
const confirmationEvents = (eventBus.publish.mock.calls as Array<[string, PublishedEvent]>)
.map(([, event]) => event)
.filter((event) => event.type === 'confirmation-request');
expect(confirmationEvents).toHaveLength(1);
expect(confirmationEvents[0].payload?.requestId).toBe('request-1');
});
});

1454
packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts
View File

File diff suppressed because it is too large Load Diff

282
packages/@n8n/instance-ai/src/runtime/__tests__/stream-runner.test.ts
View File

@ -1,282 +0,0 @@
import type { WorkSummary } from '../../stream/work-summary-accumulator';
import type * as ResumableStreamExecutor from '../resumable-stream-executor';
import { executeResumableStream } from '../resumable-stream-executor';
import { streamAgentRun } from '../stream-runner';
vi.mock('../resumable-stream-executor', async () => {
const actual = await vi.importActual<typeof ResumableStreamExecutor>(
'../resumable-stream-executor',
);
return {
...actual,
executeResumableStream: vi.fn(),
};
});
const emptyWorkSummary: WorkSummary = {
toolCalls: [],
totalToolCalls: 0,
totalToolErrors: 0,
askedClarifyingQuestion: false,
};
function createLogger() {
return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };
}
function createEventBus() {
return {
publish: vi.fn(),
subscribe: vi.fn(),
getEventsAfter: vi.fn(),
getEventsForRun: vi.fn().mockReturnValue([]),
getEventsForRuns: vi.fn().mockReturnValue([]),
};
}
async function* fromChunks(chunks: unknown[]) {
for (const chunk of chunks) {
await Promise.resolve();
yield chunk;
}
}
async function* emptyStream() {
await Promise.resolve();
yield* [];
}
async function collectAsyncIterable(stream: AsyncIterable<unknown>) {
const chunks: unknown[] = [];
for await (const chunk of stream) {
chunks.push(chunk);
}
return chunks;
}
describe('streamAgentRun', () => {
it('returns errored status when agent stream contains an error chunk', async () => {
vi.mocked(executeResumableStream).mockResolvedValue({
status: 'errored',
agentRunId: 'agent-run-1',
workSummary: emptyWorkSummary,
});
const eventBus = createEventBus();
const agent = {
stream: vi.fn().mockResolvedValue({
runId: 'agent-run-1',
fullStream: fromChunks([
{ type: 'text-delta', delta: 'Hello' },
{ type: 'error', error: new Error('Not Found') },
]),
}),
};
const result = await streamAgentRun(
agent,
'test input',
{},
{
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
signal: new AbortController().signal,
eventBus,
logger: createLogger(),
},
);
expect(result.status).toBe('errored');
expect(result.agentRunId).toBe('agent-run-1');
expect(result.workSummary).toBe(emptyWorkSummary);
});
it('returns completed status for successful streams', async () => {
vi.mocked(executeResumableStream).mockResolvedValue({
status: 'completed',
agentRunId: 'agent-run-1',
workSummary: emptyWorkSummary,
});
const eventBus = createEventBus();
const agent = {
stream: vi.fn().mockResolvedValue({
runId: 'agent-run-1',
fullStream: fromChunks([{ type: 'text-delta', delta: 'All good' }]),
}),
};
const result = await streamAgentRun(
agent,
'test input',
{},
{
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
signal: new AbortController().signal,
eventBus,
logger: createLogger(),
},
);
expect(result.status).toBe('completed');
expect(result.workSummary).toBe(emptyWorkSummary);
});
it('passes through the buffered manual confirmation event', async () => {
const mockedExecuteResumableStream = vi.mocked(executeResumableStream);
const agent = {
stream: vi.fn().mockResolvedValue({
runId: 'agent-run-1',
fullStream: emptyStream(),
}),
};
const eventBus = createEventBus();
mockedExecuteResumableStream.mockResolvedValue({
status: 'suspended',
agentRunId: 'agent-run-1',
workSummary: emptyWorkSummary,
suspension: {
requestId: 'request-1',
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
suspendPayload: {},
},
confirmationEvent: {
type: 'confirmation-request',
runId: 'run-1',
agentId: 'agent-1',
payload: {
requestId: 'request-1',
toolCallId: 'tool-call-1',
toolName: 'pause-for-user',
args: {},
severity: 'warning',
message: 'Please confirm',
},
},
});
const result = await streamAgentRun(
agent,
'hello',
{},
{
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
signal: new AbortController().signal,
eventBus,
logger: createLogger(),
},
);
expect(result.status).toBe('suspended');
expect(result.agentRunId).toBe('agent-run-1');
expect(result.workSummary).toBe(emptyWorkSummary);
expect(result.suspension?.requestId).toBe('request-1');
expect(result.confirmationEvent?.type).toBe('confirmation-request');
expect(result.confirmationEvent?.payload.requestId).toBe('request-1');
expect(mockedExecuteResumableStream).toHaveBeenCalledWith(
expect.objectContaining({
control: { mode: 'manual' },
}),
);
});
it('passes an already-normalized native stream source through to the resumable executor', async () => {
const mockedExecuteResumableStream = vi.mocked(executeResumableStream);
const streamResult = {
runId: 'agent-run-2',
fullStream: emptyStream(),
text: Promise.resolve('done'),
steps: Promise.resolve([{ text: 'done' }]),
response: Promise.resolve({ id: 'response-1' }),
usage: Promise.resolve({ inputTokens: 1, outputTokens: 2, totalTokens: 3 }),
};
const agent = {
stream: vi.fn().mockResolvedValue(streamResult),
};
const eventBus = createEventBus();
mockedExecuteResumableStream.mockResolvedValue({
status: 'completed',
agentRunId: 'agent-run-2',
text: Promise.resolve('done'),
workSummary: emptyWorkSummary,
});
await streamAgentRun(
agent,
'hello',
{},
{
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
signal: new AbortController().signal,
eventBus,
logger: createLogger(),
},
);
expect(mockedExecuteResumableStream).toHaveBeenCalledWith(
expect.objectContaining({
stream: streamResult,
}),
);
});
it('normalizes native agent readable streams for the resumable executor', async () => {
const mockedExecuteResumableStream = vi.mocked(executeResumableStream);
mockedExecuteResumableStream.mockClear();
const nativeChunk = { type: 'text-delta', delta: 'All good' };
const readable = new ReadableStream<unknown>({
start(controller) {
controller.enqueue(nativeChunk);
controller.close();
},
});
const agent = {
stream: vi.fn().mockResolvedValue({
runId: 'agent-run-1',
stream: readable,
getState: vi.fn(),
}),
};
const eventBus = createEventBus();
mockedExecuteResumableStream.mockResolvedValue({
status: 'completed',
agentRunId: 'agent-run-1',
workSummary: emptyWorkSummary,
});
await streamAgentRun(
agent,
'hello',
{},
{
threadId: 'thread-1',
runId: 'run-1',
agentId: 'agent-1',
signal: new AbortController().signal,
eventBus,
logger: createLogger(),
},
);
const call = mockedExecuteResumableStream.mock.calls[0];
expect(call).toBeDefined();
const source = call?.[0].stream;
expect(source).toEqual(
expect.objectContaining({
runId: 'agent-run-1',
}),
);
await expect(collectAsyncIterable(source.fullStream)).resolves.toEqual([nativeChunk]);
await expect(source.text).resolves.toBe('All good');
});
});

468
packages/@n8n/instance-ai/src/runtime/background-task-manager.ts
View File

@ -1,468 +0,0 @@
import type { BackgroundTaskResult, InstanceAiTraceContext } from '../types';
import type {
InstanceAiLivenessDecision,
InstanceAiLivenessPolicy,
InstanceAiLivenessTimeoutReason,
} from './liveness-policy';
export type BackgroundTaskStatus = 'running' | 'completed' | 'failed' | 'cancelled';
export interface ManagedBackgroundTask {
taskId: string;
threadId: string;
runId: string;
role: string;
agentId: string;
status: BackgroundTaskStatus;
result?: string;
error?: string;
startedAt: number;
lastActivityAt: number;
timeoutReason?: InstanceAiLivenessTimeoutReason;
abortController: AbortController;
corrections: string[];
/** Callback resolved when a new correction is queued. Re-created by the consumer after each notification. */
onCorrectionQueued?: () => void;
messageGroupId?: string;
outcome?: Record<string, unknown>;
plannedTaskId?: string;
workItemId?: string;
traceContext?: InstanceAiTraceContext;
createTraceContext?: () => Promise<InstanceAiTraceContext | undefined>;
/** Identity used for single-flight dedupe lookups; copied from the spawn options. */
dedupeKey?: BackgroundTaskDedupeKey;
/**
* The checkpoint task id this background task was spawned under, when the
* orchestrator called a detached sub-agent tool inside a
* `<planned-task-follow-up type="checkpoint">` turn. The checkpoint safety
* net uses this to tell "orchestrator exited silently" apart from
* "orchestrator handed off to an in-flight patch builder".
*/
parentCheckpointId?: string;
}
export interface BackgroundTaskDedupeKey {
/** Planned-task graph ID this background task is dispatched for. Primary dedupe key. */
plannedTaskId?: string;
/** Target workflow ID for this background task. Fallback dedupe key when there is no planned task. */
workflowId?: string;
/** Agent role (e.g. 'workflow-builder'). Scopes the workflowId fallback so different roles against the same workflow don't collide. */
role: string;
}
export interface SpawnManagedBackgroundTaskOptions {
taskId: string;
threadId: string;
runId: string;
role: string;
agentId: string;
messageGroupId?: string;
plannedTaskId?: string;
workItemId?: string;
traceContext?: InstanceAiTraceContext;
createTraceContext?: () => Promise<InstanceAiTraceContext | undefined>;
/**
* Identity for single-flight dedupe. When supplied, a spawn with the same `plannedTaskId`
* (primary) or `role + workflowId` (fallback) as a currently-running task returns
* `{ status: 'duplicate', existing }` instead of launching a second task.
*/
dedupeKey?: BackgroundTaskDedupeKey;
/**
* Link this background task to a running checkpoint in the planned-task
* graph. Set when the orchestrator spawns a detached sub-agent (builder,
* delegate) from inside a
* `<planned-task-follow-up type="checkpoint">` turn. The post-run safety
* net defers failing the checkpoint while any child with this id is still
* running, and the settlement path re-emits the checkpoint follow-up when
* the last child settles.
*/
parentCheckpointId?: string;
run: (
signal: AbortSignal,
drainCorrections: () => string[],
waitForCorrection: () => Promise<void>,
taskContext: { traceContext?: InstanceAiTraceContext },
) => Promise<string | BackgroundTaskResult>;
onLimitReached?: (errorMessage: string) => void;
onCompleted?: (task: ManagedBackgroundTask) => void | Promise<void>;
onFailed?: (task: ManagedBackgroundTask) => void | Promise<void>;
onSettled?: (task: ManagedBackgroundTask) => void | Promise<void>;
}
export type SpawnManagedBackgroundTaskResult =
| { status: 'started'; task: ManagedBackgroundTask }
| { status: 'limit-reached' }
| { status: 'duplicate'; existing: ManagedBackgroundTask };
export interface BackgroundTaskTimeoutOptions {
shouldSkipTask?: (task: ManagedBackgroundTask) => boolean;
}
export interface BackgroundTaskMessageOptions<
TTask extends ManagedBackgroundTask = ManagedBackgroundTask,
> {
formatTask?: (task: TTask) => Promise<string> | string;
}
export class BackgroundTaskManager {
private readonly tasks = new Map<string, ManagedBackgroundTask>();
private readonly callbacks = new Map<
string,
Pick<SpawnManagedBackgroundTaskOptions, 'onCompleted' | 'onFailed' | 'onSettled'>
>();
/** plannedTaskId → taskId for the currently-running task. Populated only when the caller provides a dedupeKey with plannedTaskId. */
private readonly byPlannedTaskId = new Map<string, string>();
/**
* `${role}:${workflowId}` → taskId for the currently-running task. Only
* populated (and only consulted) when the caller provides a dedupeKey
* WITHOUT a plannedTaskId. When both keys are present we treat
* plannedTaskId as the canonical identity — two distinct planned tasks may
* legitimately target the same workflow (e.g., build + later patch) and
* must not collapse into each other.
*/
private readonly byRoleAndWorkflowId = new Map<string, string>();
constructor(
private readonly maxConcurrentPerThread = 5,
/**
* Ceiling on sub-agents running across every thread on this process. Guards the
* fan-out the per-thread limit misses: a handful of runs each spawning their full
* complement. `-1` means unlimited, matching the execution concurrency limits.
*/
private readonly maxConcurrentTotal = -1,
) {}
private workflowKey(role: string, workflowId: string): string {
return `${role}:${workflowId}`;
}
private findDuplicate(
dedupeKey: BackgroundTaskDedupeKey | undefined,
): ManagedBackgroundTask | undefined {
if (!dedupeKey) return undefined;
if (dedupeKey.plannedTaskId) {
// plannedTaskId is the canonical identity when present — we must NOT
// fall back to the workflowId index, otherwise distinct planned tasks
// targeting the same (role, workflowId) would falsely collapse.
const existingId = this.byPlannedTaskId.get(dedupeKey.plannedTaskId);
if (existingId) {
const existing = this.tasks.get(existingId);
if (existing?.status === 'running') return existing;
}
return undefined;
}
if (dedupeKey.workflowId) {
const existingId = this.byRoleAndWorkflowId.get(
this.workflowKey(dedupeKey.role, dedupeKey.workflowId),
);
if (existingId) {
const existing = this.tasks.get(existingId);
if (existing?.status === 'running') return existing;
}
}
return undefined;
}
getTaskSnapshots(threadId: string): ManagedBackgroundTask[] {
return [...this.tasks.values()].filter((task) => task.threadId === threadId);
}
getRunningTasks(threadId: string): ManagedBackgroundTask[] {
return [...this.tasks.values()].filter(
(task) => task.threadId === threadId && task.status === 'running',
);
}
/** Sub-agents running across every thread on this process. */
runningTaskCount(): number {
let count = 0;
for (const task of this.tasks.values()) {
if (task.status === 'running') count++;
}
return count;
}
/**
* Return all running background tasks on this thread that were spawned
* under the given checkpoint task id. Used by the checkpoint safety net to
* defer failing a checkpoint while a detached patch sub-agent it just
* launched is still in-flight.
*/
getRunningTasksByParentCheckpoint(
threadId: string,
checkpointTaskId: string,
): ManagedBackgroundTask[] {
return [...this.tasks.values()].filter(
(task) =>
task.threadId === threadId &&
task.status === 'running' &&
task.parentCheckpointId === checkpointTaskId,
);
}
queueCorrection(
threadId: string,
taskId: string,
correction: string,
): 'queued' | 'task-completed' | 'task-not-found' {
const task = this.tasks.get(taskId);
if (task?.threadId !== threadId) return 'task-not-found';
if (task.status !== 'running') return 'task-completed';
this.touchTask(threadId, taskId);
task.corrections.push(correction);
if (task.onCorrectionQueued) {
const notify = task.onCorrectionQueued;
task.onCorrectionQueued = undefined;
notify();
}
return 'queued';
}
cancelTask(threadId: string, taskId: string): ManagedBackgroundTask | undefined {
const task = this.tasks.get(taskId);
if (task?.threadId !== threadId || task.status !== 'running') return undefined;
task.abortController.abort();
task.status = 'cancelled';
this.tasks.delete(taskId);
this.releaseDedupeIndices(task);
this.callbacks.delete(taskId);
return task;
}
cancelThread(threadId: string): ManagedBackgroundTask[] {
const cancelled: ManagedBackgroundTask[] = [];
for (const [taskId, task] of this.tasks) {
if (task.threadId !== threadId || task.status !== 'running') continue;
task.abortController.abort();
task.status = 'cancelled';
cancelled.push(task);
this.tasks.delete(taskId);
this.releaseDedupeIndices(task);
this.callbacks.delete(taskId);
}
return cancelled;
}
cancelAll(): ManagedBackgroundTask[] {
const cancelled: ManagedBackgroundTask[] = [];
for (const [taskId, task] of this.tasks) {
task.abortController.abort();
task.status = 'cancelled';
cancelled.push(task);
this.tasks.delete(taskId);
this.releaseDedupeIndices(task);
this.callbacks.delete(taskId);
}
return cancelled;
}
touchTask(threadId: string, taskId: string, at = Date.now()): boolean {
const task = this.tasks.get(taskId);
if (task?.threadId !== threadId || task.status !== 'running') return false;
task.lastActivityAt = at;
return true;
}
async timeoutTimedOutTasks(
policy: InstanceAiLivenessPolicy,
now = Date.now(),
options: BackgroundTaskTimeoutOptions = {},
): Promise<ManagedBackgroundTask[]> {
const timedOut: ManagedBackgroundTask[] = [];
for (const task of [...this.tasks.values()]) {
if (task.status !== 'running') continue;
if (options.shouldSkipTask?.(task)) continue;
const decision = policy.evaluate({
surface: 'background-task',
startedAt: task.startedAt,
lastActivityAt: task.lastActivityAt,
now,
});
if (decision.action !== 'timeout') continue;
await this.timeoutTask(task, decision);
timedOut.push(task);
}
return timedOut;
}
spawn(options: SpawnManagedBackgroundTaskOptions): SpawnManagedBackgroundTaskResult {
const duplicate = this.findDuplicate(options.dedupeKey);
if (duplicate) return { status: 'duplicate', existing: duplicate };
const runningCount = this.getRunningTasks(options.threadId).length;
if (runningCount >= this.maxConcurrentPerThread) {
options.onLimitReached?.(
`Cannot start background task: limit of ${this.maxConcurrentPerThread} concurrent tasks reached. Wait for existing tasks to complete.`,
);
return { status: 'limit-reached' };
}
if (this.maxConcurrentTotal !== -1 && this.runningTaskCount() >= this.maxConcurrentTotal) {
options.onLimitReached?.(
`Cannot start background task: this n8n instance is at its limit of ${this.maxConcurrentTotal} concurrent tasks. Wait for existing tasks to complete.`,
);
return { status: 'limit-reached' };
}
const now = Date.now();
const task: ManagedBackgroundTask = {
taskId: options.taskId,
threadId: options.threadId,
runId: options.runId,
role: options.role,
agentId: options.agentId,
status: 'running',
startedAt: now,
lastActivityAt: now,
abortController: new AbortController(),
corrections: [],
messageGroupId: options.messageGroupId,
plannedTaskId: options.plannedTaskId,
workItemId: options.workItemId,
traceContext: options.traceContext,
dedupeKey: options.dedupeKey,
parentCheckpointId: options.parentCheckpointId,
};
this.tasks.set(options.taskId, task);
this.callbacks.set(options.taskId, {
onCompleted: options.onCompleted,
onFailed: options.onFailed,
onSettled: options.onSettled,
});
if (options.dedupeKey?.plannedTaskId) {
this.byPlannedTaskId.set(options.dedupeKey.plannedTaskId, options.taskId);
} else if (options.dedupeKey?.workflowId) {
// Only index by (role, workflowId) when there is no plannedTaskId.
// Otherwise a later spawn for a different planned task targeting the
// same workflow would be wrongly matched against this one.
this.byRoleAndWorkflowId.set(
this.workflowKey(options.dedupeKey.role, options.dedupeKey.workflowId),
options.taskId,
);
}
void this.executeTask(task, options);
return { status: 'started', task };
}
private releaseDedupeIndices(task: ManagedBackgroundTask): void {
const key = task.dedupeKey;
if (!key) return;
if (key.plannedTaskId) {
if (this.byPlannedTaskId.get(key.plannedTaskId) === task.taskId) {
this.byPlannedTaskId.delete(key.plannedTaskId);
}
return;
}
if (key.workflowId) {
const wfKey = this.workflowKey(key.role, key.workflowId);
if (this.byRoleAndWorkflowId.get(wfKey) === task.taskId) {
this.byRoleAndWorkflowId.delete(wfKey);
}
}
}
private async executeTask(
task: ManagedBackgroundTask,
options: SpawnManagedBackgroundTaskOptions,
): Promise<void> {
const drainCorrections = (): string[] => {
const pending = [...task.corrections];
task.corrections.length = 0;
return pending;
};
const waitForCorrection = async (): Promise<void> =>
await new Promise<void>((resolve) => {
if (task.corrections.length > 0) {
resolve();
return;
}
task.onCorrectionQueued = resolve;
});
try {
if (!task.traceContext && options.createTraceContext) {
task.traceContext = await options.createTraceContext();
}
const raw = await options.run(
task.abortController.signal,
drainCorrections,
waitForCorrection,
{ traceContext: task.traceContext },
);
if (task.status !== 'running') return;
task.status = 'completed';
task.result = typeof raw === 'string' ? raw : raw.text;
task.outcome = typeof raw === 'string' ? undefined : raw.outcome;
await options.onCompleted?.(task);
} catch (error) {
if (task.status !== 'running') return;
if (task.abortController.signal.aborted) return;
task.status = 'failed';
task.error = error instanceof Error ? error.message : String(error);
await options.onFailed?.(task);
} finally {
try {
if (!task.abortController.signal.aborted && task.status !== 'running') {
await options.onSettled?.(task);
}
} finally {
if (task.status !== 'running') {
this.tasks.delete(task.taskId);
this.releaseDedupeIndices(task);
this.callbacks.delete(task.taskId);
}
}
}
}
private async timeoutTask(
task: ManagedBackgroundTask,
decision: Extract<InstanceAiLivenessDecision, { action: 'timeout' }>,
): Promise<void> {
if (task.status !== 'running') return;
task.abortController.abort();
task.status = 'failed';
task.timeoutReason = decision.reason;
task.error = `Background ${task.role} task timed out after ${decision.timeoutMs}ms`;
const callbacks = this.callbacks.get(task.taskId);
try {
await callbacks?.onFailed?.(task);
} finally {
try {
await callbacks?.onSettled?.(task);
} finally {
this.tasks.delete(task.taskId);
this.releaseDedupeIndices(task);
this.callbacks.delete(task.taskId);
}
}
}
}
export async function enrichMessageWithRunningTasks<
TTask extends ManagedBackgroundTask = ManagedBackgroundTask,
>(
message: string,
runningTasks: TTask[],
options: BackgroundTaskMessageOptions<TTask> = {},
): Promise<string> {
if (runningTasks.length === 0) return message;
const parts: string[] = [];
for (const task of runningTasks) {
if (options.formatTask) {
parts.push(await options.formatTask(task));
continue;
}
parts.push(`[Running task — ${task.role}, task: ${task.taskId}]`);
}
return `<running-tasks>\n${parts.join('\n')}\n</running-tasks>\n\n${message}`;
}

129
packages/@n8n/instance-ai/src/runtime/liveness-policy.ts
View File

@ -1,129 +0,0 @@
export type InstanceAiLivenessSurface =
| 'active-run'
| 'suspended-run'
| 'pending-confirmation'
| 'background-task';
export type InstanceAiLivenessTimeoutReason = 'idle_timeout' | 'max_lifetime';
export interface InstanceAiLivenessPolicyConfig {
confirmationTimeoutMs: number;
backgroundTaskIdleTimeoutMs: number;
backgroundTaskMaxLifetimeMs: number;
activeRunIdleTimeoutMs: number;
activeRunMaxLifetimeMs: number;
}
const MINUTE_MS = 60_000;
const DAY_MS = 24 * 60 * MINUTE_MS;
export const INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG = {
confirmationTimeoutMs: DAY_MS,
backgroundTaskIdleTimeoutMs: 10 * MINUTE_MS,
backgroundTaskMaxLifetimeMs: 30 * MINUTE_MS,
// Left disabled on purpose. An orchestrator awaiting a long sub-agent can be
// legitimately quiet for a while, and whether `touchRun` fires across every such
// stretch is unverified -- enabling this would risk reaping live runs.
activeRunIdleTimeoutMs: 0,
// Matches `confirmationTimeoutMs` on purpose, and must not simply be lowered.
// `startedAt` survives suspend/resume (see `suspendRun` / `activateSuspendedRun`), so
// this bounds wall-clock since the run first began -- including time parked waiting on
// a human -- not execution time. Lowering it to, say, 2h would reap a run whose user
// approved a confirmation card three hours later, right as it resumed.
//
// Once `N8N_INSTANCE_AI_MAX_CONCURRENT_RUNS` is enabled a day is too long to hold a
// slot, because a wedged run then costs a share of instance capacity rather than just
// its own thread. The fix at that point is to measure from the last activation rather
// than the original start, not to shrink this number.
activeRunMaxLifetimeMs: DAY_MS,
} satisfies InstanceAiLivenessPolicyConfig;
export function createInstanceAiLivenessPolicyConfig(
overrides: Partial<Pick<InstanceAiLivenessPolicyConfig, 'confirmationTimeoutMs'>> = {},
): InstanceAiLivenessPolicyConfig {
return {
...INSTANCE_AI_DEFAULT_LIVENESS_POLICY_CONFIG,
...overrides,
};
}
export interface InstanceAiLivenessInput {
surface: InstanceAiLivenessSurface;
startedAt: number;
lastActivityAt: number;
now?: number;
}
export type InstanceAiLivenessDecision =
| { action: 'keep-alive' }
| {
action: 'timeout';
reason: InstanceAiLivenessTimeoutReason;
surface: InstanceAiLivenessSurface;
timeoutMs: number;
elapsedMs: number;
idleMs: number;
};
export class InstanceAiLivenessPolicy {
constructor(private readonly config: InstanceAiLivenessPolicyConfig) {}
hasEnabledTimeouts(): boolean {
return Object.values(this.config).some((value) => value > 0);
}
evaluate(input: InstanceAiLivenessInput): InstanceAiLivenessDecision {
const now = input.now ?? Date.now();
const elapsedMs = Math.max(0, now - input.startedAt);
const idleMs = Math.max(0, now - input.lastActivityAt);
const limits = this.getLimits(input.surface);
if (limits.idleTimeoutMs > 0 && idleMs >= limits.idleTimeoutMs) {
return {
action: 'timeout',
reason: 'idle_timeout',
surface: input.surface,
timeoutMs: limits.idleTimeoutMs,
elapsedMs,
idleMs,
};
}
if (limits.maxLifetimeMs > 0 && elapsedMs >= limits.maxLifetimeMs) {
return {
action: 'timeout',
reason: 'max_lifetime',
surface: input.surface,
timeoutMs: limits.maxLifetimeMs,
elapsedMs,
idleMs,
};
}
return { action: 'keep-alive' };
}
private getLimits(surface: InstanceAiLivenessSurface): {
idleTimeoutMs: number;
maxLifetimeMs: number;
} {
if (surface === 'background-task') {
return {
idleTimeoutMs: this.config.backgroundTaskIdleTimeoutMs,
maxLifetimeMs: this.config.backgroundTaskMaxLifetimeMs,
};
}
if (surface === 'active-run') {
return {
idleTimeoutMs: this.config.activeRunIdleTimeoutMs,
maxLifetimeMs: this.config.activeRunMaxLifetimeMs,
};
}
return {
idleTimeoutMs: this.config.confirmationTimeoutMs,
maxLifetimeMs: 0,
};
}
}

689
packages/@n8n/instance-ai/src/runtime/resumable-stream-executor.ts
View File

@ -1,689 +0,0 @@
import { isFinishReason } from '@n8n/agents';
import type { FinishReason, StreamResult } from '@n8n/agents';
import type { InstanceAiEvent } from '@n8n/api-types';
import { isRecord } from '@n8n/utils/is-record';
import { randomUUID } from 'node:crypto';
import type { InstanceAiEventBus } from '../event-bus';
import type { Logger } from '../logger';
import type {
OrchestratorRunHandoffReason,
OrchestratorRunStopSignal,
} from './orchestrator-run-control';
import { isQuotaExhaustedError, mapAgentChunkToEvent } from '../stream/map-chunk';
import { UsageAccumulator, type RunTokenUsage } from '../stream/usage-accumulator';
import { WorkSummaryAccumulator, type WorkSummary } from '../stream/work-summary-accumulator';
import { parseSuspension, resumeAgentStream } from '../utils/stream-helpers';
import type { SuspensionInfo } from '../utils/stream-helpers';
type ConfirmationRequestEvent = Extract<InstanceAiEvent, { type: 'confirmation-request' }>;
export interface ResumableStreamSource {
runId?: string;
fullStream: AsyncIterable<unknown>;
text?: Promise<string>;
steps?: Promise<unknown[]>;
response?: Promise<unknown>;
finishReason?: Promise<string | undefined>;
request?: Promise<unknown>;
usage?: Promise<unknown>;
totalUsage?: Promise<unknown>;
}
export interface ResumableStreamContext {
threadId: string;
runId: string;
agentId: string;
eventBus: InstanceAiEventBus;
signal: AbortSignal;
logger: Logger;
onActivity?: () => void;
/** Stop consuming after the current chunk has been mapped and published. */
stopSignal?: () => OrchestratorRunStopSignal | undefined;
}
export interface ManualSuspensionControl {
mode: 'manual';
}
export interface AutoResumeControl {
mode: 'auto';
waitForConfirmation: (requestId: string) => Promise<Record<string, unknown>>;
drainCorrections?: () => string[];
/** Returns a promise that resolves when a new user correction is queued. Used to unblock
* HITL suspensions so the builder can incorporate the correction instead of waiting forever. */
waitForCorrection?: () => Promise<void>;
onSuspension?: (suspension: SuspensionInfo) => void;
buildResumeOptions?: (input: {
agentRunId: string;
suspension: SuspensionInfo;
}) => Record<string, unknown>;
}
export type ResumableStreamControl = ManualSuspensionControl | AutoResumeControl;
export interface ExecuteResumableStreamOptions {
agent: unknown;
stream: ResumableStreamSource;
context: ResumableStreamContext;
control: ResumableStreamControl;
initialAgentRunId?: string;
}
export type TraceStatus = 'completed' | 'cancelled' | 'suspended' | 'errored';
export interface ExecuteResumableStreamResult {
status: TraceStatus;
agentRunId: string;
text?: Promise<string>;
error?: unknown;
suspension?: SuspensionInfo;
confirmationEvent?: ConfirmationRequestEvent;
/** Accumulated tool call outcomes observed during stream consumption. */
workSummary: WorkSummary;
/** Accumulated token usage and cost, when the stream emitted usage. */
usage?: RunTokenUsage;
/** Reason this stream stopped early after publishing the current chunk. */
stopReason?: OrchestratorRunHandoffReason;
/** Terminal `finish` chunk's reason; `'max-iterations'` means the agent ran out of steps. */
finishReason?: FinishReason;
}
function isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {
return (
value !== null &&
typeof value === 'object' &&
typeof Reflect.get(value, Symbol.asyncIterator) === 'function'
);
}
function isReadableStream(value: unknown): value is ReadableStream<unknown> {
return (
value !== null &&
typeof value === 'object' &&
typeof Reflect.get(value, 'getReader') === 'function'
);
}
function isResumableStreamSource(value: unknown): value is ResumableStreamSource {
return isRecord(value) && isAsyncIterable(value.fullStream);
}
function isNativeStreamResult(value: unknown): value is StreamResult {
return isRecord(value) && isReadableStream(value.stream);
}
function extractContentText(content: unknown): string {
if (typeof content === 'string') {
return content;
}
if (!Array.isArray(content)) {
return '';
}
return content
.map((part) => {
if (!isRecord(part) || part.type !== 'text') {
return '';
}
return typeof part.text === 'string' ? part.text : '';
})
.join('');
}
async function collectNativeStreamText(stream: ReadableStream<unknown>): Promise<string> {
const reader = stream.getReader();
let deltaText = '';
let messageText = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done) return deltaText || messageText;
if (!isRecord(value)) {
continue;
}
if (value.type === 'text-delta' && typeof value.delta === 'string') {
deltaText += value.delta;
continue;
}
if (value.type !== 'message' || !isRecord(value.message)) {
continue;
}
if (value.message.role === 'assistant') {
messageText += extractContentText(value.message.content);
}
}
} finally {
reader.releaseLock();
}
}
async function* readableStreamToAsyncIterable(stream: ReadableStream<unknown>) {
const reader = stream.getReader();
try {
while (true) {
const { done, value } = await reader.read();
if (done) return;
yield value;
}
} finally {
reader.releaseLock();
}
}
export function normalizeStreamSource(result: unknown): ResumableStreamSource {
if (isResumableStreamSource(result)) {
return result;
}
if (isNativeStreamResult(result)) {
const [eventStream, textStream] = result.stream.tee();
return {
runId: result.runId,
fullStream: readableStreamToAsyncIterable(eventStream),
text: collectNativeStreamText(textStream),
};
}
throw new Error('Unsupported agent stream result');
}
function buildCancelledResult(
agentRunId: string,
text: Promise<string> | undefined,
workSummaryAccumulator: WorkSummaryAccumulator,
usageAccumulator: UsageAccumulator,
): ExecuteResumableStreamResult {
return {
status: 'cancelled',
agentRunId,
text,
workSummary: workSummaryAccumulator.toSummary(),
usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined,
};
}
/**
* Record a parsed suspension. The first suspension is kept (and, in auto mode,
* its confirmation awaited); any further distinct suspension before resume is
* logged and deferred.
*/
function recordSuspension(
parsedSuspension: SuspensionInfo,
current: SuspensionInfo | undefined,
control: ResumableStreamControl,
context: ResumableStreamContext,
): { suspension: SuspensionInfo; pendingConfirmation?: Promise<Record<string, unknown>> } {
if (current) {
if (!isSameSuspension(parsedSuspension, current)) {
context.logger.warn('Additional HITL suspension encountered before resume; deferring', {
threadId: context.threadId,
runId: context.runId,
activeRequestId: current.requestId,
deferredRequestId: parsedSuspension.requestId,
activeToolCallId: current.toolCallId,
deferredToolCallId: parsedSuspension.toolCallId,
});
}
return { suspension: current };
}
if (control.mode === 'auto') {
control.onSuspension?.(parsedSuspension);
return {
suspension: parsedSuspension,
pendingConfirmation: control.waitForConfirmation(parsedSuspension.requestId),
};
}
return { suspension: parsedSuspension };
}
/**
* Publish events, holding back the primary confirmation-request event in
* manual mode and de-duplicating it. Returns the updated confirmation-tracking state.
*/
function publishEvents(
events: InstanceAiEvent[],
args: {
suspension: SuspensionInfo | undefined;
confirmationEvent: ConfirmationRequestEvent | undefined;
confirmationEventPublished: boolean;
control: ResumableStreamControl;
context: ResumableStreamContext;
workSummaryAccumulator: WorkSummaryAccumulator;
},
): { confirmationEvent?: ConfirmationRequestEvent; confirmationEventPublished: boolean } {
const { suspension, control, context, workSummaryAccumulator } = args;
let confirmationEvent = args.confirmationEvent;
let confirmationEventPublished = args.confirmationEventPublished;
for (const event of events) {
workSummaryAccumulator.observe(event);
let shouldPublishEvent = true;
if (event.type === 'confirmation-request') {
const isPrimarySuspension =
event.payload.requestId === suspension?.requestId &&
event.payload.toolCallId === suspension?.toolCallId;
if (!isPrimarySuspension || confirmationEventPublished || confirmationEvent) {
shouldPublishEvent = false;
}
if (shouldPublishEvent && control.mode === 'manual') {
confirmationEvent = event;
shouldPublishEvent = false;
}
if (shouldPublishEvent) {
confirmationEventPublished = true;
}
}
if (shouldPublishEvent) {
context.eventBus.publish(context.threadId, event);
}
}
return { confirmationEvent, confirmationEventPublished };
}
interface StreamPassResult {
cancelled: boolean;
finishReason?: FinishReason;
stopReason?: OrchestratorRunHandoffReason;
suspension?: SuspensionInfo;
hasError: boolean;
error?: unknown;
pendingConfirmation?: Promise<Record<string, unknown>>;
confirmationEvent?: ConfirmationRequestEvent;
drainedCorrectionsForResume: string[];
currentResponseId: string | undefined;
nativeStepIndex: number;
}
/**
* Consume one stream until it ends (or is cancelled), publishing events,
* accumulating usage/work, and capturing the first suspension. Returns the pass
* outcome plus the updated response-id / step counters for the next pass.
*/
async function consumeStreamPass(args: {
activeStream: AsyncIterable<unknown>;
activeAgentRunId: string;
options: ExecuteResumableStreamOptions;
workSummaryAccumulator: WorkSummaryAccumulator;
usageAccumulator: UsageAccumulator;
currentResponseId: string | undefined;
nativeStepIndex: number;
}): Promise<StreamPassResult> {
const { activeStream, activeAgentRunId, options, workSummaryAccumulator, usageAccumulator } =
args;
let currentResponseId = args.currentResponseId;
let nativeStepIndex = args.nativeStepIndex;
/**
* Segment id minted for deltas that arrive before any `start-step` supplied
* a provider response id (some providers never emit one). Every delta/block
* must carry a responseId: the shared run reducer keys block replace
* semantics on segment identity, and id-less adjacent segments would merge
* into one timeline entry. Sticky for the contiguous delta run, cleared at
* the next structural fact so blocks stay exactly 1:1 with segments.
*/
let syntheticSegmentId: string | undefined;
let suspension: SuspensionInfo | undefined;
let hasError = false;
let error: unknown;
// Once we've surfaced an out-of-credits error, drop any follow-on error chunks
// (e.g. the SDK's generic "no output generated") so the user sees one clear reason.
let quotaErrorPublished = false;
let pendingConfirmation: Promise<Record<string, unknown>> | undefined;
let confirmationEvent: ConfirmationRequestEvent | undefined;
let confirmationEventPublished = false;
let finishReason: FinishReason | undefined;
const drainedCorrectionsForResume: string[] = [];
for await (const chunk of activeStream) {
if (options.context.signal.aborted) {
// A stop aborts the agent stream too: it stops generating and emits a
// terminal finish chunk carrying the run's usage. Drain the rest of the
// stream (without publishing further events) so that usage is observed
// and the cancelled run still gets billed for the tokens it consumed.
usageAccumulator.observe(chunk);
try {
for await (const remaining of activeStream) {
usageAccumulator.observe(remaining);
}
} catch (drainError) {
options.context.logger.debug('Instance AI abort drain ended early', {
threadId: options.context.threadId,
runId: options.context.runId,
error: drainError instanceof Error ? drainError.message : String(drainError),
});
}
return {
cancelled: true,
hasError,
drainedCorrectionsForResume,
currentResponseId,
nativeStepIndex,
};
}
options.context.onActivity?.();
usageAccumulator.observe(chunk);
if (isRecord(chunk) && chunk.type === 'finish' && isFinishReason(chunk.finishReason)) {
finishReason = chunk.finishReason;
}
if (isRecord(chunk) && chunk.type === 'start-step') {
nativeStepIndex += 1;
const responseRunId = activeAgentRunId || options.context.runId;
currentResponseId = `${responseRunId}:step:${nativeStepIndex}`;
syntheticSegmentId = undefined;
}
const parsedSuspension = parseSuspension(chunk);
if (parsedSuspension) {
const recorded = recordSuspension(
parsedSuspension,
suspension,
options.control,
options.context,
);
suspension = recorded.suspension;
if (recorded.pendingConfirmation) pendingConfirmation = recorded.pendingConfirmation;
}
if (isErrorChunk(chunk)) {
hasError = true;
// A quota error was already surfaced this run — swallow later error
// chunks (usage was still observed above) to avoid a confusing second
// callout. Do this before overwriting `error` so `result.error` stays the
// quota error the user saw, rather than the swallowed follow-on — otherwise
// telemetry would log a failure the user was never shown.
if (quotaErrorPublished) continue;
error = chunk.error;
if (isQuotaExhaustedError(chunk.error)) quotaErrorPublished = true;
}
const isDeltaChunk =
isRecord(chunk) && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta');
if (isDeltaChunk && !currentResponseId && !syntheticSegmentId) {
syntheticSegmentId = `${activeAgentRunId || options.context.runId}:seg:${randomUUID()}`;
}
const mappedEvent = mapAgentChunkToEvent(
options.context.runId,
options.context.agentId,
chunk,
currentResponseId ?? syntheticSegmentId,
);
// A segment boundary ends the synthetic segment; the next delta mints a
// fresh id so two different segments can never share one. finish-step
// maps to no event but is still a boundary — left sticky, a provider
// that never emits start-step would leak one id across steps, and the
// reducer's id-keyed replace could then drop an earlier block on replay.
const isFinishStep = isRecord(chunk) && chunk.type === 'finish-step';
if ((mappedEvent && !isDeltaChunk) || isFinishStep) syntheticSegmentId = undefined;
const events = mappedEvent ? [mappedEvent] : [];
const published = publishEvents(events, {
suspension,
confirmationEvent,
confirmationEventPublished,
control: options.control,
context: options.context,
workSummaryAccumulator,
});
confirmationEvent = published.confirmationEvent;
confirmationEventPublished = published.confirmationEventPublished;
if (options.control.mode === 'auto' && options.control.drainCorrections) {
const corrections = options.control.drainCorrections();
publishCorrections(options.context, corrections);
drainedCorrectionsForResume.push(...corrections);
}
const stopSignal = options.context.stopSignal?.();
if (stopSignal) {
return {
cancelled: false,
stopReason: stopSignal.reason,
finishReason,
suspension,
hasError,
error,
pendingConfirmation,
confirmationEvent,
drainedCorrectionsForResume,
currentResponseId,
nativeStepIndex,
};
}
}
return {
cancelled: false,
finishReason,
suspension,
hasError,
error,
pendingConfirmation,
confirmationEvent,
drainedCorrectionsForResume,
currentResponseId,
nativeStepIndex,
};
}
export async function executeResumableStream(
options: ExecuteResumableStreamOptions,
): Promise<ExecuteResumableStreamResult> {
let activeStream = options.stream.fullStream;
let activeAgentRunId = options.stream.runId ?? options.initialAgentRunId ?? '';
let text = options.stream.text;
const workSummaryAccumulator = new WorkSummaryAccumulator();
const usageAccumulator = new UsageAccumulator();
let currentResponseId: string | undefined;
let nativeStepIndex = 0;
while (true) {
const pass = await consumeStreamPass({
activeStream,
activeAgentRunId,
options,
workSummaryAccumulator,
usageAccumulator,
currentResponseId,
nativeStepIndex,
});
currentResponseId = pass.currentResponseId;
nativeStepIndex = pass.nativeStepIndex;
if (pass.cancelled) {
return buildCancelledResult(activeAgentRunId, text, workSummaryAccumulator, usageAccumulator);
}
const { suspension, hasError, error, pendingConfirmation, confirmationEvent } = pass;
const { drainedCorrectionsForResume } = pass;
if (options.context.signal.aborted) {
return buildCancelledResult(activeAgentRunId, text, workSummaryAccumulator, usageAccumulator);
}
if (pass.stopReason) {
return {
status: hasError ? 'errored' : 'completed',
agentRunId: activeAgentRunId,
text,
...(error !== undefined ? { error } : {}),
finishReason: pass.finishReason,
workSummary: workSummaryAccumulator.toSummary(),
usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined,
stopReason: pass.stopReason,
};
}
if (!suspension) {
return {
status: hasError ? 'errored' : 'completed',
agentRunId: activeAgentRunId,
text,
...(error !== undefined ? { error } : {}),
finishReason: pass.finishReason,
workSummary: workSummaryAccumulator.toSummary(),
usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined,
};
}
if (options.control.mode === 'manual') {
return {
status: 'suspended',
agentRunId: activeAgentRunId,
text,
suspension,
workSummary: workSummaryAccumulator.toSummary(),
usage: usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined,
...(confirmationEvent ? { confirmationEvent } : {}),
};
}
const confirmationPromise =
pendingConfirmation ?? options.control.waitForConfirmation(suspension.requestId);
const resumeData = await waitForConfirmationOrCorrection(
options.context.signal,
confirmationPromise,
options.control,
options.context,
drainedCorrectionsForResume,
);
const resumeOptions = options.control.buildResumeOptions?.({
agentRunId: activeAgentRunId,
suspension,
}) ?? {
runId: activeAgentRunId,
toolCallId: suspension.toolCallId,
};
const resumed = await resumeAgentStream(options.agent, resumeData, {
...resumeOptions,
});
const resumedSource = normalizeStreamSource(resumed);
activeAgentRunId =
(typeof resumedSource.runId === 'string' ? resumedSource.runId : '') || activeAgentRunId;
activeStream = resumedSource.fullStream;
text = resumedSource.text;
}
}
function publishCorrections(context: ResumableStreamContext, corrections: string[]): void {
for (const correction of corrections) {
context.eventBus.publish(context.threadId, {
type: 'text-delta',
runId: context.runId,
agentId: context.agentId,
// Each correction line is its own segment: a unique responseId keeps
// the run reducer's block replace semantics exact (two publishes
// sharing an id with different texts would read as one segment).
responseId: `${context.runId}:correction:${randomUUID()}`,
payload: { text: `\n[USER CORRECTION]: ${correction}\n` },
});
}
}
function buildCorrectionResumeData(corrections: string[]): Record<string, unknown> {
return {
__correctionOverride: true,
message:
'The user sent a correction while this run was active. ' +
'The confirmation has been skipped. Apply the correction and continue.',
corrections,
};
}
function isErrorChunk(chunk: unknown): chunk is { type: 'error'; error?: unknown } {
return isRecord(chunk) && chunk.type === 'error';
}
async function waitForConfirmation(
signal: AbortSignal,
confirmationPromise: Promise<Record<string, unknown>>,
): Promise<Record<string, unknown>> {
if (signal.aborted) {
throw new Error(
'The assistant was interrupted while waiting for your response. Send a new message to continue.',
);
}
let abortHandler: (() => void) | undefined;
try {
return await Promise.race([
confirmationPromise,
new Promise<never>((_, reject) => {
abortHandler = () =>
reject(
new Error(
'The assistant was interrupted while waiting for your response. Send a new message to continue.',
),
);
signal.addEventListener('abort', abortHandler, { once: true });
}),
]);
} finally {
if (abortHandler) {
signal.removeEventListener('abort', abortHandler);
}
}
}
/**
* Race the HITL confirmation promise against an incoming user correction.
* When a correction arrives first, auto-confirm the suspended tool call with
* override data so the builder can resume and incorporate the correction.
*/
async function waitForConfirmationOrCorrection(
signal: AbortSignal,
confirmationPromise: Promise<Record<string, unknown>>,
control: AutoResumeControl,
context: ResumableStreamContext,
drainedCorrectionsForResume: string[],
): Promise<Record<string, unknown>> {
if (drainedCorrectionsForResume.length > 0) {
void confirmationPromise.catch(() => undefined);
return buildCorrectionResumeData(drainedCorrectionsForResume);
}
if (!control.waitForCorrection) {
return await waitForConfirmation(signal, confirmationPromise);
}
const correctionSentinel = Object.freeze({ __correctionOverride: true });
const raced = Promise.race([
confirmationPromise,
control.waitForCorrection().then(() => correctionSentinel as Record<string, unknown>),
]);
const result = await waitForConfirmation(signal, raced);
if (result === correctionSentinel) {
const corrections = control.drainCorrections?.() ?? [];
publishCorrections(context, corrections);
return buildCorrectionResumeData(corrections);
}
return result;
}
function isSameSuspension(left: SuspensionInfo, right: SuspensionInfo): boolean {
return left.requestId === right.requestId && left.toolCallId === right.toolCallId;
}

620
packages/@n8n/instance-ai/src/runtime/run-state-registry.ts
View File

@ -3,66 +3,7 @@ import type {
InstanceAiBuildMode,
InstanceAiPromptConfiguration,
InstanceAiCredentialDestinationDecision,
InstanceAiThreadStatusResponse,
} from '@n8n/api-types';
import { nanoid } from 'nanoid';
import type { InstanceAiTraceContext, ModelConfig, OrchestrationContext } from '../types';
import type {
InstanceAiLivenessPolicy,
InstanceAiLivenessSurface,
InstanceAiLivenessTimeoutReason,
} from './liveness-policy';
import type { OrchestratorRunHandoffState } from './orchestrator-run-control';
import type { WorkflowBuildOutcome } from '../workflow-loop/workflow-loop-state';
import type { SuspendedInstanceContext } from './instance-context-state';
export interface ActiveRunState {
runId: string;
threadId: string;
abortController: AbortController;
/** Prevents an older finalizer from clearing a newer executor for the same thread. */
executionToken?: symbol;
messageGroupId?: string;
tracing?: InstanceAiTraceContext;
modelId?: ModelConfig;
startedAt?: number;
lastActivityAt?: number;
/** Owner of the run, resolved at entry so concurrency can be counted per user. */
userId?: string;
}
export interface SuspendedRunState<TUser = unknown> extends ActiveRunState {
agentRunId: string;
agent: unknown;
/** The orchestration context the agent's tools closed over. Stored so a
* resume can rebind `tracing` to the new resume trace — spans emitted
* through the suspended turn's shut-down runtime export nothing. */
orchestrationContext?: OrchestrationContext;
threadId: string;
user: TUser;
toolCallId: string;
toolName?: string;
suspendPayload?: Record<string, unknown>;
requestId: string;
createdAt: number;
/** Set when the suspended run was a planned-task checkpoint follow-up.
* Preserved across suspend/resume so the resumed run's finalizer can
* run the deadlock fallback and reschedule. */
checkpoint?: { isCheckpointFollowUp: true; checkpointTaskId: string };
/** Set when the suspended run was a planned build-workflow follow-up. */
plannedBuild?: {
isPlannedBuildFollowUp: true;
buildTaskId: string;
workItemId: string;
isSupportingWorkflowTask?: boolean;
savedOutcome?: WorkflowBuildOutcome;
};
/** Shared signal used to stop resumed orchestration after durable work is handed off. */
runHandoff?: OrchestratorRunHandoffState;
/** Keep the injection and gate results across suspension. Resumed segments build no new block. */
instanceContext?: SuspendedInstanceContext;
}
/**
* Flat confirmation payload consumed by native tool `resumeSchema`s and sub-agent HITL.
@ -99,53 +40,12 @@ export interface ConfirmationData {
connectedSlugs?: string[];
}
export interface PendingConfirmation {
resolve: (data: ConfirmationData) => void;
threadId: string;
userId: string;
createdAt: number;
startedAt?: number;
lastActivityAt?: number;
}
export interface BackgroundTaskStatusSnapshot {
taskId: string;
role: string;
agentId: string;
status: 'running' | 'completed' | 'failed' | 'cancelled';
startedAt: number;
runId: string;
messageGroupId?: string;
threadId: string;
}
export interface RunStateTimeoutDetails {
reason: InstanceAiLivenessTimeoutReason;
surface: InstanceAiLivenessSurface;
timeoutMs: number;
elapsedMs: number;
idleMs: number;
}
export interface StartRunOptions<TUser> {
threadId: string;
user: TUser;
messageGroupId?: string;
}
export interface StartedRunState extends ActiveRunState {
messageGroupId?: string;
}
export class RunStateRegistry<TUser = unknown> {
private readonly activeRuns = new Map<string, ActiveRunState>();
private readonly suspendedRuns = new Map<string, SuspendedRunState<TUser>>();
private readonly pendingConfirmations = new Map<string, PendingConfirmation>();
private readonly threadUsers = new Map<string, TUser>();
/**
* Per-thread state that follow-up turns reuse: request options the client sent
* on the user turn, and the message-group index of runs. The Agents runtime
* owns the run lifecycle itself.
*/
export class RunStateRegistry {
private readonly threadMessageGroupId = new Map<string, string>();
private readonly runIdsByMessageGroup = new Map<string, string[]>();
@ -154,8 +54,8 @@ export class RunStateRegistry<TUser = unknown> {
private readonly threadTimeZones = new Map<string, string>();
/** Computer Use entries the client reported, reused by follow-up runs. Only the
* client can see its own rollout and the device, and a resumed or background
* run has no request of its own to ask. */
* client can see its own rollout and the device, and a follow-up run has no
* request of its own to ask. */
private readonly threadComputerUseChannels = new Map<string, ComputerUseChannel[]>();
/** Build mode captured at user-run entry and reused by follow-up runs. */
@ -167,51 +67,10 @@ export class RunStateRegistry<TUser = unknown> {
string,
{ version: string; metadata?: InstanceAiPromptConfiguration }
>();
/**
* Resolves a user id from the opaque `TUser` the registry is parameterised over.
* Required rather than optional: per-user concurrency counting depends on it, and a
* missing extractor would silently report every user as holding zero runs.
*/
constructor(private readonly getUserId: (user: TUser) => string) {}
startRun(options: StartRunOptions<TUser>): StartedRunState {
const runId = `run_${nanoid()}`;
const abortController = new AbortController();
const messageGroupId = options.messageGroupId ?? `mg_${nanoid()}`;
const now = Date.now();
this.activeRuns.set(options.threadId, {
runId,
threadId: options.threadId,
abortController,
messageGroupId,
startedAt: now,
lastActivityAt: now,
userId: this.getUserId(options.user),
});
this.threadUsers.set(options.threadId, options.user);
// When creating a fresh message group (no reuse), clean up the previous
// one so runIdsByMessageGroup doesn't leak entries indefinitely.
if (!options.messageGroupId) {
const prevGroupId = this.threadMessageGroupId.get(options.threadId);
if (prevGroupId && prevGroupId !== messageGroupId) {
this.runIdsByMessageGroup.delete(prevGroupId);
}
}
this.indexRunInGroup(options.threadId, messageGroupId, runId);
return { runId, threadId: options.threadId, abortController, messageGroupId };
}
/**
* Seed the message-group indexes for a run: map the thread to its current
* group and record the run under that group. Idempotent.
*
* Called on `startRun` and re-applied on `suspendRun`/`activateSuspendedRun`
* so a run resumed after a restart (where these maps start empty) repopulates
* the group association the SSE bootstrap relies on.
*/
indexRunInGroup(threadId: string, messageGroupId: string, runId: string): void {
this.threadMessageGroupId.set(threadId, messageGroupId);
@ -223,282 +82,14 @@ export class RunStateRegistry<TUser = unknown> {
if (!groupRunIds.includes(runId)) groupRunIds.push(runId);
}
getThreadStatus(
threadId: string,
backgroundTasks: BackgroundTaskStatusSnapshot[],
): InstanceAiThreadStatusResponse {
const activeRun = this.activeRuns.get(threadId);
const suspendedRun = this.suspendedRuns.get(threadId);
const liveRun = activeRun ?? suspendedRun;
return {
hasActiveRun: activeRun !== undefined,
isSuspended: suspendedRun !== undefined,
...(liveRun ? { runId: liveRun.runId } : {}),
backgroundTasks: backgroundTasks
.filter((task) => task.threadId === threadId)
.map((task) => ({
taskId: task.taskId,
role: task.role,
agentId: task.agentId,
status: task.status,
startedAt: task.startedAt,
runId: task.runId,
messageGroupId: task.messageGroupId,
})),
};
}
hasLiveRun(threadId: string): boolean {
return this.activeRuns.has(threadId) || this.suspendedRuns.has(threadId);
}
hasActiveRun(threadId: string): boolean {
return this.activeRuns.has(threadId);
}
hasSuspendedRun(threadId: string): boolean {
return this.suspendedRuns.has(threadId);
}
getMessageGroupId(threadId: string): string | undefined {
return this.threadMessageGroupId.get(threadId);
}
getLiveMessageGroupId(
threadId: string,
backgroundTasks: BackgroundTaskStatusSnapshot[],
): string | undefined {
if (this.hasLiveRun(threadId)) {
return this.threadMessageGroupId.get(threadId);
}
const runningTask = backgroundTasks
.filter((task) => task.threadId === threadId && task.status === 'running')
.sort((a, b) => b.startedAt - a.startedAt)[0];
return runningTask?.messageGroupId ?? this.threadMessageGroupId.get(threadId);
}
getRunIdsForMessageGroup(messageGroupId: string): string[] {
return this.runIdsByMessageGroup.get(messageGroupId) ?? [];
}
getActiveRunId(threadId: string): string | undefined {
return this.activeRuns.get(threadId)?.runId;
}
/**
* Runs holding a slot on this process, which is what the instance cap admits against.
*
* Includes a run parked on an inline approval card, which stays in `activeRuns`.
* Excludes a suspended run, which leaves `activeRuns` but keeps its agent in memory.
* So this bounds concurrent execution, not resident memory.
*/
activeRunCount(): number {
return this.activeRuns.size;
}
/**
* Number of runs this user currently has executing, across all their threads.
*
* Excludes everything parked on a human, because none of it is spending anything and
* counting it would lock a user out for the whole confirmation timeout after a few
* abandoned cards. That means two things, not one:
*
* - suspended runs, which have left `activeRuns` entirely; and
* - runs blocked in `waitForConfirmation`, which stay in `activeRuns` while an inline
* approval card is open. `sweepTimedOut` skips these for the same reason.
*
* {@link activeRunCount} differs on the second case only. An inline-parked run is still
* in `activeRuns`, so it holds a slot there. A suspended run holds a slot in neither,
* although it still retains its agent -- see {@link activeRunCount}.
*
* The scan is linear in concurrently-executing runs. That stays small in practice
* regardless of the caps -- each run costs ~12-20MB, so a process cannot hold many --
* and it is negligible next to the model call that follows.
*/
activeRunCountForUser(userId: string): number {
let count = 0;
for (const [threadId, run] of this.activeRuns) {
if (run.userId !== userId) continue;
if (this.hasPendingConfirmationForThread(threadId)) continue;
count++;
}
return count;
}
getActiveRun(threadId: string): ActiveRunState | undefined {
return this.activeRuns.get(threadId);
}
getSuspendedRun(threadId: string): SuspendedRunState<TUser> | undefined {
return this.suspendedRuns.get(threadId);
}
attachTracing(threadId: string, tracing: InstanceAiTraceContext): void {
const activeRun = this.activeRuns.get(threadId);
if (!activeRun) return;
this.activeRuns.set(threadId, {
...activeRun,
threadId,
tracing,
});
}
clearActiveRun(threadId: string, executionToken?: symbol): void {
if (this.activeRuns.get(threadId)?.executionToken !== executionToken) return;
this.activeRuns.delete(threadId);
}
cancelActiveRun(threadId: string): ActiveRunState | undefined {
const active = this.activeRuns.get(threadId);
if (!active) return undefined;
this.activeRuns.delete(threadId);
return active;
}
suspendRun(threadId: string, state: SuspendedRunState<TUser>): void {
const activeRun = this.activeRuns.get(threadId);
this.activeRuns.delete(threadId);
state.startedAt = state.startedAt ?? activeRun?.startedAt ?? state.createdAt;
state.lastActivityAt = state.lastActivityAt ?? state.createdAt;
state.userId = state.userId ?? activeRun?.userId;
this.suspendedRuns.set(threadId, state);
// Re-seed group indexes: on a restart-resumed orphan these maps start
// empty, so without this the SSE bootstrap loses the group association.
if (state.messageGroupId) {
this.indexRunInGroup(threadId, state.messageGroupId, state.runId);
}
}
findSuspendedByRequestId(requestId: string): SuspendedRunState<TUser> | undefined {
for (const run of this.suspendedRuns.values()) {
if (run.requestId === requestId) return run;
}
return undefined;
}
cancelSuspendedRun(threadId: string): SuspendedRunState<TUser> | undefined {
const suspended = this.suspendedRuns.get(threadId);
if (!suspended) return undefined;
this.suspendedRuns.delete(threadId);
return suspended;
}
activateSuspendedRun(
threadId: string,
executionToken?: symbol,
): SuspendedRunState<TUser> | undefined {
const suspended = this.suspendedRuns.get(threadId);
if (!suspended) return undefined;
this.suspendedRuns.delete(threadId);
const now = Date.now();
this.activeRuns.set(threadId, {
runId: suspended.runId,
threadId,
abortController: suspended.abortController,
executionToken,
messageGroupId: suspended.messageGroupId,
tracing: suspended.tracing,
modelId: suspended.modelId,
startedAt: suspended.startedAt ?? suspended.createdAt,
lastActivityAt: now,
userId: suspended.userId ?? this.getUserId(suspended.user),
});
// Re-seed group indexes for the reactivated run (empty after a restart).
if (suspended.messageGroupId) {
this.indexRunInGroup(threadId, suspended.messageGroupId, suspended.runId);
}
return suspended;
}
registerPendingConfirmation(requestId: string, pending: PendingConfirmation): void {
this.pendingConfirmations.set(requestId, {
...pending,
startedAt: pending.startedAt ?? pending.createdAt,
lastActivityAt: pending.lastActivityAt ?? pending.createdAt,
});
}
getPendingConfirmation(requestId: string): PendingConfirmation | undefined {
return this.pendingConfirmations.get(requestId);
}
hasPendingConfirmationForThread(threadId: string): boolean {
for (const pending of this.pendingConfirmations.values()) {
if (pending.threadId === threadId) return true;
}
return false;
}
touchActiveRun(threadId: string, at = Date.now()): boolean {
const active = this.activeRuns.get(threadId);
if (!active) return false;
active.lastActivityAt = at;
return true;
}
touchSuspendedRun(threadId: string, at = Date.now()): boolean {
const suspended = this.suspendedRuns.get(threadId);
if (!suspended) return false;
suspended.lastActivityAt = at;
return true;
}
touchPendingConfirmation(requestId: string, at = Date.now()): boolean {
const pending = this.pendingConfirmations.get(requestId);
if (!pending) return false;
pending.lastActivityAt = at;
return true;
}
resolvePendingConfirmation(
requestingUserId: string,
requestId: string,
data: ConfirmationData,
): boolean {
const pending = this.pendingConfirmations.get(requestId);
if (pending?.userId !== requestingUserId) return false;
this.pendingConfirmations.delete(requestId);
pending.resolve(data);
return true;
}
cancelThread(
threadId: string,
cancelledConfirmation: ConfirmationData = { approved: false },
): {
active?: ActiveRunState;
suspended?: SuspendedRunState<TUser>;
} {
for (const [requestId, pending] of this.pendingConfirmations) {
if (pending.threadId !== threadId) continue;
pending.resolve(cancelledConfirmation);
this.pendingConfirmations.delete(requestId);
}
const active = this.activeRuns.get(threadId);
const suspended = this.suspendedRuns.get(threadId);
if (suspended) {
this.suspendedRuns.delete(threadId);
}
return { ...(active ? { active } : {}), ...(suspended ? { suspended } : {}) };
}
getThreadUser(threadId: string): TUser | undefined {
return this.threadUsers.get(threadId);
}
setTimeZone(threadId: string, timeZone: string): void {
this.threadTimeZones.set(threadId, timeZone);
}
@ -562,155 +153,8 @@ export class RunStateRegistry<TUser = unknown> {
return this.threadPromptSelections.get(threadId)?.metadata;
}
/**
* Find active/suspended runs and pending confirmations that timed out under policy.
* Returns thread IDs and request IDs that should be cancelled/rejected.
* Does NOT mutate state — the caller is responsible for cancelling.
*/
sweepTimedOut(maxAgeMs: number): {
suspendedThreadIds: string[];
confirmationRequestIds: string[];
};
sweepTimedOut(
policy: InstanceAiLivenessPolicy,
now?: number,
): {
activeThreadIds: string[];
suspendedThreadIds: string[];
confirmationRequestIds: string[];
activeTimeouts: Record<string, RunStateTimeoutDetails>;
suspendedTimeouts: Record<string, RunStateTimeoutDetails>;
confirmationTimeouts: Record<string, RunStateTimeoutDetails>;
};
sweepTimedOut(
policyOrMaxAgeMs: InstanceAiLivenessPolicy | number,
now = Date.now(),
):
| {
suspendedThreadIds: string[];
confirmationRequestIds: string[];
}
| {
activeThreadIds: string[];
suspendedThreadIds: string[];
confirmationRequestIds: string[];
activeTimeouts: Record<string, RunStateTimeoutDetails>;
suspendedTimeouts: Record<string, RunStateTimeoutDetails>;
confirmationTimeouts: Record<string, RunStateTimeoutDetails>;
} {
if (typeof policyOrMaxAgeMs === 'number') {
const maxAgeMs = policyOrMaxAgeMs;
const suspendedThreadIds: string[] = [];
for (const [threadId, run] of this.suspendedRuns) {
if (now - run.createdAt >= maxAgeMs) {
suspendedThreadIds.push(threadId);
}
}
const confirmationRequestIds: string[] = [];
for (const [reqId, pending] of this.pendingConfirmations) {
if (now - pending.createdAt >= maxAgeMs) {
confirmationRequestIds.push(reqId);
}
}
return { suspendedThreadIds, confirmationRequestIds };
}
const policy = policyOrMaxAgeMs;
const activeThreadIds: string[] = [];
const activeTimeouts: Record<string, RunStateTimeoutDetails> = {};
for (const [threadId, run] of this.activeRuns) {
if (this.hasPendingConfirmationForThread(threadId)) continue;
const startedAt = run.startedAt ?? run.lastActivityAt ?? now;
const lastActivityAt = run.lastActivityAt ?? startedAt;
const decision = policy.evaluate({
surface: 'active-run',
startedAt,
lastActivityAt,
now,
});
if (decision.action === 'timeout') {
activeThreadIds.push(threadId);
activeTimeouts[threadId] = decision;
}
}
const suspendedThreadIds: string[] = [];
const suspendedTimeouts: Record<string, RunStateTimeoutDetails> = {};
for (const [threadId, run] of this.suspendedRuns) {
const startedAt = run.startedAt ?? run.createdAt;
const lastActivityAt = run.lastActivityAt ?? run.createdAt;
const decision = policy.evaluate({
surface: 'suspended-run',
startedAt,
lastActivityAt,
now,
});
if (decision.action === 'timeout') {
suspendedThreadIds.push(threadId);
suspendedTimeouts[threadId] = decision;
}
}
const confirmationRequestIds: string[] = [];
const confirmationTimeouts: Record<string, RunStateTimeoutDetails> = {};
for (const [reqId, pending] of this.pendingConfirmations) {
const startedAt = pending.startedAt ?? pending.createdAt;
const lastActivityAt = pending.lastActivityAt ?? pending.createdAt;
const decision = policy.evaluate({
surface: 'pending-confirmation',
startedAt,
lastActivityAt,
now,
});
if (decision.action === 'timeout') {
confirmationRequestIds.push(reqId);
confirmationTimeouts[reqId] = decision;
}
}
return {
activeThreadIds,
suspendedThreadIds,
confirmationRequestIds,
activeTimeouts,
suspendedTimeouts,
confirmationTimeouts,
};
}
/**
* Auto-reject a pending confirmation by request ID.
* Returns true if the confirmation existed and was rejected.
*/
rejectPendingConfirmation(requestId: string): boolean {
const pending = this.pendingConfirmations.get(requestId);
if (!pending) return false;
this.pendingConfirmations.delete(requestId);
pending.resolve({ approved: false });
return true;
}
/** Remove a message-group entry from runIdsByMessageGroup. */
deleteMessageGroup(groupId: string): void {
this.runIdsByMessageGroup.delete(groupId);
}
/**
* Remove all per-thread state: active/suspended runs, confirmations,
* user, time zone, and message-group mappings.
* Returns the cancelled active/suspended runs so the caller can abort them.
*/
clearThread(
threadId: string,
cancelledConfirmation: ConfirmationData = { approved: false },
): {
active?: ActiveRunState;
suspended?: SuspendedRunState<TUser>;
} {
const { active, suspended } = this.cancelThread(threadId, cancelledConfirmation);
if (active) this.activeRuns.delete(threadId);
this.threadUsers.delete(threadId);
/** Remove all per-thread state. */
clearThread(threadId: string): void {
this.threadTimeZones.delete(threadId);
this.threadComputerUseChannels.delete(threadId);
this.threadBuildModes.delete(threadId);
@ -721,49 +165,9 @@ export class RunStateRegistry<TUser = unknown> {
const groupId = this.threadMessageGroupId.get(threadId);
if (groupId) this.runIdsByMessageGroup.delete(groupId);
this.threadMessageGroupId.delete(threadId);
return { ...(active ? { active } : {}), ...(suspended ? { suspended } : {}) };
}
/**
* Process-wide teardown. Returns the in-flight runs so the service can
* abort them and persist terminal snapshots where appropriate.
*
* Pending confirmations are intentionally NOT resolved — auto-resolving an
* inline HITL (`waitForConfirmation(...)`) with `{ approved: false }`
* causes the awaiting agent tool to run to completion as "denied" before
* the process exits, which then mutates the snapshot tree mid-shutdown
* and clobbers the plan/ask card the user would otherwise see on reload.
* Letting the Promises dangle is safe: the process is exiting, the
* abortController for each active run is aborted next, and the
* `instance_ai_pending_confirmations` row survives so the user can still
* see the confirmation card (and get a clear "lost on restart" error if
* they click confirm — see `handleOrphanedConfirmation`).
*
* `pendingThreadIds` is returned so the service can skip the
* publish-run-finish + terminal-snapshot treatment for runs that are
* only sitting in `activeRuns` because they're waiting on an inline
* confirmation Promise.
*/
shutdown(): {
activeRuns: Array<ActiveRunState & { promptVersion?: string }>;
suspendedRuns: Array<SuspendedRunState<TUser>>;
pendingThreadIds: string[];
} {
// Capture the selected version before clearing per-thread state.
const activeRuns = [...this.activeRuns.values()].map((run) => ({
...run,
promptVersion: this.getPromptVersion(run.threadId),
}));
const suspendedRuns = [...this.suspendedRuns.values()];
const pendingThreadIds = [
...new Set([...this.pendingConfirmations.values()].map((p) => p.threadId)),
];
this.activeRuns.clear();
this.suspendedRuns.clear();
this.pendingConfirmations.clear();
this.threadUsers.clear();
clear(): void {
this.threadTimeZones.clear();
this.threadComputerUseChannels.clear();
this.threadBuildModes.clear();
@ -772,7 +176,5 @@ export class RunStateRegistry<TUser = unknown> {
this.threadPromptSelections.clear();
this.threadMessageGroupId.clear();
this.runIdsByMessageGroup.clear();
return { activeRuns, suspendedRuns, pendingThreadIds };
}
}

119
packages/@n8n/instance-ai/src/runtime/stream-runner.ts
View File

@ -1,119 +0,0 @@
import type { InstanceAiEvent } from '@n8n/api-types';
import type { InstanceAiEventBus } from '../event-bus';
import type { Logger } from '../logger';
import type {
OrchestratorRunHandoffReason,
OrchestratorRunStopSignal,
} from './orchestrator-run-control';
import {
executeResumableStream,
normalizeStreamSource,
type ResumableStreamSource,
type TraceStatus,
} from './resumable-stream-executor';
import type { RunTokenUsage } from '../stream/usage-accumulator';
import type { WorkSummary } from '../stream/work-summary-accumulator';
import { resumeAgentStream } from '../utils/stream-helpers';
import type { SuspensionInfo } from '../utils/stream-helpers';
export interface StreamableAgent {
stream: (input: unknown, options: Record<string, unknown>) => Promise<unknown>;
}
export interface StreamRunOptions {
threadId: string;
runId: string;
agentId: string;
signal: AbortSignal;
eventBus: InstanceAiEventBus;
logger: Logger;
onActivity?: () => void;
stopSignal?: () => OrchestratorRunStopSignal | undefined;
}
export interface StreamRunResult {
status: TraceStatus;
agentRunId: string;
text?: Promise<string>;
error?: unknown;
workSummary: WorkSummary;
usage?: RunTokenUsage;
stopReason?: OrchestratorRunHandoffReason;
suspension?: SuspensionInfo;
confirmationEvent?: Extract<InstanceAiEvent, { type: 'confirmation-request' }>;
}
export async function streamAgentRun(
agent: StreamableAgent,
input: unknown,
streamOptions: Record<string, unknown>,
options: StreamRunOptions,
): Promise<StreamRunResult> {
const result = await agent.stream(input, streamOptions);
const stream = normalizeStreamSource(result);
const agentRunId = typeof stream.runId === 'string' ? stream.runId : '';
return await consumeStream(agent, stream, { ...options, agentRunId });
}
export async function resumeAgentRun(
agent: unknown,
resumeData: Record<string, unknown>,
resumeOptions: Record<string, unknown>,
options: StreamRunOptions & { agentRunId: string },
): Promise<StreamRunResult> {
const resumed = await resumeAgentStream(agent, resumeData, resumeOptions);
const stream = normalizeStreamSource(resumed);
const agentRunId = (typeof stream.runId === 'string' && stream.runId) || options.agentRunId;
return await consumeStream(agent, stream, { ...options, agentRunId });
}
async function consumeStream(
agent: unknown,
stream: ResumableStreamSource,
options: StreamRunOptions & { agentRunId: string },
): Promise<StreamRunResult> {
const result = await executeResumableStream({
agent,
stream,
context: {
threadId: options.threadId,
runId: options.runId,
agentId: options.agentId,
eventBus: options.eventBus,
signal: options.signal,
logger: options.logger,
onActivity: options.onActivity,
stopSignal: options.stopSignal,
},
control: { mode: 'manual' },
initialAgentRunId: options.agentRunId,
});
if (result.status === 'suspended' && result.suspension) {
return {
status: 'suspended',
agentRunId: result.agentRunId,
text: result.text,
workSummary: result.workSummary,
usage: result.usage,
suspension: result.suspension,
...(result.confirmationEvent ? { confirmationEvent: result.confirmationEvent } : {}),
};
}
return {
status:
result.status === 'cancelled'
? 'cancelled'
: result.status === 'errored'
? 'errored'
: 'completed',
agentRunId: result.agentRunId,
text: result.text,
...(result.error !== undefined ? { error: result.error } : {}),
workSummary: result.workSummary,
usage: result.usage,
...(result.stopReason ? { stopReason: result.stopReason } : {}),
};
}

125
packages/@n8n/instance-ai/src/storage/__tests__/terminal-outcome-storage.test.ts
View File

@ -1,125 +0,0 @@
vi.mock('../thread-patch', () => ({
getThread: vi.fn(),
patchThread: vi.fn(),
}));
import { TerminalOutcomeStorage, type TerminalOutcome } from '../terminal-outcome-storage';
import { getThread, patchThread, type PatchableThreadMemory } from '../thread-patch';
const mockedGetThread = vi.mocked(getThread);
const mockedPatchThread = vi.mocked(patchThread);
function makeMemory(): PatchableThreadMemory {
return {};
}
function makeOutcome(overrides: Partial<TerminalOutcome> = {}): TerminalOutcome {
return {
id: 'outcome-1',
threadId: 'thread-1',
runId: 'run-1',
taskId: 'task-1',
agentId: 'agent-1',
status: 'completed',
userFacingMessage: 'done',
createdAt: '2026-05-02T00:00:00.000Z',
...overrides,
};
}
function makeThread(metadata: Record<string, unknown>): Awaited<ReturnType<typeof getThread>> {
return {
id: 'thread-1',
title: 'Thread',
metadata,
resourceId: 'resource-1',
createdAt: new Date('2026-05-02T00:00:00.000Z'),
updatedAt: new Date('2026-05-02T00:00:00.000Z'),
};
}
describe('TerminalOutcomeStorage', () => {
let memory: PatchableThreadMemory;
let storage: TerminalOutcomeStorage;
beforeEach(() => {
vi.clearAllMocks();
memory = makeMemory();
storage = new TerminalOutcomeStorage(memory);
});
describe('getUndelivered()', () => {
it('returns valid undelivered outcomes when one stored entry is malformed', async () => {
const valid = makeOutcome({ id: 'outcome-valid' });
mockedGetThread.mockResolvedValue(
makeThread({
instanceAiTerminalOutcomes: {
'outcome-valid': valid,
'outcome-broken': { id: 'outcome-broken' },
},
}),
);
const result = await storage.getUndelivered('thread-1');
expect(result).toEqual([valid]);
});
it('returns empty list when metadata is missing', async () => {
mockedGetThread.mockResolvedValue(makeThread({}));
const result = await storage.getUndelivered('thread-1');
expect(result).toEqual([]);
});
it('skips delivered outcomes', async () => {
const undelivered = makeOutcome({ id: 'undelivered' });
const delivered = makeOutcome({
id: 'delivered',
deliveredAt: '2026-05-02T00:00:01.000Z',
});
mockedGetThread.mockResolvedValue(
makeThread({
instanceAiTerminalOutcomes: {
undelivered,
delivered,
},
}),
);
const result = await storage.getUndelivered('thread-1');
expect(result).toEqual([undelivered]);
});
});
describe('upsert()', () => {
it('preserves valid outcomes when an existing entry is malformed', async () => {
const valid = makeOutcome({ id: 'outcome-valid' });
const next = makeOutcome({ id: 'outcome-new' });
let captured: Record<string, unknown> | undefined;
mockedPatchThread.mockImplementation(async (_memory, args) => {
await Promise.resolve();
const patch = args.update({
metadata: {
instanceAiTerminalOutcomes: {
'outcome-valid': valid,
'outcome-broken': { not: 'an outcome' },
},
},
} as unknown as Parameters<typeof args.update>[0]);
captured = patch?.metadata?.instanceAiTerminalOutcomes as Record<string, unknown>;
return null;
});
await storage.upsert('thread-1', next);
expect(captured).toEqual({
'outcome-valid': valid,
'outcome-new': next,
});
});
});
});

2
packages/@n8n/instance-ai/src/storage/index.ts
View File

@ -5,8 +5,6 @@ export { ThreadIterationLogStorage } from './thread-iteration-log-storage';
export { ThreadTaskStorage } from './thread-task-storage';
export { PlannedTaskStorage } from './planned-task-storage';
export { getThread, patchThread } from './thread-patch';
export { TerminalOutcomeStorage } from './terminal-outcome-storage';
export type { TerminalOutcome } from './terminal-outcome-storage';
export type { PatchableThreadMemory, ThreadPatch } from './thread-patch';
export { WorkflowLoopStorage } from './workflow-loop-storage';
export type {

85
packages/@n8n/instance-ai/src/storage/terminal-outcome-storage.ts
View File

@ -1,85 +0,0 @@
import { z } from 'zod';
import { getThread, patchThread, type PatchableThreadMemory } from './thread-patch';
const METADATA_KEY = 'instanceAiTerminalOutcomes';
const terminalOutcomeStatusSchema = z.enum(['completed', 'failed', 'cancelled']);
const terminalOutcomeSchema = z.object({
id: z.string(),
threadId: z.string(),
runId: z.string(),
messageGroupId: z.string().optional(),
correlationId: z.string().optional(),
taskId: z.string(),
agentId: z.string(),
status: terminalOutcomeStatusSchema,
userFacingMessage: z.string(),
createdAt: z.string(),
deliveredAt: z.string().optional(),
});
export type TerminalOutcome = z.infer<typeof terminalOutcomeSchema>;
export class TerminalOutcomeStorage {
constructor(private readonly memory: PatchableThreadMemory) {}
async upsert(threadId: string, outcome: TerminalOutcome): Promise<void> {
await patchThread(this.memory, {
threadId,
update: ({ metadata = {} }) => {
const current = parseOutcomes(metadata[METADATA_KEY]);
return {
metadata: {
...metadata,
[METADATA_KEY]: {
...current,
[outcome.id]: outcome,
},
},
};
},
});
}
async markDelivered(threadId: string, outcomeId: string, deliveredAt: string): Promise<void> {
await patchThread(this.memory, {
threadId,
update: ({ metadata = {} }) => {
const current = parseOutcomes(metadata[METADATA_KEY]);
const outcome = current[outcomeId];
if (!outcome) return null;
return {
metadata: {
...metadata,
[METADATA_KEY]: {
...current,
[outcomeId]: {
...outcome,
deliveredAt,
},
},
},
};
},
});
}
async getUndelivered(threadId: string): Promise<TerminalOutcome[]> {
const thread = await getThread(this.memory, threadId);
const outcomes = parseOutcomes(thread?.metadata?.[METADATA_KEY]);
return Object.values(outcomes).filter((outcome) => !outcome.deliveredAt);
}
}
function parseOutcomes(raw: unknown): Record<string, TerminalOutcome> {
if (!raw || typeof raw !== 'object') return {};
const outcomes: Record<string, TerminalOutcome> = {};
for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
const parsed = terminalOutcomeSchema.safeParse(value);
if (parsed.success) outcomes[key] = parsed.data;
}
return outcomes;
}

263
packages/@n8n/instance-ai/src/stream/consume-with-hitl.ts
View File

@ -1,13 +1,14 @@
import type { StreamResult } from '@n8n/agents';
import type { InstanceAiEvent } from '@n8n/api-types';
import { isRecord } from '@n8n/utils/is-record';
import { randomUUID } from 'node:crypto';
import type { InstanceAiEventBus } from '../event-bus/event-bus.interface';
import type { Logger } from '../logger';
import {
executeResumableStream,
normalizeStreamSource,
} from '../runtime/resumable-stream-executor';
import type { RunTokenUsage } from '../stream/usage-accumulator';
import type { WorkSummary } from '../stream/work-summary-accumulator';
import { isQuotaExhaustedError, mapAgentChunkToEvent } from './map-chunk';
import { UsageAccumulator, type RunTokenUsage } from './usage-accumulator';
import { WorkSummaryAccumulator, type WorkSummary } from './work-summary-accumulator';
import { parseSuspension } from '../utils/stream-helpers';
import type { SuspensionInfo } from '../utils/stream-helpers';
type ConfirmationRequestEvent = Extract<InstanceAiEvent, { type: 'confirmation-request' }>;
@ -41,6 +42,105 @@ export type ConsumeStreamCascadingResult =
usage?: RunTokenUsage;
};
interface StreamSource {
runId?: string;
fullStream: AsyncIterable<unknown>;
text?: Promise<string>;
}
function isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {
return (
value !== null &&
typeof value === 'object' &&
typeof Reflect.get(value, Symbol.asyncIterator) === 'function'
);
}
function isReadableStream(value: unknown): value is ReadableStream<unknown> {
return (
value !== null &&
typeof value === 'object' &&
typeof Reflect.get(value, 'getReader') === 'function'
);
}
function isStreamSource(value: unknown): value is StreamSource {
return isRecord(value) && isAsyncIterable(value.fullStream);
}
function isNativeStreamResult(value: unknown): value is StreamResult {
return isRecord(value) && isReadableStream(value.stream);
}
function extractContentText(content: unknown): string {
if (typeof content === 'string') return content;
if (!Array.isArray(content)) return '';
return content
.map((part) =>
isRecord(part) && part.type === 'text' && typeof part.text === 'string' ? part.text : '',
)
.join('');
}
async function collectNativeStreamText(stream: ReadableStream<unknown>): Promise<string> {
const reader = stream.getReader();
let deltaText = '';
let messageText = '';
try {
while (true) {
const { done, value } = await reader.read();
if (done) return deltaText || messageText;
if (!isRecord(value)) continue;
if (value.type === 'text-delta' && typeof value.delta === 'string') {
deltaText += value.delta;
continue;
}
if (value.type === 'message' && isRecord(value.message)) {
if (value.message.role === 'assistant') {
messageText += extractContentText(value.message.content);
}
}
}
} finally {
reader.releaseLock();
}
}
async function* readableStreamToAsyncIterable(stream: ReadableStream<unknown>) {
const reader = stream.getReader();
try {
while (true) {
const { done, value } = await reader.read();
if (done) return;
yield value;
}
} finally {
reader.releaseLock();
}
}
function normalizeStreamSource(result: unknown): StreamSource {
if (isStreamSource(result)) return result;
if (isNativeStreamResult(result)) {
const [eventStream, textStream] = result.stream.tee();
return {
runId: result.runId,
fullStream: readableStreamToAsyncIterable(eventStream),
text: collectNativeStreamText(textStream),
};
}
throw new Error('Unsupported agent stream result');
}
function isErrorChunk(chunk: unknown): chunk is { type: 'error'; error?: unknown } {
return isRecord(chunk) && chunk.type === 'error';
}
/**
* Consume a sub-agent stream and return cleanly when it either finishes or
* hits a HITL suspension. This returns the suspension info to the caller so it can decide
@ -48,46 +148,139 @@ export type ConsumeStreamCascadingResult =
* via `ctx.suspend(payload)`, so the parent's agent run is also checkpointed
* and survives a process restart.
*
* Uses `executeResumableStream`'s manual mode, which suppresses the
* sub-agent's `confirmation-request` event publish (returning it on the
* result instead) — the caller emits the card at whatever runId is
* meaningful at its level.
* The sub-agent's primary `confirmation-request` event is not published. It
* is returned on the result instead, so the caller emits the card at whatever
* runId is meaningful at its level.
*/
export async function consumeStreamCascading(
options: ConsumeStreamCascadingOptions,
): Promise<ConsumeStreamCascadingResult> {
const stream = normalizeStreamSource(options.stream);
const result = await executeResumableStream({
agent: options.agent,
stream,
context: {
threadId: options.threadId,
runId: options.runId,
agentId: options.agentId,
eventBus: options.eventBus,
signal: options.abortSignal,
logger: options.logger,
},
control: { mode: 'manual' },
});
const agentRunId = stream.runId ?? '';
const workSummaryAccumulator = new WorkSummaryAccumulator();
const usageAccumulator = new UsageAccumulator();
const usage = () => (usageAccumulator.hasUsage() ? usageAccumulator.toUsage() : undefined);
if (result.status === 'suspended' && result.suspension) {
let currentResponseId: string | undefined;
let nativeStepIndex = 0;
// Segment id for deltas that arrive before a `start-step` (some providers
// never emit one). Cleared at the next structural chunk.
let syntheticSegmentId: string | undefined;
let suspension: SuspensionInfo | undefined;
let confirmationEvent: ConfirmationRequestEvent | undefined;
let hasError = false;
// Drop follow-on error chunks after a quota error, so the user sees one reason.
let quotaErrorPublished = false;
for await (const chunk of stream.fullStream) {
if (options.abortSignal.aborted) {
// Drain the rest so the usage of the cancelled stream is still observed.
usageAccumulator.observe(chunk);
try {
for await (const remaining of stream.fullStream) usageAccumulator.observe(remaining);
} catch (drainError) {
options.logger.debug('Instance AI abort drain ended early', {
threadId: options.threadId,
runId: options.runId,
error: drainError instanceof Error ? drainError.message : String(drainError),
});
}
return {
status: 'cancelled',
agentRunId,
text: stream.text ?? Promise.resolve(''),
workSummary: workSummaryAccumulator.toSummary(),
usage: usage(),
};
}
usageAccumulator.observe(chunk);
if (isRecord(chunk) && chunk.type === 'start-step') {
nativeStepIndex += 1;
currentResponseId = `${agentRunId || options.runId}:step:${nativeStepIndex}`;
syntheticSegmentId = undefined;
}
const parsedSuspension = parseSuspension(chunk);
if (parsedSuspension) {
if (!suspension) {
suspension = parsedSuspension;
} else if (
parsedSuspension.requestId !== suspension.requestId ||
parsedSuspension.toolCallId !== suspension.toolCallId
) {
options.logger.warn('Additional HITL suspension encountered before resume; deferring', {
threadId: options.threadId,
runId: options.runId,
activeRequestId: suspension.requestId,
deferredRequestId: parsedSuspension.requestId,
});
}
}
if (isErrorChunk(chunk)) {
hasError = true;
if (quotaErrorPublished) continue;
if (isQuotaExhaustedError(chunk.error)) quotaErrorPublished = true;
}
const isDeltaChunk =
isRecord(chunk) && (chunk.type === 'text-delta' || chunk.type === 'reasoning-delta');
if (isDeltaChunk && !currentResponseId && !syntheticSegmentId) {
syntheticSegmentId = `${agentRunId || options.runId}:seg:${randomUUID()}`;
}
const event = mapAgentChunkToEvent(
options.runId,
options.agentId,
chunk,
currentResponseId ?? syntheticSegmentId,
);
const isFinishStep = isRecord(chunk) && chunk.type === 'finish-step';
if ((event && !isDeltaChunk) || isFinishStep) syntheticSegmentId = undefined;
if (!event) continue;
workSummaryAccumulator.observe(event);
if (event.type === 'confirmation-request') {
// Hold back the primary card and drop duplicates or secondary cards.
const isPrimary =
event.payload.requestId === suspension?.requestId &&
event.payload.toolCallId === suspension?.toolCallId;
if (isPrimary && !confirmationEvent) confirmationEvent = event;
continue;
}
options.eventBus.publish(options.threadId, event);
}
if (options.abortSignal.aborted) {
return {
status: 'cancelled',
agentRunId,
text: stream.text ?? Promise.resolve(''),
workSummary: workSummaryAccumulator.toSummary(),
usage: usage(),
};
}
if (suspension) {
return {
status: 'suspended',
agentRunId: result.agentRunId,
suspension: result.suspension,
...(result.confirmationEvent ? { confirmationEvent: result.confirmationEvent } : {}),
...(result.text ? { text: result.text } : {}),
workSummary: result.workSummary,
...(result.usage ? { usage: result.usage } : {}),
agentRunId,
suspension,
...(confirmationEvent ? { confirmationEvent } : {}),
...(stream.text ? { text: stream.text } : {}),
workSummary: workSummaryAccumulator.toSummary(),
usage: usage(),
};
}
return {
status: result.status === 'suspended' ? 'errored' : result.status,
agentRunId: result.agentRunId,
text: result.text ?? stream.text ?? Promise.resolve(''),
workSummary: result.workSummary,
...(result.usage ? { usage: result.usage } : {}),
status: hasError ? 'errored' : 'completed',
agentRunId,
text: stream.text ?? Promise.resolve(''),
workSummary: workSummaryAccumulator.toSummary(),
usage: usage(),
};
}

132
packages/@n8n/instance-ai/src/tools/__tests__/task-control.tool.test.ts
View File

@ -1,5 +1,3 @@
import type { Mock } from 'vitest';
import { executeTool, parseToolInput } from '../../__tests__/tool-test-utils';
import type { OrchestrationContext } from '../../types';
import { createTaskControlTool } from '../task-control.tool';
@ -23,8 +21,6 @@ function createMockContext(overrides: Partial<OrchestrationContext> = {}): Orche
get: vi.fn(),
save: vi.fn(),
},
cancelBackgroundTask: vi.fn(),
sendCorrectionToTask: vi.fn(),
...overrides,
} as unknown as OrchestrationContext;
}
@ -83,134 +79,6 @@ describe('task-control tool', () => {
expect(result).toEqual({ saved: true });
});
});
// ── cancel-task ────────────────────────────────────────────────────────
describe('cancel-task action', () => {
it('should call cancelBackgroundTask and return success message', async () => {
const context = createMockContext();
const tool = createTaskControlTool(context);
const result = await executeTool(
tool,
{ action: 'cancel-task' as const, taskId: 'build-ABC123' },
{} as never,
);
expect(context.cancelBackgroundTask).toHaveBeenCalledWith('build-ABC123');
expect(result).toEqual({ result: 'Background task build-ABC123 cancelled.' });
});
it('should return error when cancelBackgroundTask is not available', async () => {
const context = createMockContext({
cancelBackgroundTask: undefined,
});
const tool = createTaskControlTool(context);
const result = await executeTool(
tool,
{ action: 'cancel-task' as const, taskId: 'build-XYZ' },
{} as never,
);
expect(result).toEqual({
result: 'Error: background task cancellation not available.',
});
});
});
// ── correct-task ───────────────────────────────────────────────────────
describe('correct-task action', () => {
it('should send correction and return success message', async () => {
const context = createMockContext();
(context.sendCorrectionToTask as Mock).mockReturnValue('queued');
const tool = createTaskControlTool(context);
const result = await executeTool(
tool,
{
action: 'correct-task' as const,
taskId: 'build-ABC',
correction: 'use the Projects database',
},
{} as never,
);
expect(context.sendCorrectionToTask).toHaveBeenCalledWith(
'build-ABC',
'use the Projects database',
);
expect(result).toEqual({
result:
'Correction sent to task build-ABC: "use the Projects database". ' +
'The builder will see this on its next step.',
});
});
it('should return task-not-found message when task does not exist', async () => {
const context = createMockContext();
(context.sendCorrectionToTask as Mock).mockReturnValue('task-not-found');
const tool = createTaskControlTool(context);
const result = await executeTool(
tool,
{
action: 'correct-task' as const,
taskId: 'build-GONE',
correction: 'fix the trigger',
},
{} as never,
);
expect(result).toEqual({
result: 'Task build-GONE not found. It may have already been cleaned up.',
});
});
it('should return task-completed message when task has finished', async () => {
const context = createMockContext();
(context.sendCorrectionToTask as Mock).mockReturnValue('task-completed');
const tool = createTaskControlTool(context);
const result = await executeTool(
tool,
{
action: 'correct-task' as const,
taskId: 'build-DONE',
correction: 'add error handling',
},
{} as never,
);
expect(result).toEqual({
result:
'Task build-DONE has already completed. The correction was not delivered. ' +
'Incorporate "add error handling" into a new follow-up task instead.',
});
});
it('should return error when sendCorrectionToTask is not available', async () => {
const context = createMockContext({
sendCorrectionToTask: undefined,
});
const tool = createTaskControlTool(context);
const result = await executeTool(
tool,
{
action: 'correct-task' as const,
taskId: 'build-ABC',
correction: 'fix it',
},
{} as never,
);
expect(result).toEqual({
result: 'Error: correction delivery not available.',
});
});
});
});
describe('task-control tool — checklist item contract', () => {

61
packages/@n8n/instance-ai/src/tools/task-control.tool.ts
View File

@ -1,5 +1,5 @@
/**
* Consolidated task-control tool — update-checklist + cancel-task + correct-task.
* Task-control tool — writes the lightweight visible checklist.
*/
import { Tool } from '@n8n/agents';
import { taskItemSchema } from '@n8n/api-types';
@ -33,22 +33,7 @@ const updateChecklistAction = z.object({
tasks: z.array(checklistItemSchema).describe('Ordered list of tasks'),
});
const cancelTaskAction = z.object({
action: z.literal('cancel-task').describe('Cancel a running background task by its task ID'),
taskId: z.string().describe('Task ID (e.g. build-XXXXXXXX)'),
});
const correctTaskAction = z.object({
action: z.literal('correct-task').describe('Send a correction to a running background task'),
taskId: z.string().describe('Task ID (e.g. build-XXXXXXXX)'),
correction: z
.string()
.describe("The correction message from the user (e.g. 'use the Projects database')"),
});
const inputSchema = sanitizeInputSchema(
z.discriminatedUnion('action', [updateChecklistAction, cancelTaskAction, correctTaskAction]),
);
const inputSchema = sanitizeInputSchema(z.discriminatedUnion('action', [updateChecklistAction]));
type Input = z.infer<typeof inputSchema>;
@ -69,58 +54,18 @@ async function handleUpdateChecklist(
return { saved: true };
}
async function handleCancelTask(
context: OrchestrationContext,
input: Extract<Input, { action: 'cancel-task' }>,
) {
if (!context.cancelBackgroundTask) {
return { result: 'Error: background task cancellation not available.' };
}
await context.cancelBackgroundTask(input.taskId);
return { result: `Background task ${input.taskId} cancelled.` };
}
async function handleCorrectTask(
context: OrchestrationContext,
input: Extract<Input, { action: 'correct-task' }>,
) {
if (!context.sendCorrectionToTask) {
return await Promise.resolve({ result: 'Error: correction delivery not available.' });
}
const status = context.sendCorrectionToTask(input.taskId, input.correction);
if (status === 'task-not-found') {
return await Promise.resolve({
result: `Task ${input.taskId} not found. It may have already been cleaned up.`,
});
}
if (status === 'task-completed') {
return await Promise.resolve({
result:
`Task ${input.taskId} has already completed. The correction was not delivered. ` +
`Incorporate "${input.correction}" into a new follow-up task instead.`,
});
}
return await Promise.resolve({
result: `Correction sent to task ${input.taskId}: "${input.correction}". The builder will see this on its next step.`,
});
}
// ── Tool factory ────────────────────────────────────────────────────────────
export function createTaskControlTool(context: OrchestrationContext) {
return new Tool('task-control')
.description(
'Manage tasks and background work. Use action="update-checklist" only for lightweight visible checklists that do not need scheduler-driven execution; for coordinated background tasks use create-tasks instead.',
'Manage the visible task checklist. Use action="update-checklist" only for lightweight visible checklists that do not need scheduler-driven execution; for coordinated background tasks use create-tasks instead.',
)
.input(inputSchema)
.handler(async (input: Input) => {
switch (input.action) {
case 'update-checklist':
return await handleUpdateChecklist(context, input);
case 'cancel-task':
return await handleCancelTask(context, input);
case 'correct-task':
return await handleCorrectTask(context, input);
}
})
.build();

28
packages/@n8n/instance-ai/src/types.ts
View File

@ -43,14 +43,12 @@ import type { z } from 'zod';
// Service interfaces — dependency inversion so the package stays decoupled from n8n internals.
// The backend module provides concrete implementations via InstanceAiAdapterService.
import type { WorkflowCodeSnapshotInput } from './debug/run-debug-buffer';
import type { DomainAccessTracker } from './domain-access/domain-access-tracker';
import type { InstanceAiEventBus } from './event-bus/event-bus.interface';
import type { Logger } from './logger';
import type { AgentContextInput } from './tools/agent-context.tool';
import type { McpClientManager } from './mcp/mcp-client-manager';
import type { OrchestratorRunHandoffReason } from './runtime/orchestrator-run-control';
import type { TraceStatus } from './runtime/resumable-stream-executor';
import type { IterationLog } from './storage/iteration-log';
import type { PatchableThreadMemory } from './storage/thread-patch';
import type { BuilderUsageItem } from './stream/usage-accumulator';
@ -1686,8 +1684,6 @@ export interface InstanceAiContext {
markCreated: (credentialType: string) => void;
markCreateFailed: (credentialType: string, errorCode: string) => void;
};
/** Records workflow code snapshots for the run debug buffer (dev tooling). */
recordWorkflowCodeSnapshot?: (snapshot: WorkflowCodeSnapshotInput) => void;
/**
* Setup panel v2 sink for durable `setup-items` snapshots. Wired by the host
* only while the setup panel flag is on, so its presence is the package-side
@ -2149,16 +2145,6 @@ export interface InstanceAiTraceContext {
traceWriter?: TraceWriter;
}
// ── Background task spawning ─────────────────────────────────────────────────
/** Structured result from a background task. The `text` field is the human-readable
* summary; `outcome` carries an optional typed payload consumed by the workflow
* loop controller. */
export interface BackgroundTaskResult {
text: string;
outcome?: Record<string, unknown>;
}
export interface WorkflowTaskService {
reportBuildOutcome(outcome: WorkflowBuildOutcome): Promise<WorkflowLoopAction>;
reportVerificationVerdict(verdict: VerificationResult): Promise<WorkflowLoopAction>;
@ -2184,6 +2170,9 @@ export interface WorkflowTaskService {
// ── Orchestration context (plan tools) ──────────────────────────────────────
/** Terminal status of an agent stream, used to label usage claims and traces. */
export type TraceStatus = 'completed' | 'cancelled' | 'suspended' | 'errored';
export interface OrchestrationContext {
threadId: string;
runId: string;
@ -2241,8 +2230,6 @@ export interface OrchestrationContext {
runtimeSkillCatalog?: RuntimeSkillSource;
/** OAuth2 callback URL for the n8n instance (e.g. http://localhost:5678/rest/oauth2-credential/callback) */
oauth2CallbackUrl?: string;
/** Cancel a running background task by its ID */
cancelBackgroundTask?: (taskId: string) => Promise<void>;
/** Persist and inspect dependency-aware planned tasks for this thread. */
plannedTaskService?: PlannedTaskService;
/** Run one scheduler pass after plan/task state changes. */
@ -2272,15 +2259,6 @@ export interface OrchestrationContext {
domainContext?: InstanceAiContext;
/** Thread-scoped iteration log for accumulating attempt history across retries */
iterationLog?: IterationLog;
/** Send a correction message to a running background task */
sendCorrectionToTask?: (
taskId: string,
correction: string,
) => 'queued' | 'task-completed' | 'task-not-found';
/** Mark the current orchestrator run as making progress. */
touchRun?: () => boolean;
/** Mark a running background task as making progress. */
touchBackgroundTask?: (taskId: string) => boolean;
/** Shared workflow-task state service for build / verify / credential-finalize flows */
workflowTaskService?: WorkflowTaskService;
/** IANA time zone for the current user (e.g. "Europe/Helsinki"). Propagated to sub-agents

4
packages/@n8n/instance-ai/vite.config.ts
View File

@ -27,13 +27,9 @@ const INDEX_LAZY_REQUIRE_TEST_SPECS = new Set([
'./workspace/sandbox-setup',
'./workspace/snapshot-manager',
'./runtime/run-state-registry',
'./runtime/background-task-manager',
'./runtime/terminal-response-guard',
'./runtime/resumable-stream-executor',
'./runtime/stream-runner',
'./parsers/structured-file-parser',
'./parsers/validate-attachments',
'./runtime/liveness-policy',
'./workflow-loop',
'./workflow-loop/runtime',
'./planned-tasks/planned-task-service',

272
packages/cli/src/modules/instance-ai/__tests__/instance-ai-liveness.service.test.ts
View File

@ -1,272 +0,0 @@
import type { InstanceAiEvent } from '@n8n/api-types';
import type { Logger } from '@n8n/backend-common';
import { mock } from 'vitest-mock-extended';
vi.mock('@n8n/instance-ai', async () => ({
...(await vi.importActual<Record<string, unknown>>(
'../../../../../@n8n/instance-ai/src/runtime/liveness-policy',
)),
orchestratorAgentId: (runId: string) => `orchestrator-${runId}`,
}));
import {
createInstanceAiLivenessPolicyConfig,
InstanceAiLivenessPolicy,
type InstanceAiLivenessTimeoutReason,
} from '@n8n/instance-ai';
import {
INSTANCE_AI_RUN_TIMEOUT_REASON,
InstanceAiLivenessService,
type InstanceAiLivenessSweepResult,
} from '../liveness';
import type { InstanceAiRunTimeoutDetails } from '../run-timeout-details';
type TestSuspendedRun = {
runId: string;
threadId: string;
abortController: AbortController;
};
function createLivenessService() {
const policyConfig = createInstanceAiLivenessPolicyConfig();
const policy = new InstanceAiLivenessPolicy(policyConfig);
const runState = {
sweepTimedOut: vi.fn(
(_policy: InstanceAiLivenessPolicy, _now?: number): InstanceAiLivenessSweepResult => ({
activeThreadIds: [] as string[],
suspendedThreadIds: [] as string[],
confirmationRequestIds: [] as string[],
}),
),
cancelActiveRun: vi.fn(
(_threadId: string): { runId: string; abortController: AbortController } | undefined =>
undefined,
),
cancelSuspendedRun: vi.fn((_threadId: string): TestSuspendedRun | undefined => undefined),
getActiveRunId: vi.fn((_threadId: string): string | undefined => undefined),
getPendingConfirmation: vi.fn(
(_requestId: string): { threadId: string } | undefined => undefined,
),
hasPendingConfirmationForThread: vi.fn((_threadId: string) => false),
rejectPendingConfirmation: vi.fn((_requestId: string) => true),
};
const backgroundTasks = {
timeoutTimedOutTasks: vi.fn(
async (
_policy: InstanceAiLivenessPolicy,
_now?: number,
_options?: {
shouldSkipTask?: (task: {
threadId: string;
taskId: string;
role: string;
timeoutReason?: InstanceAiLivenessTimeoutReason;
}) => boolean;
},
) =>
[] as Array<{
threadId: string;
taskId: string;
role: string;
timeoutReason?: InstanceAiLivenessTimeoutReason;
}>,
),
};
const eventBus = {
publish: vi.fn((_threadId: string, _event: InstanceAiEvent) => {}),
};
const finalizeCancelledSuspendedRun = vi.fn(
(_suspended: TestSuspendedRun, _reason: string) => {},
);
const onPendingConfirmationRejected = vi.fn((_requestId: string) => {});
const logger = mock<Logger>();
const service = new InstanceAiLivenessService<TestSuspendedRun>({
policy,
backgroundTaskIdleTimeoutMs: policyConfig.backgroundTaskIdleTimeoutMs,
runState,
backgroundTasks,
eventBus,
finalizeCancelledSuspendedRun,
onPendingConfirmationRejected,
logger,
});
return {
service,
policy,
runState,
backgroundTasks,
eventBus,
finalizeCancelledSuspendedRun,
onPendingConfirmationRejected,
logger,
};
}
describe('InstanceAiLivenessService', () => {
it('cancels timed-out run surfaces without cascading into background tasks', async () => {
const {
service,
policy,
runState,
backgroundTasks,
eventBus,
finalizeCancelledSuspendedRun,
onPendingConfirmationRejected,
} = createLivenessService();
const activeAbortController = new AbortController();
const suspendedAbortController = new AbortController();
const suspended = {
runId: 'run-suspended',
threadId: 'thread-suspended',
abortController: suspendedAbortController,
};
const activeTimeout: InstanceAiRunTimeoutDetails = {
reason: 'idle_timeout',
surface: 'active-run',
timeoutMs: 600_000,
elapsedMs: 650_000,
idleMs: 606_000,
};
const suspendedTimeout: InstanceAiRunTimeoutDetails = {
reason: 'max_lifetime',
surface: 'suspended-run',
timeoutMs: 1_800_000,
elapsedMs: 1_801_000,
idleMs: 900_000,
};
runState.sweepTimedOut.mockReturnValue({
activeThreadIds: ['thread-active'],
suspendedThreadIds: ['thread-suspended'],
confirmationRequestIds: ['request-1'],
activeTimeouts: { 'thread-active': activeTimeout },
suspendedTimeouts: { 'thread-suspended': suspendedTimeout },
});
runState.cancelActiveRun.mockReturnValue({
runId: 'run-active',
abortController: activeAbortController,
});
runState.cancelSuspendedRun.mockReturnValue(suspended);
runState.getPendingConfirmation.mockReturnValue({ threadId: 'thread-confirmation' });
runState.getActiveRunId.mockReturnValue('run-confirmation');
backgroundTasks.timeoutTimedOutTasks.mockResolvedValue([
{
threadId: 'thread-bg',
taskId: 'task-1',
role: 'workflow-builder',
timeoutReason: 'idle_timeout',
},
]);
await service.sweepTimedOutWork(123_456);
expect(runState.sweepTimedOut).toHaveBeenCalledWith(policy, 123_456);
expect(runState.cancelActiveRun).toHaveBeenCalledWith('thread-active');
expect(runState.cancelSuspendedRun).toHaveBeenCalledWith('thread-suspended');
expect(activeAbortController.signal.aborted).toBe(true);
expect(suspendedAbortController.signal.aborted).toBe(true);
expect(finalizeCancelledSuspendedRun).toHaveBeenCalledWith(
suspended,
INSTANCE_AI_RUN_TIMEOUT_REASON,
);
expect(runState.rejectPendingConfirmation).toHaveBeenCalledWith('request-1');
expect(onPendingConfirmationRejected).toHaveBeenCalledWith('request-1');
expect(backgroundTasks.timeoutTimedOutTasks).toHaveBeenCalledWith(
policy,
123_456,
expect.objectContaining({ shouldSkipTask: expect.any(Function) }),
);
const timeoutOptions = backgroundTasks.timeoutTimedOutTasks.mock.calls[0]?.[2];
runState.hasPendingConfirmationForThread.mockReturnValueOnce(true);
expect(
timeoutOptions?.shouldSkipTask?.({
threadId: 'thread-bg',
taskId: 'task-1',
role: 'workflow-builder',
}),
).toBe(true);
expect(runState.hasPendingConfirmationForThread).toHaveBeenCalledWith('thread-bg');
expect(eventBus.publish).toHaveBeenCalledWith(
'thread-active',
expect.objectContaining({ responseId: 'run-timeout:run-active' }),
);
expect(eventBus.publish).toHaveBeenCalledWith(
'thread-confirmation',
expect.objectContaining({ responseId: 'run-timeout:run-confirmation' }),
);
expect(service.consumeRunTimeout('run-active')).toEqual({
timedOut: true,
details: activeTimeout,
});
expect(service.consumeRunTimeout('run-suspended')).toEqual({
timedOut: true,
details: suspendedTimeout,
});
expect(service.hasTimedOutActiveRunThread('thread-active')).toBe(true);
});
it('lets suspended-run finalization own the timeout notice', () => {
const { service, runState, eventBus, finalizeCancelledSuspendedRun } = createLivenessService();
const abortController = new AbortController();
const suspended = {
runId: 'run-suspended',
threadId: 'thread-suspended',
abortController,
};
runState.cancelSuspendedRun.mockReturnValue(suspended);
service.cancelTimedOutSuspendedRun('thread-suspended');
expect(eventBus.publish).not.toHaveBeenCalled();
expect(finalizeCancelledSuspendedRun).toHaveBeenCalledWith(
suspended,
INSTANCE_AI_RUN_TIMEOUT_REASON,
);
expect(abortController.signal.aborted).toBe(true);
});
it('deduplicates timeout notices per run', () => {
const { service, eventBus } = createLivenessService();
service.publishRunTimeoutNotice('thread-1', 'run-1');
service.publishRunTimeoutNotice('thread-1', 'run-1');
expect(eventBus.publish).toHaveBeenCalledTimes(1);
expect(eventBus.publish).toHaveBeenCalledWith(
'thread-1',
expect.objectContaining({ responseId: 'run-timeout:run-1' }),
);
});
it('publishes a notice per run, and evicts the oldest dedupe entry past the cap', () => {
const { service, eventBus } = createLivenessService();
service.publishRunTimeoutNotice('thread-1', 'run-a');
service.publishRunTimeoutNotice('thread-1', 'run-b');
expect(eventBus.publish).toHaveBeenCalledTimes(2);
// Push `run-a` out of the capped FIFO, after which its notice can repeat —
// the accepted trade-off for a bounded guard.
for (let i = 0; i < InstanceAiLivenessService.NOTICE_DEDUPE_CACHE_SIZE; i++) {
service.publishRunTimeoutNotice('thread-1', `filler-${i}`);
}
eventBus.publish.mockClear();
service.publishRunTimeoutNotice('thread-1', 'run-a');
expect(eventBus.publish).toHaveBeenCalledTimes(1);
});
it('keeps sweeping run state when background task timeout handling fails', async () => {
const { service, backgroundTasks, logger } = createLivenessService();
backgroundTasks.timeoutTimedOutTasks.mockRejectedValue(new Error('failed'));
await service.sweepTimedOutWork();
expect(logger.warn).toHaveBeenCalledWith('Failed to sweep timed-out background tasks', {
error: 'failed',
});
});
});

49
packages/cli/src/modules/instance-ai/__tests__/instance-ai-sandbox.service.test.ts
View File

@ -32,14 +32,11 @@ import {
setupSandboxWorkspace,
withSandboxLifecycleTrace,
type InstanceAiContext,
type ManagedBackgroundTask,
} from '@n8n/instance-ai';
import {
InstanceAiSandboxService,
type InstanceAiSandboxBackgroundTasks,
type InstanceAiSandboxProxy,
type InstanceAiSandboxRunState,
type InstanceAiSandboxServiceOptions,
type InstanceAiSandboxSettings,
} from '../sandbox';
@ -49,8 +46,6 @@ const fakeUser = { id: 'user-1' } as User;
type Overrides = {
resolveTracingConfig?: InstanceAiSandboxServiceOptions['resolveTracingConfig'];
config?: Partial<InstanceAiConfig>;
runState?: Partial<InstanceAiSandboxRunState>;
backgroundTasks?: Partial<InstanceAiSandboxBackgroundTasks>;
settingsService?: Partial<InstanceAiSandboxSettings>;
aiService?: Partial<InstanceAiSandboxProxy>;
};
@ -58,15 +53,6 @@ type Overrides = {
function createSandboxService(overrides: Overrides = {}) {
const logger = { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() };
const errorReporter = { error: vi.fn(), warn: vi.fn() } as unknown as ErrorReporter;
const runState: InstanceAiSandboxRunState = {
getActiveRunId: vi.fn(() => undefined),
hasSuspendedRun: vi.fn(() => false),
...overrides.runState,
};
const backgroundTasks: InstanceAiSandboxBackgroundTasks = {
getRunningTasks: vi.fn(() => [] as ManagedBackgroundTask[]),
...overrides.backgroundTasks,
};
const settingsService: InstanceAiSandboxSettings = {
resolveDaytonaConfig: vi.fn(async () => ({ apiKey: 'test-daytona-key' })),
resolveN8nSandboxConfig: vi.fn(async () => ({})),
@ -81,14 +67,12 @@ function createSandboxService(overrides: Overrides = {}) {
config: overrides.config as InstanceAiConfig,
logger,
errorReporter,
runState,
backgroundTasks,
settingsService,
aiService,
resolveTracingConfig: overrides.resolveTracingConfig,
};
const service = new InstanceAiSandboxService(options);
return { service, logger, errorReporter, runState, backgroundTasks, settingsService, aiService };
return { service, logger, errorReporter, settingsService, aiService };
}
describe('InstanceAiSandboxService', () => {
@ -867,37 +851,6 @@ describe('InstanceAiSandboxService', () => {
}
});
it('keeps an in-use sandbox alive when the expiry timer fires', async () => {
vi.useFakeTimers();
try {
const getRunningTasks = vi.fn(() => [{ taskId: 'task-1' }] as ManagedBackgroundTask[]);
const { service } = createSandboxService({
config: { sandboxEnabled: true, sandboxProvider: 'daytona', builderSandboxTtlMs: 1000 },
backgroundTasks: { getRunningTasks },
});
const sandbox = { id: 'sandbox-1' };
const workspace = { init: vi.fn(async () => {}), destroy: vi.fn(async () => {}) };
(createSandbox as Mock).mockResolvedValue(sandbox);
(createWorkspace as Mock).mockReturnValue(workspace);
(setupSandboxWorkspace as Mock).mockResolvedValue(undefined);
await service.getOrCreateWorkspace('thread-1', fakeUser, {} as InstanceAiContext);
vi.advanceTimersByTime(1000);
// In-use sandboxes are touched (re-scheduled) rather than dropped.
const reused = await service.getOrCreateWorkspace(
'thread-1',
fakeUser,
{} as InstanceAiContext,
);
expect(reused).toBeDefined();
expect(createSandbox).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});
it('clears scheduled expiry timers on stopSandboxExpiryTimers', async () => {
vi.useFakeTimers();
const clearTimeoutSpy = vi.spyOn(global, 'clearTimeout');

306
packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts
View File

@ -1,9 +1,3 @@
const terminalOutcomeStorageMock = {
getUndelivered: vi.fn(),
markDelivered: vi.fn(),
upsert: vi.fn(),
};
// Manual mock — must be declared before any import that touches the mocked module.
vi.mock('@n8n/instance-ai', () => ({
orchestratorAgentId: (runId: string) => `orchestrator-${runId}`,
@ -92,16 +86,10 @@ vi.mock('@n8n/instance-ai', () => ({
};
}
},
TerminalOutcomeStorage: class {
getUndelivered = terminalOutcomeStorageMock.getUndelivered;
markDelivered = terminalOutcomeStorageMock.markDelivered;
upsert = terminalOutcomeStorageMock.upsert;
},
}));
import type { Mock } from 'vitest';
import type { InstanceAiEvent } from '@n8n/api-types';
import type { ManagedBackgroundTask, TerminalOutcome } from '@n8n/instance-ai';
import {
InstanceAiTerminalOutcomeService,
@ -118,28 +106,9 @@ type Deps = {
telemetry: { track: Mock };
errorReporter: { report: Mock };
logger: { warn: Mock; debug: Mock; error: Mock };
runState: { getRunIdsForMessageGroup: Mock; cancelThread: Mock };
suspendedThreads: { dropPendingConfirmationsForThread: Mock };
tracing: { finalizeRunTracing: Mock; buildMessageTraceMetadata: Mock };
publishRunFinish: Mock;
runState: { getRunIdsForMessageGroup: Mock };
};
function makeTerminalOutcome(overrides: Partial<TerminalOutcome> = {}): TerminalOutcome {
return {
id: 'group-1:task-1:completed',
threadId: 'thread-a',
runId: 'run-1',
messageGroupId: 'group-1',
correlationId: 'message-1',
taskId: 'task-1',
agentId: 'agent-builder',
status: 'completed',
userFacingMessage: 'The background workflow-builder task finished.',
createdAt: '2026-05-01T00:00:00.000Z',
...overrides,
};
}
function createService(): {
service: InstanceAiTerminalOutcomeService;
deps: Deps;
@ -159,35 +128,15 @@ function createService(): {
logger: { warn: vi.fn(), debug: vi.fn(), error: vi.fn() },
runState: {
getRunIdsForMessageGroup: vi.fn(() => ['run-1']),
cancelThread: vi.fn(),
},
suspendedThreads: { dropPendingConfirmationsForThread: vi.fn(async () => {}) },
tracing: {
finalizeRunTracing: vi.fn(async () => {}),
buildMessageTraceMetadata: vi.fn(() => ({ completion_source: 'orchestrator' })),
},
publishRunFinish: vi.fn(
(_threadId: string, runId: string, status: 'completed' | 'cancelled' | 'errored') => {
events.push({
type: 'run-finish',
runId,
agentId: 'agent-001',
payload: { status: status === 'errored' ? 'error' : status },
} as InstanceAiEvent);
},
),
};
const options = {
eventBus: deps.eventBus,
agentMemory: {},
telemetry: deps.telemetry,
errorReporter: deps.errorReporter,
logger: deps.logger,
runState: deps.runState,
suspendedThreads: deps.suspendedThreads,
tracing: deps.tracing,
publishRunFinish: deps.publishRunFinish,
} as unknown as InstanceAiTerminalOutcomeServiceOptions;
return { service: new InstanceAiTerminalOutcomeService(options), deps };
@ -195,230 +144,17 @@ function createService(): {
beforeEach(() => {
vi.clearAllMocks();
terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([]);
terminalOutcomeStorageMock.markDelivered.mockResolvedValue(undefined);
terminalOutcomeStorageMock.upsert.mockResolvedValue(undefined);
});
describe('InstanceAiTerminalOutcomeService — terminal outcome replay', () => {
it('publishes recovered background outcomes and marks them delivered', async () => {
const outcome = makeTerminalOutcome();
terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]);
const { service, deps } = createService();
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(deps.eventBus.publish).toHaveBeenCalledWith('thread-a', {
type: 'text-block',
runId: outcome.runId,
agentId: 'orchestrator-run-1',
responseId: `background-outcome:${outcome.id}`,
payload: { text: outcome.userFacingMessage },
});
expect(terminalOutcomeStorageMock.markDelivered).toHaveBeenCalledWith(
'thread-a',
outcome.id,
expect.any(String),
);
expect(deps.telemetry.track).toHaveBeenCalledWith(
'instance_ai_terminal_response_decision',
expect.objectContaining({ source: 'terminal_outcome_replay', action: 'replay_event' }),
);
});
it('deduplicates replay by response id only', async () => {
const outcome = makeTerminalOutcome({ id: 'group-1:task-2:completed' });
terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]);
const { service, deps } = createService();
// Same text under a different response id must not suppress the replay.
deps.eventBus.events.push({
type: 'text-block',
runId: outcome.runId,
agentId: 'orchestrator-run-1',
responseId: 'background-outcome:different-id',
payload: { text: outcome.userFacingMessage },
} as InstanceAiEvent);
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(deps.eventBus.publish).toHaveBeenCalledTimes(1);
expect(deps.eventBus.publish).toHaveBeenCalledWith(
'thread-a',
expect.objectContaining({ responseId: `background-outcome:${outcome.id}` }),
);
// A second replay finds the exact response id already emitted and skips it.
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(deps.eventBus.publish).toHaveBeenCalledTimes(1);
expect(deps.telemetry.track).toHaveBeenCalledWith(
'instance_ai_terminal_response_decision',
expect.objectContaining({ source: 'terminal_outcome_replay', action: 'already-emitted' }),
);
});
it('leaves the outcome undelivered when publishing the line fails', async () => {
const outcome = makeTerminalOutcome();
terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]);
const { service, deps } = createService();
deps.eventBus.publish.mockImplementation(() => {
throw new Error('bus unavailable');
});
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled();
expect(deps.logger.warn).toHaveBeenCalledWith(
'Failed to replay Instance AI terminal outcome',
expect.objectContaining({ threadId: 'thread-a', runId: outcome.runId }),
);
});
it('leaves the outcome undelivered when the drain drops the published line', async () => {
const outcome = makeTerminalOutcome();
terminalOutcomeStorageMock.getUndelivered.mockResolvedValue([outcome]);
const { service, deps } = createService();
// The publish enqueues without throwing, but the line never reaches the
// log — the shape of a dropped drain batch.
deps.eventBus.publish.mockImplementation(() => {});
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled();
expect(deps.telemetry.track).not.toHaveBeenCalledWith(
'instance_ai_terminal_response_decision',
expect.anything(),
);
});
it('checks persisted outcomes on repeated replay calls', async () => {
const { service } = createService();
await service.replayUndeliveredTerminalOutcomes('thread-a');
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(terminalOutcomeStorageMock.getUndelivered).toHaveBeenCalledTimes(2);
});
});
describe('InstanceAiTerminalOutcomeService — background outcome recording', () => {
function makeTask(overrides: Partial<ManagedBackgroundTask> = {}): ManagedBackgroundTask {
return {
taskId: 'task-1',
threadId: 'thread-a',
runId: 'run-1',
role: 'workflow-builder',
agentId: 'agent-builder',
status: 'completed',
result: 'done',
startedAt: 0,
lastActivityAt: 0,
abortController: new AbortController(),
corrections: [],
messageGroupId: 'group-1',
...overrides,
} as ManagedBackgroundTask;
}
it('persists, publishes, and marks the outcome delivered on success', async () => {
const { service, deps } = createService();
await service.recordBackgroundTerminalOutcome(makeTask());
expect(terminalOutcomeStorageMock.upsert).toHaveBeenCalledTimes(1);
expect(deps.eventBus.publish).toHaveBeenCalledWith(
'thread-a',
expect.objectContaining({
type: 'text-block',
payload: { text: 'The background workflow-builder task finished.' },
}),
);
expect(terminalOutcomeStorageMock.markDelivered).toHaveBeenCalledTimes(1);
});
it('keeps the outcome pending and replays it later when persistence fails', async () => {
terminalOutcomeStorageMock.upsert.mockRejectedValueOnce(new Error('db down'));
const { service, deps } = createService();
await service.recordBackgroundTerminalOutcome(makeTask());
expect(deps.telemetry.track).toHaveBeenCalledWith(
'instance_ai_terminal_outcome_persistence_failure',
expect.objectContaining({ phase: 'metadata' }),
);
expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled();
// The pending outcome is recovered on the next replay.
await service.replayUndeliveredTerminalOutcomes('thread-a');
expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled();
});
it('does not mark the outcome delivered when the drain drops the line', async () => {
const { service, deps } = createService();
deps.eventBus.publish.mockImplementation(() => {});
await service.recordBackgroundTerminalOutcome(makeTask());
expect(terminalOutcomeStorageMock.upsert).toHaveBeenCalledTimes(1);
expect(terminalOutcomeStorageMock.markDelivered).not.toHaveBeenCalled();
expect(deps.telemetry.track).toHaveBeenCalledWith(
'instance_ai_terminal_outcome_persistence_failure',
expect.objectContaining({ phase: 'event' }),
);
expect(deps.telemetry.track).not.toHaveBeenCalledWith(
'instance_ai_terminal_response_decision',
expect.anything(),
);
});
});
describe('InstanceAiTerminalOutcomeService — durable-log outcome lines', () => {
it('publishes the outcome line as a persisted text-block, not a trailing delta', async () => {
// A trailing delta would race the coalescer's idle flush on an immediate
// page reload; a text-block is persisted before it is emitted live.
const { deps } = createService();
const service = new InstanceAiTerminalOutcomeService({
eventBus: deps.eventBus,
agentMemory: {},
telemetry: deps.telemetry,
logger: deps.logger,
runState: deps.runState,
suspendedThreads: deps.suspendedThreads,
tracing: deps.tracing,
publishRunFinish: deps.publishRunFinish,
} as never);
await service.recordBackgroundTerminalOutcome({
taskId: 'task-dl',
threadId: 'thread-dl',
runId: 'run-dl',
role: 'workflow-builder',
agentId: 'agent-builder',
status: 'cancelled',
result: undefined,
startedAt: 0,
lastActivityAt: 0,
abortController: new AbortController(),
corrections: [],
} as never);
const line = deps.eventBus.events.find(
(event) => event.type === 'text-block' || event.type === 'text-delta',
);
expect(line?.type).toBe('text-block');
});
});
describe('InstanceAiTerminalOutcomeService — terminal response guard wiring', () => {
it('publishes fallback output before run-finish on a silent completed run', async () => {
it('publishes fallback output on a silent completed run', async () => {
const { service, deps } = createService();
await service.evaluateTerminalResponse('thread-a', 'run-1', 'completed', {
messageGroupId: 'group-1',
});
deps.publishRunFinish('thread-a', 'run-1', 'completed');
expect(deps.eventBus.events.map((event) => event.type)).toEqual(['text-delta', 'run-finish']);
expect(deps.eventBus.events.map((event) => event.type)).toEqual(['text-delta']);
});
it('reports a silent completed run so the stall is not lost', async () => {
@ -465,16 +201,15 @@ describe('InstanceAiTerminalOutcomeService — terminal response guard wiring',
expect(deps.eventBus.events).toEqual([]);
});
it('publishes fallback error before run-finish on a silent failed run', async () => {
it('publishes fallback error on a silent failed run', async () => {
const { service, deps } = createService();
await service.evaluateTerminalResponse('thread-a', 'run-1', 'errored', {
messageGroupId: 'group-1',
errorMessage: 'Safe user-facing error',
});
deps.publishRunFinish('thread-a', 'run-1', 'errored');
expect(deps.eventBus.events.map((event) => event.type)).toEqual(['error', 'run-finish']);
expect(deps.eventBus.events.map((event) => event.type)).toEqual(['error']);
});
it('forwards a structured error code onto the emitted error event', async () => {
@ -490,40 +225,15 @@ describe('InstanceAiTerminalOutcomeService — terminal response guard wiring',
expect(errorEvent?.payload).toMatchObject({ code: 'quota_exhausted' });
});
it('clears malformed confirmation suspension and finishes the run after the guard error', async () => {
it('publishes the guard error for a malformed confirmation', async () => {
const { service, deps } = createService();
const abortController = new AbortController();
const decision = await service.evaluateWaitingResponse('thread-a', 'run-1', undefined, {
messageGroupId: 'group-1',
});
expect(decision?.reason).toBe('confirmation-invalid');
const finalization = await service.finishInvalidConfirmationRun({
threadId: 'thread-a',
runId: 'run-1',
promptVersion: 'progressive@1',
abortController,
});
expect(finalization.status).toBe('error');
expect(finalization.reason).toBe('invalid_confirmation_payload');
expect(deps.publishRunFinish).toHaveBeenCalledWith(
'thread-a',
'run-1',
'errored',
expect.any(String),
'progressive@1',
);
expect(deps.runState.cancelThread).toHaveBeenCalledWith('thread-a');
expect(deps.suspendedThreads.dropPendingConfirmationsForThread).toHaveBeenCalledWith(
'thread-a',
);
expect(abortController.signal.aborted).toBe(true);
expect(deps.eventBus.events.at(-1)).toMatchObject({
type: 'run-finish',
payload: { status: 'error' },
});
expect(deps.eventBus.events.at(-1)).toMatchObject({ type: 'error' });
});
it('reads events across the message group when a group id is provided', async () => {

91
packages/cli/src/modules/instance-ai/__tests__/instance-ai-test.controller.test.ts
View File

@ -11,32 +11,21 @@ vi.mock('../eval/execution.service', () => ({
EvalExecutionService: vi.fn(),
}));
import type { ProjectRepository, UserRepository, WorkflowRepository } from '@n8n/db';
import type { WorkflowRepository } from '@n8n/db';
import type { Request, Response } from 'express';
import { mock } from 'vitest-mock-extended';
import { ForbiddenError } from '@n8n/errors';
import type { InstanceAiMemoryService } from '../instance-ai-memory.service';
import { InstanceAiTestController } from '../instance-ai-test.controller';
import type { InstanceAiService } from '../instance-ai.service';
import type { InstanceAiThreadRepository } from '../repositories/instance-ai-thread.repository';
import type { AgentExecutionThreadRepository } from '../../agents/repositories/agent-execution-thread.repository';
describe('InstanceAiTestController', () => {
const instanceAiService = mock<InstanceAiService>();
const threadRepo = mock<InstanceAiThreadRepository>();
const threadRepo = mock<AgentExecutionThreadRepository>();
const workflowRepo = mock<WorkflowRepository>();
const userRepo = mock<UserRepository>();
const memoryService = mock<InstanceAiMemoryService>();
const projectRepo = mock<ProjectRepository>();
const controller = new InstanceAiTestController(
instanceAiService,
threadRepo,
workflowRepo,
userRepo,
memoryService,
projectRepo,
);
const controller = new InstanceAiTestController(instanceAiService, threadRepo, workflowRepo);
const originalEnv = process.env;
@ -109,28 +98,29 @@ describe('InstanceAiTestController', () => {
});
describe('getIdleState', () => {
it('should report idle when there is no running work', () => {
instanceAiService.hasRunningWorkForTest.mockReturnValue(false);
it('should report idle when no Assistant thread is running', async () => {
threadRepo.find.mockResolvedValue([{ id: 't1' }] as never);
instanceAiService.getLiveRun.mockResolvedValue({ status: 'suspended', runIds: [] });
const result = controller.getIdleState();
const result = await controller.getIdleState();
expect(instanceAiService.hasRunningWorkForTest).toHaveBeenCalled();
expect(instanceAiService.getLiveRun).toHaveBeenCalledWith('t1');
expect(result).toEqual({ idle: true });
});
it('should report non-idle when there is running work', () => {
instanceAiService.hasRunningWorkForTest.mockReturnValue(true);
it('should report non-idle when an Assistant thread is running', async () => {
threadRepo.find.mockResolvedValue([{ id: 't1' }] as never);
instanceAiService.getLiveRun.mockResolvedValue({ status: 'running', runIds: [] });
const result = controller.getIdleState();
const result = await controller.getIdleState();
expect(instanceAiService.hasRunningWorkForTest).toHaveBeenCalled();
expect(result).toEqual({ idle: false });
});
it('should throw ForbiddenError when trace replay is not enabled', () => {
it('should throw ForbiddenError when trace replay is not enabled', async () => {
delete process.env.E2E_TESTS;
expect(() => controller.getIdleState()).toThrow(ForbiddenError);
await expect(controller.getIdleState()).rejects.toThrow(ForbiddenError);
});
});
@ -154,66 +144,17 @@ describe('InstanceAiTestController', () => {
});
});
describe('background timeout simulation', () => {
it('should create an owned thread and start a stuck background task', async () => {
const user = { id: 'user-1' } as never;
const simulation = {
threadId: 'thread-1',
runId: 'run-1',
messageGroupId: 'mg-1',
taskId: 'task-1',
agentId: 'agent-1',
timeoutAt: 123,
};
userRepo.findOneByOrFail.mockResolvedValue(user);
projectRepo.getPersonalProjectForUserOrFail.mockResolvedValue({
id: 'personal-project-id',
} as never);
instanceAiService.startStuckBackgroundTaskForTest.mockResolvedValue(simulation);
const result = await controller.startBackgroundTimeoutSimulation({
userId: 'user-1',
threadId: 'thread-1',
});
expect(memoryService.ensureThread).toHaveBeenCalledWith(
'user-1',
'thread-1',
'personal-project-id',
{ source: 'playwright', origin: 'internal' },
);
expect(instanceAiService.startStuckBackgroundTaskForTest).toHaveBeenCalledWith(
user,
'thread-1',
);
expect(result).toEqual(simulation);
});
it('should run the liveness sweep at a simulated time', async () => {
const result = await controller.runLivenessSweep({ now: 123 });
expect(instanceAiService.runLivenessSweepForTest).toHaveBeenCalledWith(123);
expect(result).toEqual({ ok: true });
});
});
describe('reset', () => {
it('should clear per-thread state and delete threads + workflows', async () => {
threadRepo.find.mockResolvedValue([{ id: 't1' }, { id: 't2' }] as never);
workflowRepo.find.mockResolvedValue([{ id: 'w1' }, { id: 'w2' }, { id: 'w3' }] as never);
const deleteExecute = vi.fn().mockResolvedValue(undefined);
const deleteQb = { execute: deleteExecute };
const queryBuilder = { delete: vi.fn().mockReturnValue(deleteQb) };
threadRepo.createQueryBuilder.mockReturnValue(queryBuilder as never);
const result = await controller.reset();
expect(instanceAiService.cancelAllBackgroundTasks).toHaveBeenCalled();
expect(instanceAiService.clearTraceContextsForTest).toHaveBeenCalled();
expect(instanceAiService.clearThreadState).toHaveBeenCalledWith('t1');
expect(instanceAiService.clearThreadState).toHaveBeenCalledWith('t2');
expect(queryBuilder.delete).toHaveBeenCalled();
expect(deleteExecute).toHaveBeenCalled();
expect(threadRepo.delete).toHaveBeenCalledWith({ agentId: 'n8n-assistant' });
expect(workflowRepo.delete).toHaveBeenCalledWith('w1');
expect(workflowRepo.delete).toHaveBeenCalledWith('w2');
expect(workflowRepo.delete).toHaveBeenCalledWith('w3');

18
packages/cli/src/modules/instance-ai/__tests__/instance-ai-tracing.service.test.ts
View File

@ -21,7 +21,6 @@ import {
type InstanceAiTracingAiService,
type InstanceAiTracingEventLog,
type InstanceAiTracingEventReader,
type InstanceAiTracingRunState,
} from '../tracing';
type FakeTraceRun = {
@ -53,7 +52,6 @@ function createService(
overrides: {
logger?: Partial<Logger>;
eventReader?: Partial<InstanceAiTracingEventReader>;
runState?: Partial<InstanceAiTracingRunState>;
eventLog?: Partial<InstanceAiTracingEventLog>;
aiService?: Partial<InstanceAiTracingAiService>;
} = {},
@ -63,10 +61,6 @@ function createService(
getEventsForRun: vi.fn(async () => []),
...overrides.eventReader,
};
const runState: InstanceAiTracingRunState = {
attachTracing: vi.fn(),
...overrides.runState,
};
const eventLog: InstanceAiTracingEventLog = {
findLangsmithAnchor: vi.fn(async () => undefined),
...overrides.eventLog,
@ -80,12 +74,11 @@ function createService(
const service = new InstanceAiTracingService({
logger,
eventReader,
runState,
eventLog,
aiService,
});
return { service, logger, eventReader, runState, eventLog, aiService };
return { service, logger, eventReader, eventLog, aiService };
}
describe('InstanceAiTracingService', () => {
@ -120,7 +113,7 @@ describe('InstanceAiTracingService', () => {
it('degrades instead of throwing when the events read fails', async () => {
// Most callers run inside a run's terminal catch block: a throw here
// would skip run-finish and hang the client until the liveness sweep.
// would skip run-finish and hang the client.
const { service, logger } = createService({
eventReader: {
getEventsForRun: vi.fn(async () => {
@ -132,7 +125,6 @@ describe('InstanceAiTracingService', () => {
await expect(
service.buildMessageTraceMetadata('thread-1', 'run-1', {
status: 'cancelled',
cancellationReason: 'timeout',
}),
).resolves.toEqual({
completion_source: 'orchestrator',
@ -140,7 +132,7 @@ describe('InstanceAiTracingService', () => {
// it can't be read as a genuinely empty run.
first_visible_state: 'empty',
first_visible_state_unavailable: true,
cancellation_type: 'idle_timeout',
cancellation_type: 'explicit',
});
expect(logger.warn).toHaveBeenCalledWith(
'Failed to read run events for Instance AI trace metadata',
@ -171,7 +163,7 @@ describe('InstanceAiTracingService', () => {
describe('resume trace registration', () => {
it('keeps a resume trace detached until the checkpoint claim succeeds', async () => {
const { service, runState } = createService();
const { service } = createService();
const baseTracing = makeTraceContext({
rootRun: { id: 'root-base', traceId: 'trace-base' },
});
@ -195,13 +187,11 @@ describe('InstanceAiTracingService', () => {
expect(result).toBe(resumeTracing);
expect(service.getTraceContext('run-1')).toBeUndefined();
expect(runState.attachTracing).not.toHaveBeenCalled();
service.registerTraceContext('run-1', 'thread-a', resumeTracing, 'group-1');
expect(service.getTraceContext('run-1')).toBe(resumeTracing);
expect(service.getMessageGroupId('run-1')).toBe('group-1');
expect(runState.attachTracing).toHaveBeenCalledWith('thread-a', resumeTracing);
});
});

37
packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts
View File

@ -44,7 +44,6 @@ import type {
InstanceAiEvalRestoreThreadRequest,
InstanceAiEvalThreadMemoryResponse,
InstanceAiSendMessageRequest,
InstanceAiCorrectTaskRequest,
InstanceAiConfirmRequest,
InstanceAiEnsureThreadRequest,
InstanceAiThreadMessagesQuery,
@ -1526,41 +1525,6 @@ describe('InstanceAiController', () => {
});
});
describe('cancelTask', () => {
it('should require instanceAi:message scope', () => {
expect(scopeOf('cancelTask')).toEqual({ scope: 'instanceAi:message', globalOnly: true });
});
it('should cancel the background task', async () => {
memoryService.checkThreadOwnership.mockResolvedValue('owned');
const result = await controller.cancelTask(req, res, THREAD_ID, 'task-1');
expect(result).toEqual({ ok: true });
expect(instanceAiService.routeCancelBackgroundTask).toHaveBeenCalledWith(THREAD_ID, 'task-1');
});
});
describe('correctTask', () => {
it('should require instanceAi:message scope', () => {
expect(scopeOf('correctTask')).toEqual({ scope: 'instanceAi:message', globalOnly: true });
});
it('should send correction to the task', async () => {
memoryService.checkThreadOwnership.mockResolvedValue('owned');
const payload = mock<InstanceAiCorrectTaskRequest>({ message: 'fix this' });
const result = await controller.correctTask(req, res, THREAD_ID, 'task-1', payload);
expect(result).toEqual({ ok: true });
expect(instanceAiService.routeCorrectionToTask).toHaveBeenCalledWith(
THREAD_ID,
'task-1',
'fix this',
);
});
});
describe('preference card routes', () => {
it('should require instanceAi:message scope', () => {
expect(scopeOf('undoPreference')).toEqual({
@ -2290,7 +2254,6 @@ describe('InstanceAiController', () => {
const result = await controller.getThreadMessages(req, res, THREAD_ID, query);
expect(result).toMatchObject({ nextEventId: 42 });
expect(instanceAiService.replayUndeliveredTerminalOutcomes).toHaveBeenCalledWith(THREAD_ID);
expect(memoryService.getRichMessages).toHaveBeenCalledWith(USER_ID, THREAD_ID, {
limit: 50,
page: 0,

203
packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts
View File

@ -6054,209 +6054,6 @@ describe('InstanceAiService — deterministic workflow setup follow-up', () => {
});
});
type TaskControlInternals = {
instanceSettings: { isMultiMain: boolean };
publisher: { publishCommand: Mock };
backgroundTasks: { getTaskSnapshots: Mock };
logger: { error: Mock };
runState: { hasLiveRun: Mock };
eventLog: { flush: Mock };
interruptedRunSweeper: { cancelUnfinishedRuns: Mock };
sendCorrectionToTask: Mock;
cancelBackgroundTask: Mock;
cancelRun: Mock;
clearThreadState: Mock;
routeCorrectionToTask: InstanceAiService['routeCorrectionToTask'];
routeCancelBackgroundTask: InstanceAiService['routeCancelBackgroundTask'];
routeCancelRun: InstanceAiService['routeCancelRun'];
routeClearThreadState: InstanceAiService['routeClearThreadState'];
handleRelayTaskControl: InstanceAiService['handleRelayTaskControl'];
};
function buildTaskControlService(isMultiMain: boolean): TaskControlInternals {
const service = Object.create(InstanceAiService.prototype) as unknown as TaskControlInternals;
service.instanceSettings = { isMultiMain };
service.publisher = { publishCommand: vi.fn().mockResolvedValue(undefined) };
service.backgroundTasks = { getTaskSnapshots: vi.fn(() => []) };
service.logger = { error: vi.fn() };
// Inert cancel-time zombie fallback — its behavior is covered by the
// dedicated 'routeCancelRun zombie fallback' suite; these tests are about
// routing.
service.runState = { hasLiveRun: vi.fn(() => false) };
service.eventLog = { flush: vi.fn(async () => {}) };
service.interruptedRunSweeper = { cancelUnfinishedRuns: vi.fn(async () => 0) };
service.sendCorrectionToTask = vi.fn(() => 'queued');
service.cancelBackgroundTask = vi.fn();
service.cancelRun = vi.fn();
service.clearThreadState = vi.fn(async () => {});
return service;
}
describe('InstanceAiService — cross-main task-control routing', () => {
describe('routeCorrectionToTask', () => {
it('broadcasts when the task is not local and multi-main', async () => {
const service = buildTaskControlService(true);
service.sendCorrectionToTask.mockReturnValue('task-not-found');
await service.routeCorrectionToTask('thread-a', 'task-1', 'try again');
expect(service.sendCorrectionToTask).toHaveBeenCalledWith('thread-a', 'task-1', 'try again');
expect(service.publisher.publishCommand).toHaveBeenCalledWith({
command: 'relay-instance-ai-task-control',
payload: {
threadId: 'thread-a',
taskId: 'task-1',
action: 'correct',
correction: 'try again',
},
});
});
it('does not broadcast when the correction was applied locally', async () => {
const service = buildTaskControlService(true);
service.sendCorrectionToTask.mockReturnValue('queued');
await service.routeCorrectionToTask('thread-a', 'task-1', 'try again');
expect(service.publisher.publishCommand).not.toHaveBeenCalled();
});
it('does not broadcast in single-main even on a local miss', async () => {
const service = buildTaskControlService(false);
service.sendCorrectionToTask.mockReturnValue('task-not-found');
await service.routeCorrectionToTask('thread-a', 'task-1', 'try again');
expect(service.publisher.publishCommand).not.toHaveBeenCalled();
});
});
describe('routeCancelBackgroundTask', () => {
it('broadcasts when the task is not local and multi-main', async () => {
const service = buildTaskControlService(true);
service.backgroundTasks.getTaskSnapshots.mockReturnValue([]);
await service.routeCancelBackgroundTask('thread-a', 'task-1');
expect(service.cancelBackgroundTask).toHaveBeenCalledWith('thread-a', 'task-1');
expect(service.publisher.publishCommand).toHaveBeenCalledWith({
command: 'relay-instance-ai-task-control',
payload: { threadId: 'thread-a', taskId: 'task-1', action: 'cancel-task' },
});
});
it('does not broadcast when the task is local', async () => {
const service = buildTaskControlService(true);
service.backgroundTasks.getTaskSnapshots.mockReturnValue([{ taskId: 'task-1' }]);
await service.routeCancelBackgroundTask('thread-a', 'task-1');
expect(service.cancelBackgroundTask).toHaveBeenCalledWith('thread-a', 'task-1');
expect(service.publisher.publishCommand).not.toHaveBeenCalled();
});
});
describe('routeCancelRun / routeClearThreadState', () => {
it('routeCancelRun cancels locally and always fans out in multi-main', async () => {
const service = buildTaskControlService(true);
await service.routeCancelRun('thread-a');
expect(service.cancelRun).toHaveBeenCalledWith('thread-a');
expect(service.publisher.publishCommand).toHaveBeenCalledWith({
command: 'relay-instance-ai-task-control',
payload: { threadId: 'thread-a', action: 'cancel-thread' },
});
});
it('routeCancelRun does not fan out in single-main', async () => {
const service = buildTaskControlService(false);
await service.routeCancelRun('thread-a');
expect(service.cancelRun).toHaveBeenCalledWith('thread-a');
expect(service.publisher.publishCommand).not.toHaveBeenCalled();
});
it('routeClearThreadState clears locally and fans out in multi-main', async () => {
const service = buildTaskControlService(true);
await service.routeClearThreadState('thread-a', 'owner-a');
expect(service.clearThreadState).toHaveBeenCalledWith('thread-a', 'owner-a');
expect(service.publisher.publishCommand).toHaveBeenCalledWith({
command: 'relay-instance-ai-task-control',
payload: { threadId: 'thread-a', action: 'clear-thread', userId: 'owner-a' },
});
});
});
describe('handleRelayTaskControl', () => {
it('applies a relayed correction via the local method and never re-broadcasts', async () => {
const service = buildTaskControlService(true);
await service.handleRelayTaskControl({
threadId: 'thread-a',
taskId: 'task-1',
action: 'correct',
correction: 'fix it',
});
expect(service.sendCorrectionToTask).toHaveBeenCalledWith('thread-a', 'task-1', 'fix it');
expect(service.publisher.publishCommand).not.toHaveBeenCalled();
});
it('ignores a correction relay missing its correction text', async () => {
const service = buildTaskControlService(true);
await service.handleRelayTaskControl({
threadId: 'thread-a',
taskId: 'task-1',
action: 'correct',
});
expect(service.sendCorrectionToTask).not.toHaveBeenCalled();
});
it.each(['owner-a', undefined])(
'routes task controls with owner %s to the local methods',
async (userId) => {
const service = buildTaskControlService(true);
await service.handleRelayTaskControl({
threadId: 'thread-a',
taskId: 'task-1',
action: 'cancel-task',
});
await service.handleRelayTaskControl({ threadId: 'thread-a', action: 'cancel-thread' });
await service.handleRelayTaskControl({
threadId: 'thread-a',
action: 'clear-thread',
userId,
});
expect(service.cancelBackgroundTask).toHaveBeenCalledWith('thread-a', 'task-1');
expect(service.cancelRun).toHaveBeenCalledWith('thread-a');
expect(service.clearThreadState).toHaveBeenCalledWith('thread-a', userId);
expect(service.publisher.publishCommand).not.toHaveBeenCalled();
},
);
it('swallows and logs errors from a local action (no unhandled rejection on the sibling main)', async () => {
const service = buildTaskControlService(true);
service.clearThreadState.mockRejectedValue(new Error('db exploded'));
await expect(
service.handleRelayTaskControl({ threadId: 'thread-a', action: 'clear-thread' }),
).resolves.toBeUndefined();
expect(service.logger.error).toHaveBeenCalledWith(
'Failed to apply relayed Instance AI task-control',
expect.objectContaining({ threadId: 'thread-a', action: 'clear-thread' }),
);
});
});
});
describe('InstanceAiService — clearThreadState agent-builder cleanup', () => {
type Internals = {
threadPushRef: Map<string, string>;

119
packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts
View File

@ -1,119 +0,0 @@
import type { User } from '@n8n/db';
import { RunDebugBuffer, RunStateRegistry } from '@n8n/instance-ai';
import { InstanceAiService } from '../instance-ai.service';
type RunDebugGatingInternals = {
instanceAiConfig: { runDebugEnabled: boolean };
aiConfig: { modelStreamIdleTimeoutMs: number; modelStreamFirstOutputTimeoutMs: number };
runDebugBuffer: RunDebugBuffer;
runState: RunStateRegistry<User>;
buildOrchestratorAgentStreamOptions: (
user: User,
threadId: string,
runId: string,
signal: AbortSignal,
) => Record<string, unknown>;
buildOrchestratorResumeAgentOptions: (
user: User,
threadId: string,
runId: string,
agentRunId: string,
toolCallId: string,
signal: AbortSignal,
) => Record<string, unknown>;
getRunDebug: (runId: string) => ReturnType<RunDebugBuffer['get']>;
};
function createRunDebugGatingService(runDebugEnabled: boolean): RunDebugGatingInternals {
const service = Object.create(InstanceAiService.prototype) as RunDebugGatingInternals;
service.instanceAiConfig = { runDebugEnabled };
service.aiConfig = { modelStreamIdleTimeoutMs: 90_000, modelStreamFirstOutputTimeoutMs: 180_000 };
service.runDebugBuffer = new RunDebugBuffer();
service.runState = new RunStateRegistry((user: User) => user.id);
return service;
}
describe('InstanceAiService run debug gating', () => {
const user = { id: 'user-1' } as User;
const threadId = 'thread-1';
const runId = 'run-1';
const signal = new AbortController().signal;
it('does not attach step hooks or create run records when run debug is disabled', () => {
const service = createRunDebugGatingService(false);
const streamOptions = service.buildOrchestratorAgentStreamOptions(
user,
threadId,
runId,
signal,
);
const resumeOptions = service.buildOrchestratorResumeAgentOptions(
user,
threadId,
runId,
'agent-run-1',
'tool-call-1',
signal,
);
expect(streamOptions.onStepStart).toBeUndefined();
expect(streamOptions.onStepEnd).toBeUndefined();
expect(streamOptions.onStepFinish).toBeUndefined();
expect(resumeOptions.onStepStart).toBeUndefined();
expect(resumeOptions.onStepEnd).toBeUndefined();
expect(resumeOptions.onStepFinish).toBeUndefined();
expect(service.getRunDebug(runId)).toBeUndefined();
// Both terminal paths opt into raw-usage recovery so stopped/errored runs bill.
expect(streamOptions.recoverUsageOnAbort).toBe(true);
expect(resumeOptions.recoverUsageOnAbort).toBe(true);
expect(streamOptions.modelStreamIdleTimeoutMs).toBe(90_000);
expect(resumeOptions.modelStreamIdleTimeoutMs).toBe(90_000);
expect(streamOptions.modelStreamFirstOutputTimeoutMs).toBe(180_000);
expect(resumeOptions.modelStreamFirstOutputTimeoutMs).toBe(180_000);
// Both paths must carry the run's signal, or a stop cannot reach the agent
// loop and its sub-agents (AGENT-453).
expect(streamOptions.abortSignal).toBe(signal);
expect(resumeOptions.abortSignal).toBe(signal);
// Both paths must carry the request-level Anthropic cache directive; without it
// on resume, HITL turns reprocess the whole conversation uncached (INS-759).
const cacheDirective = { anthropic: { cacheControl: { type: 'ephemeral' } } };
expect(streamOptions.providerOptions).toEqual(cacheDirective);
expect(resumeOptions.providerOptions).toEqual(cacheDirective);
});
it('attaches step hooks and creates run records when run debug is enabled', () => {
const service = createRunDebugGatingService(true);
const streamOptions = service.buildOrchestratorAgentStreamOptions(
user,
threadId,
runId,
signal,
);
const resumeOptions = service.buildOrchestratorResumeAgentOptions(
user,
threadId,
runId,
'agent-run-1',
'tool-call-1',
signal,
);
expect(typeof streamOptions.onStepStart).toBe('function');
expect(typeof streamOptions.onStepEnd).toBe('function');
expect(typeof streamOptions.onStepFinish).toBe('function');
expect(typeof resumeOptions.onStepStart).toBe('function');
expect(typeof resumeOptions.onStepEnd).toBe('function');
expect(typeof resumeOptions.onStepFinish).toBe('function');
expect(service.getRunDebug(runId)).toEqual(
expect.objectContaining({
runId,
threadId,
steps: [],
workflowCode: [],
}),
);
});
});

23
packages/cli/src/modules/instance-ai/__tests__/run-trace-metadata.test.ts
View File

@ -96,27 +96,4 @@ describe('buildInstanceAiRunTraceMetadata', () => {
first_tool_name: 'workflows',
});
});
it('records timeout cancellation type and idle tail without extra fields', () => {
const metadata = buildInstanceAiRunTraceMetadata([], {
status: 'cancelled',
cancellationReason: 'timeout',
runTimeout: {
timedOut: true,
details: {
reason: 'idle_timeout',
surface: 'active-run',
timeoutMs: 600_000,
elapsedMs: 650_200,
idleMs: 606_400.4,
},
},
});
expect(metadata).toEqual({
first_visible_state: 'empty',
cancellation_type: 'idle_timeout',
idle_tail_ms: 606_400,
});
});
});

313
packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts
View File

@ -1,38 +1,22 @@
import type { InstanceAiErrorEvent, InstanceAiEvent } from '@n8n/api-types';
import type { Logger } from '@n8n/backend-common';
import type { User } from '@n8n/db';
import {
InstanceAiTerminalResponseGuard,
orchestratorAgentId,
TerminalOutcomeStorage,
type InstanceAiTraceContext,
type ManagedBackgroundTask,
type PatchableThreadMemory,
type RunStateRegistry,
type TerminalOutcome,
type TerminalResponseDecision,
type TerminalResponseStatus,
type WorkSummary,
} from '@n8n/instance-ai';
import { getErrorMessage } from '@n8n/utils/errors/get-error-message';
import { OperationalError } from 'n8n-workflow';
import type { Telemetry } from '@/telemetry';
import type { InProcessEventBus } from './event-bus/in-process-event-bus';
import type { InstanceAiErrorReporterService } from './instance-ai-error-reporter.service';
import type {
InstanceAiTracingService,
MessageTraceFinalization,
} from './tracing/instance-ai-tracing.service';
type InstanceAiErrorCode = NonNullable<InstanceAiErrorEvent['payload']['code']>;
function getBackgroundOutcomeResponseId(outcome: TerminalOutcome): string {
return `background-outcome:${outcome.id}`;
}
// The slice of each collaborator the terminal-outcome coordinator actually
// uses. Anchored to the concrete types via `Pick` so the signatures stay in
// sync with the source.
@ -50,68 +34,28 @@ export type InstanceAiTerminalOutcomeTelemetry = Pick<Telemetry, 'track'>;
export type InstanceAiTerminalOutcomeErrorReporter = Pick<InstanceAiErrorReporterService, 'report'>;
export type InstanceAiTerminalOutcomeRunState = Pick<
RunStateRegistry<User>,
'getRunIdsForMessageGroup' | 'cancelThread'
>;
export interface InstanceAiTerminalOutcomeSuspendedThreads {
dropPendingConfirmationsForThread(threadId: string): Promise<void>;
}
export type InstanceAiTerminalOutcomeTracing = Pick<
InstanceAiTracingService,
'finalizeRunTracing' | 'buildMessageTraceMetadata'
>;
export type InstanceAiTerminalOutcomeRunState = Pick<RunStateRegistry, 'getRunIdsForMessageGroup'>;
export interface InstanceAiTerminalOutcomeServiceOptions {
eventBus: InstanceAiTerminalOutcomeEventBus;
agentMemory: PatchableThreadMemory;
telemetry: InstanceAiTerminalOutcomeTelemetry;
errorReporter: InstanceAiTerminalOutcomeErrorReporter;
logger: Logger;
runState: InstanceAiTerminalOutcomeRunState;
suspendedThreads: InstanceAiTerminalOutcomeSuspendedThreads;
tracing: InstanceAiTerminalOutcomeTracing;
/**
* Publishes the run-finish event plus the success heartbeat. Owned by the
* run loop (`InstanceAiService`), which also emits it on the foreground path.
*/
publishRunFinish: (
threadId: string,
runId: string,
status: 'completed' | 'cancelled' | 'errored',
reason?: string,
promptVersion?: string,
) => void;
}
/**
* Owns the terminal-response guard and the durable replay of background task
* outcomes for Instance AI conversations.
* Owns the terminal-response guard for Instance AI conversations.
*
* Two responsibilities live here:
*
* 1. **Terminal-response guard.** Whenever a run reaches a terminal state
* (completed / cancelled / errored) or starts waiting on a confirmation,
* it consults {@link InstanceAiTerminalResponseGuard} against the run's
* emitted events and publishes a fallback line when the agent went silent,
* suppresses duplicate completions, and flags malformed confirmations.
*
* 2. **Terminal-outcome durability.** Background tasks finish out of band from
* the foreground run, so their user-facing summary is persisted to
* {@link TerminalOutcomeStorage} and published as a durable text-block, then
* replayed on reconnect so a closed SSE stream never drops the result.
* Whenever a run reaches a terminal state (completed / cancelled / errored) or
* starts waiting on a confirmation, it consults
* {@link InstanceAiTerminalResponseGuard} against the run's emitted events and
* publishes a fallback line when the agent went silent, suppresses duplicate
* completions, and flags malformed confirmations.
*/
export class InstanceAiTerminalOutcomeService {
private readonly pendingTerminalOutcomes = new Map<string, TerminalOutcome>();
private terminalOutcomeStorage?: TerminalOutcomeStorage;
private readonly eventBus: InstanceAiTerminalOutcomeEventBus;
private readonly agentMemory: PatchableThreadMemory;
private readonly telemetry: InstanceAiTerminalOutcomeTelemetry;
private readonly errorReporter: InstanceAiTerminalOutcomeErrorReporter;
@ -120,22 +64,12 @@ export class InstanceAiTerminalOutcomeService {
private readonly runState: InstanceAiTerminalOutcomeRunState;
private readonly suspendedThreads: InstanceAiTerminalOutcomeSuspendedThreads;
private readonly tracing: InstanceAiTerminalOutcomeTracing;
private readonly publishRunFinish: InstanceAiTerminalOutcomeServiceOptions['publishRunFinish'];
constructor(options: InstanceAiTerminalOutcomeServiceOptions) {
this.eventBus = options.eventBus;
this.agentMemory = options.agentMemory;
this.telemetry = options.telemetry;
this.errorReporter = options.errorReporter;
this.logger = options.logger;
this.runState = options.runState;
this.suspendedThreads = options.suspendedThreads;
this.tracing = options.tracing;
this.publishRunFinish = options.publishRunFinish;
}
async evaluateTerminalResponse(
@ -257,237 +191,4 @@ export class InstanceAiTerminalOutcomeService {
this.eventBus.publish(threadId, decision.event);
}
}
private createTerminalOutcomeStorage(): TerminalOutcomeStorage {
this.terminalOutcomeStorage ??= new TerminalOutcomeStorage(this.agentMemory);
return this.terminalOutcomeStorage;
}
async finishInvalidConfirmationRun(args: {
threadId: string;
runId: string;
promptVersion?: string;
abortController: AbortController;
tracing?: InstanceAiTraceContext;
}): Promise<MessageTraceFinalization> {
this.runState.cancelThread(args.threadId);
void this.suspendedThreads.dropPendingConfirmationsForThread(args.threadId);
args.abortController.abort();
await this.tracing.finalizeRunTracing(args.runId, args.tracing, {
status: 'error',
reason: 'invalid_confirmation_payload',
});
this.publishRunFinish(
args.threadId,
args.runId,
'errored',
'I need your input to continue, but I could not display the prompt. Please try again.',
args.promptVersion,
);
return {
status: 'error',
reason: 'invalid_confirmation_payload',
metadata: await this.tracing.buildMessageTraceMetadata(args.threadId, args.runId, {
status: 'error',
}),
};
}
private buildBackgroundTerminalOutcome(task: ManagedBackgroundTask): TerminalOutcome {
const status =
task.status === 'failed' ? 'failed' : task.status === 'cancelled' ? 'cancelled' : 'completed';
const userFacingMessage =
status === 'completed'
? `The background ${task.role} task finished.`
: status === 'cancelled'
? `The background ${task.role} task was cancelled.`
: `The background ${task.role} task failed before I could complete that part.`;
return {
id: `${task.messageGroupId ?? task.runId}:${task.taskId}:${status}`,
threadId: task.threadId,
runId: task.runId,
messageGroupId: task.messageGroupId,
correlationId: task.messageGroupId,
taskId: task.taskId,
agentId: task.agentId,
status,
userFacingMessage,
createdAt: new Date().toISOString(),
};
}
async replayUndeliveredTerminalOutcomes(threadId: string): Promise<void> {
const storage = this.createTerminalOutcomeStorage();
const noOutcomes: TerminalOutcome[] = [];
const persistedOutcomes = await storage.getUndelivered(threadId).catch((error) => {
this.logger.warn('Failed to load undelivered Instance AI terminal outcomes', {
threadId,
error: getErrorMessage(error),
});
return noOutcomes;
});
const inMemoryOutcomes = [...this.pendingTerminalOutcomes.values()].filter(
(outcome) => outcome.threadId === threadId,
);
const outcomes = new Map<string, TerminalOutcome>();
for (const outcome of [...persistedOutcomes, ...inMemoryOutcomes]) {
outcomes.set(outcome.id, outcome);
}
const persistedOutcomeIds = new Set(persistedOutcomes.map((outcome) => outcome.id));
for (const outcome of outcomes.values()) {
const responseId = getBackgroundOutcomeResponseId(outcome);
let delivery: 'published' | 'already-emitted' | 'dropped' = 'dropped';
try {
delivery = await this.publishTerminalOutcomeLine(outcome, responseId);
} catch (error) {
this.logger.warn('Failed to replay Instance AI terminal outcome', {
threadId,
runId: outcome.runId,
taskId: outcome.taskId,
error: getErrorMessage(error),
});
}
// Left undelivered on purpose: the next replay retries it.
if (delivery === 'dropped') continue;
const action = delivery === 'published' ? 'replay_event' : 'already-emitted';
if (persistedOutcomeIds.has(outcome.id)) {
await storage
.markDelivered(threadId, outcome.id, new Date().toISOString())
.catch((error) => {
this.logger.warn('Failed to mark Instance AI terminal outcome as delivered', {
threadId,
runId: outcome.runId,
taskId: outcome.taskId,
error: getErrorMessage(error),
});
});
}
this.pendingTerminalOutcomes.delete(outcome.id);
this.telemetry.track('instance_ai_terminal_response_decision', {
thread_id: threadId,
run_id: outcome.runId,
message_group_id: outcome.messageGroupId,
task_id: outcome.taskId,
source: 'terminal_outcome_replay',
status: outcome.status,
action,
visibility_source: 'background-outcome',
});
}
}
/**
* Publish the outcome line as a durable text-block and read it back.
* `publish` only enqueues — the drain persists asynchronously and settles
* flush waiters even when it had to drop a batch — so only the read-back
* makes the line trustworthy as a delivery record. 'dropped' means it never
* reached the log; the caller must leave the outcome undelivered so a later
* replay retries it.
*/
private async publishTerminalOutcomeLine(
outcome: TerminalOutcome,
responseId: string,
): Promise<'published' | 'already-emitted' | 'dropped'> {
const isOutcomeLine = (event: InstanceAiEvent) => event.responseId === responseId;
const alreadyPublished = (
await this.eventBus.getEventsForRun(outcome.threadId, outcome.runId)
).some(isOutcomeLine);
if (alreadyPublished) return 'already-emitted';
this.eventBus.publish(outcome.threadId, {
type: 'text-block',
runId: outcome.runId,
agentId: orchestratorAgentId(outcome.runId),
responseId,
payload: { text: outcome.userFacingMessage },
});
// The adapter's read settles the thread's drain before querying, so the
// block is either in the log by now or was dropped.
const durable = (await this.eventBus.getEventsForRun(outcome.threadId, outcome.runId)).some(
isOutcomeLine,
);
return durable ? 'published' : 'dropped';
}
async recordBackgroundTerminalOutcome(task: ManagedBackgroundTask): Promise<void> {
const outcome = this.buildBackgroundTerminalOutcome(task);
let persisted = false;
try {
await this.createTerminalOutcomeStorage().upsert(task.threadId, outcome);
persisted = true;
} catch (error) {
this.pendingTerminalOutcomes.set(outcome.id, outcome);
this.logger.warn('Failed to persist Instance AI terminal outcome', {
threadId: task.threadId,
runId: task.runId,
taskId: task.taskId,
error: getErrorMessage(error),
});
this.telemetry.track('instance_ai_terminal_outcome_persistence_failure', {
thread_id: task.threadId,
run_id: task.runId,
task_id: task.taskId,
status: outcome.status,
phase: 'metadata',
});
}
const responseId = getBackgroundOutcomeResponseId(outcome);
let delivery: 'published' | 'already-emitted' | 'dropped' = 'dropped';
try {
delivery = await this.publishTerminalOutcomeLine(outcome, responseId);
} catch (error) {
this.logger.warn('Failed to publish Instance AI terminal outcome line', {
threadId: task.threadId,
runId: task.runId,
taskId: task.taskId,
error: getErrorMessage(error),
});
}
if (delivery === 'dropped') {
// Leave the outcome undelivered — the metadata row (or the pending-map
// entry when the upsert failed too) makes the next replay retry it.
this.telemetry.track('instance_ai_terminal_outcome_persistence_failure', {
thread_id: task.threadId,
run_id: task.runId,
task_id: task.taskId,
status: outcome.status,
phase: 'event',
});
return;
}
this.telemetry.track('instance_ai_terminal_response_decision', {
thread_id: task.threadId,
run_id: task.runId,
message_group_id: task.messageGroupId,
task_id: task.taskId,
source: 'background_outcome',
status: outcome.status,
action: delivery === 'published' ? 'emit' : 'already-emitted',
visibility_source: 'background-outcome',
});
if (!persisted) return;
try {
await this.createTerminalOutcomeStorage().markDelivered(
task.threadId,
outcome.id,
new Date().toISOString(),
);
this.pendingTerminalOutcomes.delete(outcome.id);
} catch (error) {
this.logger.warn('Failed to mark Instance AI terminal outcome as delivered', {
threadId: task.threadId,
runId: task.runId,
taskId: task.taskId,
error: getErrorMessage(error),
});
}
}
}

46
packages/cli/src/modules/instance-ai/instance-ai-test.controller.ts
View File

@ -1,11 +1,9 @@
import { ProjectRepository, UserRepository, WorkflowRepository } from '@n8n/db';
import { Body, Delete, Get, Param, Post, RestController } from '@n8n/decorators';
import { WorkflowRepository } from '@n8n/db';
import { Delete, Get, Param, Post, RestController } from '@n8n/decorators';
import type { Request, Response } from 'express';
import { v4 as uuidv4 } from 'uuid';
import { ForbiddenError } from '@n8n/errors';
import { InstanceAiMemoryService } from './instance-ai-memory.service';
import { InstanceAiService } from './instance-ai.service';
import { AgentExecutionThreadRepository } from '../agents/repositories/agent-execution-thread.repository';
import { ASSISTANT_AGENT_ID } from './assistant-turn-options';
@ -20,9 +18,6 @@ export class InstanceAiTestController {
private readonly instanceAiService: InstanceAiService,
private readonly threadRepo: AgentExecutionThreadRepository,
private readonly workflowRepo: WorkflowRepository,
private readonly userRepo: UserRepository,
private readonly memoryService: InstanceAiMemoryService,
private readonly projectRepo: ProjectRepository,
) {}
@Post('/test/tool-trace', { skipAuth: true })
@ -44,30 +39,18 @@ export class InstanceAiTestController {
}
@Get('/test/idle', { skipAuth: true })
getIdleState() {
async getIdleState() {
this.assertTraceReplayEnabled();
return { idle: !this.instanceAiService.hasRunningWorkForTest() };
}
@Post('/test/background-timeout/start', { skipAuth: true })
async startBackgroundTimeoutSimulation(@Body payload: { userId: string; threadId?: string }) {
this.assertTraceReplayEnabled();
const threadId = payload.threadId ?? uuidv4();
const user = await this.userRepo.findOneByOrFail({ id: payload.userId });
const personalProject = await this.projectRepo.getPersonalProjectForUserOrFail(user.id);
await this.memoryService.ensureThread(user.id, threadId, personalProject.id, {
source: 'playwright',
origin: 'internal',
const threads = await this.threadRepo.find({
where: { agentId: ASSISTANT_AGENT_ID },
select: ['id'],
});
return await this.instanceAiService.startStuckBackgroundTaskForTest(user, threadId);
}
@Post('/test/liveness-sweep', { skipAuth: true })
async runLivenessSweep(@Body payload: { now?: number } = {}) {
this.assertTraceReplayEnabled();
await this.instanceAiService.runLivenessSweepForTest(payload.now);
return { ok: true };
for (const { id } of threads) {
if ((await this.instanceAiService.getLiveRun(id)).status === 'running') {
return { idle: false };
}
}
return { idle: true };
}
@Delete('/test/tool-trace/:slug', { skipAuth: true })
@ -86,14 +69,13 @@ export class InstanceAiTestController {
* test's recorded responses (observed: a follow-up test's recording referencing
* the previous test's workflow name).
*
* This endpoint cancels background tasks, clears per-thread in-memory state,
* and deletes all thread + workflow rows.
* This endpoint clears per-thread in-memory state and deletes all thread +
* workflow rows.
*/
@Post('/test/reset', { skipAuth: true })
async reset() {
this.assertTraceReplayEnabled();
this.instanceAiService.cancelAllBackgroundTasks();
this.instanceAiService.clearTraceContextsForTest();
const threads = await this.threadRepo.find({

80
packages/cli/src/modules/instance-ai/instance-ai.controller.ts
View File

@ -11,7 +11,6 @@ import {
InstanceAiSendMessageRequest,
InstanceAiEventsQuery,
instanceAiGatewayKeySchema,
InstanceAiCorrectTaskRequest,
InstanceAiEnsureThreadRequest,
InstanceAiPersistPendingAgentRequest,
InstanceAiThreadMessagesQuery,
@ -154,11 +153,6 @@ export class InstanceAiController {
}
}
private requireRunDebugEnabled(): void {
if (!this.instanceAiService.isRunDebugEnabled()) {
throw new NotFoundError('Run debug is not enabled');
}
}
// Each BrotliCompress stream allocates ~8.6 MB of native memory for its
// dictionary, and the compression middleware retains streams via closures on
// the response object for the lifetime of the HTTP keep-alive connection.
@ -339,12 +333,7 @@ export class InstanceAiController {
req.once('close', cleanup);
res.once('finish', cleanup);
// 2. Re-publish any terminal outcomes that never reached the client.
if (ownership === 'owned') {
await this.instanceAiService.replayUndeliveredTerminalOutcomes(threadId);
}
// 3. Set SSE headers.
// 2. Set SSE headers.
// Disable response compression — SSE streams small chunks where compression
// overhead exceeds the benefit, and each Brotli compressor retains ~8.6 MB
// of native memory for the lifetime of the connection.
@ -355,7 +344,7 @@ export class InstanceAiController {
res.setHeader('X-Accel-Buffering', 'no');
res.flushHeaders();
// 4. Determine replay cursor
// 3. Determine replay cursor
// Last-Event-ID header (browser auto-reconnect) takes precedence over query param.
// Both are validated as non-negative integers; invalid values fall back to 0.
const headerValue = req.headers['last-event-id'];
@ -363,7 +352,7 @@ export class InstanceAiController {
const cursor =
Number.isFinite(parsedHeader) && parsedHeader >= 0 ? parsedHeader : (query.lastEventId ?? 0);
// 5. Collect live message groups.
// 4. Collect live message groups.
// Multiple groups can be active simultaneously when a background task
// from an older turn outlives its original turn.
const liveRun = await this.instanceAiService.getLiveRun(threadId);
@ -410,7 +399,7 @@ export class InstanceAiController {
);
};
// 6. Replay missed events from the durable log — survives restarts and is
// 5. Replay missed events from the durable log — survives restarts and is
// valid on any main (the table is in the shared DB). The reads are
// async, so live events can land mid-bootstrap: buffer them across
// every await (the
@ -553,7 +542,7 @@ export class InstanceAiController {
bootstrapping = false;
// 7. Keep-alive
// 6. Keep-alive
keepAlive = setInterval(() => {
res.write(': ping\n\n');
res.flush?.();
@ -636,35 +625,6 @@ export class InstanceAiController {
return { ok: true };
}
@Post('/chat/:threadId/tasks/:taskId/cancel')
@GlobalScope('instanceAi:message')
async cancelTask(
req: AuthenticatedRequest,
_res: Response,
@Param('threadId') threadId: string,
@Param('taskId') taskId: string,
) {
this.requireInstanceAiEnabled();
await this.assertThreadAccess(req.user.id, threadId);
await this.instanceAiService.routeCancelBackgroundTask(threadId, taskId);
return { ok: true };
}
@Post('/chat/:threadId/tasks/:taskId/correct')
@GlobalScope('instanceAi:message')
async correctTask(
req: AuthenticatedRequest,
_res: Response,
@Param('threadId') threadId: string,
@Param('taskId') taskId: string,
@Body payload: InstanceAiCorrectTaskRequest,
) {
this.requireInstanceAiEnabled();
await this.assertThreadAccess(req.user.id, threadId);
await this.instanceAiService.routeCorrectionToTask(threadId, taskId, payload.message);
return { ok: true };
}
// ── Preference card (the save_user_preference result in the chat) ────────
//
// The thread check is the ownership boundary. The row check is inside
@ -1033,7 +993,6 @@ export class InstanceAiController {
) {
this.requireInstanceAiEnabled();
await this.assertThreadAccess(req.user.id, threadId);
await this.instanceAiService.replayUndeliveredTerminalOutcomes(threadId);
// ?raw=true returns the old format for the thread inspector
if (query.raw === 'true') {
@ -1100,35 +1059,6 @@ export class InstanceAiController {
return await this.instanceAiService.getThreadStatus(threadId);
}
@Get('/debug/runs/:runId')
@GlobalScope('instanceAi:message')
async getRunDebug(req: AuthenticatedRequest, _res: Response, @Param('runId') runId: string) {
this.requireInstanceAiEnabled();
this.requireRunDebugEnabled();
const record = this.instanceAiService.getRunDebug(runId);
if (!record) {
throw new NotFoundError('Run debug record not found');
}
await this.assertThreadAccess(req.user.id, record.threadId);
return record;
}
@Get('/debug/threads/:threadId/runs')
@GlobalScope('instanceAi:message')
async listThreadDebugRuns(
req: AuthenticatedRequest,
_res: Response,
@Param('threadId') threadId: string,
) {
this.requireInstanceAiEnabled();
this.requireRunDebugEnabled();
await this.assertThreadAccess(req.user.id, threadId);
return {
threadId,
runs: this.instanceAiService.listThreadDebugRuns(threadId),
};
}
// ── Evaluation endpoints ──────────────────────────────────────────────────
// Runs for minutes; the eval client (N8nClient) disables undici's 300s timeout for it.

655
packages/cli/src/modules/instance-ai/instance-ai.service.ts
View File

@ -48,7 +48,6 @@ import {
type InstanceAiConfig,
} from '@n8n/config';
import { UserRepository, type User } from '@n8n/db';
import { OnPubSubEvent } from '@n8n/decorators';
import { Container, Service } from '@n8n/di';
import {
CONCISE_PROMPT_VERSION,
@ -69,17 +68,13 @@ import {
threadProvenanceMetadata,
createInternalOperationTraceContext,
emitAgentSnapshotTraceEvent,
createInstanceAiLivenessPolicyConfig,
InstanceAiLivenessPolicy,
McpClientManager,
createDomainAccessTracker,
BackgroundTaskManager,
MemoryTaskRegistry,
classifyAttachments,
buildAttachmentManifest,
getDateTimeSection,
isParseableAttachment,
enrichMessageWithBackgroundTasks,
isQuotaExhaustedError,
PlannedTaskCoordinator,
PlannedTaskStorage,
@ -88,7 +83,6 @@ import {
RunStateRegistry,
shutdownProductTelemetryProviders,
tokenUsageToBuilderUsageItems,
RunDebugBuffer,
truncateToTitle,
generateTitleForRun,
patchThread,
@ -102,7 +96,6 @@ import {
saveAgentBuilderTarget,
type DomainAccessTracker,
type InstanceAiContext,
type ManagedBackgroundTask,
type McpServerConfig,
type ModelConfig,
type AgentSnapshotArtifact,
@ -113,7 +106,6 @@ import {
type PlannedTaskService,
type PlannedWorkflowVerification,
type ServiceProxyConfig,
type SuspendedRunState,
type WorkflowBuildOutcome,
type ProjectSummary,
type WorkflowLoopWorkItemRecord,
@ -123,7 +115,6 @@ import {
type WorkSummary,
deriveInstanceContextReach,
type RunTokenUsage,
type RunDebugRecord,
WorkflowTaskCoordinator,
WorkflowLoopStorage,
ThreadTaskStorage,
@ -137,7 +128,7 @@ import { getErrorMessage } from '@n8n/utils/errors/get-error-message';
import { isRecord } from '@n8n/utils/is-record';
import { lazyImport } from '@n8n/utils/lazy-import';
import { setSchemaBaseDirs } from '@n8n/workflow-sdk';
import { ErrorReporter, InstanceSettings } from 'n8n-core';
import { ErrorReporter } from 'n8n-core';
import { OperationalError, UnexpectedError, UserError } from 'n8n-workflow';
import { nanoid } from 'nanoid';
@ -148,8 +139,6 @@ import { InstanceAiAgentContextAdapterService } from '@/modules/agents/instance-
import { modelStreamStallOptions } from '@/modules/agents/model-stream-stall-options';
import { userHasScopes } from '@/permissions.ee/check-access';
import { Push } from '@/push';
import { Publisher } from '@/scaling/pubsub/publisher.service';
import type { PubSubCommandMap } from '@/scaling/pubsub/pubsub.event-map';
import {
AI_PREFERENCES_CLEARED_BLOCK,
AiPreferenceService,
@ -186,7 +175,6 @@ import {
import { InstanceAiGatewayService } from './instance-ai-gateway.service';
import { InstanceAiMemoryService } from './instance-ai-memory.service';
import { InstanceAiModelService } from './instance-ai-model.service';
import { InstanceAiRunProbe } from './instance-ai-run-probe';
import { InstanceAiSettingsService } from './instance-ai-settings.service';
import { InstanceAiTemporaryWorkflowService } from './instance-ai-temporary-workflow.service';
import { InstanceAiTerminalOutcomeService } from './instance-ai-terminal-outcome.service';
@ -213,7 +201,6 @@ import {
} from './internal-messages';
import { loadOnboardingSkill } from './onboarding';
import { ONBOARDING_OPENING } from './onboarding-opening';
import { INSTANCE_AI_RUN_TIMEOUT_REASON, InstanceAiLivenessService } from './liveness';
import { InstanceAiMcpRegistryService } from './mcp';
import { runMetricsModelLabel } from './observability';
import {
@ -520,14 +507,6 @@ function buildSuspensionTraceOutputs(runId: string, suspension: SuspensionInfo |
};
}
/**
* Upper bound on how long `shutdown()` will wait for in-flight executeRun /
* processResumedStream promises to drain after their abortControllers fire.
* Sized well below n8n's `gracefulShutdownTimeoutInS` (30s default) so a
* stuck agent can't burn the whole budget here.
*/
const INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS = 5 * 1000;
function isTextMessagePart(part: unknown): part is { type: 'text'; text: string } {
return (
typeof part === 'object' &&
@ -539,8 +518,20 @@ function isTextMessagePart(part: unknown): part is { type: 'text'; text: string
);
}
/** Planned tasks the scheduler may start in one tick. */
const MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD = 5;
/** The finished build task a workflow-verification follow-up reports on. */
type WorkflowVerificationSourceTask = {
taskId: string;
role: string;
status: 'completed';
result?: string;
error?: string;
plannedTaskId?: string;
workItemId?: string;
};
/**
* Circuit breaker for machine-started follow-up runs (verification, synthesize,
* replan, …). A follow-up that dies before the agent can settle its trigger
@ -613,9 +604,7 @@ export class InstanceAiService {
private readonly formBaseUrl: string;
private readonly runState = new RunStateRegistry<User>((user) => user.id);
private readonly backgroundTasks: BackgroundTaskManager;
private readonly runState = new RunStateRegistry();
private readonly memoryTaskRegistry = new MemoryTaskRegistry();
@ -669,12 +658,6 @@ export class InstanceAiService {
private readonly terminalOutcome: InstanceAiTerminalOutcomeService;
private readonly liveness: InstanceAiLivenessService<SuspendedRunState<User>>;
/** Owns DB persistence of suspended runs + orphan-confirmation restoration. */
private readonly runDebugBuffer = new RunDebugBuffer();
/** Default IANA timezone for the instance (from GENERIC_TIMEZONE env var). */
private readonly defaultTimeZone: string;
@ -684,27 +667,9 @@ export class InstanceAiService {
private readonly taskProjector: WorkflowVerificationTaskProjector;
/**
* In-flight `executeRun` / `processResumedStream` promises. Tracked so
* `shutdown()` can drain them before n8n closes the DB connection — the
* SDK's abort-driven `cleanupRun` and `executeRun`'s `finally` block
* both write to the DB during teardown, and racing the connection close
* surfaces as `DriverAlreadyReleasedError` for callers.
*/
private readonly inFlightExecutions = new Set<Promise<unknown>>();
/**
* Run IDs whose post-stream terminal handling should be skipped when their
* abort fires. Populated by `shutdown()` for runs that were sitting on an
* inline HITL confirmation and for suspended runs, and drained by
* `shouldPreserveHitlOnShutdown(runId)`.
*/
private readonly preserveHitlOnShutdown = new Set<string>();
constructor(
logger: Logger,
globalConfig: GlobalConfig,
private readonly instanceSettings: InstanceSettings,
private readonly adapterService: InstanceAiAdapterService,
private readonly eventBus: InProcessEventBus,
private readonly eventLog: DurableEventLog,
@ -727,10 +692,8 @@ export class InstanceAiService {
ssrfProtectionService: SsrfProtectionService,
private readonly eventService: EventService,
private readonly evalCredentialAllowlists: EvalThreadCredentialAllowlistService,
runProbe: InstanceAiRunProbe,
private readonly modelService: InstanceAiModelService,
private readonly creditService: InstanceAiCreditService,
private readonly publisher: Publisher,
private readonly instanceAiErrorReporter: InstanceAiErrorReporterService,
private readonly push: Push,
private readonly conversationHistoryService: InstanceAiConversationHistoryService,
@ -739,7 +702,6 @@ export class InstanceAiService {
private readonly aiUsageService: AiUsageService,
) {
this.logger = logger.scoped('instance-ai');
runProbe.registerActiveRunCountProvider(() => this.runState.activeRunCount());
this.workflowObligations = new WorkflowVerificationObligationService(
this.agentMemory,
(threadId) => this.runState.isSetupPanelEnabled(threadId),
@ -752,26 +714,6 @@ export class InstanceAiService {
);
this.instanceAiConfig = globalConfig.instanceAi;
this.aiConfig = globalConfig.ai;
this.backgroundTasks = new BackgroundTaskManager(
MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD,
this.instanceAiConfig.maxConcurrentSubAgents,
);
const livenessPolicyConfig = createInstanceAiLivenessPolicyConfig({
confirmationTimeoutMs: this.instanceAiConfig.confirmationTimeout,
});
this.liveness = new InstanceAiLivenessService<SuspendedRunState<User>>({
policy: new InstanceAiLivenessPolicy(livenessPolicyConfig),
backgroundTaskIdleTimeoutMs: livenessPolicyConfig.backgroundTaskIdleTimeoutMs,
runState: this.runState,
backgroundTasks: this.backgroundTasks,
eventBus: this.eventBus,
logger: this.logger,
// Suspended turns live in Agents checkpoints now. Their TTL owns expiry.
finalizeCancelledSuspendedRun: () => {},
onPendingConfirmationRejected: (requestId) => {
void requestId;
},
});
this.tracing = new InstanceAiTracingService({
logger: this.logger,
// `first_visible_state` has to see the run's streamed text, which lives
@ -779,7 +721,6 @@ export class InstanceAiService {
eventReader: {
getEventsForRun: async (threadId, runId) => await this.readRunEvents(threadId, [runId]),
},
runState: this.runState,
eventLog: this.eventLogRepository,
aiService: this.aiService,
});
@ -787,8 +728,6 @@ export class InstanceAiService {
config: this.instanceAiConfig,
logger: this.logger,
errorReporter: this.errorReporter,
runState: this.runState,
backgroundTasks: this.backgroundTasks,
settingsService: this.settingsService,
aiService: this.aiService,
resolveTracingConfig: async (threadId, userId) => {
@ -807,19 +746,10 @@ export class InstanceAiService {
getEventsForRun: async (threadId, runId) => await this.readRunEvents(threadId, [runId]),
getEventsForRuns: async (threadId, runIds) => await this.readRunEvents(threadId, runIds),
},
agentMemory: this.agentMemory,
telemetry: this.telemetry,
errorReporter: this.instanceAiErrorReporter,
logger: this.logger,
runState: this.runState,
// Pending cards are suspended Agents checkpoints; nothing separate to drop.
suspendedThreads: { dropPendingConfirmationsForThread: async () => {} },
tracing: this.tracing,
publishRunFinish: (threadId, runId, status, reason, promptVersion) => {
this.publishRunFinish(threadId, runId, status, reason, undefined, undefined, {
promptVersion,
});
},
});
this.defaultTimeZone = globalConfig.generic.timezone;
const restEndpoint = globalConfig.endpoints.rest;
@ -842,7 +772,6 @@ export class InstanceAiService {
});
});
});
this.liveness.start();
}
private async createProxyRunConfig(user: Pick<User, 'id'>): Promise<{
@ -1164,10 +1093,6 @@ export class InstanceAiService {
});
}
isRunDebugEnabled(): boolean {
return this.instanceAiConfig.runDebugEnabled;
}
/** What observational memory holds for a thread: the live observations and the
* compaction cursor. An eval asserts on these rows instead of parsing the
* rendered system prompt. Refuses a thread the caller does not own, so the
@ -1197,21 +1122,6 @@ export class InstanceAiService {
};
}
getRunDebug(runId: string) {
return this.runDebugBuffer.get(runId);
}
listThreadDebugRuns(threadId: string) {
return this.runDebugBuffer.listByThread(threadId).map((record: RunDebugRecord) => ({
runId: record.runId,
threadId: record.threadId,
startedAt: record.startedAt,
stepCount: record.steps.length,
workflowCodeCount: record.workflowCode.length,
label: record.label,
}));
}
clearTraceContextsForTest(): void {
this.tracing.clearTraceContextsForTest();
}
@ -1246,7 +1156,6 @@ export class InstanceAiService {
) {
throw new BadRequestError(`Unknown Instance AI prompt version "${promptVersion}"`);
}
this.liveness.clearThreadState(threadId);
const { runId } = await this.enqueueAssistantTurn(user, threadId, message, {
runId: `run_${nanoid()}`,
messageGroupId: `mg_${nanoid()}`,
@ -1270,109 +1179,6 @@ export class InstanceAiService {
void this.cancelAwaitingApprovalPlan(threadId);
}
/** Send a correction message to a running background task. */
sendCorrectionToTask(
threadId: string,
taskId: string,
correction: string,
): 'queued' | 'task-completed' | 'task-not-found' {
return this.backgroundTasks.queueCorrection(threadId, taskId, correction);
}
/** Cancel a single background task by ID. */
cancelBackgroundTask(threadId: string, taskId: string): void {
const task = this.backgroundTasks.cancelTask(threadId, taskId);
if (!task) return;
void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
this.eventBus.publish(threadId, {
type: 'agent-completed',
runId: task.runId,
agentId: task.agentId,
payload: { role: task.role, result: '', status: 'cancelled' },
});
void this.terminalOutcome.recordBackgroundTerminalOutcome(task);
const user = this.runState.getThreadUser(threadId);
if (user) {
void this.handlePlannedTaskSettlement(user, task, 'cancelled');
}
}
// ── Cross-main task-control routing ───────────────────────────────────────
// User actions (correct/cancel/clear) can land on a different main than the
// one running the task/run. Each `route*` method applies the action locally
// and, when the target isn't local (or is thread-wide), broadcasts so the
// owning main applies it. Broadcast + local-gate — no shared ownership store.
// `applyTaskControlLocally` is the single action → local-method mapping,
// shared by the route entry points and the `@OnPubSubEvent` relay handler
// (which never re-broadcasts, so there's no loop). These wrappers are the
// controller entry points; internal callers keep using the local methods
// directly so they don't trigger cross-main broadcasts.
private broadcastTaskControl(payload: PubSubCommandMap['relay-instance-ai-task-control']): void {
if (!this.instanceSettings.isMultiMain) return;
void this.publisher
.publishCommand({ command: 'relay-instance-ai-task-control', payload })
.catch((error: unknown) =>
this.logger.error('Failed to relay Instance AI task-control to sibling mains', {
threadId: payload.threadId,
action: payload.action,
error,
}),
);
}
/** Apply a task-control action to this main's local slice of the thread.
* Returns whether the action's target was found locally: task-scoped
* actions report a local hit to gate re-broadcast, thread-wide actions
* always report a miss so they fan out to every main. */
private async applyTaskControlLocally({
threadId,
taskId,
action,
correction,
userId,
}: PubSubCommandMap['relay-instance-ai-task-control']): Promise<boolean> {
switch (action) {
case 'correct':
// A relay without its correction text is malformed: nothing to apply.
if (!taskId || correction === undefined) return true;
return this.sendCorrectionToTask(threadId, taskId, correction) !== 'task-not-found';
case 'cancel-task': {
if (!taskId) return true;
const isLocal = this.backgroundTasks
.getTaskSnapshots(threadId)
.some((task) => task.taskId === taskId);
this.cancelBackgroundTask(threadId, taskId);
return isLocal;
}
case 'cancel-thread':
// A thread's run + tasks can be spread across mains, so always fan out.
this.cancelRun(threadId);
return false;
case 'clear-thread':
await this.clearThreadState(threadId, userId);
return false;
}
}
private async routeTaskControl(
payload: PubSubCommandMap['relay-instance-ai-task-control'],
): Promise<void> {
const foundLocally = await this.applyTaskControlLocally(payload);
if (!foundLocally) this.broadcastTaskControl(payload);
}
async routeCorrectionToTask(threadId: string, taskId: string, correction: string): Promise<void> {
await this.routeTaskControl({ threadId, taskId, action: 'correct', correction });
}
async routeCancelBackgroundTask(threadId: string, taskId: string): Promise<void> {
await this.routeTaskControl({ threadId, taskId, action: 'cancel-task' });
}
/** Stop the running turn, or cancel the suspended one. Works from any main. */
async routeCancelRun(user: User, threadId: string): Promise<void> {
const thread = await this.systemAgents.getThread(ASSISTANT_AGENT_ID, user, threadId);
@ -1390,163 +1196,9 @@ export class InstanceAiService {
}
}
/** Thread deletion clears this main's in-memory state only. */
async routeClearThreadState(threadId: string, userId?: string): Promise<void> {
await this.routeTaskControl({
threadId,
action: 'clear-thread',
...(userId ? { userId } : {}),
});
}
/** Apply a task-control action relayed from another main to this main's local
* slice of the thread. Never re-broadcasts. Not self-sent, so this never
* fires on the originating main. */
@OnPubSubEvent('relay-instance-ai-task-control', { instanceType: 'main' })
async handleRelayTaskControl(
payload: PubSubCommandMap['relay-instance-ai-task-control'],
): Promise<void> {
// Guard the whole handler: it runs as a fire-and-forget pubsub listener, so
// a throw (e.g. from the async clearThreadState) would surface as an
// unhandled rejection on this sibling main instead of being contained.
try {
await this.applyTaskControlLocally(payload);
} catch (error) {
this.logger.error('Failed to apply relayed Instance AI task-control', {
threadId: payload.threadId,
taskId: payload.taskId,
action: payload.action,
error,
});
}
}
/** Cancel all background tasks across all threads. Test-only. */
cancelAllBackgroundTasks(): number {
const cancelled = this.backgroundTasks.cancelAll();
for (const task of cancelled) {
void this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
}
return cancelled.length;
}
async startStuckBackgroundTaskForTest(
user: User,
threadId: string,
): Promise<{
threadId: string;
runId: string;
messageGroupId: string;
taskId: string;
agentId: string;
timeoutAt: number;
}> {
const messageId = `msg_${nanoid()}`;
const messageText = 'I started a background workflow-builder task.';
const { runId, messageGroupId } = this.runState.startRun({ threadId, user });
if (!messageGroupId) {
throw new UnexpectedError('Failed to create message group for timeout simulation');
}
const taskId = `task_${nanoid()}`;
const agentId = `agent_${nanoid()}`;
this.eventBus.publish(threadId, {
type: 'run-start',
runId,
agentId: orchestratorAgentId(runId),
userId: user.id,
payload: { messageId, messageGroupId },
});
this.eventBus.publish(threadId, {
type: 'text-delta',
runId,
agentId: orchestratorAgentId(runId),
responseId: `test-background-start:${runId}`,
payload: { text: messageText },
});
this.eventBus.publish(threadId, {
type: 'agent-spawned',
runId,
agentId,
payload: {
parentId: orchestratorAgentId(runId),
role: 'workflow-builder',
tools: [],
taskId,
kind: 'builder',
title: 'Building workflow',
subtitle: 'Timeout simulation',
goal: 'Simulate a stuck background task timeout',
},
});
// Persist the assistant message so the UI can render it after navigation.
await this.agentMemory.saveMessages({
threadId,
resourceId: user.id,
messages: [
{
id: messageId,
createdAt: new Date(),
type: 'llm',
role: 'assistant',
content: [{ type: 'text', text: messageText }],
},
],
});
const outcome = this.backgroundTasks.spawn({
taskId,
threadId,
runId,
role: 'workflow-builder',
agentId,
messageGroupId,
run: async (signal) =>
await new Promise<string>((resolve) => {
signal.addEventListener('abort', () => resolve('aborted'), { once: true });
}),
onFailed: (task) => {
this.eventBus.publish(threadId, {
type: 'agent-completed',
runId,
agentId,
payload: {
role: task.role,
result: '',
error: task.error ?? 'Unknown error',
},
});
},
onSettled: async (task) => {
await this.terminalOutcome.recordBackgroundTerminalOutcome(task);
},
});
if (outcome.status !== 'started') {
throw new UnexpectedError('Failed to start stuck background task simulation');
}
this.runState.clearActiveRun(threadId);
this.eventBus.publish(threadId, {
type: 'run-finish',
runId,
agentId: orchestratorAgentId(runId),
userId: user.id,
payload: { status: 'completed' },
});
return {
threadId,
runId,
messageGroupId,
taskId,
agentId,
timeoutAt: outcome.task.lastActivityAt + this.liveness.backgroundTaskIdleTimeoutMs + 1,
};
}
async runLivenessSweepForTest(now?: number): Promise<void> {
await this.liveness.sweepTimedOutWork(now);
await this.clearThreadState(threadId, userId);
}
// ── Gateway lifecycle (delegated to LocalGatewayRegistry) ───────────────
@ -1561,17 +1213,6 @@ export class InstanceAiService {
return this.tracing.getTraceEvents(slug);
}
hasRunningWorkForTest(): boolean {
const threadIds = new Set(this.tracing.getTrackedThreadIds());
for (const threadId of threadIds) {
if (this.runState.getActiveRunId(threadId)) return true;
if (this.backgroundTasks.getRunningTasks(threadId).length > 0) return true;
}
return false;
}
activateTraceSlug(slug: string): void {
this.tracing.activateTraceSlug(slug);
}
@ -1585,31 +1226,7 @@ export class InstanceAiService {
* Must be called when a thread is deleted so the maps don't leak.
*/
async clearThreadState(threadId: string, userId?: string): Promise<void> {
this.liveness.clearThreadState(threadId);
// Clear run-state registry entries (active/suspended runs, confirmations,
// user, time zone, and message-group mappings).
const { active, suspended } = this.runState.clearThread(threadId);
if (active) {
active.abortController.abort();
await this.tracing.finalizeRunTracing(active.runId, active.tracing, {
status: 'cancelled',
reason: 'thread_cleared',
});
}
if (suspended) {
suspended.abortController.abort();
await this.tracing.finalizeRunTracing(suspended.runId, suspended.tracing, {
status: 'cancelled',
reason: 'thread_cleared',
});
}
// Cancel background tasks belonging to this thread
for (const task of this.backgroundTasks.cancelThread(threadId)) {
task.abortController.abort();
await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
}
this.runState.clearThread(threadId);
await this.tracing.finalizeRemainingMessageTraceRoots(threadId, {
status: 'cancelled',
reason: 'thread_cleared',
@ -1646,79 +1263,6 @@ export class InstanceAiService {
}
async shutdown(): Promise<void> {
this.liveness.shutdown();
const { activeRuns, suspendedRuns, pendingThreadIds } = this.runState.shutdown();
const threadsWithPendingHitl = new Set(pendingThreadIds);
for (const run of activeRuns) {
// Runs holding an inline HITL confirmation (`create-tasks`,
// sub-agent `ask-user`) sit in `activeRuns` because the orchestrator
// is alive — it's just awaiting the in-process Promise. Their
// `instance_ai_pending_confirmations` row survives the restart and
// `handleOrphanedConfirmation` will issue the user-visible
// `restart_lost_confirmation` UserError + `run-finish` when (if)
// the user clicks confirm. If we publish run-finish here, the fold
// would render the plan/ask card as cancelled before the user has a
// chance to see it on reload.
if (threadsWithPendingHitl.has(run.threadId)) {
await this.tracing.finalizeRunTracing(run.runId, run.tracing, {
status: 'cancelled',
reason: 'service_shutdown',
});
// Record the policy *before* the abort fires so the run's catch
// handler (which runs synchronously off the abort) sees the
// flag. The catch path consults `shouldPreserveHitlOnShutdown`
// and skips the terminal-fallback / run-finish writes that
// would otherwise overwrite the plan/ask card.
this.preserveHitlOnShutdown.add(run.runId);
run.abortController.abort();
continue;
}
// Truly mid-stream run: the durable run-finish is all history needs —
// the fold derives the terminal tree from the log after restart.
this.publishRunFinish(
run.threadId,
run.runId,
'cancelled',
'service_shutdown',
undefined,
undefined,
{ promptVersion: run.promptVersion },
);
await this.tracing.finalizeRunTracing(run.runId, run.tracing, {
status: 'cancelled',
reason: 'service_shutdown',
});
run.abortController.abort();
}
for (const run of suspendedRuns) {
// Suspended runs are recoverable from the checkpoint store + pending
// confirmation index, so leave the run-finish unpublished and the
// snapshot untouched. We only need to abort the in-process stream;
// the DB rows are intentionally preserved across restart. The flag
// keeps the card publish alive if the abort lands before the card
// reaches the log.
await this.tracing.finalizeRunTracing(run.runId, run.tracing, {
status: 'cancelled',
reason: 'service_shutdown',
});
this.preserveHitlOnShutdown.add(run.runId);
run.abortController.abort();
}
for (const task of this.backgroundTasks.cancelAll()) {
task.abortController.abort();
await this.tracing.finalizeBackgroundTaskTracing(task, 'cancelled');
}
// Drain the now-aborted executeRun / processResumedStream promises
// before returning. Each one's finally block does DB work
// (`schedulePlannedTasks`, `dropPendingConfirmationsForThread`) and
// the SDK's abort-driven `cleanupRun` issues `checkpointStore.delete`,
// all of which would race the connection close in `exitSuccessFully`
// and surface as `DriverAlreadyReleasedError` otherwise. Bounded so
// a hung agent can't block n8n's own graceful-shutdown deadline.
await this.drainInFlightExecutions(INSTANCE_AI_SHUTDOWN_DRAIN_TIMEOUT_MS);
this.instanceAiErrorReporter.endAllRuns();
const threadsWithTraces = new Set(this.tracing.getTrackedThreadIds());
@ -1739,6 +1283,7 @@ export class InstanceAiService {
// teardown paths.
this.domainAccessTrackersByThread.clear();
this.runState.clear();
this.tracing.clear();
// Flush in-flight drains + open coalesce buffers so the tail of every
@ -1755,23 +1300,6 @@ export class InstanceAiService {
this.logger.debug('Instance AI service shut down');
}
private async drainInFlightExecutions(timeoutMs: number): Promise<void> {
if (this.inFlightExecutions.size === 0) return;
const drain = Promise.allSettled([...this.inFlightExecutions]);
const timeout = new Promise<'timeout'>((resolve) => {
setTimeout(() => resolve('timeout'), timeoutMs).unref();
});
const outcome = await Promise.race([drain.then(() => 'drained' as const), timeout]);
if (outcome === 'timeout') {
this.logger.warn('Timed out waiting for in-flight Instance AI runs to drain', {
timeoutMs,
stillInFlight: this.inFlightExecutions.size,
});
}
}
/**
* One prune pass: expire stale checkpoints, hard-delete tombstones past the
* GC horizon, drop expired pending confirmations, and delete expired
@ -1989,10 +1517,7 @@ export class InstanceAiService {
private buildWorkflowVerificationFollowUpMessage(input: {
obligation: WorkflowVerificationObligation;
outcome?: WorkflowBuildOutcome;
sourceTask?: Pick<
ManagedBackgroundTask,
'taskId' | 'role' | 'status' | 'result' | 'error' | 'plannedTaskId' | 'workItemId'
>;
sourceTask?: WorkflowVerificationSourceTask;
}): string {
const payload = {
obligation: input.obligation,
@ -2011,15 +1536,6 @@ export class InstanceAiService {
return { memory, taskStorage, plannedTaskService };
}
/**
* Replays any undelivered background-task outcomes for a thread so a
* reconnecting client never misses a result that completed while its stream
* was closed. Delegates to {@link InstanceAiTerminalOutcomeService}.
*/
async replayUndeliveredTerminalOutcomes(threadId: string): Promise<void> {
await this.terminalOutcome.replayUndeliveredTerminalOutcomes(threadId);
}
private async syncPlannedTasksToUi(threadId: string, graph: PlannedTaskGraph): Promise<void> {
const { taskStorage } = await this.createPlannedTaskState();
const tasks = await this.taskProjector.projectPlannedTaskList(threadId, graph);
@ -2258,12 +1774,6 @@ export class InstanceAiService {
});
this.domainAccessTrackersByThread.set(threadId, domainTracker);
context.domainAccessTracker = domainTracker;
if (this.isRunDebugEnabled()) {
context.recordWorkflowCodeSnapshot = (snapshot) => {
this.runDebugBuffer.ensure(runId, threadId);
this.runDebugBuffer.recordWorkflowCode(runId, snapshot);
};
}
browserMcpServer?.setDomainGate({
tracker: domainTracker,
@ -2441,14 +1951,9 @@ export class InstanceAiService {
runtimeSkills,
runtimeSkillCatalog: allRuntimeSkills,
oauth2CallbackUrl: this.oauth2CallbackUrl,
cancelBackgroundTask: async (taskId) => this.cancelBackgroundTask(threadId, taskId),
touchRun: () => this.runState.touchActiveRun(threadId),
touchBackgroundTask: (taskId) => this.backgroundTasks.touchTask(threadId, taskId),
plannedTaskService,
schedulePlannedTasks: async () => await this.schedulePlannedTasks(user, threadId),
iterationLog,
sendCorrectionToTask: (taskId, correction) =>
this.sendCorrectionToTask(threadId, taskId, correction),
workflowTaskService: workflowTasks,
workspace: runtimeWorkspace,
workspaceRoot,
@ -2678,43 +2183,6 @@ export class InstanceAiService {
}
}
private async handlePlannedTaskSettlement(
user: User,
task: ManagedBackgroundTask,
status: 'succeeded' | 'failed' | 'cancelled',
{ reschedule = true }: { reschedule?: boolean } = {},
): Promise<void> {
if (!task.plannedTaskId) return;
const { plannedTaskService } = await this.createPlannedTaskState();
let graph: PlannedTaskGraph | null = null;
if (status === 'succeeded') {
graph = await plannedTaskService.markSucceeded(task.threadId, task.plannedTaskId, {
result: task.result,
outcome: task.outcome,
});
} else if (status === 'failed') {
graph = await plannedTaskService.markFailed(task.threadId, task.plannedTaskId, {
error: task.error,
});
} else {
graph = await plannedTaskService.markCancelled(task.threadId, task.plannedTaskId, {
error: task.error,
});
}
if (graph) {
await this.syncPlannedTasksToUi(task.threadId, graph);
}
// Callers that tore the whole thread down (user stop, run timeout,
// shutdown) opt out: the scheduler must not start a follow-up run. The
// graph stays intact so the next user message can pick it back up.
if (!reschedule) return;
await this.schedulePlannedTasks(user, task.threadId);
}
/**
* Setup panel v2 ground truth at run start: re-analyze the workflows this
* thread announced and tell the agent what is open and what the user
@ -3055,7 +2523,8 @@ export class InstanceAiService {
private createPlannedTaskRunGate(): PlannedTaskRunGate {
return {
hasLiveRun: (threadId) => this.runState.hasLiveRun(threadId),
// The Agents runtime queues follow-ups behind a live turn.
hasLiveRun: () => false,
};
}
@ -3159,10 +2628,7 @@ export class InstanceAiService {
private toWorkflowVerificationSourceTask(
verification: PlannedWorkflowVerification,
): Pick<
ManagedBackgroundTask,
'taskId' | 'role' | 'status' | 'result' | 'error' | 'plannedTaskId' | 'workItemId'
> {
): WorkflowVerificationSourceTask {
return {
taskId: verification.task.backgroundTaskId ?? verification.task.id,
role: 'workflow-builder',
@ -3228,11 +2694,7 @@ export class InstanceAiService {
await this.syncPlannedTasksToUi(threadId, graph);
const availableSlots = Math.max(
0,
MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD -
this.backgroundTasks.getRunningTasks(threadId).length,
);
const availableSlots = MAX_CONCURRENT_BACKGROUND_TASKS_PER_THREAD;
const pendingWorkflowVerification =
await this.workflowObligations.findPendingPlannedWorkflowVerification(threadId, graph);
const action = await plannedTaskService.tick(threadId, {
@ -3717,7 +3179,7 @@ export class InstanceAiService {
attachmentManifest = buildAttachmentManifest(classifiedAttachments);
}
const turnHadFileAttachments = nonStructuredAttachments.length > 0;
const enrichedMessage = await this.buildMessageWithRunningTasks(threadId, message);
const enrichedMessage = message;
const messageBody =
!message && hasParseableAttachment
? `The user attached file(s) without a message. Inspect the first parseable attachment with parse-file and provide a concise summary.\n\n${attachmentManifest}`
@ -4063,7 +3525,7 @@ export class InstanceAiService {
}
if (status === 'suspended') {
this.emitRunMetrics(threadId, 'suspended', {
this.emitRunMetrics('suspended', {
modelId,
workSummary: result.workSummary,
usage: result.usage,
@ -4281,37 +3743,16 @@ export class InstanceAiService {
const graph = await plannedTaskService.getGraph(threadId);
const task = graph?.tasks.find((t) => t.id === checkpointTaskId);
if (task && task.status === 'running') {
// If the orchestrator spawned a detached sub-agent inside this
// checkpoint's turn (builder, research, data-table) and
// that child is still running, leave the checkpoint running. The
// child's settlement path re-emits `orchestrate-checkpoint` so the
// orchestrator re-enters the same checkpoint context and can then
// call `complete-checkpoint`.
const inflightChildren = this.backgroundTasks.getRunningTasksByParentCheckpoint(
this.logger.warn('Checkpoint run ended without reporting completion — marking failed', {
threadId,
checkpointTaskId,
);
if (inflightChildren.length > 0) {
this.logger.debug(
'Checkpoint run ended with in-flight child tasks — deferring finalization',
{
threadId,
checkpointTaskId,
inflightTaskIds: inflightChildren.map((t) => t.taskId),
},
);
} else {
this.logger.warn('Checkpoint run ended without reporting completion — marking failed', {
threadId,
checkpointTaskId,
});
await plannedTaskService.markCheckpointFailed(threadId, checkpointTaskId, {
error: 'Checkpoint run ended without reporting completion',
});
const nextGraph = await plannedTaskService.getGraph(threadId);
if (nextGraph) {
await this.syncPlannedTasksToUi(threadId, nextGraph);
}
});
await plannedTaskService.markCheckpointFailed(threadId, checkpointTaskId, {
error: 'Checkpoint run ended without reporting completion',
});
const nextGraph = await plannedTaskService.getGraph(threadId);
if (nextGraph) {
await this.syncPlannedTasksToUi(threadId, nextGraph);
}
}
} catch (error) {
@ -4815,17 +4256,6 @@ export class InstanceAiService {
}
}
private async buildMessageWithRunningTasks(threadId: string, message: string): Promise<string> {
return await enrichMessageWithBackgroundTasks(
message,
this.backgroundTasks.getRunningTasks(threadId),
{
formatTask: async (task: ManagedBackgroundTask) =>
`[Running task — ${task.role}]: taskId=${task.taskId}`,
},
);
}
private trackConfirmationRequest(
userId: string,
threadId: string,
@ -4953,7 +4383,7 @@ export class InstanceAiService {
status: effectiveStatus,
...(userId ? { user_id: userId } : {}),
});
this.emitBrowserCredentialSetupOutcomes(threadId, runId, status, reason);
this.emitBrowserCredentialSetupOutcomes(threadId, runId, status);
if (status === 'errored') {
this.telemetry.track('Builder generation errored', {
thread_id: threadId,
@ -5037,15 +4467,14 @@ export class InstanceAiService {
* Emit one terminal telemetry event per browser-assisted credential setup
* attempt of the finished run (NODE-5511). Consumes the pending record so
* every attempt yields exactly one success or failure event. When the flow
* itself never failed, the run's own termination (user stop, timeout,
* stream error) is reported as the error code so aborts aren't counted as
* itself never failed, the run's own termination (user stop or stream
* error) is reported as the error code so aborts aren't counted as
* flow failures.
*/
private emitBrowserCredentialSetupOutcomes(
threadId: string,
runId: string,
runStatus: 'completed' | 'cancelled' | 'errored',
runFinishReason?: string,
): void {
const browserSetup = this.pendingBrowserCredentialSetups.get(runId);
if (!browserSetup) return;
@ -5055,9 +4484,7 @@ export class InstanceAiService {
? 'not_attempted'
: runStatus === 'errored'
? 'run_errored'
: runFinishReason === INSTANCE_AI_RUN_TIMEOUT_REASON
? 'run_timed_out'
: 'run_cancelled';
: 'run_cancelled';
for (const attempt of browserSetup.attempts) {
this.telemetry.track('Instance AI Browser Use credential setup completed', {
user_id: browserSetup.userId,
@ -5112,7 +4539,7 @@ export class InstanceAiService {
contextReach: options?.contextReach,
},
);
this.emitRunMetrics(threadId, status, options);
this.emitRunMetrics(status, options);
if (status === 'completed' && options?.userId && options?.modelId) {
void this.refineTitleIfNeeded(threadId, options.userId, options.modelId);
}
@ -5181,16 +4608,12 @@ export class InstanceAiService {
/** Emit a typed event consumed by the Prometheus Instance AI metrics collector. */
private emitRunMetrics(
threadId: string,
status: 'completed' | 'cancelled' | 'errored' | 'suspended',
options?: { modelId?: ModelConfig; workSummary?: WorkSummary; usage?: RunTokenUsage },
): void {
const startedAt = this.runState.getActiveRun(threadId)?.startedAt;
this.eventService.emit('instance-ai-run-finished', {
status: status === 'errored' ? 'error' : status,
// Duration is reported once, by the run's terminal event.
durationMs:
status !== 'suspended' && startedAt !== undefined ? Date.now() - startedAt : undefined,
durationMs: undefined,
model: runMetricsModelLabel(options?.modelId),
toolCalls: options?.workSummary?.totalToolCalls ?? 0,
toolErrors: options?.workSummary?.totalToolErrors ?? 0,

14
packages/cli/src/modules/instance-ai/liveness/index.ts
View File

@ -1,14 +0,0 @@
export {
INSTANCE_AI_RUN_TIMEOUT_REASON,
InstanceAiLivenessService,
type InstanceAiLivenessBackgroundTasks,
type InstanceAiConsumedRunTimeout,
type InstanceAiLivenessEventBus,
type InstanceAiLivenessPendingConfirmation,
type InstanceAiLivenessRunState,
type InstanceAiLivenessServiceOptions,
type InstanceAiLivenessSweepResult,
type InstanceAiLivenessTimedOutActiveRun,
type InstanceAiLivenessTimedOutSuspendedRun,
type InstanceAiLivenessTimedOutTask,
} from './instance-ai-liveness.service';

274
packages/cli/src/modules/instance-ai/liveness/instance-ai-liveness.service.ts
View File

@ -1,274 +0,0 @@
import type { InstanceAiEvent } from '@n8n/api-types';
import type { Logger } from '@n8n/backend-common';
import { Time } from '@n8n/constants';
import {
orchestratorAgentId,
type InstanceAiLivenessPolicy,
type InstanceAiLivenessTimeoutReason,
} from '@n8n/instance-ai';
import { getErrorMessage } from '@n8n/utils/errors/get-error-message';
import type { InstanceAiRunTimeoutDetails } from '../run-timeout-details';
export const INSTANCE_AI_RUN_TIMEOUT_REASON = 'timeout';
const RUN_TIMEOUT_MESSAGE =
'The run stopped making progress, so I cancelled it. You can retry or adjust the request.';
export type InstanceAiLivenessTimedOutActiveRun = {
runId: string;
abortController: AbortController;
};
export type InstanceAiLivenessTimedOutSuspendedRun = {
runId: string;
threadId: string;
abortController: AbortController;
};
export type InstanceAiLivenessTimedOutTask = {
threadId: string;
taskId: string;
role: string;
timeoutReason?: InstanceAiLivenessTimeoutReason;
};
export type InstanceAiLivenessPendingConfirmation = {
threadId: string;
};
export type InstanceAiLivenessSweepResult = {
activeThreadIds: string[];
suspendedThreadIds: string[];
confirmationRequestIds: string[];
activeTimeouts?: Record<string, InstanceAiRunTimeoutDetails>;
suspendedTimeouts?: Record<string, InstanceAiRunTimeoutDetails>;
confirmationTimeouts?: Record<string, InstanceAiRunTimeoutDetails>;
};
export type InstanceAiConsumedRunTimeout = {
timedOut: boolean;
details?: InstanceAiRunTimeoutDetails;
};
export type InstanceAiLivenessRunState<
TSuspendedRun extends InstanceAiLivenessTimedOutSuspendedRun,
> = {
sweepTimedOut: (policy: InstanceAiLivenessPolicy, now?: number) => InstanceAiLivenessSweepResult;
cancelActiveRun: (threadId: string) => InstanceAiLivenessTimedOutActiveRun | undefined;
cancelSuspendedRun: (threadId: string) => TSuspendedRun | undefined;
getActiveRunId: (threadId: string) => string | undefined;
getPendingConfirmation: (requestId: string) => InstanceAiLivenessPendingConfirmation | undefined;
hasPendingConfirmationForThread: (threadId: string) => boolean;
rejectPendingConfirmation: (requestId: string) => boolean;
};
export type InstanceAiLivenessBackgroundTasks = {
timeoutTimedOutTasks: (
policy: InstanceAiLivenessPolicy,
now?: number,
options?: {
shouldSkipTask?: (task: InstanceAiLivenessTimedOutTask) => boolean;
},
) => Promise<InstanceAiLivenessTimedOutTask[]>;
};
export type InstanceAiLivenessEventBus = {
publish: (threadId: string, event: InstanceAiEvent) => void;
};
export type InstanceAiLivenessServiceOptions<
TSuspendedRun extends InstanceAiLivenessTimedOutSuspendedRun,
> = {
policy: InstanceAiLivenessPolicy;
backgroundTaskIdleTimeoutMs: number;
runState: InstanceAiLivenessRunState<TSuspendedRun>;
backgroundTasks: InstanceAiLivenessBackgroundTasks;
eventBus: InstanceAiLivenessEventBus;
finalizeCancelledSuspendedRun: (suspended: TSuspendedRun, reason: string) => void;
onPendingConfirmationRejected?: (requestId: string) => void;
logger: Logger;
};
export class InstanceAiLivenessService<
TSuspendedRun extends InstanceAiLivenessTimedOutSuspendedRun,
> {
private timeoutInterval?: NodeJS.Timeout;
private readonly timedOutRunIds = new Map<string, InstanceAiRunTimeoutDetails | undefined>();
private readonly timedOutActiveRunThreads = new Set<string>();
/**
* Runs whose timeout notice has already been published, so a run cancelled
* by the sweep and then finalised by its own abort handler only gets the
* notice once. Capped FIFO (see {@link NOTICE_DEDUPE_CACHE_SIZE}): dedupe
* only has to hold across a single run's cancellation window, and a run
* lives on one main for one process lifetime.
*/
private readonly noticedRunIds = new Set<string>();
/** Max retained run ids in {@link noticedRunIds}; oldest evicted first. */
static readonly NOTICE_DEDUPE_CACHE_SIZE = 1000;
constructor(private readonly options: InstanceAiLivenessServiceOptions<TSuspendedRun>) {}
get backgroundTaskIdleTimeoutMs(): number {
return this.options.backgroundTaskIdleTimeoutMs;
}
start(): void {
if (!this.options.policy.hasEnabledTimeouts()) return;
this.timeoutInterval = setInterval(() => {
void this.sweepTimedOutWork();
}, Time.minutes.toMilliseconds);
}
shutdown(): void {
if (this.timeoutInterval) {
clearInterval(this.timeoutInterval);
this.timeoutInterval = undefined;
}
this.timedOutRunIds.clear();
this.timedOutActiveRunThreads.clear();
this.noticedRunIds.clear();
}
clearThreadState(threadId: string): void {
this.timedOutActiveRunThreads.delete(threadId);
}
markRunTimedOut(runId: string, details?: InstanceAiRunTimeoutDetails): void {
this.timedOutRunIds.set(runId, details);
}
consumeRunTimedOut(runId: string): boolean {
const timedOut = this.timedOutRunIds.has(runId);
if (timedOut) this.timedOutRunIds.delete(runId);
return timedOut;
}
consumeRunTimeout(runId: string): InstanceAiConsumedRunTimeout {
const timedOut = this.timedOutRunIds.has(runId);
if (!timedOut) return { timedOut: false };
const details = this.timedOutRunIds.get(runId);
this.timedOutRunIds.delete(runId);
return details ? { timedOut: true, details } : { timedOut: true };
}
hasTimedOutActiveRunThread(threadId: string): boolean {
return this.timedOutActiveRunThreads.has(threadId);
}
async sweepTimedOutWork(now = Date.now()): Promise<void> {
const {
activeThreadIds,
suspendedThreadIds,
confirmationRequestIds,
activeTimeouts,
suspendedTimeouts,
confirmationTimeouts,
} = this.options.runState.sweepTimedOut(this.options.policy, now);
for (const threadId of activeThreadIds) {
this.options.logger.debug('Cancelling timed-out active run', { threadId });
this.cancelTimedOutActiveRun(threadId, activeTimeouts?.[threadId]);
}
for (const threadId of suspendedThreadIds) {
this.options.logger.debug('Auto-rejecting timed-out suspended run', { threadId });
this.cancelTimedOutSuspendedRun(threadId, suspendedTimeouts?.[threadId]);
}
for (const reqId of confirmationRequestIds) {
this.options.logger.debug('Auto-rejecting timed-out sub-agent confirmation', {
requestId: reqId,
});
const pending = this.options.runState.getPendingConfirmation(reqId);
if (pending) {
const runId = this.options.runState.getActiveRunId(pending.threadId);
if (runId) {
this.markRunTimedOut(runId, confirmationTimeouts?.[reqId]);
this.publishRunTimeoutNotice(pending.threadId, runId);
}
}
this.options.runState.rejectPendingConfirmation(reqId);
this.options.onPendingConfirmationRejected?.(reqId);
}
try {
const timedOutTasks = await this.options.backgroundTasks.timeoutTimedOutTasks(
this.options.policy,
now,
{
shouldSkipTask: (task) =>
this.options.runState.hasPendingConfirmationForThread(task.threadId),
},
);
for (const task of timedOutTasks) {
this.options.logger.debug('Timed out background task', {
threadId: task.threadId,
taskId: task.taskId,
role: task.role,
timeoutReason: task.timeoutReason,
});
}
} catch (error) {
this.options.logger.warn('Failed to sweep timed-out background tasks', {
error: getErrorMessage(error),
});
}
}
cancelTimedOutActiveRun(threadId: string, details?: InstanceAiRunTimeoutDetails): void {
const active = this.options.runState.cancelActiveRun(threadId);
if (!active) return;
this.markRunTimedOut(active.runId, details);
this.timedOutActiveRunThreads.add(threadId);
this.publishRunTimeoutNotice(threadId, active.runId);
active.abortController.abort();
}
cancelTimedOutSuspendedRun(threadId: string, details?: InstanceAiRunTimeoutDetails): void {
const suspended = this.options.runState.cancelSuspendedRun(threadId);
if (!suspended) return;
this.markRunTimedOut(suspended.runId, details);
suspended.abortController.abort();
this.options.finalizeCancelledSuspendedRun(suspended, INSTANCE_AI_RUN_TIMEOUT_REASON);
}
publishRunTimeoutNotice(threadId: string, runId: string): void {
// Deduped locally rather than by scanning the run's events: the notice is
// a delta, and under the durable log deltas are ephemeral (never stored,
// persisted only once their segment coalesces), so a read-back cannot see
// a notice published moments earlier.
if (this.noticedRunIds.has(runId)) return;
// Recorded only after the publish lands: marking first would suppress the
// notice permanently if publish threw.
this.options.eventBus.publish(threadId, {
type: 'text-delta',
runId,
agentId: orchestratorAgentId(runId),
responseId: `run-timeout:${runId}`,
payload: { text: RUN_TIMEOUT_MESSAGE },
});
this.rememberNoticedRunId(runId);
}
/**
* Record a notified run id, evicting the oldest once the cap is reached. A
* Set keeps insertion order, so the first value is the oldest entry.
*/
private rememberNoticedRunId(runId: string): void {
this.noticedRunIds.add(runId);
if (this.noticedRunIds.size > InstanceAiLivenessService.NOTICE_DEDUPE_CACHE_SIZE) {
const oldest = this.noticedRunIds.values().next().value;
if (oldest !== undefined) this.noticedRunIds.delete(oldest);
}
}
}

9
packages/cli/src/modules/instance-ai/run-timeout-details.ts
View File

@ -1,9 +0,0 @@
import type { InstanceAiLivenessSurface, InstanceAiLivenessTimeoutReason } from '@n8n/instance-ai';
export type InstanceAiRunTimeoutDetails = {
reason: InstanceAiLivenessTimeoutReason;
surface: InstanceAiLivenessSurface;
timeoutMs: number;
elapsedMs: number;
idleMs: number;
};

34
packages/cli/src/modules/instance-ai/run-trace-metadata.ts
View File

@ -1,9 +1,4 @@
import type { InstanceAiEvent } from '@n8n/api-types';
import type { InstanceAiLivenessTimeoutReason } from '@n8n/instance-ai';
import type { InstanceAiRunTimeoutDetails } from './run-timeout-details';
const RUN_TIMEOUT_REASON = 'timeout';
export type InstanceAiFirstVisibleState =
| 'assistant_text'
@ -12,17 +7,9 @@ export type InstanceAiFirstVisibleState =
| 'task_card'
| 'empty';
export type InstanceAiCancellationType = 'explicit' | InstanceAiLivenessTimeoutReason;
export type InstanceAiRunTimeoutTraceContext = {
timedOut: boolean;
details?: InstanceAiRunTimeoutDetails;
};
export type InstanceAiRunTraceMetadataOptions = {
status: 'completed' | 'cancelled' | 'error';
cancellationReason?: string;
runTimeout?: InstanceAiRunTimeoutTraceContext;
};
type FirstVisibleSummary = {
@ -80,18 +67,6 @@ function getFirstVisibleSummary(events: InstanceAiEvent[]): FirstVisibleSummary
return withFirstToolName(sawToolCall ? 'tool_call' : 'empty', firstToolName);
}
function getCancellationType(
options: InstanceAiRunTraceMetadataOptions,
): InstanceAiCancellationType | undefined {
if (options.status !== 'cancelled') return undefined;
if (options.runTimeout?.timedOut || options.cancellationReason === RUN_TIMEOUT_REASON) {
return options.runTimeout?.details?.reason ?? 'idle_timeout';
}
return 'explicit';
}
export function buildInstanceAiRunTraceMetadata(
events: InstanceAiEvent[],
options: InstanceAiRunTraceMetadataOptions,
@ -105,13 +80,8 @@ export function buildInstanceAiRunTraceMetadata(
metadata.first_tool_name = firstVisible.firstToolName;
}
const cancellationType = getCancellationType(options);
if (cancellationType) {
metadata.cancellation_type = cancellationType;
}
if (options.runTimeout?.details?.idleMs !== undefined) {
metadata.idle_tail_ms = Math.round(options.runTimeout.details.idleMs);
if (options.status === 'cancelled') {
metadata.cancellation_type = 'explicit';
}
return metadata;

2
packages/cli/src/modules/instance-ai/sandbox/index.ts
View File

@ -1,8 +1,6 @@
export {
InstanceAiSandboxService,
type InstanceAiSandboxBackgroundTasks,
type InstanceAiSandboxProxy,
type InstanceAiSandboxRunState,
type InstanceAiSandboxServiceOptions,
type InstanceAiSandboxSettings,
type RuntimeSandboxEntry,

28
packages/cli/src/modules/instance-ai/sandbox/instance-ai-sandbox.service.ts
View File

@ -12,7 +12,6 @@ import {
type ServiceProxyConfig,
type InstanceAiContext,
type Logger,
type ManagedBackgroundTask,
type SandboxConfig,
} from '@n8n/instance-ai';
import type { ErrorReporter } from 'n8n-core';
@ -130,17 +129,6 @@ function withThreadScopedSandboxIdentity(config: SandboxConfig, threadId: string
};
}
/** Thread-run state the sandbox lifecycle consults to know when a workspace is still in use. */
export type InstanceAiSandboxRunState = {
getActiveRunId: (threadId: string) => string | undefined;
hasSuspendedRun: (threadId: string) => boolean;
};
/** Background-task view the sandbox lifecycle consults to know when a workspace is still in use. */
export type InstanceAiSandboxBackgroundTasks = {
getRunningTasks: (threadId: string) => ManagedBackgroundTask[];
};
/** Settings collaborator that resolves provider credentials from admin config. */
export type InstanceAiSandboxSettings = {
resolveDaytonaConfig: () => Promise<{ apiUrl?: string; apiKey?: string }>;
@ -164,8 +152,6 @@ export type InstanceAiSandboxServiceOptions = {
config: InstanceAiConfig;
logger: Logger;
errorReporter: ErrorReporter;
runState: InstanceAiSandboxRunState;
backgroundTasks: InstanceAiSandboxBackgroundTasks;
settingsService: InstanceAiSandboxSettings;
aiService: InstanceAiSandboxProxy;
resolveTracingConfig?: (
@ -359,7 +345,7 @@ export class InstanceAiSandboxService {
if (existing) {
if (
existing.configFingerprint !== cacheState.fingerprint ||
(this.isSandboxEntryExpired(existing) && !this.isSandboxInUse(threadId))
this.isSandboxEntryExpired(existing)
) {
this.evictSandboxEntry(
threadId,
@ -653,14 +639,6 @@ export class InstanceAiSandboxService {
this.scheduleSandboxExpiry(threadId, entry);
}
private isSandboxInUse(threadId: string): boolean {
return Boolean(
this.options.runState.getActiveRunId(threadId) ||
this.options.runState.hasSuspendedRun(threadId) ||
this.options.backgroundTasks.getRunningTasks(threadId).length > 0,
);
}
private scheduleSandboxExpiry(threadId: string, entry: RuntimeSandboxEntry): void {
if (this.sandboxTtlMs <= 0) return;
if (entry.cleanupTimer) clearTimeout(entry.cleanupTimer);
@ -673,10 +651,6 @@ export class InstanceAiSandboxService {
entry.cleanupTimer = setTimeout(() => {
const current = this.sandboxes.get(threadId);
if (current !== entry) return;
if (this.isSandboxInUse(threadId)) {
this.touchSandboxEntry(threadId, entry);
return;
}
this.evictSandboxEntry(threadId, entry, 'idle', true);
}, delay);
entry.cleanupTimer.unref();

1
packages/cli/src/modules/instance-ai/tracing/index.ts
View File

@ -2,7 +2,6 @@ export {
InstanceAiTracingService,
type InstanceAiTracingAiService,
type InstanceAiTracingEventReader,
type InstanceAiTracingRunState,
type InstanceAiTracingEventLog,
type InstanceAiTracingServiceOptions,
type MessageTraceFinalization,

83
packages/cli/src/modules/instance-ai/tracing/instance-ai-tracing.service.ts
View File

@ -8,9 +8,7 @@ import {
submitLangsmithUserFeedback,
type BrowserExtensionTraceContext,
type InstanceAiTraceContext,
type ManagedBackgroundTask,
type ModelConfig,
type RunStateRegistry,
type ServiceProxyConfig,
} from '@n8n/instance-ai';
import { getErrorMessage } from '@n8n/utils/errors/get-error-message';
@ -63,8 +61,6 @@ export type InstanceAiTracingEventReader = {
getEventsForRun: (threadId: string, runId: string) => Promise<InstanceAiEvent[]>;
};
export type InstanceAiTracingRunState = Pick<RunStateRegistry<User>, 'attachTracing'>;
export type InstanceAiTracingEventLog = Pick<InstanceAiEventLogRepository, 'findLangsmithAnchor'>;
export type InstanceAiTracingAiService = Pick<AiService, 'isProxyEnabled' | 'getClient'>;
@ -72,7 +68,6 @@ export type InstanceAiTracingAiService = Pick<AiService, 'isProxyEnabled' | 'get
export type InstanceAiTracingServiceOptions = {
logger: Logger;
eventReader: InstanceAiTracingEventReader;
runState: InstanceAiTracingRunState;
eventLog: InstanceAiTracingEventLog;
aiService: InstanceAiTracingAiService;
};
@ -84,7 +79,7 @@ export type InstanceAiTracingServiceOptions = {
* ID that started an orchestration turn) and the test-only trace replay state.
* Responsible for creating resume trace contexts, finalizing message- and
* run-level trace roots, releasing trace clients, and submitting LangSmith user
* feedback. Collaborators (run state, event bus, event log, AI service)
* feedback. Collaborators (event reader, event log, AI service)
* are supplied via the options bag because the run-context registry it manages
* is process-local and not suitable for dependency injection.
*/
@ -107,8 +102,6 @@ export class InstanceAiTracingService {
private readonly eventReader: InstanceAiTracingEventReader;
private readonly runState: InstanceAiTracingRunState;
private readonly eventLog: InstanceAiTracingEventLog;
private readonly aiService: InstanceAiTracingAiService;
@ -116,7 +109,6 @@ export class InstanceAiTracingService {
constructor(options: InstanceAiTracingServiceOptions) {
this.logger = options.logger;
this.eventReader = options.eventReader;
this.runState = options.runState;
this.eventLog = options.eventLog;
this.aiService = options.aiService;
}
@ -184,7 +176,6 @@ export class InstanceAiTracingService {
messageGroupId?: string,
): void {
this.storeTraceContext(runId, threadId, tracing, messageGroupId);
this.runState.attachTracing(threadId, tracing);
}
async createOrchestratorResumeTraceContext(options: {
@ -297,7 +288,6 @@ export class InstanceAiTracingService {
...(options.cancellationReason !== undefined
? { cancellationReason: options.cancellationReason }
: {}),
...(options.runTimeout !== undefined ? { runTimeout: options.runTimeout } : {}),
};
// The events read hits the DB under the durable log, and most callers run
@ -374,57 +364,6 @@ export class InstanceAiTracingService {
this.traceContextsByRunId.clear();
}
async finalizeDetachedTraceRun(
taskId: string,
traceContext: InstanceAiTraceContext | undefined,
options: {
status: 'completed' | 'failed' | 'cancelled';
outputs?: Record<string, unknown>;
error?: string;
metadata?: Record<string, unknown>;
},
): Promise<void> {
if (!traceContext) return;
try {
if (
traceContext.actorRun.id !== traceContext.rootRun.id &&
traceContext.actorRun.endTime === undefined
) {
await traceContext.finishRun(traceContext.actorRun, {
outputs: {
status: options.status,
...options.outputs,
},
metadata: {
final_status: options.status,
...options.metadata,
},
...(options.error ? { error: options.error } : {}),
});
}
await traceContext.finishRun(traceContext.rootRun, {
outputs: {
status: options.status,
...options.outputs,
},
metadata: {
final_status: options.status,
...options.metadata,
},
...(options.error ? { error: options.error } : {}),
});
} catch (error) {
this.logger.warn('Failed to finalize Instance AI detached trace run', {
taskId,
traceRunId: traceContext.rootRun.id,
error: getErrorMessage(error),
});
} finally {
releaseTraceClient(traceContext.rootRun.traceId);
}
}
async finalizeRunTracing(
runId: string,
tracing: InstanceAiTraceContext | undefined,
@ -461,26 +400,6 @@ export class InstanceAiTracingService {
}
}
async finalizeBackgroundTaskTracing(
task: ManagedBackgroundTask,
status: 'completed' | 'failed' | 'cancelled',
): Promise<void> {
await this.finalizeDetachedTraceRun(task.taskId, task.traceContext, {
status,
outputs: {
taskId: task.taskId,
agentId: task.agentId,
role: task.role,
...(task.result ? { result: task.result } : {}),
},
...(status === 'failed' && task.error ? { error: task.error } : {}),
metadata: {
...(task.plannedTaskId ? { planned_task_id: task.plannedTaskId } : {}),
...(task.workItemId ? { work_item_id: task.workItemId } : {}),
},
});
}
async submitLangsmithFeedback(
user: User,
threadId: string,

72
packages/cli/src/modules/instance-ai/workflow-verification-task-projector.ts
View File

@ -5,7 +5,6 @@ import {
orchestratorAgentId,
ThreadTaskStorage,
WorkflowLoopStorage,
type ManagedBackgroundTask,
type PlannedTaskGraph,
type PlannedTaskRecord,
type WorkflowBuildOutcome,
@ -280,61 +279,6 @@ export class WorkflowVerificationTaskProjector {
return verifyRow(task.id, obligation);
}
/** Sync a direct builder's checklist rows from its background-task lifecycle. */
async syncFromBackgroundTask(task: ManagedBackgroundTask): Promise<void> {
if (task.plannedTaskId || task.parentCheckpointId || task.role !== 'workflow-builder') return;
const items = await this.directItemsFromTask(task);
if (!items) return;
await this.upsertItems(task.threadId, task.runId, items);
}
private async directItemsFromTask(task: ManagedBackgroundTask): Promise<TaskItem[] | undefined> {
const build = { id: task.taskId, description: BUILD_DESCRIPTION };
const verify = { id: verifyRowId(task.taskId), description: VERIFY_DESCRIPTION };
switch (task.status) {
case 'running':
return [
{ ...build, detail: DETAIL.building, status: 'in_progress' },
{ ...verify, detail: DETAIL.waitingForBuild, status: 'todo' },
];
case 'failed':
return [
{ ...build, detail: task.error ?? DETAIL.blocked, status: 'failed' },
{ ...verify, detail: DETAIL.buildIncomplete, status: 'cancelled' },
];
case 'cancelled':
return [
{ ...build, detail: task.error ?? DETAIL.cancelled, status: 'cancelled' },
{ ...verify, detail: DETAIL.buildIncomplete, status: 'cancelled' },
];
case 'completed':
return await this.directItemsFromCompletedTask(task, build, verify);
}
}
private async directItemsFromCompletedTask(
task: ManagedBackgroundTask,
build: { id: string; description: string },
verify: { id: string; description: string },
): Promise<TaskItem[] | undefined> {
const outcome = parseWorkflowBuildOutcome(task.outcome);
const workItemId = task.workItemId ?? outcome?.workItemId;
if (!workItemId) {
return [
{ ...build, detail: DETAIL.submitted, status: 'done' },
{ ...verify, detail: DETAIL.noWorkflow, status: 'cancelled' },
];
}
const obligation = await this.obligations.getObligation(task.threadId, workItemId, {
source: 'direct',
});
if (!obligation) return undefined;
return directTaskItems(task.taskId, outcome, obligation);
}
/** Re-derive every direct builder's checklist rows from workflow-loop storage. */
async syncFromWorkflowLoop(threadId: string, runId: string): Promise<void> {
try {
@ -382,22 +326,6 @@ export class WorkflowVerificationTaskProjector {
}
}
private async upsertItems(threadId: string, runId: string, items: TaskItem[]): Promise<void> {
try {
const taskStorage = new ThreadTaskStorage(this.agentMemory);
const existing = (await taskStorage.get(threadId)) ?? { tasks: [] };
const { tasks, changed } = upsertTaskItemsInOrder(existing.tasks, items);
if (!changed) return;
await this.saveAndPublish(taskStorage, threadId, runId, { tasks });
} catch (error) {
this.logger.warn('Failed to update direct workflow builder task checklist items', {
threadId,
taskIds: items.map((item) => item.id),
error: getErrorMessage(error),
});
}
}
private async saveAndPublish(
taskStorage: ThreadTaskStorage,
threadId: string,

1
packages/cli/src/scaling/constants.ts
View File

@ -54,7 +54,6 @@ export const IMMEDIATE_COMMANDS = new Set<PubSub.Command['command']>([
'cancel-agent-background-job',
'wake-agent-background-job',
'relay-instance-ai-event',
'relay-instance-ai-task-control',
'agent-chat-subscription-changed',
'agent-chat-integration-changed',
// Correlated request/response pairs: the subscriber debounces by command name,

14
packages/cli/src/scaling/pubsub/pubsub.event-map.ts
View File

@ -238,20 +238,6 @@ export type PubSubCommandMap = {
storedEvent: { id?: number; event: InstanceAiEvent };
};
/**
* Relay an Instance AI task-control action (correction / cancel / clear) to
* sibling mains. The action may target a background task or run held
* in another main's in-memory state. Broadcast + local-gate: every main applies
* it only to its own local slice of the thread.
*/
'relay-instance-ai-task-control': {
threadId: string;
userId?: string;
taskId?: string;
action: 'correct' | 'cancel-task' | 'cancel-thread' | 'clear-thread';
correction?: string;
};
/**
* Relay human message events between main instances.
* Used for cross-client synchronization when a user sends a message

2
packages/cli/src/scaling/pubsub/pubsub.types.ts
View File

@ -70,7 +70,6 @@ export namespace PubSub {
export type WakeAgentBackgroundJob = ToCommand<'wake-agent-background-job'>;
export type RelayChatStreamEvent = ToCommand<'relay-chat-stream-event'>;
export type RelayInstanceAiEvent = ToCommand<'relay-instance-ai-event'>;
export type RelayInstanceAiTaskControl = ToCommand<'relay-instance-ai-task-control'>;
export type RelayChatHumanMessage = ToCommand<'relay-chat-human-message'>;
export type RelayChatMessageEdit = ToCommand<'relay-chat-message-edit'>;
export type ClearTestWebhooks = ToCommand<'clear-test-webhooks'>;
@ -122,7 +121,6 @@ export namespace PubSub {
| Commands.WakeAgentBackgroundJob
| Commands.RelayChatStreamEvent
| Commands.RelayInstanceAiEvent
| Commands.RelayInstanceAiTaskControl
| Commands.RelayChatHumanMessage
| Commands.RelayChatMessageEdit
| Commands.ClearTestWebhooks