github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-10-17 05:29:44 +02:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity Graph

refactor(core): Remove legacy Instance AI eval agent (no-changelog) (#36740)

Browse Source
This commit is contained in:
Predrag Ristic 2026-09-08 10:14:32 +00:00 committed by GitHub
parent 2550501caa
commit 418f85e9eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
95 changed files with 503 additions and 12148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
packages/@n8n/api-types/src/schemas/__tests__/agent-run-reducer.test.ts
View File

@ -494,16 +494,21 @@ describe('agent-run-reducer', () => {
makeToolCall('run-1', 'root', 'tc-legacy-builder', 'build-workflow-with-agent'),
);
reduceEvent(state, makeToolCall('run-1', 'root', 'tc-research', 'research-with-agent'));
reduceEvent(state, makeToolCall('run-1', 'root', 'tc-eval-setup', 'eval-setup-with-agent'));
reduceEvent(state, makeToolCall('run-1', 'root', 'tc-skill', 'load_skill'));
expect(state.toolCallsById['tc-builder'].renderHint).toBe('builder');
expect(state.toolCallsById['tc-legacy-builder'].renderHint).toBe('builder');
expect(state.toolCallsById['tc-research'].renderHint).toBe('researcher');
expect(state.toolCallsById['tc-eval-setup'].renderHint).toBe('eval-setup');
expect(state.toolCallsById['tc-skill'].renderHint).toBe('skill');
});
it('keeps the eval-setup render hint when replaying a historical tool call', () => {
const state = stateWithRun('run-1', 'root');
reduceEvent(state, makeToolCall('run-1', 'root', 'tc-eval-setup', 'eval-setup-with-agent'));
expect(state.toolCallsById['tc-eval-setup'].renderHint).toBe('eval-setup');
});
it('tool-input-start announces a pending tool call before its args stream', () => {
const state = stateWithRun('run-1', 'root');
reduceEvent(state, {
@ -644,6 +649,23 @@ describe('agent-run-reducer', () => {
});
});
it('hydrates the kind from a historical eval-setup agent event', () => {
const state = stateWithRun('run-1', 'root');
reduceEvent(state, {
type: 'agent-spawned',
runId: 'run-1',
agentId: 'legacy-eval',
payload: {
parentId: 'root',
role: 'evaluation setup',
tools: ['workflows'],
kind: 'eval-setup',
},
});
expect(state.agentsById['legacy-eval'].kind).toBe('eval-setup');
});
it('agent-spawned with unknown parent is silently dropped', () => {
const state = stateWithRun('run-1', 'root');
reduceEvent(state, makeAgentSpawned('run-1', 'orphan', 'unknown-parent'));

19
packages/@n8n/api-types/src/schemas/__tests__/instance-ai.schema.test.ts
View File

@ -101,6 +101,25 @@ describe('instanceAiEventSchema', () => {
expect(instanceAiEventSchema.parse(event)).toEqual(event);
});
it('parses historical eval-setup agent events', () => {
const event = {
type: 'agent-spawned',
runId: 'run-legacy-eval',
agentId: 'agent-legacy-eval',
payload: {
parentId: 'agent-root',
role: 'evaluation setup',
tools: ['workflows'],
taskId: 'task-legacy-eval',
kind: 'eval-setup',
title: 'Setting up evaluations',
targetResource: { type: 'workflow', id: 'workflow-1' },
},
};
expect(instanceAiEventSchema.parse(event)).toEqual(event);
});
it('keeps setup-items durable (not ephemeral) so snapshots survive refresh', () => {
expect(INSTANCE_AI_EPHEMERAL_EVENT_TYPES.has('setup-items')).toBe(false);
});

3
packages/@n8n/instance-ai/README.md
View File

@ -14,8 +14,7 @@ database entities, settings, and n8n service adapters live in
Instance AI is built around a deep-agent loop:
- An orchestrator agent receives the user's request and maintains the plan.
- Most work runs in the orchestrator. Eval setup uses a focused background agent.
- Eval data preparation runs synchronously in the orchestrator.
- Evaluation setup runs in the orchestrator through the config-evals skill.
- Domain tools read and update n8n resources through backend adapters.
- Observational memory condenses long conversations.
- Workflow building runs in a sandbox workspace, validates generated TypeScript,

57
packages/@n8n/instance-ai/docs/architecture.md
View File

@ -7,9 +7,9 @@ natural language interface to workflows, executions, credentials, and nodes —
the goal that most users never need to interact with workflows directly.
The system follows the **deep agent architecture** — an orchestrator with explicit
planning, orchestrator-led workflow building, a specialized eval-setup background
agent, observational memory, and structured prompts. The LLM controls the
execution loop; the architecture provides the primitives.
planning, orchestrator-led workflow building, observational memory, and
structured prompts. The LLM controls the execution loop; the architecture
provides the primitives.
The system is LLM-agnostic and designed to work with any capable language model.
@ -38,17 +38,10 @@ graph TB
OrcAgent --> DirectTools[Domain Tools]
OrcAgent --> MCPTools[MCP Tools]
OrcAgent --> Memory[Memory System]
OrcAgent --> EvalSetupTool[eval-setup-with-agent]
end
subgraph BackgroundAgent ["Detached Domain-Task Agent"]
EvalSetupTool -->|spawns| EvalSetupAgent[Eval Setup Agent]
EvalSetupAgent --> EvalTools[Workflow + Node Tools]
end
subgraph EventSystem ["Event System"]
OrcAgent -->|publishes| EventBus
EvalSetupAgent -->|publishes| EventBus
EventBus --> DurableLog[Durable Event Log]
DurableLog --> EventsTable[(instance_ai_events)]
end
@ -106,16 +99,9 @@ Plans are stored in thread-scoped storage.
### 2. Orchestrator-Led Execution
Most work runs in the orchestrator itself: workflow building via the
Work runs in the orchestrator itself: workflow building via the
`workflow-builder` skill and `build-workflow`, data-table operations, web
research, credential setup with Computer Use, and MCP tools.
The only detached domain-task agent launched by an orchestration tool is the
**eval-setup agent**
(`eval-setup-with-agent`). It patches workflows with EvaluationTrigger and
Evaluation nodes after the user approves an eval proposal. It receives a
focused tool subset, publishes events directly to the event bus, and cannot
spawn further agents.
research, credential setup with Computer Use, config-based evaluations, and MCP tools.
### 3. Observational Memory
@ -126,8 +112,7 @@ that the orchestrator must send to the model during a long loop.
### 4. Structured System Prompt
The orchestrator's system prompt covers planning discipline, loop behavior, and
tool usage guidelines. The eval-setup background agent gets a focused, task-specific
prompt.
tool usage guidelines.
## Agent Hierarchy
@ -139,7 +124,6 @@ graph TD
O -->|direct| T2[executions]
O -->|direct| T3[credentials]
O -->|direct| T5[data-tables]
O -->|eval-setup-with-agent| S5[Eval Setup Agent]
S3 -->|kind: build-workflow| S4[Orchestrator Follow-Up]
S3 -->|kind: checkpoint| S6[Orchestrator Follow-Up]
@ -147,12 +131,10 @@ graph TD
S4 -->|tools| T8[nodes]
S4 -->|tools| T9[workspace files]
S4 -->|tools| T10
S5 -->|tools| T11[workflows + nodes]
style O fill:#f9f,stroke:#333
style S3 fill:#ffa,stroke:#333
style S4 fill:#bbf,stroke:#333
style S5 fill:#bbf,stroke:#333
style S6 fill:#bbf,stroke:#333
```
@ -163,6 +145,7 @@ graph TD
- Workflow building (`workflow-builder` skill + workspace files + `build-workflow`)
- Verification and credential application (verify-built-workflow, apply-workflow-credentials)
- Data-table work (`data-table-manager` skill + `data-tables` / `parse-file`)
- Config-based evaluations (`config-evals` skill + `eval-config`)
**Planned tasks** (`planning` skill + `create-tasks`):
- Dependency-aware task graphs with parallel execution
@ -172,10 +155,6 @@ graph TD
- Workflow runtime verification is tracked separately as a workflow-loop
obligation, so routine "verify workflow" checkpoints are not required
**Eval setup** (`eval-setup-with-agent`):
- Detached background agent that patches eval nodes into an existing workflow
- Triggered after `evals(action="propose")` returns `shouldDelegateToEvalSetupAgent: true`
## Package Responsibilities
### `@n8n/instance-ai` (Core)
@ -183,8 +162,8 @@ graph TD
The agent package — framework-agnostic business logic.
- **Agent factory** (`agent/`) — creates orchestrator instances with tools, memory, MCP, and tool search
- **Sub-agent support** (`tools/orchestration/`, `agent/`) — creates the eval-setup background agent and its shared briefing and persistence protocol
- **Orchestration tools** (`tools/orchestration/`) — `create-tasks`, `task-control`, `complete-checkpoint`, `eval-setup-with-agent`, `eval-data`, `verify-built-workflow`, `report-verification-verdict`, `apply-workflow-credentials`, `build-agent`, `get-session`
- **Sub-agent support** (`agent/`) — shared protocol for embedded specialist agents
- **Orchestration tools** (`tools/orchestration/`) — `create-tasks`, `task-control`, `complete-checkpoint`, `verify-built-workflow`, `report-verification-verdict`, `apply-workflow-credentials`, `build-agent`, `get-session`
- **Domain tools** (`tools/`) — native tools across workflows, executions, credentials, nodes, data tables, workspace, and web research
- **Knowledge base** (`knowledge-base/`, `workspace/`) — best-practices guides and curated templates materialized in the builder sandbox for workspace tools to read
- **Runtime** (`runtime/`) — stream execution engine, resumable streams with HITL suspension, background task manager, run state registry
@ -271,13 +250,12 @@ intentional:
- MCP server configuration can change between requests
- User context (permissions) is request-scoped
- Memory is handled externally (storage-backed), not in-agent
- Background agents (eval-setup) are created within the request lifecycle
### 3. Pub/Sub Streaming
The event bus decouples agent execution from event delivery:
- All agents (orchestrator + eval-setup background agent) publish to a per-thread channel
- Agent runs publish to a per-thread channel
- Frontend subscribes via SSE with `Last-Event-ID` for reconnect/replay
- All events carry `runId` (correlates to triggering message) and `agentId`
- Durable SSE facts use monotonically increasing per-thread `id` values for replay
@ -285,7 +263,6 @@ The event bus decouples agent execution from event delivery:
- Coalesced step-level facts are appended to the `instance_ai_events` table —
the only replay source, so ids survive restarts — while token deltas remain
live-only and are not retained
- No need to pipe sub-agent streams through orchestrator tool execution
- One active run per thread (additional `POST /chat` is rejected while active)
- Cancellation via `POST /instance-ai/chat/:threadId/cancel` (idempotent)
@ -314,15 +291,15 @@ streamAgentRun() → agent.stream() → executeResumableStream()
The `executeResumableStream()` loop consumes agent chunks, translates them to
canonical `InstanceAiEvent` schema, publishes to the event bus, and handles HITL
suspension/resume cycles. Two control modes:
suspension/resume cycles. It supports two control modes:
- **Manual** — returns suspension to caller (used by the orchestrator's main run)
- **Auto** — waits for confirmation and resumes automatically (used by the eval-setup background agent)
- **Auto** — waits for confirmation and resumes automatically
### Background Task Manager
Long-running eval-setup tasks run as background tasks with concurrency limits
(default: 5 per thread). Features:
The generic background task manager enforces concurrency limits (default: 5 per
thread). Features:
- **Correction queueing** — users can steer running tasks mid-flight via
`task-control(action="correct-task")`
@ -433,8 +410,7 @@ The cli's `InstanceAiService` holds one manager instance and passes it to
The embedded Agent Builder receives the same per-run, approval-wrapped MCP tool
registry as the orchestrator. Builder-native tool names remain reserved, so an
MCP connector cannot shadow configuration or lifecycle tools. Specialized
background agents such as eval setup remain isolated from MCP tools.
MCP connector cannot shadow configuration or lifecycle tools.
The local Computer Use server is separate from external MCP configuration. Its
browser tools are available directly to the orchestrator and are guided by the
@ -470,9 +446,6 @@ allowing the user to approve or deny access to specific hosts.
- **Domain access gating** — external URL fetches require per-domain user approval
- **Memory isolation** — messages, observations, plans, and event history are
thread-scoped. Cross-user isolation is enforced.
- **Sub-agent containment** — the eval-setup background agent cannot spawn further
agents, receives only its wired tool subset (no MCP tools), and has a bounded
`maxIterations`. A mandatory protocol prevents cascading delegation.
- **MCP tool isolation** — MCP tools are name-checked against reserved domain tool
names to prevent shadowing. Schema sanitization converts unsupported shapes
for provider compatibility.

10
packages/@n8n/instance-ai/docs/e2e-tests.md
View File

@ -114,8 +114,8 @@ would fail when replay produces different IDs or other runtime data.
### Shared State Across Runs
A single test may trigger multiple n8n "runs" — the orchestrator run, a planned
task follow-up, or an eval-setup background task. The `TraceIndex` and
A single test may trigger multiple n8n "runs" — the orchestrator run or a planned
task follow-up. The `TraceIndex` and
`IdRemapper` are shared across all runs within one test (keyed by the test slug),
so cursor positions and ID mappings persist correctly.
@ -212,8 +212,8 @@ The `TraceIndex` groups events by `agentRole` with independent cursors per role.
```
orchestrator: [nodes, build-workflow, executions-suspend, executions-resume]
^cursor=0
eval-setup: [workflows(action="get-json"), workflows(action="update")]
^cursor=0
agent-builder: [read_config, write_config]
^cursor=0
```
When a tool is called, `traceIndex.next(role, toolName)` advances that role's cursor and validates the tool name matches. A mismatch means the agent diverged from the recorded path — the test fails with a clear error.
@ -412,7 +412,7 @@ LLM responses are frozen — the replay serves the exact same bytes regardless o
3. **Per-role trace cursors** — The `TraceIndex` groups events by `agentRole` with independent cursors. This handles interleaved orchestrator and sub-agent calls naturally, without requiring a single global sequence that breaks when parallelism changes.
4. **Shared state across runs** — The `TraceIndex` and `IdRemapper` are shared across all runs within one test (orchestrator run, planned follow-up, eval-setup background task). This means a workflowId learned in run 1 is available for remapping in run 2.
4. **Shared state across runs** — The `TraceIndex` and `IdRemapper` are shared across all runs within one test (orchestrator run and planned follow-ups). This means a workflowId learned in run 1 is available for remapping in run 2.
5. **Reduced request-body matching** — During recording, full LLM request bodies are replaced with regex anchors for the agent type and stable turn context. Volatile IDs and dynamic prompt context are normalized so replay does not depend on the complete conversation or raw tool output.

16
packages/@n8n/instance-ai/docs/memory.md
View File

@ -10,9 +10,7 @@ The memory system serves two purposes:
- **Conversation history** — the stored messages for the current thread
(thread-scoped)
The eval-setup background agent does not share the orchestrator's observational
memory. It receives a briefing and uses a dedicated persistence wrapper for
checkpoint and suspension state.
Embedded specialist agents do not share the orchestrator's observational memory.
## Tiers
@ -75,16 +73,10 @@ All memory is thread-scoped (isolated per conversation):
- **Observational memory** — compressed operational history
- **Plan** — the current execution plan
### Sub-agent memory
### Embedded agent memory
The eval-setup background agent receives its task briefing and optional
`conversationContext` from `eval-setup-with-agent`. It does not read or write
the orchestrator's observational memory. Its agent persistence stores the
checkpoint data needed for resume and HITL handling.
An `IterationLog` database adapter is available through the dedicated
`instance_ai_iteration_logs` table. The briefing builder can include these
entries when supplied by a caller.
The Agent Builder owns its session state separately from the orchestrator's
observational memory. Generic background-task state stays thread-scoped.
### Cross-user isolation

79
packages/@n8n/instance-ai/docs/streaming-protocol.md
View File

@ -3,9 +3,8 @@
## Overview
Instance AI uses a pub/sub event bus to deliver agent events to the frontend
in real-time. All agents — the orchestrator and eval-setup background agent —
publish events to a per-thread channel. The frontend subscribes independently
via SSE.
in real-time. Agent runs publish events to a per-thread channel. The frontend
subscribes independently via SSE.
The protocol is designed for minimal time-to-first-token, progressive rendering
of multi-agent activity, and resilient reconnection.
@ -52,11 +51,9 @@ data: {"type":"tool-call","runId":"run_abc","agentId":"agent-001","payload":{"to
```
Event IDs are monotonically increasing integers per thread channel. They are
unique within that thread. With the durable log enabled, the shared database
assigns the per-thread sequence. With the durable log disabled, multi-main
deployments use a shared Redis sequence. In both modes, IDs and replay cursors
are valid against any main. Live-only ephemeral frames do not include an `id:`
field when the durable log is enabled.
unique within that thread. The shared database assigns the per-thread sequence,
so IDs and replay cursors are valid against any main. Live-only ephemeral frames
do not include an `id:` field.
## Event Schema
@ -78,7 +75,7 @@ The `runId` correlates all events started by one user message. This includes
events from detached work that continues after the orchestrator responds. The
POST endpoint returns the `runId`, and every event carries it.
The `agentId` identifies which agent branch (orchestrator or background agent) the
The `agentId` identifies which agent branch (orchestrator or child agent) the
event belongs to. The frontend uses this to render an agent activity tree.
For the full TypeScript type definitions, see
@ -208,8 +205,7 @@ A tool has failed.
### `agent-spawned`
The orchestrator has started a detached background agent (for example via
`eval-setup-with-agent`).
The orchestrator has started a child or embedded specialist agent.
```json
{
@ -218,19 +214,21 @@ The orchestrator has started a detached background agent (for example via
"agentId": "agent-002",
"payload": {
"parentId": "agent-001",
"role": "eval-setup",
"tools": ["workflows", "nodes"]
"role": "agent-builder",
"tools": [],
"kind": "agent-builder",
"title": "Building agent"
}
}
```
The frontend adds a new node to the agent activity tree under the parent.
For this event type, `agentId` is the spawned background agent ID; `payload.parentId`
links it to the orchestrator.
For this event type, `agentId` is the child agent ID; `payload.parentId` links it
to the orchestrator. Historical detached-agent events use the same shape.
### `agent-completed`
A background agent has finished its work.
A child or embedded specialist agent has finished its work.
```json
{
@ -238,13 +236,13 @@ A background agent has finished its work.
"runId": "run_abc123",
"agentId": "agent-002",
"payload": {
"role": "eval-setup",
"result": "Added evaluation nodes to workflow wf-123"
"role": "agent-builder",
"result": "Updated the support agent"
}
}
```
The frontend marks the background agent node as completed.
The frontend marks the child agent node as completed.
### `confirmation-request`
@ -427,25 +425,22 @@ The four statuses are `completed`, `cancelled`, `error` and `interrupted`.
← run-finish {runId: "r1", agentId: "a1", payload: {status: "completed"}}
```
### Eval Setup Background Agent
### Agent Builder Child Agent
```
← run-start {runId: "r1", agentId: "a1", payload: {messageId: "m1"}}
← tool-call {runId: "r1", agentId: "a1", payload: {toolCallId: "tc1", toolName: "eval-setup-with-agent", args: {workflowId: "wf-123", task: "Add evaluation nodes"}}}
← agent-spawned {runId: "r1", agentId: "a2", payload: {parentId: "a1", role: "eval-setup", tools: ["workflows", "nodes"], taskId: "task-1"}}
← tool-result {runId: "r1", agentId: "a1", payload: {toolCallId: "tc1", result: {result: "Eval setup started (task: task-1).", taskId: "task-1"}}}
← text-delta {runId: "r1", agentId: "a1", payload: {text: "Evaluation setup has started."}}
← run-finish {runId: "r1", agentId: "a1", payload: {status: "completed"}}
← tool-call {runId: "r1", agentId: "a2", payload: {toolCallId: "tc2", toolName: "workflows", args: {action: "get-json", workflowId: "wf-123"}}}
← tool-call {runId: "r1", agentId: "a1", payload: {toolCallId: "tc1", toolName: "build-agent", args: {name: "Support agent", message: "Add a support task"}}}
← agent-spawned {runId: "r1", agentId: "a2", payload: {parentId: "a1", role: "agent-builder", tools: [], kind: "agent-builder"}}
← tool-call {runId: "r1", agentId: "a2", payload: {toolCallId: "tc2", toolName: "read_config", args: {}}}
← tool-result {runId: "r1", agentId: "a2", payload: {toolCallId: "tc2", result: {...}}}
← tool-call {runId: "r1", agentId: "a2", payload: {toolCallId: "tc3", toolName: "workflows", args: {action: "update", workflowId: "wf-123", workflow: {...}}}}
← tool-call {runId: "r1", agentId: "a2", payload: {toolCallId: "tc3", toolName: "write_config", args: {...}}}
← tool-result {runId: "r1", agentId: "a2", payload: {toolCallId: "tc3", result: {...}}}
← agent-completed {runId: "r1", agentId: "a2", payload: {role: "eval-setup", result: "Added evaluation nodes"}}
← agent-completed {runId: "r1", agentId: "a2", payload: {role: "agent-builder", result: "Updated the support agent"}}
← tool-result {runId: "r1", agentId: "a1", payload: {toolCallId: "tc1", result: {ok: true, agentId: "agent-123", configUpdated: true}}}
← text-delta {runId: "r1", agentId: "a1", payload: {text: "The support agent is ready."}}
← run-finish {runId: "r1", agentId: "a1", payload: {status: "completed"}}
```
Because eval setup is detached, its events can interleave with orchestrator
events or continue after the orchestrator's `run-finish` event.
## Event Bus
### Architecture
@ -552,21 +547,19 @@ The frontend renders events as a collapsible tree grouped by `agentId`:
🤖 Orchestrator
├── 💭 "Let me check what credentials are available..."
├── 🔧 credentials → [slack-bot, weather-api]
├── 📋 create-tasks: build → configure evaluation
├── 🔧 build-workflow → wf-123
├── 🔧 executions(run) wf-123
├── 📋 create-tasks: build → verify
├── 🔧 build-agent → agent-123
│
├── 🤖 Eval setup
│ ├── 🔧 workflows(get-json) → wf-123
│ ├── 🔧 nodes(type-definition) → evaluation nodes
│ ├── 🔧 workflows(update) → wf-123
│ └── ✅ "Added evaluation nodes"
├── 🤖 Agent Builder
│ ├── 🔧 read_config → agent-123
│ ├── 🔧 write_config → agent-123
│ └── ✅ "Updated the support agent"
│
└── 💬 "Done! Your workflow runs daily at 8am..."
└── 💬 "The support agent is ready."
```
The eval-setup section is collapsible. Users can inspect its tool activity or
view only the summary.
Child-agent sections are collapsible. Users can inspect their tool activity or
view only the summary. Stored historical child-agent events use the same tree.
## Session Restore
@ -588,7 +581,7 @@ replaying all SSE events.
"hasActiveRun": false,
"isSuspended": false,
"backgroundTasks": [
{ "taskId": "t1", "role": "eval-setup", "agentId": "agent-002", "status": "running", "startedAt": 1709300000 }
{ "taskId": "t1", "role": "builder", "agentId": "agent-002", "status": "running", "startedAt": 1709300000 }
]
}
```

139
packages/@n8n/instance-ai/docs/tools.md
View File

@ -15,7 +15,7 @@ live in `src/tools/tool-ids.ts`.
| Tool | Actions |
|------|---------|
| `workflows` | 14 |
| `workflows` | 12 |
| `data-tables` | 11 |
| `workspace` | 8 |
| `executions` | 7 |
@ -25,7 +25,6 @@ live in `src/tools/tool-ids.ts`.
| `conversation-history` | 2 |
| `task-control` | 3 |
| `research` | 2 |
| `evals` | 4 |
| `eval-config` | 6 |
| `n8n-docs` | 3 |
| `agents` | 1 |
@ -152,44 +151,6 @@ task must exist, have kind `checkpoint`, and be in the `running` state.
**Returns**: `{ result: string, ok: boolean }`
### `eval-data`
Populate the evaluation data table for a workflow that already has evaluation
nodes. This tool is synchronous. It does not start a sub-agent and does not use
HITL. It imports execution-history rows when at least 10 valid rows exist.
Otherwise, it generates 10 synthetic input rows and leaves expected-output
columns for the user to complete. It inserts at most 25 rows.
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `workflowId` | string | yes | Workflow whose evaluation table is populated |
| `projectId` | string | no | Project scope for the data table |
**Returns**: `{ status: "imported" | "generated" | "skipped", rowCount?,
source?, reason?, expectedOutputsNeedUserReview?, expectedOutputColumns?, table? }`
### `eval-setup-with-agent`
Start the detached eval-setup agent after the evaluation proposal is approved.
The agent normally receives `workflows` with only `get-json` and `update`, plus
the full `nodes` domain tool. It does not receive credentials, data tables,
workspace, the sandbox knowledge base, or MCP tools. Its persistence wrapper
supports checkpoint and suspension state.
The two-action workflow restriction is applied when workflow updates are
available and not blocked. If the host omits workflow permissions or blocks
workflow updates, the restricted replacement is skipped and the agent retains
the full workflow tool. Individual workflow action handlers then apply the
permissions available in that context.
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `workflowId` | string | yes | Workflow to add evaluations to |
| `task` | string | yes | Exact task returned by `evals(action="propose")` |
| `conversationContext` | string | no | Thread summary that anchors dataset design |
**Returns**: `{ result: string, taskId: string }`
### `get-session` *(conditional)*
Read a resolved Agent preview session — title, session number and transcript.
@ -268,12 +229,14 @@ Atomically apply real credentials to previously-mocked workflow nodes.
**Returns**: `{ updatedNodes: string[] }`
## `workflows` (14 actions)
## `workflows` (12 actions)
The full domain surface has up to fourteen actions. Version actions are
registered only when their backend methods are available. The orchestrator
surface excludes the raw `get-json` and `update` actions. The eval-setup agent
gets only those two raw actions.
The domain surface has up to twelve actions. Version actions are registered only
when their backend methods are available. Use `get` to inspect a workflow. Use
`get-as-code`, workspace edits, and `build-workflow` to change a workflow.
The internal `getAsWorkflowJSON` and `updateFromWorkflowJSON` service methods
remain available to compiler, setup, validation, credential, and verification
flows. They are not model-facing actions.
### `workflows(action="list")`
@ -462,19 +425,6 @@ List version history for a workflow (metadata only).
**Returns**: `{ versions: [{ versionId, name, description, authors, createdAt, autosaved, isActive, isCurrentDraft }] }`
### `workflows(action="get-json")`
Get the full `WorkflowJSON` for workspace-file edits. Write it to a
`.workflow.json` file, edit the file, then save with `build-workflow`. Pass
`versionId` to read a past version instead of the current draft.
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `workflowId` | string | yes | Workflow ID |
| `versionId` | string | no | Read this version instead of the current draft |
**Returns**: full `WorkflowJSON`.
### `workflows(action="validate")`
Return the per-node configuration issues a human would see as red warning
@ -487,17 +437,6 @@ workflow is configured correctly before suggesting the user run or publish it.
| `workflowId` | string | yes | Workflow ID |
| `ignoreIssues` | array | no | Issue categories to skip: `parameters`, `credentials`, `input`, `execution`, `typeUnknown`, `aiGateway`, `chatModel` |
### `workflows(action="update")`
Raw update escape hatch. Saves a complete modified `WorkflowJSON` back to the
workflow, replacing the full definition. Prefer the workspace-file path
(`get-json` -> edit -> `build-workflow`) for ordinary edits.
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `workflowId` | string | yes | Workflow ID |
| `workflow` | object | yes | Full `WorkflowJSON` — name, nodes and connections must all be included |
### `workflows(action="restore-version")` *(conditional — requires license)*
Restore a workflow to a previous version (overwrites current draft). HITL
@ -901,26 +840,6 @@ plain text / markdown → passthrough.
## Evaluation Tools
### `evals` (4 actions)
Set up on-canvas evaluations for workflows that contain AI nodes. The tool is
deferred and becomes available through tool search.
All four actions require `workflowId`. When a workflow has more than one AI
node, pass `targetAgentNodeName` to select the target.
| Action | Additional fields | Result and behavior |
|--------|-------------------|---------------------|
| `offer` | `projectId?` | Checks eligibility. Returns `eligible`, a reason when ineligible, AI node names, and a user-facing message when eligible. |
| `recommend-metric` | — | Suggests one metric and suspends for approval. Returns `{ approved: true, metricId }` or `{ approved: false }`. |
| `select-metrics` | — | Shows the multi-select metric picker after a recommendation is denied. Returns `chosenMetricIds` and the answers. |
| `propose` | `projectId?`, `metrics?`, `datasetChoice?`, `existingDataTableId?` | Builds the eval-setup task and creates, links, or defers the dataset. A successful result sets `shouldDelegateToEvalSetupAgent: true` and returns the task, workflow ID, dataset details, and a newly created table artifact when applicable. |
`datasetChoice` is `create-empty`, `link-existing`, or `later`; it defaults to
`create-empty`. `link-existing` requires `existingDataTableId`. A proposal can
also add generated pin data for referenced tool nodes before it returns the
eval-setup task.
### `eval-config` (6 actions, conditional)
Manage config-based evaluations without adding evaluation nodes to the canvas.
@ -1207,37 +1126,15 @@ thrown tool error — when the service is unavailable or a lookup fails.
## Tool Distribution
The orchestrator receives the safe orchestrator domain surface and all
registered orchestration tools. It does not receive raw workflow JSON read or
update actions. It currently receives the full six-action `nodes` tool. The
eval-setup background agent receives only its explicitly wired tool subset,
except for the workflow-tool permission fallback described above.
The orchestrator receives the safe native domain tools and orchestration tools
from `src/tools/index.ts`. Its workflow tool omits raw workflow JSON reads and
full-definition replacements. It receives the full six-action `nodes` tool.
External and local MCP tools are added after their names are checked against the
native tools active for the current request.
| Tool | Orchestrator | Eval-setup background agent |
|---------------|:---:|:---:|
| Orchestration tools (`create-tasks`, `task-control`, etc.) | ✅ | ❌ |
| `n8n-docs` | ✅ | ❌ |
| `evals` | ✅ (search/load) | ❌ |
| `eval-config` | ✅ (conditional, search/load) | ❌ |
| `workflows` | ✅ (without `get-json` or `update`) | ✅ (`get-json` and `update` normally; full tool when permissions are missing or updates are blocked) |
| `executions` | ✅ | ❌ |
| `credentials` | ✅ | ❌ |
| `nodes` | ✅ (full domain tool) | ✅ (full domain tool) |
| `data-tables` | ✅ (direct, via `data-table-manager` skill) | ❌ |
| `workspace` | ✅ | ❌ |
| `ask-user` | ✅ | ❌ |
| `parse-file` | ✅ (when the turn has a parseable attachment) | ❌ |
| `research` | ✅ | ❌ |
| `conversation-history` | ✅ (experiment `109_instance_ai_conversation_history`, via `conversationHistoryService`) | ❌ |
| `agents` and `build-agent` | ✅ (when the Agents module supplies the builder delegate) | ❌ |
| Knowledge base (via runtime workspace tools) | ✅ | ❌ |
| Sandbox-backed internals (`build-workflow` TypeScript compilation, `materialize-node-type`) | ✅ | ❌ |
| External MCP tools | ✅ (when configured) | ❌ |
| Local gateway MCP tools, including Computer Use browser tools | ✅ (when connected and allowed) | ❌ |
The embedded Agent Builder is separate from the eval-setup column. It inherits
the orchestrator's safe MCP connector tools. Eval setup does not receive MCP
tools.
The embedded Agent Builder uses the agents-module builder's own tool surface
through `build-agent`. It does not receive the Instance AI domain registry. It
inherits the orchestrator's safe MCP connector tools.
---
@ -1264,8 +1161,8 @@ existing domain.
2. Add its id to `DOMAIN_TOOL_IDS` or `ORCHESTRATION_TOOL_IDS` in
`src/tools/tool-ids.ts`
3. Export a factory that takes the service context and returns an `@n8n/agents` tool
4. Register it in `src/tools/index.ts` — `createAllTools`,
`createOrchestratorDomainTools`, and/or `createOrchestrationTools`
4. Register it in `src/tools/index.ts` with `createOrchestratorDomainTools` or
`createOrchestrationTools`
5. Decide whether it belongs in `ALWAYS_LOADED_TOOL_NAMES`. Everything not in
that set is normally reached through `search_tools` + `load_tool`. Tools in
`CHECKPOINT_FOLLOW_UP_TOOL_NAMES` are also loaded directly during checkpoint

2
packages/@n8n/instance-ai/evaluations/__tests__/event-parser.test.ts
View File

@ -372,7 +372,7 @@ describe('extractOutcomeFromEvents', () => {
// eval-config create → ref is the owning workflow id from the args.
call('tc-2', 'eval-config', { action: 'create', workflowId: 'wf-1', name: 'My eval' }),
resultEvent('tc-2', { config: { id: 'cfg-1', workflowId: 'wf-1' } }),
// A spawn whose targetResource is a workflow (e.g. eval-setup) contributes no agent ref.
// A spawn whose targetResource is a workflow contributes no agent ref.
spawn('a2', { type: 'workflow', id: 'wf-2' }),
// A spawn with no targetResource contributes nothing.
spawn('a3'),

19
packages/@n8n/instance-ai/evaluations/discovery/runner.ts
View File

@ -46,10 +46,8 @@ import {
normalizeStreamSource,
} from '../../src/runtime/resumable-stream-executor';
import { loadInstanceAiRuntimeSkillSource } from '../../src/skills/runtime-skills';
import { createAllTools } from '../../src/tools';
import type {
InstanceAiContext,
InstanceAiToolRegistry,
LocalGatewayStatus,
ModelConfig,
OrchestrationContext,
@ -142,9 +140,8 @@ export async function runDiscoveryScenario(
});
// `OrchestrationContext` is required for the orchestrator to receive tools like
// `create-tasks`, `eval-setup-with-agent`, and runtime skills. We provide stubs
// for the heavy fields: discovery scenarios measure first-step tool-call
// decisions, not background execution.
// `create-tasks` and runtime skills. Discovery scenarios measure first-step
// tool-call decisions, not background execution.
const orchestrationContext = createStubOrchestrationContext({
context,
modelId: options.modelId,
@ -280,13 +277,6 @@ interface StubOrchestrationContextOptions {
function createStubOrchestrationContext(
opts: StubOrchestrationContextOptions,
): OrchestrationContext {
// Domain tools are passed to background agents such as eval-setup.
// Discovery scenarios measure the orchestrator's first-step dispatch decision;
// background execution is out of scope. We still populate domainTools faithfully
// so any background agent that does spawn has a coherent toolset (avoids hitting
// "no tools" errors that would confuse the diagnostic comment).
const domainTools: InstanceAiToolRegistry = createAllTools(opts.context);
const taskStorage: TaskStorage = {
// eslint-disable-next-line @typescript-eslint/require-await
get: async (): Promise<TaskList | null> => null,
@ -302,14 +292,9 @@ function createStubOrchestrationContext(
modelId: opts.modelId,
eventBus: opts.eventBus,
logger: silentLogger(),
domainTools,
runtimeSkills: loadInstanceAiRuntimeSkillSource(),
abortSignal: opts.abortSignal,
taskStorage,
// Discovery evals assert first-dispatch intent only. Production starts a
// detached background task here; the harness accepts the spawn so the tool
// can publish its `agent-spawned` event without executing the background agent.
spawnBackgroundTask: ({ taskId, agentId }) => ({ status: 'started', taskId, agentId }),
// Surface the localMcpServer so Computer Use browser tools are available to the
// orchestrator.
...(opts.context.localMcpServer ? { localMcpServer: opts.context.localMcpServer } : {}),

2
packages/@n8n/instance-ai/scripts/print-prompts.ts
View File

@ -2,7 +2,7 @@
// ---------------------------------------------------------------------------
// Print Prompts CLI
//
// Renders the final system prompt for the main Instance Agent and the eval-setup
// Renders the final system prompt for the main Instance Agent and the generic
// sub-agent prompt template, then writes one markdown file per agent variant into
// `.output/prompts/<agent>/<variant>.md` (gitignored). Useful for
// auditing the full prompt verbatim, diffing prompts across branches, or

2
packages/@n8n/instance-ai/skills/planned-task-runtime/SKILL.md
View File

@ -156,7 +156,7 @@ and let replan take over.
## Background task completed
When `<background-task-completed>` is present, a detached background task
finished (for example eval setup). The `result` field holds the task's
finished. The `result` field holds the task's
authoritative summary of what was actually done. **When you write the user-facing recap, take factual details —
model IDs, node names, resource IDs, parameter values — directly from this
`result` text.** Do not substitute values from conversation history or training

6
packages/@n8n/instance-ai/src/__tests__/index.test.ts
View File

@ -37,9 +37,6 @@ vi.mock('../tools/web-research/sanitize-web-content', () => ({
wrapUntrustedData: (content: string, source: string) =>
`<untrusted_data source="${source}">${content}</untrusted_data>`,
}));
vi.mock('../tools', () => ({
createAllTools: () => ['all-tools'],
}));
vi.mock('../tools/orchestration/agent-persistence', () => ({
SUB_AGENT_RESOURCE_PREFIX: 'instance-ai-subagent',
createSubAgentResourceIdPrefix: (threadId: string) => `instance-ai-subagent:${threadId}:`,
@ -188,7 +185,6 @@ describe('@n8n/instance-ai public entrypoint', () => {
const entrypoint = await import('../index.js');
expect(entrypoint.MAX_STEPS.ORCHESTRATOR).toBeGreaterThan(0);
expect(entrypoint.createAllTools).toEqual(expect.any(Function));
expect(entrypoint.createInstanceAgent).toEqual(expect.any(Function));
expect(entrypoint.createLazyRuntimeWorkspace).toEqual(expect.any(Function));
expect(entrypoint.getParseableAttachmentMimeTypes).toEqual(expect.any(Function));
@ -218,8 +214,6 @@ describe('@n8n/instance-ai public entrypoint', () => {
expect(entrypoint.wrapUntrustedData('hello', 'https://example.com')).toContain(
'<untrusted_data source="https://example.com">',
);
expect(call(entrypoint.createAllTools)).toEqual(['all-tools']);
expect(entrypoint.PURE_REPLAY_TOOLS.has('web-search')).toBe(true);
expect(entrypoint.createSubAgentResourceIdPrefix('thread-1')).toBe(
'instance-ai-subagent:thread-1:',

118
packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts
View File

@ -51,20 +51,25 @@ const mockBuiltTool = (name: string, marker?: string) => ({
});
vi.mock('../../tools', () => ({
createAllTools: vi.fn(
(context: { runLabel?: string }) =>
new Map([
['workflows', mockBuiltTool(`workflows-${context.runLabel ?? 'unknown'}`)],
['evals', mockBuiltTool(`evals-${context.runLabel ?? 'unknown'}`)],
['research', mockBuiltTool(`research-${context.runLabel ?? 'unknown'}`)],
['nodes', mockBuiltTool(`nodes-${context.runLabel ?? 'unknown'}`)],
]),
getActiveOrchestratorDomainToolNames: vi.fn(
(context: {
evaluationConfigService?: unknown;
mcpService?: unknown;
conversationHistoryService?: unknown;
currentUserAttachments?: unknown[];
}) => {
const names = ['workflows', 'research', 'n8n-docs', 'nodes', 'executions', 'build-workflow'];
if (context.evaluationConfigService) names.push('eval-config');
if (context.mcpService) names.push('mcp-servers');
if (context.conversationHistoryService) names.push('conversation-history');
if (context.currentUserAttachments?.length) names.push('parse-file');
return new Set(names);
},
),
createOrchestratorDomainTools: vi.fn(
(context: { runLabel?: string }) =>
new Map([
['workflows', mockBuiltTool(`workflows-${context.runLabel ?? 'unknown'}`)],
['evals', mockBuiltTool(`evals-${context.runLabel ?? 'unknown'}`)],
['research', mockBuiltTool(`research-${context.runLabel ?? 'unknown'}`)],
['n8n-docs', mockBuiltTool(`n8n-docs-${context.runLabel ?? 'unknown'}`)],
['nodes', mockBuiltTool(`nodes-${context.runLabel ?? 'unknown'}`)],
@ -187,8 +192,6 @@ describe('createInstanceAgent', () => {
const secondRunAttachedTools = getAttachedTools(1);
expect(attachedTools['create-tasks-run-1']).toBeUndefined();
expect(deferredTools['create-tasks-run-1']).toMatchObject({ name: 'create-tasks-run-1' });
expect(attachedTools['evals-run-1']).toBeUndefined();
expect(deferredTools['evals-run-1']).toMatchObject({ name: 'evals-run-1' });
expect(attachedTools['plan-run-1']).toBeUndefined();
expect(attachedTools['research-run-1']).toMatchObject({ name: 'research-run-1' });
expect(attachedTools['build-workflow-run-1']).toMatchObject({
@ -542,6 +545,7 @@ describe('createInstanceAgent', () => {
const buildWith = async (toolName = 'mcp_linear_create_issue') => {
const linearTool = { ...mockBuiltTool(toolName), mcpServerName: 'mcp_linear' };
const orchestrationContext: Record<string, unknown> = { runId: 'connected-mcp' };
await createInstanceAgent({
modelId: 'test-model',
@ -550,11 +554,13 @@ describe('createInstanceAgent', () => {
logger: mockLogger,
mcpService: { search: vi.fn() },
},
orchestrationContext: { runId: 'connected-mcp' },
orchestrationContext,
memoryConfig: {},
mcpServers,
mcpManager: createMcpManagerStub(new Map([[toolName, linearTool]])),
} as never);
return orchestrationContext;
};
// The tools capture their context by value, so the inventory has to be in place
@ -569,21 +575,23 @@ describe('createInstanceAgent', () => {
});
it('reports no tools for a service whose only tool has an unsafe name', async () => {
await buildWith('mcp_linear.create_issue');
const orchestrationContext = await buildWith('mcp_linear.create_issue');
expect(lastDomainToolContext()).toEqual([
{ slug: 'linear', toolNames: [] },
{ slug: 'notion', toolNames: [] },
]);
expect(orchestrationContext.mcpTools).toBeUndefined();
});
it('reports no tools for a service whose only tool collides with a domain tool', async () => {
await buildWith('workflows');
const orchestrationContext = await buildWith('workflows');
expect(lastDomainToolContext()).toEqual([
{ slug: 'linear', toolNames: [] },
{ slug: 'notion', toolNames: [] },
]);
expect(orchestrationContext.mcpTools).toBeUndefined();
});
});
@ -652,9 +660,7 @@ describe('createInstanceAgent', () => {
['github_workflows', mockBuiltTool('github_workflows', 'github-workflows')],
['custom_plan', mockBuiltTool('custom_plan', 'custom-plan')],
]);
const orchestrationContext: Record<string, unknown> = {
runId: 'local-priority',
};
const orchestrationContext: Record<string, unknown> = { runId: 'local-priority' };
createToolsFromLocalMcpServer.mockReturnValue(localTools);
await createInstanceAgent({
@ -676,12 +682,12 @@ describe('createInstanceAgent', () => {
string,
ReturnType<typeof mockBuiltTool>
>;
expect(agentTools.shared_tool).toMatchObject({ marker: 'local-shared' });
expect(agentTools.github_workflows).toMatchObject({ marker: 'github-workflows' });
expect(agentTools.custom_plan).toMatchObject({ marker: 'custom-plan' });
expect(mcpContextTools.get('shared_tool')).toMatchObject({ marker: 'local-shared' });
expect(mcpContextTools.get('github_workflows')).toMatchObject({ marker: 'github-workflows' });
expect(mcpContextTools.get('custom_plan')).toMatchObject({ marker: 'custom-plan' });
});
it('keeps native orchestrator-only tools when an MCP tool claims the same name', async () => {
@ -699,6 +705,7 @@ describe('createInstanceAgent', () => {
modelId: 'test-model',
context: {
runLabel: 'reserved-names',
conversationHistoryService: {},
localGatewayStatus: undefined,
licenseHints: undefined,
localMcpServer,
@ -713,31 +720,72 @@ describe('createInstanceAgent', () => {
expect(attachedTools['conversation-history']).toMatchObject({ marker: 'native-history' });
});
it('defers evals behind tool search so the first prompt stays smaller', async () => {
const memoryConfig = {} as never;
it('keeps MCP tools whose conditional native counterparts are inactive', async () => {
const orchestrationContext: Record<string, unknown> = { runId: 'inactive-native-tools' };
await createInstanceAgent({
modelId: 'test-model',
context: {
runLabel: 'evals-test',
runLabel: 'inactive-native-tools',
localGatewayStatus: undefined,
licenseHints: undefined,
localMcpServer: undefined,
logger: mockLogger,
},
orchestrationContext: {
runId: 'evals-test',
},
memoryConfig,
mcpManager: createMcpManagerStub(),
orchestrationContext,
memoryConfig: {},
mcpManager: createMcpManagerStub(
new Map([
['eval-config', mockBuiltTool('eval-config')],
['parse_file', mockBuiltTool('parse_file')],
['mcp-servers', mockBuiltTool('mcp-servers')],
]),
),
} as never);
const attachedTools = getAttachedTools();
const deferredTools = getDeferredTools();
expect(attachedTools['evals-evals-test']).toBeUndefined();
expect(deferredTools['evals-evals-test']).toMatchObject({
name: 'evals-evals-test',
expect(getDeferredTools()).toMatchObject({
'eval-config': { name: 'eval-config' },
parse_file: { name: 'parse_file' },
});
expect(getAttachedTools()['mcp-servers']).toMatchObject({ name: 'mcp-servers' });
expect(orchestrationContext.mcpTools).toEqual(
new Map([
['eval-config', expect.objectContaining({ name: 'eval-config' })],
['parse_file', expect.objectContaining({ name: 'parse_file' })],
['mcp-servers', expect.objectContaining({ name: 'mcp-servers' })],
]),
);
});
it('rejects normalized MCP collisions with active conditional native tools', async () => {
const orchestrationContext: Record<string, unknown> = { runId: 'active-native-tools' };
await createInstanceAgent({
modelId: 'test-model',
context: {
runLabel: 'active-native-tools',
localGatewayStatus: undefined,
licenseHints: undefined,
localMcpServer: undefined,
logger: mockLogger,
evaluationConfigService: {},
mcpService: {},
currentUserAttachments: [{ type: 'file' }],
},
orchestrationContext,
memoryConfig: {},
mcpManager: createMcpManagerStub(
new Map([
['eval-config', mockBuiltTool('eval-config')],
['parse_file', mockBuiltTool('parse_file')],
['mcp-servers', mockBuiltTool('mcp-servers')],
]),
),
} as never);
const deferredTools = getDeferredTools();
expect(deferredTools['eval-config']).toBeUndefined();
expect(deferredTools.parse_file).toBeUndefined();
expect(deferredTools['mcp-servers']).toBeUndefined();
expect(orchestrationContext.mcpTools).toBeUndefined();
});
it('configures observational memory on the Memory builder when provided', async () => {
@ -858,8 +906,8 @@ describe('createInstanceAgent', () => {
});
it('sticks Modal endpoint requests to the conversation thread via Modal-Session-ID', async () => {
// Sub-agents (build-agent, eval-setup, etc.) reuse context.modelId / orchestration
// context.modelId, so mutating the sticky header here covers those paths too.
// Embedded specialist agents reuse context.modelId / orchestration context.modelId,
// so mutating the sticky header here covers those paths too.
const modalModel = {
id: 'custom/moonshotai/Kimi-K3' as const,
url: 'https://n8ngmbh--ep-kimi-k3-server.us-west.modal.direct/v1',

43
packages/@n8n/instance-ai/src/agent/instance-agent.ts
View File

@ -16,14 +16,13 @@ import { getSystemPrompt } from './system-prompt';
import { listConnectedMcpServices } from '../mcp/connected-mcp-services';
import { hasRuntimeSkills } from '../skills/runtime-skills';
import { createToolRegistry, mergeToolRegistries, toolRegistryValues } from '../tool-registry';
import { createOrchestratorDomainTools, createOrchestrationTools } from '../tools';
import { createToolsFromLocalMcpServer } from '../tools/filesystem/create-tools-from-mcp-server';
import {
ALWAYS_LOADED_TOOL_NAMES,
CHECKPOINT_FOLLOW_UP_TOOL_NAMES,
DOMAIN_TOOL_IDS,
ORCHESTRATION_TOOL_IDS,
} from '../tools/tool-ids';
createOrchestratorDomainTools,
createOrchestrationTools,
getActiveOrchestratorDomainToolNames,
} from '../tools';
import { createToolsFromLocalMcpServer } from '../tools/filesystem/create-tools-from-mcp-server';
import { ALWAYS_LOADED_TOOL_NAMES, CHECKPOINT_FOLLOW_UP_TOOL_NAMES } from '../tools/tool-ids';
import { buildAgentTraceInputs, mergeTraceRunInputs } from '../tracing/langsmith-tracing';
import type {
CreateInstanceAgentOptions,
@ -94,7 +93,6 @@ export async function createInstanceAgent(
// explicit child runs that land on the active trace — orchestration tools
// (e.g. verify) already get it via OrchestrationContext.
const domainContext: InstanceAiContext = { ...context, tracing: orchestrationContext?.tracing };
// Load MCP tools (cached by config hash inside the manager — only spawns
// processes / opens connections on first call or config change). The manager
// returns per-server connection failures alongside the tools so they travel
@ -142,31 +140,12 @@ export async function createInstanceAgent(
? createOrchestrationTools(orchestrationContext)
: createToolRegistry();
// Keep MCP tools from shadowing native tools. Reserve every native tool id —
// orchestrator-only tools (e.g. conversation-history) are built after this
// guard runs, and the later merge is last-write-wins.
// Keep MCP tools from shadowing domain or orchestration tools.
const reservedToolNames = new Set<string>([
...Object.values(DOMAIN_TOOL_IDS),
...Object.values(ORCHESTRATION_TOOL_IDS),
...getActiveOrchestratorDomainToolNames(domainContext),
...orchestrationTools.keys(),
]);
// Store all MCP tools on orchestrationContext for sub-agents.
const allMcpTools = createToolRegistry();
const mcpContextToolNames = createClaimedToolNames(reservedToolNames);
addSafeMcpTools(allMcpTools, rawLocalMcpTools, {
source: 'local gateway MCP',
claimedToolNames: mcpContextToolNames,
warn: warnSkippedMcpTool,
});
addSafeMcpTools(allMcpTools, mcpTools, {
source: 'external MCP',
claimedToolNames: mcpContextToolNames,
warn: warnSkippedMcpTool,
});
if (orchestrationContext && allMcpTools.size > 0) {
orchestrationContext.mcpTools = allMcpTools;
}
const claimedOrchestratorToolNames = createClaimedToolNames(reservedToolNames);
const safeLocalMcpTools = createToolRegistry();
addSafeMcpTools(safeLocalMcpTools, rawLocalMcpTools, {
@ -180,6 +159,10 @@ export async function createInstanceAgent(
claimedToolNames: claimedOrchestratorToolNames,
warn: warnSkippedMcpTool,
});
if (orchestrationContext) {
const builderMcpTools = mergeToolRegistries(safeLocalMcpTools, safeMcpTools);
if (builderMcpTools.size > 0) orchestrationContext.mcpTools = builderMcpTools;
}
const orchestratorDomainTools = createOrchestratorDomainTools({
...domainContext,

95
packages/@n8n/instance-ai/src/agent/sub-agent-briefing.ts
View File

@ -1,95 +0,0 @@
import { formatPreviousAttempts, type IterationLog } from '../storage/iteration-log';
// ── Types ───────────────────────────────────────────────────────────────────
export interface RunningTaskSummary {
taskId: string;
role: string;
goal?: string;
}
export interface SubAgentBriefingInput {
/** The core task description. */
task: string;
/** Brief summary of the conversation so far. */
conversationContext?: string;
/** Structured artifacts (IDs, data). */
artifacts?: Record<string, unknown>;
/** Additional context blocks (e.g., sandbox instructions, workflowId notes). */
additionalContext?: string;
/** Requirements block. */
requirements?: string;
/** Iteration log + task key for retry context. */
iteration?: {
log: IterationLog;
threadId: string;
taskKey: string;
};
/** Currently running background tasks in this thread. */
runningTasks?: RunningTaskSummary[];
}
// ── Builder ─────────────────────────────────────────────────────────────────
/**
* Build a structured XML-formatted briefing for a sub-agent.
*
* Sub-agent spawn sites use this instead of ad-hoc string concatenation.
* The XML structure gives the LLM clear section boundaries and makes the
* briefing parseable.
*/
export async function buildSubAgentBriefing(input: SubAgentBriefingInput): Promise<string> {
const parts: string[] = [];
// Core task — always present
parts.push(`<task>\n${input.task}\n</task>`);
// Conversation context — what the user discussed, decisions made
if (input.conversationContext) {
parts.push(`<conversation-context>\n${input.conversationContext}\n</conversation-context>`);
}
// Structured artifacts — IDs, data, references
if (input.artifacts && Object.keys(input.artifacts).length > 0) {
parts.push(`<artifacts>\n${JSON.stringify(input.artifacts)}\n</artifacts>`);
}
// Additional context — sandbox paths, workflowId notes, etc.
if (input.additionalContext) {
parts.push(input.additionalContext);
}
// Requirements block
if (input.requirements) {
parts.push(input.requirements);
}
// Thread state — what else is happening in parallel
if (input.runningTasks && input.runningTasks.length > 0) {
const taskLines = input.runningTasks
.map(
(t) =>
` <running-task taskId="${t.taskId}" role="${t.role}">${t.goal ?? ''}</running-task>`,
)
.join('\n');
parts.push(`<thread-state>\n${taskLines}\n</thread-state>`);
}
// Iteration context — previous attempt history
if (input.iteration) {
try {
const entries = await input.iteration.log.getForTask(
input.iteration.threadId,
input.iteration.taskKey,
);
const formatted = formatPreviousAttempts(entries);
if (formatted) {
parts.push(formatted);
}
} catch {
// Non-fatal — iteration log is best-effort
}
}
return parts.join('\n\n');
}

2
packages/@n8n/instance-ai/src/agent/system-prompt.ts
View File

@ -64,7 +64,7 @@ function getToolDiscoverySection(
${mcpSearchGuidance}When the available tools do not cover the user's request, remember that you have access to more tools. Use \`search_tools\` with keyword queries to find relevant tools, then \`load_tool\` to activate them. Loaded tools persist for the rest of the conversation. When a loaded skill names a tool you do not see, search for that tool name and load it before proceeding.
Examples: ${mcpExamples}search "create tasks" for \`create-tasks\`, search "eval" for \`evals\`.
Example: ${mcpExamples}search "create tasks" for \`create-tasks\`.
For questions about n8n itself — how a node behaves, the shape of its output, what a parameter does, product semantics — prefer \`n8n-docs\` and the node type definitions, both already loaded and needing no search, over web search, which is for third-party services and APIs.
`;

2
packages/@n8n/instance-ai/src/constants/max-steps.ts
View File

@ -11,8 +11,6 @@ export const MAX_STEPS = {
BROWSER: 300,
/** Legacy limit retained for compatibility; workflow builds run in the orchestrator. */
BUILDER: 60,
/** Eval setup background agent — patches eval nodes into an existing workflow. */
EVAL_SETUP: 30,
/** Legacy limit retained for compatibility; research runs in the orchestrator. */
RESEARCH: 25,
} as const;

7
packages/@n8n/instance-ai/src/index.ts
View File

@ -25,7 +25,6 @@ import type * as RuntimeSkillsMod from './skills/runtime-skills';
import type * as StorageMod from './storage';
import type * as MapChunkMod from './stream/map-chunk';
import type * as UsageAccumulatorMod from './stream/usage-accumulator';
import type * as ToolsMod from './tools';
import type * as AgentPersistenceMod from './tools/orchestration/agent-persistence';
import type * as SanitizeWebContentMod from './tools/web-research/sanitize-web-content';
import type * as AgentSnapshotEventMod from './tracing/agent-snapshot-event';
@ -108,7 +107,6 @@ const loadSystemPrompt = lazyModule(
const loadSanitizeWebContent = lazyModule(
() => require('./tools/web-research/sanitize-web-content') as typeof SanitizeWebContentMod,
);
const loadTools = lazyModule(() => require('./tools') as typeof ToolsMod);
const loadAgentPersistence = lazyModule(
() => require('./tools/orchestration/agent-persistence') as typeof AgentPersistenceMod,
);
@ -343,9 +341,6 @@ export const resolveAIAReasoning: typeof AiaModelDefaultsMod.resolveAIAReasoning
export const getDateTimeSection: typeof SystemPromptMod.getDateTimeSection = lazyFunction(
() => loadSystemPrompt().getDateTimeSection,
);
export const createAllTools: typeof ToolsMod.createAllTools = lazyFunction(
() => loadTools().createAllTools,
);
export const createSubAgentResourceIdPrefix: typeof AgentPersistenceMod.createSubAgentResourceIdPrefix =
lazyFunction(() => loadAgentPersistence().createSubAgentResourceIdPrefix);
export declare const SUB_AGENT_RESOURCE_PREFIX: typeof AgentPersistenceMod.SUB_AGENT_RESOURCE_PREFIX;
@ -664,8 +659,6 @@ export type {
PlannedTaskSchedulerAction,
PlannedTaskService,
OrchestrationContext,
SpawnBackgroundTaskOptions,
SpawnBackgroundTaskResult,
BackgroundTaskResult,
InstanceAiToolTraceOptions,
InstanceAiTraceContext,

7
packages/@n8n/instance-ai/src/runtime/__tests__/confirmation-payload.test.ts
View File

@ -8,7 +8,6 @@ import type { ZodType } from 'zod';
import { domainGatingResumeSchema } from '../../domain-access';
import { credentialsResumeSchema } from '../../tools/credentials.tool';
import { evalsResumeSchema } from '../../tools/evals/evals.tool';
import { gatewayConfirmationResumeSchema } from '../../tools/filesystem/create-tools-from-mcp-server';
import { planResumeSchema } from '../../tools/orchestration/plan.tool';
import { askUserResumeSchema } from '../../tools/shared/ask-user.tool';
@ -65,7 +64,6 @@ describe('confirmation payload → tool resume schema contract', () => {
instanceAiApprovalResumeSchema,
],
['plan', planResumeSchema],
['evals', evalsResumeSchema],
];
/** Cards rendered by a dedicated component, all of which offer a whole-card
@ -114,10 +112,7 @@ describe('confirmation payload → tool resume schema contract', () => {
{ questionId: 'q3', selectedOptions: [], skipped: true },
],
},
targets: [
['ask-user', askUserResumeSchema],
['evals', evalsResumeSchema],
],
targets: [['ask-user', askUserResumeSchema]],
},
{
label: 'credential selection',

52
packages/@n8n/instance-ai/src/stream/__tests__/consume-with-hitl.test.ts
View File

@ -1,9 +1,5 @@
import type { InstanceAiEventBus } from '../../event-bus/event-bus.interface';
import {
consumeStreamCascading,
consumeStreamWithHitl,
requireCompletedHitlText,
} from '../consume-with-hitl';
import { consumeStreamCascading } from '../consume-with-hitl';
async function* fromChunks(chunks: unknown[]) {
for (const chunk of chunks) {
@ -23,52 +19,6 @@ function createLogger() {
return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };
}
describe('consumeStreamWithHitl', () => {
it('preserves errored status from the resumable stream executor', async () => {
const result = await consumeStreamWithHitl({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([{ type: 'error', error: new Error('boom') }]),
text: Promise.resolve('partial'),
},
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
logger: createLogger(),
threadId: 'thread-1',
abortSignal: new AbortController().signal,
waitForConfirmation: vi.fn(),
});
expect(result.status).toBe('errored');
expect(result.agentRunId).toBe('agent-run-1');
await expect(requireCompletedHitlText(result, 'Test sub-agent')).rejects.toThrow(
'Test sub-agent failed while streaming',
);
});
it('returns completed text when the stream completed', async () => {
const result = await consumeStreamWithHitl({
agent: {},
stream: {
runId: 'agent-run-1',
fullStream: fromChunks([{ type: 'text-delta', delta: 'done' }]),
text: Promise.resolve('done'),
},
runId: 'run-1',
agentId: 'agent-1',
eventBus: createEventBus(),
logger: createLogger(),
threadId: 'thread-1',
abortSignal: new AbortController().signal,
waitForConfirmation: vi.fn(),
});
await expect(requireCompletedHitlText(result, 'Test sub-agent')).resolves.toBe('done');
});
});
describe('consumeStreamCascading', () => {
const finishChunk = {
type: 'finish',

106
packages/@n8n/instance-ai/src/stream/consume-with-hitl.ts
View File

@ -5,7 +5,6 @@ import type { Logger } from '../logger';
import {
executeResumableStream,
normalizeStreamSource,
type TraceStatus,
} from '../runtime/resumable-stream-executor';
import type { RunTokenUsage } from '../stream/usage-accumulator';
import type { WorkSummary } from '../stream/work-summary-accumulator';
@ -13,107 +12,6 @@ import type { SuspensionInfo } from '../utils/stream-helpers';
type ConfirmationRequestEvent = Extract<InstanceAiEvent, { type: 'confirmation-request' }>;
export interface ConsumeWithHitlOptions {
agent: unknown;
stream: unknown;
runId: string;
agentId: string;
eventBus: InstanceAiEventBus;
logger: Logger;
threadId: string;
abortSignal: AbortSignal;
waitForConfirmation?: (requestId: string) => Promise<Record<string, unknown>>;
/** Drain queued user corrections (mid-flight steering for background tasks). */
drainCorrections?: () => string[];
/** Returns a promise that resolves when a new user correction is queued.
* Used to unblock HITL suspensions when a correction arrives mid-confirmation. */
waitForCorrection?: () => Promise<void>;
/** Max iterations for the agent; passed to native stream resume so resumed streams keep the same limit. */
maxIterations?: number;
/** Additional options to preserve when resuming a suspended stream. */
resumeOptions?: Record<string, unknown>;
/** Native agent persistence owner for suspended sub-agent state. */
persistence?: { threadId: string; resourceId: string };
}
export interface ConsumeWithHitlResult {
/** Final native stream consumption status. */
status: TraceStatus;
/** Native sub-agent run ID. */
agentRunId: string;
/** Promise that resolves to the agent's full text output (including post-resume text). */
text: Promise<string>;
/** Accumulated tool call outcomes observed during stream consumption. */
workSummary: WorkSummary;
}
export async function requireCompletedHitlText(
result: ConsumeWithHitlResult,
agentLabel: string,
): Promise<string> {
if (result.status === 'completed') {
return await result.text;
}
const reason =
result.status === 'cancelled'
? 'was cancelled'
: result.status === 'errored'
? 'failed while streaming'
: `ended with unexpected status "${result.status}"`;
throw new Error(`${agentLabel} ${reason}`);
}
/**
* Consume a sub-agent stream with HITL suspend/resume support.
* Detects `tool-call-suspended` chunks, waits for user confirmation,
* and resumes the stream. Used by delegate, builder, and other agent tools.
*
* Returns `{ text }` — a promise for the agent's full text output.
* When HITL occurred, this returns the resumed stream's text (not the original).
*/
export async function consumeStreamWithHitl(
options: ConsumeWithHitlOptions,
): Promise<ConsumeWithHitlResult> {
if (!options.waitForConfirmation) {
throw new Error('Sub-agent tool requires confirmation but no HITL handler is available');
}
const stream = normalizeStreamSource(options.stream);
const result = await executeResumableStream({
agent: options.agent,
stream,
context: {
threadId: options.threadId,
runId: options.runId,
agentId: options.agentId,
eventBus: options.eventBus,
signal: options.abortSignal,
logger: options.logger,
},
control: {
mode: 'auto',
waitForConfirmation: options.waitForConfirmation,
drainCorrections: options.drainCorrections,
waitForCorrection: options.waitForCorrection,
buildResumeOptions: ({ agentRunId, suspension }) => ({
runId: agentRunId,
toolCallId: suspension.toolCallId,
...(options.maxIterations ? { maxIterations: options.maxIterations } : {}),
...(options.resumeOptions ?? {}),
...(options.persistence ? { persistence: options.persistence } : {}),
}),
},
});
return {
status: result.status,
agentRunId: result.agentRunId,
text: result.text ?? stream.text ?? Promise.resolve(''),
workSummary: result.workSummary,
};
}
export interface ConsumeStreamCascadingOptions {
agent: unknown;
stream: unknown;
@ -145,9 +43,7 @@ export type ConsumeStreamCascadingResult =
/**
* Consume a sub-agent stream and return cleanly when it either finishes or
* hits a HITL suspension. Unlike `consumeStreamWithHitl` (which transparently
* bridges sub-agent suspensions to a parent `waitForConfirmation` Promise),
* this returns the suspension info to the caller so the caller can decide
* hits a HITL suspension. This returns the suspension info to the caller so it can decide
* how to handle it — e.g. cascade the suspension up to its own SDK suspend
* via `ctx.suspend(payload)`, so the parent's agent run is also checkpointed
* and survives a process restart.

71
packages/@n8n/instance-ai/src/tools/__tests__/index.test.ts
View File

@ -1,4 +1,8 @@
import { createAllTools, createOrchestrationTools, createOrchestratorDomainTools } from '..';
import {
createOrchestrationTools,
createOrchestratorDomainTools,
getActiveOrchestratorDomainToolNames,
} from '..';
import { isParseableAttachment } from '../../parsers/structured-file-parser';
import type { InstanceAiContext } from '../../types';
import { ALWAYS_LOADED_TOOL_NAMES } from '../tool-ids';
@ -53,18 +57,6 @@ vi.mock('../orchestration/complete-checkpoint.tool', () => ({
createCompleteCheckpointTool: vi.fn(() => ({ id: 'complete-checkpoint' })),
}));
vi.mock('../evals/evals.tool', () => ({
createEvalsTool: vi.fn(() => ({ id: 'evals' })),
}));
vi.mock('../orchestration/eval-setup-agent.tool', () => ({
createEvalSetupAgentTool: vi.fn(() => ({ id: 'eval-setup-with-agent' })),
}));
vi.mock('../orchestration/eval-data-agent.tool', () => ({
createEvalDataAgentTool: vi.fn(() => ({ id: 'eval-data' })),
}));
vi.mock('../orchestration/plan.tool', () => ({
createPlanTool: vi.fn(() => ({ id: 'create-tasks' })),
}));
@ -99,9 +91,7 @@ vi.mock('../workflows/build-workflow.tool', () => ({
}));
vi.mock('../workflows.tool', () => ({
createWorkflowsTool: vi.fn((_context: unknown, options?: unknown) => ({
id: options ? 'workflows-orchestrator' : 'workflows',
})),
createWorkflowsTool: vi.fn(() => ({ id: 'workflows' })),
}));
vi.mock('../workspace.tool', () => ({
@ -129,35 +119,13 @@ describe('domain tool construction', () => {
vi.mocked(isParseableAttachment).mockReturnValue(false);
});
it('creates the native full domain tool map', () => {
const context = makeContext();
const domainTools = createAllTools(context);
expect(Object.fromEntries(domainTools)).toMatchObject({
workflows: { id: 'workflows' },
evals: { id: 'evals' },
executions: { id: 'executions' },
credentials: { id: 'credentials' },
'data-tables': { id: 'data-tables' },
workspace: { id: 'workspace' },
research: { id: 'research' },
'n8n-docs': { id: 'n8n-docs' },
nodes: { id: 'nodes' },
'ask-user': { id: 'ask-user' },
'build-workflow': { id: 'build-workflow' },
});
expect(domainTools.has('templates')).toBe(false);
});
it('creates the native orchestrator domain tool map', async () => {
const context = makeContext();
const orchestratorTools = createOrchestratorDomainTools(context);
expect(Object.fromEntries(orchestratorTools)).toMatchObject({
workflows: { id: 'workflows-orchestrator' },
evals: { id: 'evals' },
workflows: { id: 'workflows' },
executions: { id: 'executions' },
credentials: { id: 'credentials' },
'data-tables': { id: 'data-tables' },
@ -169,11 +137,12 @@ describe('domain tool construction', () => {
'build-workflow': { id: 'build-workflow' },
});
expect(orchestratorTools.has('templates')).toBe(false);
expect(orchestratorTools.has('evals')).toBe(false);
const { createWorkflowsTool } = await import('../workflows.tool.js');
const { createNodesTool } = await import('../nodes.tool.js');
const { createDataTablesTool } = await import('../data-tables.tool.js');
expect(createWorkflowsTool).toHaveBeenCalledWith(context, 'orchestrator');
expect(createWorkflowsTool).toHaveBeenCalledWith(context);
expect(createNodesTool).toHaveBeenCalledWith(context);
expect(createDataTablesTool).toHaveBeenCalledWith(context);
});
@ -197,7 +166,6 @@ describe('domain tool construction', () => {
],
});
expect(createAllTools(context).get('parse-file')).toMatchObject({ id: 'parse-file' });
expect(createOrchestratorDomainTools(context).get('parse-file')).toMatchObject({
id: 'parse-file',
});
@ -206,14 +174,12 @@ describe('domain tool construction', () => {
it('gates the eval-config tool on the config-evals flag (evaluationConfigService presence)', () => {
// Flag off: adapter leaves evaluationConfigService unset → tool absent.
const disabled = makeContext();
expect(createAllTools(disabled).get('eval-config')).toBeUndefined();
expect(createOrchestratorDomainTools(disabled).get('eval-config')).toBeUndefined();
// Flag on: adapter wires evaluationConfigService → tool exposed.
const enabled = makeContext({
evaluationConfigService: {} as InstanceAiContext['evaluationConfigService'],
});
expect(createAllTools(enabled).get('eval-config')).toBeDefined();
expect(createOrchestratorDomainTools(enabled).get('eval-config')).toBeDefined();
});
@ -224,8 +190,21 @@ describe('domain tool construction', () => {
const enabled = makeContext({ mcpService: {} as InstanceAiContext['mcpService'] });
expect(createOrchestratorDomainTools(enabled).get('mcp-servers')).toBeDefined();
// Orchestrator only — a sub-agent can't offer the user a connection.
expect(createAllTools(enabled).get('mcp-servers')).toBeUndefined();
});
it('reports the same active names that the domain registry exposes', () => {
vi.mocked(isParseableAttachment).mockReturnValue(true);
const context = makeContext({
evaluationConfigService: {} as InstanceAiContext['evaluationConfigService'],
mcpService: {} as InstanceAiContext['mcpService'],
currentUserAttachments: [
{ type: 'file', data: '', mimeType: 'text/csv', fileName: 'input.csv' },
],
});
expect(getActiveOrchestratorDomainToolNames(context)).toEqual(
new Set(createOrchestratorDomainTools(context).keys()),
);
});
it('never defers mcp-servers behind search_tools', () => {
@ -252,6 +231,8 @@ describe('domain tool construction', () => {
expect(orchestrationTools.has('create-tasks')).toBe(true);
expect(orchestrationTools.has('plan')).toBe(false);
expect(orchestrationTools.has('delegate')).toBe(false);
expect(orchestrationTools.has('eval-setup-with-agent')).toBe(false);
expect(orchestrationTools.has('eval-data')).toBe(false);
});
it('registers build-agent only when a builder delegate is present on the domain context', () => {

4
packages/@n8n/instance-ai/src/tools/__tests__/provider-schema.test.ts
View File

@ -1,7 +1,7 @@
import type { BuiltTool } from '@n8n/agents';
import { z } from 'zod';
import { createAllTools, createOrchestrationTools, createOrchestratorDomainTools } from '..';
import { createOrchestrationTools, createOrchestratorDomainTools } from '..';
import type { InstanceAiContext, OrchestrationContext } from '../../types';
function createCallableService() {
@ -46,7 +46,6 @@ function createOrchestrationContext(): OrchestrationContext {
modelId: 'test-model',
eventBus: {},
logger: domainContext.logger,
domainTools: createAllTools(domainContext),
abortSignal: new AbortController().signal,
taskStorage: {},
workflowTaskService: {
@ -78,7 +77,6 @@ describe('provider-facing tool schemas', () => {
const domainContext = createInstanceAiContext();
const orchestrationContext = createOrchestrationContext();
const toolSets = [
createAllTools(domainContext),
createOrchestratorDomainTools(domainContext),
createOrchestrationTools(orchestrationContext),
];

2
packages/@n8n/instance-ai/src/tools/__tests__/task-control.tool.test.ts
View File

@ -1,7 +1,6 @@
import type { Mock } from 'vitest';
import { executeTool } from '../../__tests__/tool-test-utils';
import { createToolRegistry } from '../../tool-registry';
import type { OrchestrationContext } from '../../types';
import { createTaskControlTool } from '../task-control.tool';
@ -19,7 +18,6 @@ function createMockContext(overrides: Partial<OrchestrationContext> = {}): Orche
subscribe: vi.fn(),
},
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() } as never,
domainTools: createToolRegistry(),
abortSignal: new AbortController().signal,
taskStorage: {
get: vi.fn(),

2
packages/@n8n/instance-ai/src/tools/__tests__/workflows.get-as-code.integration.test.ts
View File

@ -83,7 +83,7 @@ describe('workflows get-as-code integration', () => {
workflowVersionId: 'v-stale',
workflowChecksum: 'checksum-stale',
});
const tool = createWorkflowsTool(context, 'full');
const tool = createWorkflowsTool(context);
const result = await executeTool<GetAsCodeResult>(tool, {
action: 'get-as-code',

2
packages/@n8n/instance-ai/src/tools/__tests__/workflows.tool.setup-integration.test.ts
View File

@ -47,7 +47,7 @@ async function openSetupCard(preferNewCredentials?: string[]) {
const context = createContext();
const suspend = vi.fn();
await executeTool(
createWorkflowsTool(context, 'full'),
createWorkflowsTool(context),
{
action: 'setup',
workflowId: 'wf1',

967
packages/@n8n/instance-ai/src/tools/__tests__/workflows.tool.test.ts
View File

File diff suppressed because it is too large Load Diff

3
packages/@n8n/instance-ai/src/tools/attachments/__tests__/parse-file-registration.test.ts
View File

@ -11,8 +11,7 @@ function toBase64(content: string): string {
}
/**
* Mirrors the conditional shared by createAllTools and
* createOrchestratorDomainTools:
* Mirrors the conditional in createOrchestratorDomainTools:
* context.currentUserAttachments?.some(isParseableAttachment)
*/
function wouldRegisterParseTool(attachments?: InstanceAiFileAttachment[]): boolean {

87
packages/@n8n/instance-ai/src/tools/evals/__tests__/analyze-agent-input-columns.service.test.ts
View File

@ -1,87 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { analyzeAgentInputColumns } from '../analyze-agent-input-columns.service';
type TestNode = Partial<WorkflowJSON['nodes'][number]>;
const wf = (nodes: TestNode[]): WorkflowJSON =>
({
name: 't',
nodes,
connections: {},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
describe('analyzeAgentInputColumns', () => {
it('extracts $json.<field> from agent prompt parameter', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [0, 0],
id: 'a',
},
]);
const result = analyzeAgentInputColumns(workflow, 'Agent');
expect(result).toEqual({ agentNodeName: 'Agent', inputColumns: ['user_query'] });
});
it('aggregates references across multiple parameters', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: '={{ $json.user_query }}',
options: { systemMessage: '={{ $json.system_prompt }}' },
},
position: [0, 0],
id: 'a',
},
]);
const result = analyzeAgentInputColumns(workflow, 'Agent');
expect(result.inputColumns.sort()).toEqual(['system_prompt', 'user_query']);
});
it('falls back to ["input"] when no $json refs are found', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: 'literal prompt with no expressions' },
position: [0, 0],
id: 'a',
},
]);
const result = analyzeAgentInputColumns(workflow, 'Agent');
expect(result.inputColumns).toEqual(['input']);
});
it('throws when the named agent node does not exist', () => {
const workflow = wf([]);
expect(() => analyzeAgentInputColumns(workflow, 'Missing')).toThrow(/not found/i);
});
it('matches both item.json.<field> and $json.<field>', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: '={{ $json.q }}',
options: { systemMessage: '={{ $input.item.json.context }}' },
},
position: [0, 0],
id: 'a',
},
]);
const result = analyzeAgentInputColumns(workflow, 'Agent');
expect(result.inputColumns.sort()).toEqual(['context', 'q']);
});
});

65
packages/@n8n/instance-ai/src/tools/evals/__tests__/apply-pin-data.service.test.ts
View File

@ -1,65 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { applyPinData } from '../apply-pin-data.service';
const wf = (pinData?: WorkflowJSON['pinData']): WorkflowJSON =>
({
name: 't',
nodes: [],
connections: {},
pinData,
settings: {},
}) as unknown as WorkflowJSON;
describe('applyPinData', () => {
it('returns the workflow unchanged when generated map is empty', () => {
const original = wf({ Existing: [{ json: { x: 1 } }] });
const result = applyPinData(original, {});
expect(result).toBe(original);
});
it('adds new pinData entries to a workflow that has none', () => {
const result = applyPinData(wf(), {
'Telegram Trigger': [{ json: { chat_id: '1' } }],
});
expect(result.pinData).toEqual({
'Telegram Trigger': [{ json: { chat_id: '1' } }],
});
});
it('preserves existing pinData on the same node (skip-if-set)', () => {
const original = wf({
'Telegram Trigger': [{ json: { chat_id: 'user-set' } }],
});
const result = applyPinData(original, {
'Telegram Trigger': [{ json: { chat_id: 'generated' } }],
});
expect(result.pinData).toEqual({
'Telegram Trigger': [{ json: { chat_id: 'user-set' } }],
});
});
it('merges new entries with existing pinData on different nodes', () => {
const original = wf({ Webhook: [{ json: { id: 'w' } }] });
const result = applyPinData(original, {
'Telegram Trigger': [{ json: { chat_id: '1' } }],
});
expect(result.pinData).toEqual({
Webhook: [{ json: { id: 'w' } }],
'Telegram Trigger': [{ json: { chat_id: '1' } }],
});
});
it('does not mutate the input workflow object', () => {
const original = wf();
const beforeKeys = Object.keys(original.pinData ?? {});
applyPinData(original, { 'Telegram Trigger': [{ json: { chat_id: '1' } }] });
expect(Object.keys(original.pinData ?? {})).toEqual(beforeKeys);
});
it('returns the same reference when every generated entry was skipped', () => {
const original = wf({ Existing: [{ json: { x: 1 } }] });
const result = applyPinData(original, { Existing: [{ json: { x: 999 } }] });
expect(result).toBe(original);
});
});

90
packages/@n8n/instance-ai/src/tools/evals/__tests__/column-ref-utils.test.ts
View File

@ -1,90 +0,0 @@
import {
collectStrings,
currentJsonExpression,
extractJsonColumnRefs,
extractNamedRefMatches,
nodeItemJsonExpression,
} from '../column-ref-utils';
describe('column-ref-utils', () => {
describe('collectStrings', () => {
it('walks nested objects and arrays', () => {
const v = { a: 'x', b: ['y', { c: 'z' }] };
expect(collectStrings(v).sort()).toEqual(['x', 'y', 'z']);
});
});
describe('extractJsonColumnRefs', () => {
it('extracts $json.field references', () => {
expect(extractJsonColumnRefs('hello {{ $json.user_query }}')).toEqual(['user_query']);
});
it('extracts $json["field"] references', () => {
expect(extractJsonColumnRefs('hello {{ $json["user-query"] }}')).toEqual(['user-query']);
});
it('does not treat bare item.json.field references as direct input columns', () => {
expect(extractJsonColumnRefs('{{ item.json.expected_response }}')).toEqual([]);
});
it('extracts $input.item.json.field references', () => {
expect(extractJsonColumnRefs('$input.item.json.input_text')).toEqual(['input_text']);
});
it('extracts $input.first().json.field references', () => {
expect(extractJsonColumnRefs('$input.first().json.input_text')).toEqual(['input_text']);
});
it('does not treat named node refs as direct input columns', () => {
expect(extractJsonColumnRefs("={{ $('Source').item.json.user_query }}")).toEqual([]);
});
it('dedupes across patterns', () => {
expect(extractJsonColumnRefs('$json.x and $input.item.json.x')).toEqual(['x']);
});
it('returns [] when no patterns match', () => {
expect(extractJsonColumnRefs('plain text {{ $now }}')).toEqual([]);
});
});
describe('extractNamedRefMatches', () => {
it('extracts named refs from item, first, $items, $node dot, and bracket field access', () => {
const text = [
'={{ $("Quoted \\"Node\\"").first().json["field-key"] }}',
"={{ $items('Other\\'s Node')[0].json.foo }}",
'={{ $node.Source.item.json.bar }}',
'={{ $node["Legacy"].json["baz qux"] }}',
].join('\n');
expect(extractNamedRefMatches(text)).toEqual([
{
nodeName: 'Quoted "Node"',
field: 'field-key',
originalExpression: '$("Quoted \\"Node\\"").first().json["field-key"]',
},
{
nodeName: "Other's Node",
field: 'foo',
originalExpression: "$items('Other\\'s Node')[0].json.foo",
},
{
nodeName: 'Source',
field: 'bar',
originalExpression: '$node.Source.item.json.bar',
},
{
nodeName: 'Legacy',
field: 'baz qux',
originalExpression: '$node["Legacy"].json["baz qux"]',
},
]);
});
});
describe('expression formatting', () => {
it('formats field access with dot notation only when safe', () => {
expect(currentJsonExpression('safe_name')).toBe('$json.safe_name');
expect(currentJsonExpression('unsafe-name')).toBe('$json["unsafe-name"]');
});
it('escapes node names and field names for generated expressions', () => {
expect(nodeItemJsonExpression('Voice "or" Text', 'message-id')).toBe(
'$("Voice \\"or\\" Text").item.json["message-id"]',
);
});
});
});

136
packages/@n8n/instance-ai/src/tools/evals/__tests__/create-empty-eval-data-table.test.ts
View File

@ -1,136 +0,0 @@
import type { DataTableSummary, InstanceAiContext } from '../../../types';
import { createEmptyEvalDataTable } from '../ensure-eval-data-table.service';
const dataTableSummary = (id: string, name: string): DataTableSummary => ({
id,
name,
columns: [],
createdAt: '2026-01-01T00:00:00.000Z',
updatedAt: '2026-01-01T00:00:00.000Z',
});
const createContext = (
dataTableService: Pick<InstanceAiContext['dataTableService'], 'create' | 'insertRows'>,
): InstanceAiContext =>
({
dataTableService,
}) as unknown as InstanceAiContext;
describe('createEmptyEvalDataTable', () => {
it('creates a string-typed table with the requested columns', async () => {
const create = vi
.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['create']>
) => ReturnType<InstanceAiContext['dataTableService']['create']>
>()
.mockResolvedValue(dataTableSummary('dt-1', 'Wf — eval samples'));
const insertRows =
vi.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['insertRows']>
) => ReturnType<InstanceAiContext['dataTableService']['insertRows']>
>();
const ctx = createContext({ create, insertRows });
const result = await createEmptyEvalDataTable(ctx, {
workflowName: 'Wf',
projectId: 'p1',
columns: ['user_query', 'expected_response'],
});
expect(create).toHaveBeenCalledWith(
'Wf — eval samples',
[
{ name: 'user_query', type: 'string' },
{ name: 'expected_response', type: 'string' },
],
{ projectId: 'p1' },
);
expect(insertRows).not.toHaveBeenCalled();
expect(result).toEqual({ id: 'dt-1', name: 'Wf — eval samples' });
});
it('normalizes requested columns to valid DataTable names', async () => {
const create = vi
.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['create']>
) => ReturnType<InstanceAiContext['dataTableService']['create']>
>()
.mockResolvedValue(dataTableSummary('dt-1', 'Wf — eval samples'));
const ctx = createContext({
create,
insertRows:
vi.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['insertRows']>
) => ReturnType<InstanceAiContext['dataTableService']['insertRows']>
>(),
});
await createEmptyEvalDataTable(ctx, {
workflowName: 'Wf',
columns: ['User Query', 'expected-response', '1st result', 'User Query'],
});
expect(create).toHaveBeenCalledWith(
'Wf — eval samples',
[
{ name: 'user_query', type: 'string' },
{ name: 'expected_response', type: 'string' },
{ name: 'column_1st_result', type: 'string' },
],
undefined,
);
});
it('retries with a nanoid suffix on name collision', async () => {
const create = vi
.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['create']>
) => ReturnType<InstanceAiContext['dataTableService']['create']>
>()
.mockRejectedValueOnce(new Error('Data table already exists'))
.mockResolvedValueOnce(dataTableSummary('dt-2', 'Wf — eval samples (abc12)'));
const ctx = createContext({
create,
insertRows:
vi.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['insertRows']>
) => ReturnType<InstanceAiContext['dataTableService']['insertRows']>
>(),
});
const result = await createEmptyEvalDataTable(ctx, {
workflowName: 'Wf',
columns: ['x'],
});
expect(create).toHaveBeenCalledTimes(2);
expect(result.id).toBe('dt-2');
});
it('rethrows non-collision errors', async () => {
const create = vi
.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['create']>
) => ReturnType<InstanceAiContext['dataTableService']['create']>
>()
.mockRejectedValueOnce(new Error('database down'));
const ctx = createContext({
create,
insertRows:
vi.fn<
(
...args: Parameters<InstanceAiContext['dataTableService']['insertRows']>
) => ReturnType<InstanceAiContext['dataTableService']['insertRows']>
>(),
});
await expect(
createEmptyEvalDataTable(ctx, { workflowName: 'Wf', columns: ['x'] }),
).rejects.toThrow('database down');
});
});

173
packages/@n8n/instance-ai/src/tools/evals/__tests__/describe-metric-for-workflow.service.test.ts
View File

@ -1,173 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import {
describeMetricForWorkflow,
recommendedMetricId,
} from '../describe-metric-for-workflow.service';
import type { MetricId } from '../metric-catalog';
type TestNode = Partial<WorkflowJSON['nodes'][number]>;
type RecommendedMetricId = Exclude<MetricId, 'helpfulness'>;
const baseAgent: TestNode = {
name: 'Chef Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'a',
};
const wf = (nodes: TestNode[], connections: WorkflowJSON['connections'] = {}): WorkflowJSON =>
({
name: 't',
nodes,
connections,
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
describe('describeMetricForWorkflow', () => {
describe('correctness', () => {
it('mentions the agent name in a generic comparison', () => {
const result = describeMetricForWorkflow(wf([baseAgent]), 'Chef Agent', 'correctness');
expect(result).toContain('Chef Agent');
expect(result).toMatch(/ground-truth/i);
});
});
describe('tool_use', () => {
it('returns the no-tools message when agent has none', () => {
const result = describeMetricForWorkflow(wf([baseAgent]), 'Chef Agent', 'tool_use');
expect(result).toMatch(/Add tools/i);
});
it('lists the tool names when agent has tools', () => {
const workflow = wf(
[
baseAgent,
{
name: 'Calculator',
type: '@n8n/n8n-nodes-langchain.toolCalculator',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'c',
},
{
name: 'Recipe Search',
type: '@n8n/n8n-nodes-langchain.toolHttpRequest',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'r',
},
],
{
Calculator: { ai_tool: [[{ node: 'Chef Agent', type: 'ai_tool', index: 0 }]] },
'Recipe Search': { ai_tool: [[{ node: 'Chef Agent', type: 'ai_tool', index: 0 }]] },
},
);
const result = describeMetricForWorkflow(workflow, 'Chef Agent', 'tool_use');
expect(result).toMatch(/Calculator/);
expect(result).toMatch(/Recipe Search/);
expect(result).toMatch(/picks correctly/i);
});
});
describe('relevance', () => {
it('returns the RAG-fallback message when no retriever is present', () => {
const result = describeMetricForWorkflow(wf([baseAgent]), 'Chef Agent', 'relevance');
expect(result).toMatch(/RAG/);
});
it('lists the retriever name when one is wired to the agent', () => {
const workflow = wf(
[
baseAgent,
{
name: 'Pinecone Store',
type: '@n8n/n8n-nodes-langchain.vectorStorePinecone',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'v',
},
],
{
'Pinecone Store': {
ai_vectorStore: [[{ node: 'Chef Agent', type: 'ai_vectorStore', index: 0 }]],
},
},
);
const result = describeMetricForWorkflow(workflow, 'Chef Agent', 'relevance');
expect(result).toMatch(/Pinecone Store/);
expect(result).toMatch(/retrieved context/i);
});
});
describe('helpfulness', () => {
it('returns a generic description', () => {
const result = describeMetricForWorkflow(wf([baseAgent]), 'Chef Agent', 'helpfulness');
expect(result).toMatch(/addresses the user/i);
});
});
describe('unknown metric', () => {
it('returns empty string for unknown ids', () => {
expect(describeMetricForWorkflow(wf([baseAgent]), 'Chef Agent', 'nope')).toBe('');
});
});
});
describe('recommendedMetricId', () => {
it('returns tool_use when the agent has tools', () => {
const workflow = wf(
[
baseAgent,
{
name: 'Calculator',
type: '@n8n/n8n-nodes-langchain.toolCalculator',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'c',
},
],
{ Calculator: { ai_tool: [[{ node: 'Chef Agent', type: 'ai_tool', index: 0 }]] } },
);
expect(recommendedMetricId(workflow, 'Chef Agent')).toBe('tool_use');
});
it('returns relevance when the agent has a retriever but no tools', () => {
const workflowRetrieverOnly = wf(
[
baseAgent,
{
name: 'Pinecone',
type: '@n8n/n8n-nodes-langchain.vectorStorePinecone',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'v',
},
],
{
Pinecone: {
ai_vectorStore: [[{ node: 'Chef Agent', type: 'ai_vectorStore', index: 0 }]],
},
},
);
expect(recommendedMetricId(workflowRetrieverOnly, 'Chef Agent')).toBe('relevance');
});
it('returns correctness for a plain agent with no tools or retrievers', () => {
expect(recommendedMetricId(wf([baseAgent]), 'Chef Agent')).toBe('correctness');
});
it('returns a narrowed metric id type', () => {
const metricId: RecommendedMetricId = recommendedMetricId(wf([baseAgent]), 'Chef Agent');
expect(metricId).toBe('correctness');
});
});

385
packages/@n8n/instance-ai/src/tools/evals/__tests__/detect-agent-named-refs.service.test.ts
View File

@ -1,385 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { detectAgentNamedRefs } from '../detect-agent-named-refs.service';
const wf = (
nodes: WorkflowJSON['nodes'],
connections?: WorkflowJSON['connections'],
): WorkflowJSON =>
({
name: 't',
nodes,
connections: connections ?? {},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
describe('detectAgentNamedRefs', () => {
it('returns [] when agent has no named refs', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [0, 0],
id: 'a',
},
]);
expect(detectAgentNamedRefs(workflow, 'Agent')).toEqual([]);
});
it("detects single $('Name').item.json.field reference", () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: "={{ $('Voice or Text').item.json.text }}" },
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toEqual([
{
nodeName: 'Voice or Text',
field: 'text',
originalExpression: "$('Voice or Text').item.json.text",
column: 'text',
targetNodeName: 'Agent',
},
]);
});
it('detects double-quoted form $("Name").item.json.field', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $("Voice or Text").item.json.text }}' },
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toEqual([
{
nodeName: 'Voice or Text',
field: 'text',
originalExpression: '$("Voice or Text").item.json.text',
column: 'text',
targetNodeName: 'Agent',
},
]);
});
it('detects legacy $node["Name"].json.field form', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $node["Voice or Text"].json.text }}' },
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result[0].nodeName).toBe('Voice or Text');
expect(result[0].field).toBe('text');
expect(result[0].column).toBe('text');
expect(result[0].targetNodeName).toBe('Agent');
});
it('aggregates references across multiple agent parameters', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: "={{ $('Voice or Text').item.json.text }}",
options: { systemMessage: "={{ $('Memory Buffer').item.json.context }}" },
},
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toHaveLength(2);
expect(result.map((r) => r.nodeName).sort()).toEqual(['Memory Buffer', 'Voice or Text']);
expect(result.map((r) => r.column).sort()).toEqual(['context', 'text']);
expect(result.every((r) => r.targetNodeName === 'Agent')).toBe(true);
});
it('dedups same (node, field) appearing multiple times', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: "={{ $('Voice or Text').item.json.text }}",
options: {
systemMessage: "Use this value: {{ $('Voice or Text').item.json.text }} now",
},
},
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toHaveLength(1);
expect(result[0].column).toBe('text');
expect(result[0].targetNodeName).toBe('Agent');
});
it('disambiguates collision: same field name from different nodes uses node-slug prefix', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: "={{ $('Voice or Text').item.json.text }}",
options: { systemMessage: "={{ $('Memory Buffer').item.json.text }}" },
},
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toHaveLength(2);
const cols = result.map((r) => r.column).sort();
expect(cols).toEqual(['memory_buffer_text', 'voice_or_text_text']);
expect(result.every((r) => r.targetNodeName === 'Agent')).toBe(true);
});
it('deduplicates column names after slugging source node names', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: "={{ $('A-B').item.json.value }}",
options: { systemMessage: "={{ $('A_B').item.json.value }}" },
},
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toHaveLength(2);
expect(result.map((r) => r.column).sort()).toEqual(['a_b_value', 'a_b_value_1']);
});
it('throws if the agent node is not in the workflow', () => {
expect(() => detectAgentNamedRefs(wf([]), 'Missing')).toThrow(/not found/i);
});
it('does not return $json.<x> direct refs (only named refs)', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {
text: '={{ $json.user_query }}',
options: { systemMessage: "={{ $('Memory').item.json.context }}" },
},
position: [0, 0],
id: 'a',
},
]);
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toHaveLength(1);
expect(result[0].nodeName).toBe('Memory');
expect(result[0].field).toBe('context');
expect(result[0].targetNodeName).toBe('Agent');
});
});
describe('sub-component scanning', () => {
it('detects named refs in a memory node connected via ai_memory', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [0, 0],
id: 'a',
},
{
name: 'Postgres Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
typeVersion: 1,
parameters: { sessionIdExpression: "={{ $('Sender ID').item.json.id }}" },
position: [0, -100],
id: 'm',
},
],
connections: {
'Postgres Memory': { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toEqual([
{
nodeName: 'Sender ID',
field: 'id',
originalExpression: "$('Sender ID').item.json.id",
column: 'id',
targetNodeName: 'Postgres Memory',
},
]);
});
it('detects named refs in a tool node connected via ai_tool', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'a',
},
{
name: 'HTTP Tool',
type: '@n8n/n8n-nodes-langchain.toolHttpRequest',
typeVersion: 1,
parameters: { url: "={{ $('Webhook').item.json.endpoint }}" },
position: [0, 100],
id: 't',
},
],
connections: {
'HTTP Tool': { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toEqual([
{
nodeName: 'Webhook',
field: 'endpoint',
originalExpression: "$('Webhook').item.json.endpoint",
column: 'endpoint',
targetNodeName: 'HTTP Tool',
},
]);
});
it('shares column when same source-field referenced from agent AND a sub-component', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: "={{ $('Voice').item.json.text }}" },
position: [0, 0],
id: 'a',
},
{
name: 'Memory',
type: '@n8n/n8n-nodes-langchain.memoryBufferWindow',
typeVersion: 1,
parameters: { sessionIdExpression: "={{ $('Voice').item.json.text }}" },
position: [0, 100],
id: 'm',
},
],
connections: {
Memory: { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectAgentNamedRefs(workflow, 'Agent');
// Two NamedRef entries — one per target — sharing the same column.
expect(result).toHaveLength(2);
expect(result.every((r) => r.column === 'text')).toBe(true);
expect(result.map((r) => r.targetNodeName).sort()).toEqual(['Agent', 'Memory']);
});
it('disambiguates collision when agent and sub-component reference different sources with same field name', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: "={{ $('Voice or Text').item.json.text }}" },
position: [0, 0],
id: 'a',
},
{
name: 'Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
typeVersion: 1,
parameters: { sessionIdExpression: "={{ $('Sender ID').item.json.text }}" },
position: [0, 100],
id: 'm',
},
],
connections: {
Memory: { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectAgentNamedRefs(workflow, 'Agent');
expect(result).toHaveLength(2);
const cols = result.map((r) => r.column).sort();
expect(cols).toEqual(['sender_id_text', 'voice_or_text_text']);
});
it('ignores nodes that are not connected to the agent via ai_* connections', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [0, 0],
id: 'a',
},
{
name: 'Unrelated Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
typeVersion: 1,
parameters: { sessionIdExpression: "={{ $('SomeNode').item.json.id }}" },
position: [200, 0],
id: 'm',
},
],
connections: {}, // Memory NOT connected to agent
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectAgentNamedRefs(workflow, 'Agent');
// Memory's named ref is ignored because Memory isn't a sub-component of Agent.
expect(result).toEqual([]);
});
});

134
packages/@n8n/instance-ai/src/tools/evals/__tests__/detect-ai-nodes.test.ts
View File

@ -1,134 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { detectAiNodes } from '../detect-ai-nodes';
function wf(
nodes: Array<{ name: string; type: string; parameters?: Record<string, unknown> }>,
): WorkflowJSON {
return {
name: 'Test',
nodes: nodes.map((n, i) => ({
id: `id-${i}`,
name: n.name,
type: n.type,
typeVersion: 1,
position: [0, 0] as [number, number],
parameters: n.parameters ?? {},
})),
connections: {},
} as unknown as WorkflowJSON;
}
describe('detectAiNodes', () => {
it('returns langchain node names when workflow contains AI nodes', () => {
const result = detectAiNodes(
wf([
{ name: 'Start', type: 'n8n-nodes-base.manualTrigger' },
{ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent' },
]),
);
expect(result.aiNodeNames).toEqual(['Agent']);
});
it('returns an empty list when no langchain nodes are present', () => {
const result = detectAiNodes(
wf([
{ name: 'Start', type: 'n8n-nodes-base.manualTrigger' },
{ name: 'HTTP', type: 'n8n-nodes-base.httpRequest' },
]),
);
expect(result.aiNodeNames).toEqual([]);
});
it('collects only root agent names — sub-components (memory, models, tools) are excluded', () => {
const result = detectAiNodes(
wf([
{ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent' },
{ name: 'Memory', type: '@n8n/n8n-nodes-langchain.memoryBufferWindow' },
{ name: 'Chat Model', type: '@n8n/n8n-nodes-langchain.lmChatOpenAi' },
{ name: 'Search Tool', type: '@n8n/n8n-nodes-langchain.toolSerpApi' },
{ name: 'Embeddings', type: '@n8n/n8n-nodes-langchain.embeddingsOpenAi' },
{ name: 'HTTP', type: 'n8n-nodes-base.httpRequest' },
]),
);
expect(result.aiNodeNames).toEqual(['Agent']);
});
it('collects multiple root agents in a multi-agent workflow', () => {
const result = detectAiNodes(
wf([
{ name: 'Categorizer Agent', type: '@n8n/n8n-nodes-langchain.agent' },
{ name: 'Chat Model', type: '@n8n/n8n-nodes-langchain.lmChatOpenAi' },
{ name: 'Responder Agent', type: '@n8n/n8n-nodes-langchain.agent' },
]),
);
expect(result.aiNodeNames).toEqual(['Categorizer Agent', 'Responder Agent']);
});
it('flags alreadyConfigured when an EvaluationTrigger is present', () => {
const result = detectAiNodes(
wf([
{ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent' },
{ name: 'EvalTrigger', type: 'n8n-nodes-base.evaluationTrigger' },
]),
);
expect(result.alreadyConfigured).toBe(true);
});
it('flags alreadyConfigured when an Evaluation node is present', () => {
const result = detectAiNodes(
wf([
{ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent' },
{ name: 'Eval', type: 'n8n-nodes-base.evaluation' },
]),
);
expect(result.alreadyConfigured).toBe(true);
});
it('does not flag alreadyConfigured for a clean AI workflow', () => {
const result = detectAiNodes(wf([{ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent' }]));
expect(result.alreadyConfigured).toBe(false);
});
describe('fallback for unrecognized langchain nodes', () => {
it('detects @n8n/n8n-nodes-langchain.agentTool as a fallback AI node', () => {
// agentTool is an agent variant — its local name starts with "agent",
// which is not a sub-component prefix, so the fallback path includes it.
const result = detectAiNodes(
wf([{ name: 'AgentTool', type: '@n8n/n8n-nodes-langchain.agentTool' }]),
);
expect(result.isAiWorkflow).toBe(true);
expect(result.aiNodeNames).toEqual(['AgentTool']);
});
it('falls back to any non-sub-component langchain node when no root matches', () => {
// Hypothetical future node type not in the allow-list, not matching a sub-component prefix.
const result = detectAiNodes(
wf([{ name: 'Future', type: '@n8n/n8n-nodes-langchain.futureRootType' }]),
);
expect(result.isAiWorkflow).toBe(true);
expect(result.aiNodeNames).toEqual(['Future']);
});
it('does not fall back when only sub-components are present', () => {
const result = detectAiNodes(
wf([
{ name: 'Memory', type: '@n8n/n8n-nodes-langchain.memoryBufferWindow' },
{ name: 'Chat Model', type: '@n8n/n8n-nodes-langchain.lmChatOpenAi' },
]),
);
expect(result.isAiWorkflow).toBe(false);
expect(result.aiNodeNames).toEqual([]);
});
it('prefers root types over fallback candidates when both present', () => {
const result = detectAiNodes(
wf([
{ name: 'Future', type: '@n8n/n8n-nodes-langchain.futureRootType' },
{ name: 'Real Agent', type: '@n8n/n8n-nodes-langchain.agent' },
]),
);
expect(result.aiNodeNames).toEqual(['Real Agent']);
});
});
});

237
packages/@n8n/instance-ai/src/tools/evals/__tests__/detect-tool-refs.service.test.ts
View File

@ -1,237 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { detectToolRefs } from '../detect-tool-refs.service';
const node = (overrides: Partial<WorkflowJSON['nodes'][number]>): WorkflowJSON['nodes'][number] =>
({
name: 'Node',
type: 'n8n-nodes-base.set',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'n',
...overrides,
}) as WorkflowJSON['nodes'][number];
const wf = (
nodes: WorkflowJSON['nodes'],
connections?: WorkflowJSON['connections'],
): WorkflowJSON =>
({
name: 't',
nodes,
connections: connections ?? {},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
describe('detectToolRefs', () => {
it('returns [] when the agent has no sub-components', () => {
const workflow = wf([node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' })]);
expect(detectToolRefs(workflow, 'Agent')).toEqual([]);
});
it("ignores refs that live on the agent's own parameters", () => {
const workflow = wf([
node({
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
id: 'a',
parameters: { text: "={{ $('Telegram Trigger').item.json.message }}" },
}),
node({ name: 'Telegram Trigger', type: 'n8n-nodes-base.telegramTrigger', id: 't' }),
]);
expect(detectToolRefs(workflow, 'Agent')).toEqual([]);
});
it('detects a named ref inside an ai_tool sub-component', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({
name: 'HTTP Tool',
type: '@n8n/n8n-nodes-langchain.toolHttpRequest',
id: 'h',
parameters: { url: "={{ $('Webhook').item.json.endpoint }}" },
}),
node({ name: 'Webhook', type: 'n8n-nodes-base.webhook', id: 'w' }),
],
connections: {
'HTTP Tool': { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
expect(detectToolRefs(workflow, 'Agent')).toEqual([
{ sourceNodeName: 'Webhook', field: 'endpoint', toolNodeName: 'HTTP Tool' },
]);
});
it('detects a named ref inside an ai_memory sub-component', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({
name: 'Postgres Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
id: 'm',
parameters: { sessionIdExpression: "={{ $('Telegram Trigger').item.json.chat_id }}" },
}),
node({ name: 'Telegram Trigger', type: 'n8n-nodes-base.telegramTrigger', id: 't' }),
],
connections: {
'Postgres Memory': { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
expect(detectToolRefs(workflow, 'Agent')).toEqual([
{
sourceNodeName: 'Telegram Trigger',
field: 'chat_id',
toolNodeName: 'Postgres Memory',
},
]);
});
it('emits one entry per unique (source, field, tool) triplet, deduped within a tool', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({
name: 'Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
id: 'm',
parameters: {
sessionIdExpression: "={{ $('Trigger').item.json.id }}",
contextWindow: "Use {{ $('Trigger').item.json.id }} for context",
},
}),
node({ name: 'Trigger', type: 'n8n-nodes-base.webhook', id: 't' }),
],
connections: {
Memory: { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectToolRefs(workflow, 'Agent');
expect(result).toHaveLength(1);
expect(result[0]).toEqual({
sourceNodeName: 'Trigger',
field: 'id',
toolNodeName: 'Memory',
});
});
it('skips refs whose source node does not exist in the workflow', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({
name: 'Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
id: 'm',
parameters: { sessionIdExpression: "={{ $('Ghost Node').item.json.id }}" },
}),
],
connections: {
Memory: { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
expect(detectToolRefs(workflow, 'Agent')).toEqual([]);
});
it('skips refs to nodes downstream of the agent (those execute during eval)', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({ name: 'Downstream', type: 'n8n-nodes-base.set', id: 'd' }),
node({
name: 'Tool',
type: '@n8n/n8n-nodes-langchain.toolHttpRequest',
id: 't',
parameters: { url: "={{ $('Downstream').item.json.value }}" },
}),
],
connections: {
Agent: { main: [[{ node: 'Downstream', type: 'main', index: 0 }]] },
Tool: { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
expect(detectToolRefs(workflow, 'Agent')).toEqual([]);
});
it('skips refs that point to the agent itself', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({
name: 'Tool',
type: '@n8n/n8n-nodes-langchain.toolHttpRequest',
id: 't',
parameters: { url: "={{ $('Agent').item.json.output }}" },
}),
],
connections: {
Tool: { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
expect(detectToolRefs(workflow, 'Agent')).toEqual([]);
});
it('aggregates refs from multiple sub-components', () => {
const workflow: WorkflowJSON = {
name: 't',
nodes: [
node({ name: 'Agent', type: '@n8n/n8n-nodes-langchain.agent', id: 'a' }),
node({
name: 'Tool',
type: '@n8n/n8n-nodes-langchain.toolHttpRequest',
id: 'h',
parameters: { url: "={{ $('Webhook').item.json.url }}" },
}),
node({
name: 'Memory',
type: '@n8n/n8n-nodes-langchain.memoryPostgres',
id: 'm',
parameters: { sessionIdExpression: "={{ $('Webhook').item.json.session }}" },
}),
node({ name: 'Webhook', type: 'n8n-nodes-base.webhook', id: 'w' }),
],
connections: {
Tool: { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
Memory: { ai_memory: [[{ node: 'Agent', type: 'ai_memory', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const result = detectToolRefs(workflow, 'Agent');
expect(result).toHaveLength(2);
expect(result.map((r) => r.toolNodeName).sort()).toEqual(['Memory', 'Tool']);
expect(result.every((r) => r.sourceNodeName === 'Webhook')).toBe(true);
});
it('throws if the agent node is not in the workflow', () => {
expect(() => detectToolRefs(wf([]), 'Missing')).toThrow(/not found/i);
});
});

383
packages/@n8n/instance-ai/src/tools/evals/__tests__/eval-data-requirements.service.test.ts
View File

@ -1,383 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { analyzeEvalDataRequirements } from '../eval-data-requirements.service';
const wf = (
nodes: WorkflowJSON['nodes'],
connections: WorkflowJSON['connections'] = {},
): WorkflowJSON => ({
name: 't',
nodes,
connections,
pinData: {},
settings: {},
});
function workflowWithEvalTopology(): WorkflowJSON {
return {
id: 'w1',
name: 'Support Classifier',
nodes: [
{
id: 'main-trigger',
name: 'Chat Trigger',
type: '@n8n/n8n-nodes-langchain.chatTrigger',
typeVersion: 1,
position: [0, 0],
parameters: {},
},
{
id: 'eval-trigger',
name: 'Eval Trigger',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 4.7,
position: [0, 240],
parameters: {
source: 'dataTable',
dataTableId: { __rl: true, mode: 'id', value: 'dt-1' },
},
},
{
id: 'agent',
name: 'Classifier Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
position: [480, 0],
parameters: {
text: '={{ $json.user_message }}',
},
},
{
id: 'check',
name: 'Eval Check',
type: 'n8n-nodes-base.evaluation',
typeVersion: 4.8,
position: [740, 0],
parameters: { operation: 'checkIfEvaluating' },
},
{
id: 'outputs',
name: 'Eval Set Outputs',
type: 'n8n-nodes-base.evaluation',
typeVersion: 4.8,
position: [980, -120],
parameters: {
operation: 'setOutputs',
outputs: {
values: [
{
outputName: 'actual_answer',
outputValue: '={{ $json.output }}',
},
],
},
},
},
{
id: 'metrics',
name: 'Eval Set Metrics',
type: 'n8n-nodes-base.evaluation',
typeVersion: 4.8,
position: [1200, -120],
parameters: {
operation: 'setMetrics',
metric: 'correctness',
expectedAnswer: "={{ $('Eval Trigger').item.json.expected_answer }}",
actualAnswer: '={{ $json.output }}',
},
},
{
id: 'side-effect',
name: 'Send Slack',
type: 'n8n-nodes-base.slack',
typeVersion: 2,
position: [980, 120],
parameters: {},
},
],
connections: {
'Chat Trigger': {
main: [[{ node: 'Classifier Agent', type: 'main', index: 0 }]],
},
'Eval Trigger': {
main: [[{ node: 'Classifier Agent', type: 'main', index: 0 }]],
},
'Classifier Agent': {
main: [[{ node: 'Eval Check', type: 'main', index: 0 }]],
},
'Eval Check': {
main: [
[{ node: 'Eval Set Outputs', type: 'main', index: 0 }],
[{ node: 'Send Slack', type: 'main', index: 0 }],
],
},
'Eval Set Outputs': {
main: [[{ node: 'Eval Set Metrics', type: 'main', index: 0 }]],
},
},
};
}
describe('analyzeEvalDataRequirements', () => {
it('finds the eval DataTable and data columns from eval topology', () => {
const result = analyzeEvalDataRequirements(workflowWithEvalTopology());
expect(result.targets).toHaveLength(1);
expect(result.targets[0]).toMatchObject({
dataTableId: 'dt-1',
evaluationTriggerName: 'Eval Trigger',
targetNodeName: 'Classifier Agent',
inputColumns: ['user_message'],
expectedOutputColumns: ['expected_answer'],
actualOutputColumns: ['actual_answer'],
expectedToActualPairs: [{ expectedColumn: 'expected_answer', actualField: 'output' }],
});
});
it('returns no targets when eval nodes are absent', () => {
const workflow = {
name: 'No evals',
nodes: [],
connections: {},
};
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets).toEqual([]);
expect(result.reason).toContain('EvaluationTrigger');
});
it('uses analyzeAgentInputColumns to derive inputColumns from agent parameters', () => {
const workflow = wf(
[
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [200, 0],
id: 'a',
},
],
{
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
},
);
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets[0].inputColumns).toEqual(['user_query']);
expect(result.targets[0].targetNodeName).toBe('Agent');
});
it('uses reachable root AI nodes as targets even when they are not agents', () => {
const workflow = wf(
[
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'LLM Chain',
type: '@n8n/n8n-nodes-langchain.chainLlm',
typeVersion: 1,
parameters: { prompt: '={{ $json.user_query }}' },
position: [200, 0],
id: 'c',
},
],
{
EvalTrig: { main: [[{ node: 'LLM Chain', type: 'main', index: 0 }]] },
},
);
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets[0].targetNodeName).toBe('LLM Chain');
expect(result.targets[0].inputColumns).toEqual(['user_query']);
});
it('extracts expectedToActualPairs from setMetrics nodes', () => {
const workflow = wf(
[
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [200, 0],
id: 'a',
},
{
name: 'Metric',
type: 'n8n-nodes-base.evaluation',
typeVersion: 1,
parameters: {
operation: 'setMetrics',
expectedAnswer: "={{ $('EvalTrig').item.json.expected_response }}",
actualAnswer: '={{ $json.output }}',
},
position: [400, 0],
id: 'm',
},
],
{
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
Agent: { main: [[{ node: 'Metric', type: 'main', index: 0 }]] },
},
);
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets[0].expectedToActualPairs).toEqual([
{ expectedColumn: 'expected_response', actualField: 'output' },
]);
});
it('extracts expected output columns from $json references in expectedAnswer', () => {
const workflow = wf(
[
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [200, 0],
id: 'a',
},
{
name: 'Metric',
type: 'n8n-nodes-base.evaluation',
typeVersion: 1,
parameters: {
operation: 'setMetrics',
expectedAnswer: '={{ $json.expected_response }}',
actualAnswer: '={{ $json.output }}',
},
position: [400, 0],
id: 'm',
},
],
{
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
Agent: { main: [[{ node: 'Metric', type: 'main', index: 0 }]] },
},
);
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets[0].expectedOutputColumns).toEqual(['expected_response']);
expect(result.targets[0].expectedToActualPairs).toEqual([
{ expectedColumn: 'expected_response', actualField: 'output' },
]);
});
it('does not treat actualAnswer named refs as expected output columns', () => {
const workflow = wf(
[
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [200, 0],
id: 'a',
},
{
name: 'Metric',
type: 'n8n-nodes-base.evaluation',
typeVersion: 1,
parameters: {
operation: 'setMetrics',
expectedAnswer: "={{ $('EvalTrig').item.json.expected_response }}",
actualAnswer: "={{ $('Agent').item.json.output }}",
},
position: [400, 0],
id: 'm',
},
],
{
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
Agent: { main: [[{ node: 'Metric', type: 'main', index: 0 }]] },
},
);
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets[0].expectedOutputColumns).toEqual(['expected_response']);
});
it('returns empty inputColumns when no root AI node is reachable from the trigger', () => {
const workflow = wf(
[
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-2' } },
position: [0, 0],
id: 't',
},
{
name: 'SomeNode',
type: 'n8n-nodes-base.set',
typeVersion: 1,
parameters: { value: '={{ $json.col }}' },
position: [200, 0],
id: 's',
},
],
{
EvalTrig: { main: [[{ node: 'SomeNode', type: 'main', index: 0 }]] },
},
);
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets).toHaveLength(1);
expect(result.targets[0].inputColumns).toEqual([]);
expect(result.targets[0].targetNodeName).toBeUndefined();
});
it('returns no targets when workflow nodes are absent', () => {
const workflow = {
name: 'Malformed workflow',
connections: {},
} as WorkflowJSON;
const result = analyzeEvalDataRequirements(workflow);
expect(result.targets).toEqual([]);
expect(result.reason).toContain('EvaluationTrigger');
});
});

1073
packages/@n8n/instance-ai/src/tools/evals/__tests__/evals.tool.test.ts
View File

File diff suppressed because it is too large Load Diff

349
packages/@n8n/instance-ai/src/tools/evals/__tests__/extract-rows-from-history.service.test.ts
View File

@ -1,349 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import type { ExecutionSummary, InstanceAiContext, NodeOutputResult } from '../../../types';
import { extractRowsFromExecutionHistory } from '../extract-rows-from-history.service';
type ExecutionService = InstanceAiContext['executionService'];
const buildContext = (
executionService: Partial<Pick<ExecutionService, 'list' | 'getNodeOutput'>> = {},
): InstanceAiContext =>
({
executionService: {
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValue([]),
getNodeOutput:
vi.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>(),
...executionService,
},
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
}) as unknown as InstanceAiContext;
const executionSummary = (
id: string,
status: ExecutionSummary['status'] = 'success',
): ExecutionSummary => ({
id,
workflowId: 'w1',
workflowName: 'Workflow',
status,
startedAt: '2026-01-01T00:00:00.000Z',
mode: 'manual',
});
const nodeOutput = (nodeName: string, json: Record<string, unknown>): NodeOutputResult => ({
nodeName,
items: [{ json }],
totalItems: 1,
returned: { from: 0, to: 0 },
});
const buildWorkflow = (): WorkflowJSON =>
({
name: 't',
nodes: [
{
name: 'Trigger',
type: 'n8n-nodes-base.webhook',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {},
position: [200, 0],
id: 'a',
},
],
connections: {
Trigger: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
describe('extractRowsFromExecutionHistory', () => {
it('returns 0 rows when the workflow has no executions', async () => {
const ctx = buildContext();
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [],
});
expect(result.rows).toEqual([]);
expect(result.scannedExecutions).toBe(0);
});
it('extracts agent-input rows from successful executions', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([executionSummary('e1'), executionSummary('e2')]),
getNodeOutput: vi
.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>()
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'hello' }))
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'world' })),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [],
});
expect(result.rows).toEqual([{ user_query: 'hello' }, { user_query: 'world' }]);
expect(result.scannedExecutions).toBe(2);
});
it('deduplicates exact-match rows from execution history', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([
executionSummary('e1'),
executionSummary('e2'),
executionSummary('e3'),
]),
getNodeOutput: vi
.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>()
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'hello' }))
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'hello' }))
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'world' })),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [],
});
expect(result.rows).toEqual([{ user_query: 'hello' }, { user_query: 'world' }]);
expect(result.scannedExecutions).toBe(3);
});
it('skips executions where the projected record is missing a required column', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([executionSummary('e1'), executionSummary('e2')]),
getNodeOutput: vi
.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>()
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'hello', context: 'c' }))
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 'world' })),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query', 'context'],
expectedToActualPairs: [],
});
expect(result.rows).toEqual([{ user_query: 'hello', context: 'c' }]);
});
it('coerces non-string column values to JSON strings', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([executionSummary('e1')]),
getNodeOutput: vi
.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>()
.mockResolvedValueOnce(nodeOutput('Trigger', { payload: { nested: 1 } })),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['payload'],
expectedToActualPairs: [],
});
expect(result.rows).toEqual([{ payload: '{"nested":1}' }]);
});
it('stops at the 25-row cap', async () => {
const summaries = Array.from({ length: 30 }, (_, i) => executionSummary(`e${i}`));
const outputs = summaries.map((_, i) => nodeOutput('Trigger', { user_query: `q${i}` }));
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce(summaries),
getNodeOutput: vi.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>(async () => await Promise.resolve(outputs.shift() ?? nodeOutput('Trigger', {}))),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [],
});
expect(result.rows).toHaveLength(25);
});
it('lists only successful executions', async () => {
const list = vi
.fn<(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>>()
.mockResolvedValue([executionSummary('e1')]);
const ctx = buildContext({
list,
getNodeOutput: vi
.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>()
.mockResolvedValueOnce(nodeOutput('Trigger', { user_query: 's' })),
});
await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [],
});
expect(list).toHaveBeenCalledTimes(1);
expect(list).toHaveBeenCalledWith({ workflowId: 'w1', status: 'success', limit: 100 });
});
it('returns 0 rows and skips silently when getNodeOutput throws for an execution', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([executionSummary('e1')]),
getNodeOutput: vi
.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>()
.mockRejectedValueOnce(new Error('boom')),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [],
});
expect(result.rows).toEqual([]);
expect(result.scannedExecutions).toBe(0);
});
it('extracts expected columns from agent output when expectedToActualPairs are provided', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([executionSummary('e1')]),
getNodeOutput: vi.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>(
async (_id, nodeName) =>
await Promise.resolve(
nodeName === 'Trigger'
? nodeOutput('Trigger', { user_query: 'hi' })
: nodeOutput('Agent', { output: 'hello world' }),
),
),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [{ expectedColumn: 'expected_response', actualField: 'output' }],
});
expect(result.rows).toEqual([{ user_query: 'hi', expected_response: 'hello world' }]);
});
it('skips execution if the agent output is missing the actualField', async () => {
const ctx = buildContext({
list: vi
.fn<
(...args: Parameters<ExecutionService['list']>) => ReturnType<ExecutionService['list']>
>()
.mockResolvedValueOnce([executionSummary('e1')]),
getNodeOutput: vi.fn<
(
...args: Parameters<ExecutionService['getNodeOutput']>
) => ReturnType<ExecutionService['getNodeOutput']>
>(
async (_id, nodeName) =>
await Promise.resolve(
nodeName === 'Trigger'
? nodeOutput('Trigger', { user_query: 'hi' })
: nodeOutput('Agent', {}),
),
),
});
const result = await extractRowsFromExecutionHistory(ctx, {
workflow: buildWorkflow(),
workflowId: 'w1',
agentNodeName: 'Agent',
inputColumns: ['user_query'],
expectedToActualPairs: [{ expectedColumn: 'expected_response', actualField: 'output' }],
});
expect(result.rows).toEqual([]);
});
});

247
packages/@n8n/instance-ai/src/tools/evals/__tests__/format-eval-setup-task.test.ts
View File

@ -1,247 +0,0 @@
import { currentJsonExpression, nodeItemJsonExpression } from '../column-ref-utils';
import { formatEvalSetupTask } from '../format-eval-setup-task';
const BASE = {
workflowId: 'w1',
workflowName: 'Telegram AI Q&A Bot',
detectedAiNodes: ['General Agent'],
datasetChoice: 'create-empty' as const,
suggestedInputColumns: ['user_message'],
suggestedOutputColumns: ['agent_response'],
enabledMetrics: [
{
id: 'correctness',
name: 'Correctness',
kind: 'llm-judge' as const,
description: 'Factual correctness of the response.',
prompt: 'Judge if the response is correct.',
cannedMetricKey: 'correctness',
defaultEnabled: true,
},
],
};
describe('formatEvalSetupTask — PRODUCTION ADAPTER section', () => {
it('omits the section when namedRefs is empty or undefined', () => {
const task = formatEvalSetupTask(BASE);
expect(task).not.toMatch(/PRODUCTION ADAPTER/);
});
it('emits the PRODUCTION ADAPTER section when namedRefs is non-empty', () => {
const task = formatEvalSetupTask({
...BASE,
namedRefs: [
{
nodeName: 'Voice or Text',
field: 'text',
originalExpression: "$('Voice or Text').item.json.text",
column: 'text',
targetNodeName: 'General Agent',
},
],
});
expect(task).toMatch(/PRODUCTION ADAPTER/);
expect(task).toMatch(/Voice or Text/);
expect(task).toMatch(/name: "text"/);
});
it('groups rewrites by target node when refs span agent + sub-component', () => {
const task = formatEvalSetupTask({
...BASE,
suggestedInputColumns: ['text', 'sender_id'],
detectedAiNodes: ['Agent'],
namedRefs: [
{
nodeName: 'Voice or Text',
field: 'text',
originalExpression: "$('Voice or Text').item.json.text",
column: 'text',
targetNodeName: 'Agent',
},
{
nodeName: 'Sender ID',
field: 'id',
originalExpression: "$('Sender ID').item.json.id",
column: 'sender_id',
targetNodeName: 'Postgres Memory',
},
],
});
expect(task).toMatch(/In `Agent`:[\s\S]*Replace `\$\('Voice or Text'\)/);
expect(task).toMatch(/In `Postgres Memory`:[\s\S]*Replace `\$\('Sender ID'\)/);
});
it('agent rewrites use $json.<col> form', () => {
const task = formatEvalSetupTask({
...BASE,
detectedAiNodes: ['Chef Agent'],
namedRefs: [
{
nodeName: 'Voice or Text',
field: 'text',
originalExpression: "$('Voice or Text').item.json.text",
column: 'text',
targetNodeName: 'Chef Agent',
},
],
});
expect(task).toMatch(
/Replace `\$\('Voice or Text'\)\.item\.json\.text` with `\{\{ \$json\.text \}\}`/,
);
});
it("sub-component rewrites use $('<AgentName>').item.json.<col> form, not $json", () => {
const task = formatEvalSetupTask({
...BASE,
detectedAiNodes: ['Chef Agent'],
suggestedInputColumns: ['sender_id'],
namedRefs: [
{
nodeName: 'Sender ID',
field: 'id',
originalExpression: "$('Sender ID').item.json.id",
column: 'sender_id',
targetNodeName: 'Postgres Memory',
},
],
});
expect(task).toContain(
`Replace \`$('Sender ID').item.json.id\` with \`{{ ${nodeItemJsonExpression('Chef Agent', 'sender_id')} }}\``,
);
// Sub-component must not fall back to plain $json.<col>
expect(task).not.toMatch(/In `Postgres Memory`:[\s\S]*\{\{ \$json\.sender_id \}\}/);
});
it('escapes generated adapter assignments and rewrite expressions', () => {
const sourceNodeName = 'Voice "or" Text';
const sourceField = 'message-id';
const agentNodeName = "Chef's Agent";
const column = 'user-message';
const task = formatEvalSetupTask({
...BASE,
detectedAiNodes: ['Other Agent', agentNodeName],
targetAgentNodeName: agentNodeName,
suggestedInputColumns: [column],
namedRefs: [
{
nodeName: sourceNodeName,
field: sourceField,
originalExpression: '$("Voice \\"or\\" Text").item.json["message-id"]',
column,
targetNodeName: agentNodeName,
},
{
nodeName: 'Sender',
field: 'sender-id',
originalExpression: '$("Sender").item.json["sender-id"]',
column: 'sender-id',
targetNodeName: 'Postgres Memory',
},
],
});
expect(task).toContain(
`value: ${JSON.stringify(`={{ ${nodeItemJsonExpression(sourceNodeName, sourceField)} }}`)}`,
);
expect(task).toContain(`with \`{{ ${currentJsonExpression('user_message')} }}\``);
expect(task).toContain(`with \`{{ ${nodeItemJsonExpression(agentNodeName, 'sender_id')} }}\``);
});
});
describe('formatEvalSetupTask — dataset and setOutputs instructions', () => {
it('normalizes DataTable columns and rewrites expressions to the normalized names', () => {
const task = formatEvalSetupTask({
...BASE,
suggestedInputColumns: ['User Query'],
suggestedOutputColumns: ['expected-response'],
enabledMetrics: [
{
id: 'correctness',
name: 'Correctness',
kind: 'llm-judge',
description: 'Factual correctness of the response.',
prompt: 'Judge if the response is correct.',
cannedMetricKey: 'correctness',
defaultEnabled: true,
},
],
namedRefs: [
{
nodeName: 'Webhook',
field: 'User Query',
originalExpression: '$("Webhook").item.json["User Query"]',
column: 'User Query',
targetNodeName: 'General Agent',
},
],
});
expect(task).toContain('Create an empty DataTable named "Telegram AI Q&A Bot — eval samples"');
expect(task).toContain('Columns to create as strings:\n- user_query\n- expected_response');
expect(task).toContain(
`value: ${JSON.stringify(`={{ ${nodeItemJsonExpression('Webhook', 'User Query')} }}`)}`,
);
expect(task).toContain(
'Replace `$("Webhook").item.json["User Query"]` with `{{ $json.user_query }}`',
);
expect(task).toContain("expectedAnswer: ={{ $('Eval Trigger').item.json.expected_response }}");
expect(task).not.toContain('Telegram AI Q&A Bot eval dataset');
expect(task).not.toContain('- User Query');
expect(task).not.toContain('- expected-response');
});
it('keeps dataTableId unconfigured when the user chose to wire data later', () => {
const task = formatEvalSetupTask({
...BASE,
datasetChoice: 'later',
});
expect(task).toContain('Leave setOutputs dataTableId empty until the user selects a DataTable');
expect(task).not.toContain("value: '<same as EvaluationTrigger>'");
});
});
describe('formatEvalSetupTask — metric instructions', () => {
it('maps tool_use to the native toolsUsed metric parameter', () => {
const task = formatEvalSetupTask({
...BASE,
enabledMetrics: [
{
id: 'tool_use',
name: 'Tool use',
kind: 'llm-judge',
description: 'Tool choice',
prompt: 'Check tools.',
cannedMetricKey: 'tool_use',
defaultEnabled: false,
},
],
});
expect(task).toContain("metric: 'toolsUsed'");
expect(task).toContain('expectedTools');
expect(task).toContain('intermediateSteps');
expect(task).not.toContain("metric: 'tool_use'");
});
it('does not instruct builders to use a non-existent relevance metric parameter', () => {
const task = formatEvalSetupTask({
...BASE,
enabledMetrics: [
{
id: 'relevance',
name: 'Relevance',
kind: 'llm-judge',
description: 'Context relevance',
prompt: 'Check relevance.',
cannedMetricKey: 'relevance',
defaultEnabled: false,
},
],
});
expect(task).toContain("metric: 'helpfulness'");
expect(task).toContain('There is no native `relevance` metric option');
expect(task).not.toContain("metric: 'relevance'");
});
});

627
packages/@n8n/instance-ai/src/tools/evals/__tests__/generate-sample-rows.service.test.ts
View File

@ -1,627 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import type { Mock, MockedFunction } from 'vitest';
vi.mock('../../../utils/eval-agents', () => {
return {
createEvalAgent: vi.fn(),
extractText: vi.fn(),
HAIKU_MODEL: 'test-haiku-model',
};
});
import { createEvalAgent, extractText } from '../../../utils/eval-agents';
import {
distributeRowCount,
extractAgentContext,
generateSampleRows,
runBatch,
} from '../generate-sample-rows.service';
import type { AgentContext, SampleRowFacet } from '../generate-sample-rows.service';
const mockCreateEvalAgent = createEvalAgent as MockedFunction<typeof createEvalAgent>;
const mockExtractText = extractText as MockedFunction<typeof extractText>;
function setupAgentMock(responseText: string) {
const generate = vi.fn().mockResolvedValue({ messages: [] });
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(responseText);
}
type GenerateArg = Array<{ content: Array<{ text: string }> }>;
type GenerateMock = Mock<(...args: [GenerateArg]) => Promise<{ messages: [] }>>;
function createGenerateMock(): GenerateMock {
return vi
.fn<(arg: GenerateArg) => Promise<{ messages: [] }>>()
.mockResolvedValue({ messages: [] });
}
function getPromptText(generate: GenerateMock): string {
const firstCall = generate.mock.calls[0];
if (!firstCall) throw new Error('Expected generate to be called');
return firstCall[0][0].content[0].text;
}
const WF: WorkflowJSON = { name: 'Test', nodes: [], connections: {} } as unknown as WorkflowJSON;
const mockLogger = { warn: vi.fn() };
describe('generateSampleRows', () => {
beforeEach(() => vi.clearAllMocks());
it('returns parsed input rows from valid JSON across batches', async () => {
setupAgentMock(JSON.stringify([{ input: 'q1' }]));
const rows = await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input', 'expected_output'],
rowCount: 5,
});
expect(rows.length).toBeGreaterThan(0);
expect(rows[0]).toEqual({ input: 'q1', expected_output: '' });
});
it('coerces non-string cell values to strings', async () => {
setupAgentMock(JSON.stringify([{ input: 42, expected_output: true }]));
const rows = await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input', 'expected_output'],
rowCount: 5,
});
expect(rows[0]).toEqual({ input: '42', expected_output: '' });
});
it('fills missing columns with empty strings', async () => {
setupAgentMock(JSON.stringify([{ input: 'q1' }]));
const rows = await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input', 'expected_output'],
rowCount: 5,
});
expect(rows[0]).toEqual({ input: 'q1', expected_output: '' });
});
it('returns single fallback row when every batch fails to parse', async () => {
setupAgentMock('not json');
const rows = await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input', 'expected_output'],
rowCount: 5,
});
expect(rows).toEqual([{ input: '', expected_output: '' }]);
});
it('returns single fallback row when every batch rejects', async () => {
const generate = vi.fn().mockRejectedValue(new Error('API down'));
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
const rows = await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input', 'expected_output'],
rowCount: 5,
});
expect(rows).toEqual([{ input: '', expected_output: '' }]);
});
});
describe('distributeRowCount', () => {
it('distributes 25 evenly across 5 facets', () => {
expect(distributeRowCount(25)).toEqual([5, 5, 5, 5, 5]);
});
it('puts the remainder in the leading facets', () => {
expect(distributeRowCount(23)).toEqual([5, 5, 5, 4, 4]);
});
it('handles small counts with trailing zeros', () => {
expect(distributeRowCount(8)).toEqual([2, 2, 2, 1, 1]);
expect(distributeRowCount(3)).toEqual([1, 1, 1, 0, 0]);
});
it('treats zero and negative inputs as zero', () => {
expect(distributeRowCount(0)).toEqual([0, 0, 0, 0, 0]);
expect(distributeRowCount(-1)).toEqual([0, 0, 0, 0, 0]);
});
});
const BATCH_FACET: SampleRowFacet = {
length: 'short',
edgeMode: 'happy',
instructions: 'Produce typical inputs.',
};
const BATCH_CONTEXT: AgentContext = {
workflowName: 'Workflow',
agentNodeName: 'Agent',
systemPrompt: 'Be helpful.',
promptTemplate: undefined,
connectedToolNames: [],
};
describe('runBatch', () => {
beforeEach(() => vi.clearAllMocks());
it('returns parsed input rows on success', async () => {
setupAgentMock(JSON.stringify([{ input: 'q1' }]));
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input', 'expected_output'],
});
expect(rows).toEqual([{ input: 'q1', expected_output: '' }]);
});
it('caps over-generated rows to the requested batch size', async () => {
setupAgentMock(JSON.stringify([{ input: 'q1' }, { input: 'q2' }, { input: 'q3' }]));
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 2,
context: BATCH_CONTEXT,
columns: ['input'],
});
expect(rows).toEqual([{ input: 'q1' }, { input: 'q2' }]);
});
it('does not send expected output columns to the sample-row agent', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([{ input: 'q1' }]));
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input', 'expected_output'],
});
const promptText = getPromptText(generate);
expect(promptText).toContain('Columns: input');
expect(promptText).not.toContain('expected_output');
});
it('returns empty array when generate throws (does not propagate)', async () => {
const generate = vi.fn().mockRejectedValue(new Error('API down'));
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input'],
});
expect(rows).toEqual([]);
});
it('logs and returns empty array when parsing fails', async () => {
const logger = { warn: vi.fn<(a: string, b?: Record<string, unknown>) => undefined>() };
setupAgentMock('not json');
const rows = await runBatch({
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input'],
logger,
});
expect(rows).toEqual([]);
expect(logger.warn).toHaveBeenCalledTimes(1);
const [message, metadata] = logger.warn.mock.calls[0];
expect(message).toBe('generate-sample-rows: batch generation failed');
expect(metadata).toEqual(expect.objectContaining({ rowCount: 1, reason: 'invalid_json' }));
});
it('returns empty array when JSON is malformed', async () => {
setupAgentMock('not json');
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input'],
});
expect(rows).toEqual([]);
});
it('returns empty array when schema validation fails', async () => {
setupAgentMock(JSON.stringify({ not: 'an array' }));
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input'],
});
expect(rows).toEqual([]);
});
it('strips markdown fences from the response', async () => {
setupAgentMock('```json\n[{"input":"q","expected_output":"a"}]\n```');
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['input', 'expected_output'],
});
expect(rows).toEqual([{ input: 'q', expected_output: '' }]);
});
it('passes facet instructions, rowCount, and agent context into the prompt', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 7,
context: BATCH_CONTEXT,
columns: ['input'],
});
const promptText = getPromptText(generate);
expect(promptText).toContain('Produce typical inputs.');
expect(promptText).toContain('7');
expect(promptText).toContain('Be helpful.');
});
it('returns empty array immediately when rowCount is zero', async () => {
const generate = vi.fn();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
const rows = await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 0,
context: BATCH_CONTEXT,
columns: ['input'],
});
expect(rows).toEqual([]);
expect(generate).not.toHaveBeenCalled();
});
describe('realExamples few-shot block', () => {
function captureBatchPrompt(generate: GenerateMock): string {
return getPromptText(generate);
}
it('injects a reference-not-seed block when examples are provided', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['user_query'],
realExamples: [{ user_query: 'how do I refund an order?' }],
});
const promptText = captureBatchPrompt(generate);
expect(promptText).toContain('Recent real inputs the agent has received in production');
expect(promptText).toContain('REFERENCE, not seeds');
expect(promptText).toContain('how do I refund an order?');
expect(promptText).toMatch(/Do NOT copy or paraphrase them/);
});
it('omits the block entirely when realExamples is undefined', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['user_query'],
});
expect(captureBatchPrompt(generate)).not.toContain('Recent real inputs');
});
it('filters examples to the requested columns and drops rows that lack all of them', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['user_query'],
realExamples: [
{ user_query: 'real one', expected_response: 'should not leak' },
{ unrelated: 'dropped' },
],
});
const promptText = captureBatchPrompt(generate);
expect(promptText).toContain('real one');
expect(promptText).not.toContain('should not leak');
expect(promptText).not.toContain('dropped');
expect(promptText).not.toContain('unrelated');
});
it('caps the example list at 10 entries', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
const examples = Array.from({ length: 13 }, (_, i) => ({ user_query: `q${i}` }));
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['user_query'],
realExamples: examples,
});
const promptText = captureBatchPrompt(generate);
expect(promptText).toContain('q0');
expect(promptText).toContain('q9');
expect(promptText).not.toContain('q10');
expect(promptText).not.toContain('q12');
});
it('truncates values longer than 300 characters with an ellipsis', async () => {
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
const longValue = 'x'.repeat(500);
await runBatch({
logger: mockLogger,
facet: BATCH_FACET,
rowCount: 1,
context: BATCH_CONTEXT,
columns: ['user_query'],
realExamples: [{ user_query: longValue }],
});
const promptText = captureBatchPrompt(generate);
expect(promptText).toMatch(/x{300}…/);
expect(promptText).not.toMatch(/x{301}/);
});
});
});
describe('extractAgentContext', () => {
const AGENT_TYPE = '@n8n/n8n-nodes-langchain.agent';
const TOOL_TYPE = '@n8n/n8n-nodes-langchain.toolHttpRequest';
function buildWorkflow(): WorkflowJSON {
return {
name: 'Support bot',
nodes: [
{
id: '1',
name: 'Agent A',
type: AGENT_TYPE,
typeVersion: 1,
position: [0, 0],
parameters: {
text: '={{ $json.message }}',
options: { systemMessage: 'You are a polite support agent.' },
},
},
{
id: '2',
name: 'Lookup tool',
type: TOOL_TYPE,
typeVersion: 1,
position: [0, 0],
parameters: {},
},
],
connections: {
'Lookup tool': {
ai_tool: [[{ node: 'Agent A', type: 'ai_tool', index: 0 }]],
},
},
} as unknown as WorkflowJSON;
}
it('returns context for the named agent', () => {
const ctx = extractAgentContext(buildWorkflow(), 'Agent A');
expect(ctx).toEqual({
workflowName: 'Support bot',
agentNodeName: 'Agent A',
systemPrompt: 'You are a polite support agent.',
promptTemplate: '={{ $json.message }}',
connectedToolNames: ['Lookup tool'],
});
});
it('returns undefined when the agent name does not match any node', () => {
const ctx = extractAgentContext(buildWorkflow(), 'Missing Agent');
expect(ctx).toBeUndefined();
});
it('caps an oversized system prompt at 2000 characters', () => {
const wf = buildWorkflow();
const huge = 'x'.repeat(5000);
(
wf.nodes[0] as unknown as { parameters: { options: { systemMessage: string } } }
).parameters.options.systemMessage = huge;
const ctx = extractAgentContext(wf, 'Agent A');
expect(ctx?.systemPrompt).toHaveLength(2000);
});
it('omits prompt template and tools when none are present', () => {
const wf: WorkflowJSON = {
name: 'Bare',
nodes: [
{
id: '1',
name: 'Agent',
type: AGENT_TYPE,
typeVersion: 1,
position: [0, 0],
parameters: {},
},
],
connections: {},
} as unknown as WorkflowJSON;
const ctx = extractAgentContext(wf, 'Agent');
expect(ctx).toEqual({
workflowName: 'Bare',
agentNodeName: 'Agent',
systemPrompt: undefined,
promptTemplate: undefined,
connectedToolNames: [],
});
});
});
describe('generateSampleRows orchestration', () => {
beforeEach(() => vi.clearAllMocks());
it('dispatches one batch per non-empty facet', async () => {
const generate = vi.fn().mockResolvedValue({ messages: [] });
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([{ input: 'r' }]));
await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input'],
rowCount: 25,
});
expect(generate).toHaveBeenCalledTimes(5);
});
it('skips facets that get zero rows', async () => {
const generate = vi.fn().mockResolvedValue({ messages: [] });
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([{ input: 'r' }]));
await generateSampleRows({ logger: mockLogger, workflow: WF, columns: ['input'], rowCount: 3 });
expect(generate).toHaveBeenCalledTimes(3);
});
it('isolates failures: one batch rejects, others still produce rows', async () => {
const responses = [
Promise.resolve({ messages: [] }),
Promise.reject(new Error('mid batch fail')),
Promise.resolve({ messages: [] }),
Promise.resolve({ messages: [] }),
Promise.resolve({ messages: [] }),
];
const generate = vi.fn().mockImplementation(async () => await responses.shift());
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText
.mockReturnValueOnce(JSON.stringify([{ input: 'a' }]))
.mockReturnValueOnce(JSON.stringify([{ input: 'c' }]))
.mockReturnValueOnce(JSON.stringify([{ input: 'd' }]))
.mockReturnValueOnce(JSON.stringify([{ input: 'e' }]));
const rows = await generateSampleRows({
logger: mockLogger,
workflow: WF,
columns: ['input'],
rowCount: 5,
});
const inputs = rows.map((r) => r.input).sort();
expect(inputs).toEqual(['a', 'c', 'd', 'e']);
});
it('uses the default rowCount of 25 when not specified', async () => {
const generate = vi.fn().mockResolvedValue({ messages: [] });
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(
JSON.stringify(Array.from({ length: 5 }, () => ({ input: 'x' }))),
);
const rows = await generateSampleRows({ logger: mockLogger, workflow: WF, columns: ['input'] });
expect(rows).toHaveLength(25);
});
it('uses targetAgentNodeName context when provided', async () => {
const wf: WorkflowJSON = {
name: 'wf',
nodes: [
{
id: '1',
name: 'AgentX',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
position: [0, 0],
parameters: { options: { systemMessage: 'TARGETED-AGENT-SYSTEM' } },
},
],
connections: {},
} as unknown as WorkflowJSON;
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
await generateSampleRows({
logger: mockLogger,
workflow: wf,
columns: ['input'],
rowCount: 5,
targetAgentNodeName: 'AgentX',
});
const promptText = getPromptText(generate);
expect(promptText).toContain('TARGETED-AGENT-SYSTEM');
});
it('falls back to first detected agent when no targetAgentNodeName', async () => {
const wf: WorkflowJSON = {
name: 'wf',
nodes: [
{
id: '1',
name: 'FirstAgent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
position: [0, 0],
parameters: { options: { systemMessage: 'FIRST-AGENT-SYSTEM' } },
},
{
id: '2',
name: 'SecondAgent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
position: [0, 0],
parameters: { options: { systemMessage: 'SECOND-AGENT-SYSTEM' } },
},
],
connections: {},
} as unknown as WorkflowJSON;
const generate = createGenerateMock();
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(JSON.stringify([]));
await generateSampleRows({ logger: mockLogger, workflow: wf, columns: ['input'], rowCount: 5 });
const promptText = getPromptText(generate);
expect(promptText).toContain('FIRST-AGENT-SYSTEM');
expect(promptText).not.toContain('SECOND-AGENT-SYSTEM');
});
});

169
packages/@n8n/instance-ai/src/tools/evals/__tests__/generate-tool-ref-pin-data.service.test.ts
View File

@ -1,169 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import type { MockedFunction } from 'vitest';
vi.mock('../../../utils/eval-agents', async () => {
const actual: object = await vi.importActual('../../../utils/eval-agents');
return { ...actual, createEvalAgent: vi.fn(), extractText: vi.fn() };
});
import { createEvalAgent, extractText } from '../../../utils/eval-agents';
import type { ToolRef } from '../detect-tool-refs.service';
import { generateToolRefPinData } from '../generate-tool-ref-pin-data.service';
const mockCreateEvalAgent = createEvalAgent as MockedFunction<typeof createEvalAgent>;
const mockExtractText = extractText as MockedFunction<typeof extractText>;
function setupAgentMock(responseText: string) {
const generate = vi.fn().mockResolvedValue({ messages: [] });
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
mockExtractText.mockReturnValue(responseText);
}
const wf = (nodes: WorkflowJSON['nodes']): WorkflowJSON =>
({
name: 't',
nodes,
connections: {},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
const triggerNode: WorkflowJSON['nodes'][number] = {
name: 'Telegram Trigger',
type: 'n8n-nodes-base.telegramTrigger',
typeVersion: 1,
parameters: { updates: ['message'] },
position: [0, 0],
id: 't',
} as WorkflowJSON['nodes'][number];
const agentNode: WorkflowJSON['nodes'][number] = {
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: 'hi' },
position: [100, 0],
id: 'a',
} as WorkflowJSON['nodes'][number];
describe('generateToolRefPinData', () => {
beforeEach(() => vi.clearAllMocks());
it('returns {} when no refs are passed', async () => {
const result = await generateToolRefPinData({
workflow: wf([agentNode]),
agentNodeName: 'Agent',
refs: [],
});
expect(result).toEqual({});
expect(mockCreateEvalAgent).not.toHaveBeenCalled();
});
it('returns parsed pinData for each requested source node', async () => {
setupAgentMock(
JSON.stringify({
'Telegram Trigger': [{ json: { message: { chat: { id: '123' } } } }],
}),
);
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'message', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(result).toEqual({
'Telegram Trigger': [{ json: { message: { chat: { id: '123' } } } }],
});
});
it('strips markdown code fences around the JSON output', async () => {
setupAgentMock('```json\n{"Telegram Trigger":[{"json":{"chat_id":"42"}}]}\n```');
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'chat_id', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(result['Telegram Trigger']).toEqual([{ json: { chat_id: '42' } }]);
});
it('returns {} on malformed LLM output', async () => {
setupAgentMock('not json at all');
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'chat_id', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(result).toEqual({});
});
it('returns {} on schema-invalid LLM output', async () => {
setupAgentMock(JSON.stringify({ 'Telegram Trigger': 'not-an-array' }));
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'chat_id', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(result).toEqual({});
});
it('drops entries for nodes that were not requested', async () => {
setupAgentMock(
JSON.stringify({
'Telegram Trigger': [{ json: { chat_id: '1' } }],
Unrelated: [{ json: { foo: 'bar' } }],
}),
);
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'chat_id', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(Object.keys(result)).toEqual(['Telegram Trigger']);
});
it('omits nodes the LLM failed to populate (no throw)', async () => {
setupAgentMock(JSON.stringify({ 'Other Trigger': [{ json: { x: 1 } }] }));
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'chat_id', toolNodeName: 'Memory' },
{ sourceNodeName: 'Other Trigger', field: 'x', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, { ...triggerNode, name: 'Other Trigger', id: 'o' }, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(Object.keys(result)).toEqual(['Other Trigger']);
});
it('returns {} when the LLM call throws', async () => {
const generate = vi.fn().mockRejectedValue(new Error('boom'));
mockCreateEvalAgent.mockReturnValue({ generate } as unknown as ReturnType<
typeof createEvalAgent
>);
const refs: ToolRef[] = [
{ sourceNodeName: 'Telegram Trigger', field: 'chat_id', toolNodeName: 'Memory' },
];
const result = await generateToolRefPinData({
workflow: wf([triggerNode, agentNode]),
agentNodeName: 'Agent',
refs,
});
expect(result).toEqual({});
});
});

75
packages/@n8n/instance-ai/src/tools/evals/__tests__/metric-catalog.test.ts
View File

@ -1,75 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { proposeDefaultMetricIds } from '../metric-catalog';
const wf = (
nodes: WorkflowJSON['nodes'],
connections: WorkflowJSON['connections'] = {},
): WorkflowJSON => ({
name: 't',
nodes,
connections,
pinData: {},
settings: {},
});
describe('metric-catalog', () => {
describe('proposeDefaultMetricIds', () => {
it('returns ["correctness"] when the agent has no tools', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'a',
},
]);
expect(proposeDefaultMetricIds(workflow, 'Agent')).toEqual(['correctness']);
});
it('does not infer tools from agent parameters', () => {
const workflow = wf([
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { tools: [{ name: 'Calculator' }] },
position: [0, 0],
id: 'a',
},
]);
expect(proposeDefaultMetricIds(workflow, 'Agent')).toEqual(['correctness']);
});
it('returns ["correctness", "tool_use"] when an ai_tool connection feeds the agent', () => {
const workflow = wf(
[
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: {},
position: [0, 0],
id: 'a',
},
{
name: 'CalcTool',
type: '@n8n/n8n-nodes-langchain.toolCalculator',
typeVersion: 1,
parameters: {},
position: [200, 0],
id: 't',
},
],
{
CalcTool: { ai_tool: [[{ node: 'Agent', type: 'ai_tool', index: 0 }]] },
},
);
expect(proposeDefaultMetricIds(workflow, 'Agent')).toEqual(['correctness', 'tool_use']);
});
});
});

29
packages/@n8n/instance-ai/src/tools/evals/analyze-agent-input-columns.service.ts
View File

@ -1,29 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { collectStrings, extractJsonColumnRefs, nodeHasName, unique } from './column-ref-utils';
export interface AgentInputColumns {
agentNodeName: string;
inputColumns: string[];
}
const FALLBACK_COLUMN = 'input';
export function analyzeAgentInputColumns(
workflow: WorkflowJSON,
agentNodeName: string,
): AgentInputColumns {
const node = (workflow.nodes ?? []).find((n) => nodeHasName(n) && n.name === agentNodeName);
if (!node) {
throw new Error(`Agent node "${agentNodeName}" not found in workflow`);
}
const refs = unique(
collectStrings(node.parameters).flatMap((text) => extractJsonColumnRefs(text)),
);
if (refs.length === 0) {
return { agentNodeName, inputColumns: [FALLBACK_COLUMN] };
}
return { agentNodeName, inputColumns: refs };
}

26
packages/@n8n/instance-ai/src/tools/evals/apply-pin-data.service.ts
View File

@ -1,26 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import type { ToolRefPinData } from './generate-tool-ref-pin-data.service';
/**
* Returns a new workflow with the generated pinData entries merged in.
*
* - If the workflow already has pinData for a given node, the existing entry
* is preserved (we never overwrite user-supplied or previously-generated
* pinData — re-running setup is idempotent).
* - If every generated entry was skipped (already-set), the input reference
* is returned unchanged so callers can short-circuit a workflow update.
*/
export function applyPinData(workflow: WorkflowJSON, generated: ToolRefPinData): WorkflowJSON {
const existing = workflow.pinData ?? {};
const additions: Record<string, Array<{ json: Record<string, unknown> }>> = {};
for (const [nodeName, items] of Object.entries(generated)) {
if (nodeName in existing) continue;
additions[nodeName] = items;
}
if (Object.keys(additions).length === 0) return workflow;
return {
...workflow,
pinData: { ...existing, ...additions },
};
}

308
packages/@n8n/instance-ai/src/tools/evals/column-ref-utils.ts
View File

@ -1,308 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
export type WorkflowNode = WorkflowJSON['nodes'][number];
export function unique(values: string[]): string[] {
return [...new Set(values.filter((value) => value.length > 0))];
}
export function nodeTypeEndsWith(node: WorkflowNode, suffix: string): boolean {
return node.type === suffix || node.type.endsWith(`.${suffix}`);
}
export function nodeHasName(node: WorkflowNode): node is WorkflowNode & { name: string } {
return typeof node.name === 'string' && node.name.length > 0;
}
export function collectStrings(value: unknown): string[] {
if (typeof value === 'string') return [value];
if (Array.isArray(value)) return value.flatMap(collectStrings);
if (!isRecord(value)) return [];
return Object.values(value).flatMap(collectStrings);
}
const JS_IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
const JS_IDENTIFIER_START = /^[A-Za-z_$]$/;
const JS_IDENTIFIER_PART = /^[A-Za-z0-9_$]$/;
function charAt(text: string, index: number): string {
return text[index] ?? '';
}
function isIdentifierPart(char: string): boolean {
return JS_IDENTIFIER_PART.test(char);
}
function skipWhitespace(text: string, start: number): number {
let index = start;
while (/\s/.test(charAt(text, index))) index++;
return index;
}
function readIdentifier(
text: string,
start: number,
): { value: string; endIndex: number } | undefined {
const first = charAt(text, start);
if (!JS_IDENTIFIER_START.test(first)) return undefined;
let endIndex = start + 1;
while (JS_IDENTIFIER_PART.test(charAt(text, endIndex))) endIndex++;
return { value: text.slice(start, endIndex), endIndex };
}
function readQuotedString(
text: string,
start: number,
): { value: string; endIndex: number } | undefined {
const quote = charAt(text, start);
if (quote !== "'" && quote !== '"') return undefined;
let value = '';
for (let index = start + 1; index < text.length; index++) {
const char = charAt(text, index);
if (char === '\\') {
const next = charAt(text, index + 1);
if (next.length === 0) return undefined;
value += next;
index++;
continue;
}
if (char === quote) {
return { value, endIndex: index + 1 };
}
value += char;
}
return undefined;
}
function readFieldAccessor(
text: string,
start: number,
): { field: string; endIndex: number } | undefined {
if (charAt(text, start) === '.') {
const identifier = readIdentifier(text, start + 1);
if (!identifier) return undefined;
return { field: identifier.value, endIndex: identifier.endIndex };
}
if (charAt(text, start) !== '[') return undefined;
const quoteStart = skipWhitespace(text, start + 1);
const quoted = readQuotedString(text, quoteStart);
if (!quoted) return undefined;
const closeBracketIndex = skipWhitespace(text, quoted.endIndex);
if (charAt(text, closeBracketIndex) !== ']') return undefined;
return { field: quoted.value, endIndex: closeBracketIndex + 1 };
}
const ITEM_JSON_ACCESSOR_PATTERNS = [
'.item.json',
'.first().json',
'.last().json',
'.all().json',
] as const;
function readItemJsonField(
text: string,
start: number,
): { field: string; endIndex: number } | undefined {
for (const accessor of ITEM_JSON_ACCESSOR_PATTERNS) {
if (!text.startsWith(accessor, start)) continue;
return readFieldAccessor(text, start + accessor.length);
}
const itemMatching = /^\.itemMatching\(\d+\)\.json/.exec(text.slice(start));
if (itemMatching) {
return readFieldAccessor(text, start + itemMatching[0].length);
}
const allItem = /^\.all\(\)\[\d+\]\.json/.exec(text.slice(start));
if (allItem) {
return readFieldAccessor(text, start + allItem[0].length);
}
return undefined;
}
function readDirectJsonField(
text: string,
start: number,
): { field: string; endIndex: number } | undefined {
if (!text.startsWith('.json', start)) return undefined;
return readFieldAccessor(text, start + '.json'.length);
}
function readItemsJsonField(
text: string,
start: number,
): { field: string; endIndex: number } | undefined {
const itemIndex = /^\[\d+\]/.exec(text.slice(start));
const jsonStart = start + (itemIndex?.[0].length ?? 0);
return readDirectJsonField(text, jsonStart);
}
function readNamedNodeJsonField(
text: string,
start: number,
): { field: string; endIndex: number } | undefined {
return readItemJsonField(text, start) ?? readDirectJsonField(text, start);
}
export function jsonFieldAccessor(field: string): string {
return JS_IDENTIFIER.test(field) ? `.${field}` : `[${JSON.stringify(field)}]`;
}
export function currentJsonExpression(field: string): string {
return `$json${jsonFieldAccessor(field)}`;
}
export function nodeItemJsonExpression(nodeName: string, field: string): string {
return `$(${JSON.stringify(nodeName)}).item.json${jsonFieldAccessor(field)}`;
}
export function extractJsonColumnRefs(text: string): string[] {
const refs: string[] = [];
for (let index = 0; index < text.length; index++) {
if (text.startsWith('$json', index) && !isIdentifierPart(charAt(text, index + 5))) {
const match = readFieldAccessor(text, index + 5);
if (match) {
refs.push(match.field);
index = match.endIndex - 1;
}
continue;
}
if (text.startsWith('$input', index) && !isIdentifierPart(charAt(text, index + 6))) {
const match = readItemJsonField(text, index + 6) ?? readItemsJsonField(text, index + 6);
if (match) {
refs.push(match.field);
index = match.endIndex - 1;
}
}
}
return unique(refs);
}
/**
* Match named cross-node references in expressions.
* Captures (sourceNodeName, field) pairs for:
* - $('Name').item.json.field
* - $("Name").item.json.field
* - $("Name").first().json["field-name"]
* - $items("Name")[0].json.field
* - $node["Name"].json.field and $node.Name.item.json.field (legacy)
*/
export interface NamedRefMatch {
nodeName: string;
field: string;
originalExpression: string;
}
function readDollarNodeRef(text: string, start: number): NamedRefMatch | undefined {
const quoteStart = skipWhitespace(text, start + '$('.length);
const nodeName = readQuotedString(text, quoteStart);
if (!nodeName) return undefined;
const closeParenIndex = skipWhitespace(text, nodeName.endIndex);
if (charAt(text, closeParenIndex) !== ')') return undefined;
const field = readItemJsonField(text, closeParenIndex + 1);
if (!field) return undefined;
return {
nodeName: nodeName.value,
field: field.field,
originalExpression: text.slice(start, field.endIndex),
};
}
function readItemsNodeRef(text: string, start: number): NamedRefMatch | undefined {
const quoteStart = skipWhitespace(text, start + '$items('.length);
const nodeName = readQuotedString(text, quoteStart);
if (!nodeName) return undefined;
const closeParenIndex = text.indexOf(')', nodeName.endIndex);
if (closeParenIndex < 0) return undefined;
const field = readItemsJsonField(text, closeParenIndex + 1);
if (!field) return undefined;
return {
nodeName: nodeName.value,
field: field.field,
originalExpression: text.slice(start, field.endIndex),
};
}
function readNodeBracketRef(text: string, start: number): NamedRefMatch | undefined {
const quoteStart = skipWhitespace(text, start + '$node['.length);
const nodeName = readQuotedString(text, quoteStart);
if (!nodeName) return undefined;
const closeBracketIndex = skipWhitespace(text, nodeName.endIndex);
if (charAt(text, closeBracketIndex) !== ']') return undefined;
const field = readNamedNodeJsonField(text, closeBracketIndex + 1);
if (!field) return undefined;
return {
nodeName: nodeName.value,
field: field.field,
originalExpression: text.slice(start, field.endIndex),
};
}
function readNodeDotRef(text: string, start: number): NamedRefMatch | undefined {
const nodeName = readIdentifier(text, start + '$node.'.length);
if (!nodeName) return undefined;
const field = readNamedNodeJsonField(text, nodeName.endIndex);
if (!field) return undefined;
return {
nodeName: nodeName.value,
field: field.field,
originalExpression: text.slice(start, field.endIndex),
};
}
function readNamedRefAt(text: string, start: number): NamedRefMatch | undefined {
if (text.startsWith('$items(', start)) return readItemsNodeRef(text, start);
if (text.startsWith('$(', start)) return readDollarNodeRef(text, start);
if (text.startsWith('$node[', start)) return readNodeBracketRef(text, start);
if (text.startsWith('$node.', start)) return readNodeDotRef(text, start);
return undefined;
}
export function extractNamedRefMatches(text: string): NamedRefMatch[] {
const matches: NamedRefMatch[] = [];
for (let index = 0; index < text.length; index++) {
const match = readNamedRefAt(text, index);
if (match) {
matches.push(match);
index += match.originalExpression.length - 1;
}
}
return matches;
}
/**
* Returns names of nodes connected into `agentNodeName` via any `ai_*`
* connection type — i.e. tools, memory, output parsers, etc.
*/
export function connectionSlotTargetsNode(slot: unknown, targetNodeName: string): boolean {
if (!Array.isArray(slot)) return false;
for (const inner of slot) {
if (!Array.isArray(inner)) continue;
for (const conn of inner) {
if (isRecord(conn) && conn.node === targetNodeName) return true;
}
}
return false;
}
export function findAgentSubComponents(workflow: WorkflowJSON, agentNodeName: string): string[] {
const subs = new Set<string>();
for (const [sourceName, byType] of Object.entries(workflow.connections ?? {})) {
if (!isRecord(byType)) continue;
for (const [connType, slot] of Object.entries(byType)) {
if (connType.startsWith('ai_') && connectionSlotTargetsNode(slot, agentNodeName)) {
subs.add(sourceName);
}
}
}
return [...subs];
}

128
packages/@n8n/instance-ai/src/tools/evals/describe-metric-for-workflow.service.ts
View File

@ -1,128 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { connectionSlotTargetsNode } from './column-ref-utils';
import type { MetricId } from './metric-catalog';
type RecommendedMetricId = Exclude<MetricId, 'helpfulness'>;
/**
* Walks `workflow.connections` to find all source nodes that connect into
* `targetNodeName` via the given connection type (e.g. 'ai_tool', 'ai_languageModel').
* Returns the source node names. Empty array if none.
*/
function findSourceNodesByConnectionType(
workflow: WorkflowJSON,
targetNodeName: string,
connectionType: string,
): string[] {
const sources: string[] = [];
const connections = workflow.connections ?? {};
for (const [sourceName, byType] of Object.entries(connections)) {
if (!isRecord(byType)) continue;
if (connectionSlotTargetsNode(byType[connectionType], targetNodeName)) {
sources.push(sourceName);
}
}
return sources;
}
/**
* Returns names of langchain "tool" nodes wired into the agent via `ai_tool`.
* Includes both pure tool nodes and Agent-as-tool patterns.
*/
function getAgentToolNames(workflow: WorkflowJSON, agentNodeName: string): string[] {
return findSourceNodesByConnectionType(workflow, agentNodeName, 'ai_tool');
}
/**
* Returns names of retriever / vector-store nodes wired into the agent via any
* ai_* connection type. Detection is by node TYPE, not connection type — we
* check whether the source node type matches retriever/vectorStore patterns.
*/
function getAgentRetrieverNames(workflow: WorkflowJSON, agentNodeName: string): string[] {
const connections = workflow.connections ?? {};
const candidates = new Set<string>();
for (const [sourceName, byType] of Object.entries(connections)) {
if (!isRecord(byType)) continue;
for (const [connType, slot] of Object.entries(byType)) {
if (connType.startsWith('ai_') && connectionSlotTargetsNode(slot, agentNodeName)) {
candidates.add(sourceName);
}
}
}
const nodeTypes = new Map<string, string>();
for (const node of workflow.nodes ?? []) {
if (typeof node.name === 'string' && typeof node.type === 'string') {
nodeTypes.set(node.name, node.type);
}
}
return [...candidates].filter((name) => {
const type = (nodeTypes.get(name) ?? '').toLowerCase();
return /\.(retriever|vectorstore)/.test(type);
});
}
function formatNameList(names: string[]): string {
if (names.length === 0) return '';
if (names.length === 1) return `\`${names[0]}\``;
if (names.length === 2) return `\`${names[0]}\`, \`${names[1]}\``;
const last = names[names.length - 1];
const head = names
.slice(0, -1)
.map((n) => `\`${n}\``)
.join(', ');
return `${head}, \`${last}\``;
}
/**
* Returns a one-line description of how `metricId` applies to the given
* workflow's agent. Inspects connections to find tools and retrievers.
*/
export function describeMetricForWorkflow(
workflow: WorkflowJSON,
agentNodeName: string,
metricId: string,
): string {
switch (metricId) {
case 'correctness':
return `Compares ${agentNodeName}'s response to a ground-truth column in your dataset`;
case 'tool_use': {
const tools = getAgentToolNames(workflow, agentNodeName);
if (tools.length === 0) {
return 'Add tools to your agent first to use this metric';
}
return `Verifies the agent picks correctly between ${formatNameList(tools)}`;
}
case 'relevance': {
const retrievers = getAgentRetrieverNames(workflow, agentNodeName);
if (retrievers.length === 0) {
return 'Rates retrieved context relevance — best for RAG workflows';
}
return `Rates whether retrieved context from ${formatNameList(retrievers)} matches the user query`;
}
case 'helpfulness':
return "Rates whether the agent's response addresses the user's intent";
default:
return '';
}
}
/**
* Returns the ID of the metric most workflow-specific for `agentNodeName`:
* - If the agent has tools → `tool_use`.
* - Else if the agent has a retriever/vector store → `relevance`.
* - Else → `correctness`.
*
* Used to mark a single option with " (recommended)" in the selection widget.
*/
export function recommendedMetricId(
workflow: WorkflowJSON,
agentNodeName: string,
): RecommendedMetricId {
if (getAgentToolNames(workflow, agentNodeName).length > 0) return 'tool_use';
if (getAgentRetrieverNames(workflow, agentNodeName).length > 0) return 'relevance';
return 'correctness';
}

111
packages/@n8n/instance-ai/src/tools/evals/detect-agent-named-refs.service.ts
View File

@ -1,111 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import {
collectStrings,
extractNamedRefMatches,
findAgentSubComponents,
nodeHasName,
type NamedRefMatch,
} from './column-ref-utils';
export interface NamedRef {
/** Source node name extracted from $('NodeName') reference. */
nodeName: string;
/** Field after .item.json.<field>. */
field: string;
/** Original expression substring, useful for the rewrite step (no $('… surrounding). E.g., "$('Voice or Text').item.json.text". */
originalExpression: string;
/** Proposed dataset column name. Equals `field` when unique; otherwise `<node_slug>_<field>` to disambiguate. */
column: string;
/** Which node has the parameter that references nodeName. Either the agent itself or a sub-component (memory, tool, ...). */
targetNodeName: string;
}
function slug(name: string): string {
const result = name
.toLowerCase()
.replace(/[^a-z0-9]+/g, '_')
.replace(/^_+|_+$/g, '');
return result.length > 0 ? result : 'node';
}
type RawMatch = NamedRefMatch;
export function detectAgentNamedRefs(workflow: WorkflowJSON, agentNodeName: string): NamedRef[] {
const node = (workflow.nodes ?? []).find((n) => nodeHasName(n) && n.name === agentNodeName);
if (!node) {
throw new Error(`Agent node "${agentNodeName}" not found in workflow`);
}
const targets = [agentNodeName, ...findAgentSubComponents(workflow, agentNodeName)];
const nodesByName = new Map(
(workflow.nodes ?? []).filter(nodeHasName).map((n) => [n.name, n] as const),
);
// First pass: collect, per target, the de-duplicated set of (source, field) raw matches.
const targetMatches = new Map<string, RawMatch[]>();
for (const target of targets) {
const tNode = nodesByName.get(target);
if (!tNode) continue;
const matches = collectStrings(tNode.parameters).flatMap((text) =>
extractNamedRefMatches(text),
);
const dedup = new Map<string, RawMatch>();
for (const m of matches) {
const key = `${m.nodeName}\x00${m.field}`;
if (!dedup.has(key)) dedup.set(key, m);
}
if (dedup.size > 0) targetMatches.set(target, [...dedup.values()]);
}
// Second pass: compute column names with collision detection across all unique source-field pairs.
const allPairs = new Map<string, { nodeName: string; field: string }>();
for (const matches of targetMatches.values()) {
for (const m of matches) {
const key = `${m.nodeName}\x00${m.field}`;
if (!allPairs.has(key)) {
allPairs.set(key, { nodeName: m.nodeName, field: m.field });
}
}
}
const byField = new Map<string, Array<{ nodeName: string; field: string }>>();
for (const p of allPairs.values()) {
const arr = byField.get(p.field) ?? [];
arr.push(p);
byField.set(p.field, arr);
}
const columnsByPair = new Map<string, string>();
const usedColumns = new Set<string>();
function uniqueColumn(base: string): string {
let candidate = base;
let suffix = 1;
while (usedColumns.has(candidate)) {
candidate = `${base}_${suffix}`;
suffix++;
}
usedColumns.add(candidate);
return candidate;
}
for (const p of allPairs.values()) {
const group = byField.get(p.field) ?? [];
const base = group.length === 1 ? p.field : `${slug(p.nodeName)}_${p.field}`;
columnsByPair.set(`${p.nodeName}\x00${p.field}`, uniqueColumn(base));
}
// Third pass: emit one NamedRef per (target, source, field).
const result: NamedRef[] = [];
for (const [target, matches] of targetMatches) {
for (const m of matches) {
const column = columnsByPair.get(`${m.nodeName}\x00${m.field}`);
if (!column) continue;
result.push({
nodeName: m.nodeName,
field: m.field,
originalExpression: m.originalExpression,
column,
targetNodeName: target,
});
}
}
return result;
}

106
packages/@n8n/instance-ai/src/tools/evals/detect-ai-nodes.ts
View File

@ -1,106 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
const LANGCHAIN_TYPE_PREFIX = '@n8n/n8n-nodes-langchain.';
const EVALUATION_TYPES = new Set<string>([
'n8n-nodes-base.evaluation',
'n8n-nodes-base.evaluationTrigger',
]);
// Explicit allow-list of root AI nodes — agents and chains that consume an LLM
// and produce the output we want to evaluate. Sub-components (chat models,
// memory, embeddings, tools, parsers, vector stores, retrievers, document
// loaders, text splitters, triggers) hang off a root and are NOT themselves
// evaluation targets. An allow-list avoids the substring traps that bite
// heuristics — e.g. `chainLlm` contains `lm`, which a substring deny-list
// would mistake for a chat-model sub-component.
const ROOT_AI_TYPES = new Set<string>([
// Agents and chains.
`${LANGCHAIN_TYPE_PREFIX}agent`,
`${LANGCHAIN_TYPE_PREFIX}openAiAssistant`,
`${LANGCHAIN_TYPE_PREFIX}chainLlm`,
`${LANGCHAIN_TYPE_PREFIX}chainRetrievalQa`,
`${LANGCHAIN_TYPE_PREFIX}chainSummarization`,
`${LANGCHAIN_TYPE_PREFIX}informationExtractor`,
`${LANGCHAIN_TYPE_PREFIX}sentimentAnalysis`,
`${LANGCHAIN_TYPE_PREFIX}textClassifier`,
// Vendor LLM nodes — workflows that wire directly to a model without a chain
// wrapper are still AI workflows and should be offered eval suites.
`${LANGCHAIN_TYPE_PREFIX}openAi`,
`${LANGCHAIN_TYPE_PREFIX}anthropic`,
`${LANGCHAIN_TYPE_PREFIX}googleGemini`,
`${LANGCHAIN_TYPE_PREFIX}ollama`,
`${LANGCHAIN_TYPE_PREFIX}alibabaCloud`,
`${LANGCHAIN_TYPE_PREFIX}miniMax`,
`${LANGCHAIN_TYPE_PREFIX}moonshot`,
]);
// Sub-component local-name prefixes (lowercased). Used only in the fallback
// path to filter out nodes that are clearly supporting components rather than
// workflow-level AI roots.
const SUB_COMPONENT_PREFIXES = [
'lm',
'embedding',
'memory',
'tool',
'outputparser',
'document',
'textsplitter',
'vectorstore',
'retriever',
'chattrigger',
'manualchattrigger',
];
export interface DetectAiNodesResult {
isAiWorkflow: boolean;
aiNodeNames: string[];
alreadyConfigured: boolean;
}
function isRootAgentType(type: string): boolean {
return ROOT_AI_TYPES.has(type);
}
function isAnyLangchainNode(type: string): boolean {
return type.startsWith(LANGCHAIN_TYPE_PREFIX);
}
// Returns true when the node's local name (after the prefix) starts with a
// known sub-component prefix. These nodes hang off a root and should not
// themselves be offered as evaluation targets.
function isLikelySubComponent(type: string): boolean {
if (!type.startsWith(LANGCHAIN_TYPE_PREFIX)) return false;
const localPart = type.slice(LANGCHAIN_TYPE_PREFIX.length).toLowerCase();
return SUB_COMPONENT_PREFIXES.some((prefix) => localPart.startsWith(prefix));
}
export function detectAiNodes(workflow: WorkflowJSON): DetectAiNodesResult {
const aiNodeNames: string[] = [];
const fallbackCandidates: string[] = [];
let alreadyConfigured = false;
for (const node of workflow.nodes ?? []) {
if (!node.name) continue;
if (isRootAgentType(node.type)) {
aiNodeNames.push(node.name);
} else if (isAnyLangchainNode(node.type) && !isLikelySubComponent(node.type)) {
// Catches future / less-common langchain root types not yet in
// ROOT_AI_TYPES. Better to offer evals on a maybe-relevant workflow
// than to silently skip a new model or chain variant.
fallbackCandidates.push(node.name);
}
if (EVALUATION_TYPES.has(node.type)) {
alreadyConfigured = true;
}
}
// Primary root types always take priority. Fall back to unrecognised
// langchain nodes only when no explicit root was found.
const finalAiNodeNames = aiNodeNames.length > 0 ? aiNodeNames : fallbackCandidates;
return {
isAiWorkflow: finalAiNodeNames.length > 0,
aiNodeNames: finalAiNodeNames,
alreadyConfigured,
};
}

94
packages/@n8n/instance-ai/src/tools/evals/detect-tool-refs.service.ts
View File

@ -1,94 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import {
collectStrings,
extractNamedRefMatches,
findAgentSubComponents,
nodeHasName,
} from './column-ref-utils';
export interface ToolRef {
/** Source node name extracted from the expression (the node we'll pin). */
sourceNodeName: string;
/** Field on the source node referenced after `.item.json.` */
field: string;
/** Sub-component (tool/memory/...) whose parameter holds the expression. */
toolNodeName: string;
}
/**
* Walks `main` connections downstream from `startNodeName`, returning the set
* of reachable node names (excluding the start itself). Used to identify
* nodes that will execute during an eval run (and therefore should NOT be
* pinned, since pinned data would shadow their real output).
*/
function reachableDownstreamMain(workflow: WorkflowJSON, startNodeName: string): Set<string> {
const reached = new Set<string>();
const queue: string[] = [startNodeName];
while (queue.length > 0) {
const current = queue.shift()!;
const slot = workflow.connections?.[current]?.main;
if (!Array.isArray(slot)) continue;
for (const inner of slot) {
if (!Array.isArray(inner)) continue;
for (const conn of inner) {
if (!isRecord(conn) || typeof conn.node !== 'string') continue;
if (reached.has(conn.node)) continue;
reached.add(conn.node);
queue.push(conn.node);
}
}
}
return reached;
}
/**
* Detects named cross-node references inside the AI agent's sub-components
* (tools, memory, output parsers — anything wired in via `ai_*`). Filters
* out refs that:
* - point to nodes not present in the workflow,
* - point to the agent itself,
* - point to nodes downstream of the agent (those execute during eval).
*
* Each remaining ref names a node whose data must be supplied via `pinData`
* for eval runs to see realistic context.
*/
export function detectToolRefs(workflow: WorkflowJSON, agentNodeName: string): ToolRef[] {
const agent = (workflow.nodes ?? []).find((n) => nodeHasName(n) && n.name === agentNodeName);
if (!agent) {
throw new Error(`Agent node "${agentNodeName}" not found in workflow`);
}
const subComponents = findAgentSubComponents(workflow, agentNodeName);
if (subComponents.length === 0) return [];
const nodesByName = new Map(
(workflow.nodes ?? []).filter(nodeHasName).map((n) => [n.name, n] as const),
);
const downstream = reachableDownstreamMain(workflow, agentNodeName);
const seen = new Set<string>();
const result: ToolRef[] = [];
for (const toolNodeName of subComponents) {
const toolNode = nodesByName.get(toolNodeName);
if (!toolNode) continue;
const matches = collectStrings(toolNode.parameters).flatMap((text) =>
extractNamedRefMatches(text),
);
for (const m of matches) {
if (m.nodeName === agentNodeName) continue;
if (downstream.has(m.nodeName)) continue;
if (!nodesByName.has(m.nodeName)) continue;
const key = `${toolNodeName}\x00${m.nodeName}\x00${m.field}`;
if (seen.has(key)) continue;
seen.add(key);
result.push({
sourceNodeName: m.nodeName,
field: m.field,
toolNodeName,
});
}
}
return result;
}

110
packages/@n8n/instance-ai/src/tools/evals/ensure-eval-data-table.service.ts
View File

@ -1,110 +0,0 @@
import { nanoid } from 'nanoid';
import type { InstanceAiContext, DataTableSummary } from '../../types';
export interface CreateEmptyEvalDataTableInput {
workflowName: string;
projectId?: string;
columns: string[];
}
const NAME_COLLISION_RE = /already exists/i;
const DATA_TABLE_COLUMN_MAX_LENGTH = 63;
export function formatEvalDataTableName(workflowName: string): string {
return `${workflowName} — eval samples`;
}
export function formatEvalDataTableColumnName(columnName: string): string {
const normalized = columnName
.trim()
.toLowerCase()
.replace(/[^a-z0-9_]+/g, '_')
.replace(/_+/g, '_')
.replace(/^_+|_+$/g, '');
const withValidStart = /^[a-z]/.test(normalized) ? normalized : `column_${normalized || 'value'}`;
const truncated = withValidStart.slice(0, DATA_TABLE_COLUMN_MAX_LENGTH).replace(/_+$/g, '');
return truncated || 'column';
}
function withUniqueSuffix(baseName: string, index: number): string {
const suffix = `_${index}`;
return `${baseName.slice(0, DATA_TABLE_COLUMN_MAX_LENGTH - suffix.length).replace(/_+$/g, '')}${suffix}`;
}
export function formatEvalDataTableColumnNameMap(columns: string[]): Map<string, string> {
const byRawColumn = new Map<string, string>();
const usedNames = new Set<string>();
for (const column of columns) {
if (byRawColumn.has(column)) continue;
const baseName = formatEvalDataTableColumnName(column);
let candidate = baseName;
let index = 2;
while (usedNames.has(candidate)) {
candidate = withUniqueSuffix(baseName, index);
index++;
}
byRawColumn.set(column, candidate);
usedNames.add(candidate);
}
return byRawColumn;
}
export function formatEvalDataTableColumnNames(columns: string[]): string[] {
return [...formatEvalDataTableColumnNameMap(columns).values()];
}
async function createWithUniqueName(
ctx: InstanceAiContext,
baseName: string,
columns: Array<{ name: string; type: 'string' }>,
options: { projectId?: string } | undefined,
): Promise<DataTableSummary> {
try {
return await ctx.dataTableService.create(baseName, columns, options);
} catch (error) {
const msg = error instanceof Error ? error.message : String(error);
if (!NAME_COLLISION_RE.test(msg)) throw error;
for (let attempt = 0; attempt < 3; attempt++) {
const suffixedName = `${baseName} (${nanoid(5)})`;
try {
return await ctx.dataTableService.create(suffixedName, columns, options);
} catch (retryError) {
const retryMsg = retryError instanceof Error ? retryError.message : String(retryError);
if (!NAME_COLLISION_RE.test(retryMsg)) throw retryError;
}
}
throw error;
}
}
/**
* Create a fresh DataTable for a workflow's eval setup, with the requested
* columns and zero rows. Population is the responsibility of `eval-data`,
* invoked separately after the user confirms via
* `evals(action="offer-data-population")`.
*
* Returns the DataTable's id, name, and projectId. The projectId is needed
* downstream so the artifacts panel can fetch the table via the project-scoped
* API — without it the preview silently fails.
*/
export async function createEmptyEvalDataTable(
ctx: InstanceAiContext,
input: CreateEmptyEvalDataTableInput,
): Promise<{ id: string; name: string; projectId?: string }> {
const dt = await createWithUniqueName(
ctx,
formatEvalDataTableName(input.workflowName),
formatEvalDataTableColumnNames(input.columns).map((n) => ({
name: n,
type: 'string' as const,
})),
input.projectId ? { projectId: input.projectId } : undefined,
);
return {
id: dt.id,
name: dt.name,
...(dt.projectId ? { projectId: dt.projectId } : {}),
};
}

2
packages/@n8n/instance-ai/src/tools/evals/eval-config.tool.ts
View File

@ -2,7 +2,7 @@
* Config-based evaluation tool — create, read, and mutate an `EvaluationConfig`
* attached to a workflow via the evaluation-config API. This is the simplified,
* off-canvas eval form (name + start/end node + judged metrics + a Data Table
* dataset), distinct from the on-canvas eval nodes the `evals` tool wires.
* dataset), distinct from manually authored on-canvas evaluation nodes.
*
* The dataset is a Data Table the agent creates/populates via the `data-tables`
* tool; here it is only linked by id.

246
packages/@n8n/instance-ai/src/tools/evals/eval-data-requirements.service.ts
View File

@ -1,246 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { analyzeAgentInputColumns } from './analyze-agent-input-columns.service';
import {
extractJsonColumnRefs,
extractNamedRefMatches,
nodeHasName,
nodeTypeEndsWith,
unique,
type WorkflowNode,
} from './column-ref-utils';
import { detectAiNodes } from './detect-ai-nodes';
export interface EvalDataTarget {
dataTableId: string;
evaluationTriggerName: string;
targetNodeName?: string;
targetAgentNodeName?: string;
inputColumns: string[];
expectedOutputColumns: string[];
actualOutputColumns: string[];
metricNodeNames: string[];
/**
* Pairs from setMetrics nodes: for each metric, the expected_* column
* (read from the eval trigger row via `expectedAnswer`) and the agent
* output field (read from $json via `actualAnswer`). Used by eval-data
* to populate expected_* columns from a past execution's agent output.
*/
expectedToActualPairs: Array<{ expectedColumn: string; actualField: string }>;
}
export interface EvalDataRequirements {
targets: EvalDataTarget[];
reason?: string;
}
function readOperation(node: WorkflowNode): string | undefined {
const parameters = node.parameters;
if (!isRecord(parameters)) return undefined;
const operation = parameters.operation;
return typeof operation === 'string' ? operation : undefined;
}
function readDataTableId(node: WorkflowNode): string | undefined {
const parameters = node.parameters;
if (!isRecord(parameters)) return undefined;
const dataTableId = parameters.dataTableId;
if (typeof dataTableId === 'string') return dataTableId;
if (!isRecord(dataTableId)) return undefined;
const value = dataTableId.value;
return typeof value === 'string' && value.length > 0 ? value : undefined;
}
function childNames(workflow: WorkflowJSON, sourceName: string, outputIndex?: number): string[] {
const connectionsByType = workflow.connections?.[sourceName];
if (!connectionsByType || typeof connectionsByType !== 'object') return [];
const main = (connectionsByType as Record<string, unknown>).main;
if (!Array.isArray(main)) return [];
const slots =
outputIndex === undefined
? main
: [main[outputIndex]].filter((slot): slot is unknown[] => Array.isArray(slot));
return unique(
slots.flatMap((slot) => {
if (!Array.isArray(slot)) return [];
return slot.flatMap((connection) => {
if (!isRecord(connection)) return [];
const node = connection.node;
return typeof node === 'string' ? [node] : [];
});
}),
);
}
function collectReachableNodeNames(workflow: WorkflowJSON, startName: string): string[] {
const seen = new Set<string>();
const queue = [startName];
while (queue.length > 0) {
const current = queue.shift();
if (!current || seen.has(current)) continue;
seen.add(current);
for (const child of childNames(workflow, current)) {
if (!seen.has(child)) queue.push(child);
}
}
seen.delete(startName);
return [...seen];
}
function nodeByName(workflow: WorkflowJSON): Map<string, WorkflowNode> {
return new Map(workflow.nodes.filter(nodeHasName).map((node) => [node.name, node]));
}
function isAiAgentNode(node: WorkflowNode | undefined): boolean {
return Boolean(node?.type.includes('n8n-nodes-langchain.agent'));
}
function extractEvalTriggerColumnRefs(text: string): string[] {
return unique([
...extractJsonColumnRefs(text),
...extractNamedRefMatches(text).map((match) => match.field),
]);
}
function expectedColumnsFromMetricNodes(nodes: WorkflowNode[]): string[] {
return unique(
nodes.flatMap((node) => {
if (!nodeTypeEndsWith(node, 'evaluation') || readOperation(node) !== 'setMetrics') return [];
const parameters = node.parameters;
if (!isRecord(parameters)) return [];
const expectedRaw = parameters.expectedAnswer;
return typeof expectedRaw === 'string' ? extractEvalTriggerColumnRefs(expectedRaw) : [];
}),
);
}
function actualColumnsFromSetOutputs(nodes: WorkflowNode[]): string[] {
return unique(
nodes.flatMap((node) => {
if (!nodeTypeEndsWith(node, 'evaluation') || readOperation(node) !== 'setOutputs') return [];
const parameters = node.parameters;
if (!isRecord(parameters)) return [];
const outputs = parameters.outputs;
if (!isRecord(outputs)) return [];
const values = outputs.values;
if (!Array.isArray(values)) return [];
return values.flatMap((value) => {
if (!isRecord(value)) return [];
const outputName = value.outputName;
return typeof outputName === 'string' ? [outputName] : [];
});
}),
);
}
function pairsFromMetricNodes(
nodes: WorkflowNode[],
): Array<{ expectedColumn: string; actualField: string }> {
const pairs: Array<{ expectedColumn: string; actualField: string }> = [];
for (const node of nodes) {
if (!nodeTypeEndsWith(node, 'evaluation') || readOperation(node) !== 'setMetrics') continue;
const parameters = node.parameters;
if (!isRecord(parameters)) continue;
const expectedRaw = parameters.expectedAnswer;
const actualRaw = parameters.actualAnswer;
if (typeof expectedRaw !== 'string' || typeof actualRaw !== 'string') continue;
const expectedRefs = extractEvalTriggerColumnRefs(expectedRaw).filter((ref) =>
ref.startsWith('expected'),
);
const actualRefs = extractJsonColumnRefs(actualRaw);
const len = Math.min(expectedRefs.length, actualRefs.length);
for (let i = 0; i < len; i++) {
pairs.push({ expectedColumn: expectedRefs[i], actualField: actualRefs[i] });
}
}
// Dedup by expectedColumn (last one wins). Multiple setMetrics nodes may
// reference the same expected column; the actualField mapping should be
// consistent in that case.
const map = new Map<string, string>();
for (const p of pairs) map.set(p.expectedColumn, p.actualField);
return [...map.entries()].map(([expectedColumn, actualField]) => ({
expectedColumn,
actualField,
}));
}
function firstReachableAgentName(
workflow: WorkflowJSON,
evalTriggerName: string,
): string | undefined {
const byName = nodeByName(workflow);
return collectReachableNodeNames(workflow, evalTriggerName).find((name) =>
isAiAgentNode(byName.get(name)),
);
}
function firstReachableTargetNodeName(
workflow: WorkflowJSON,
evalTriggerName: string,
): string | undefined {
const reachableNodeNames = collectReachableNodeNames(workflow, evalTriggerName);
const reachableNameSet = new Set(reachableNodeNames);
const reachableNodes = (workflow.nodes ?? []).filter(
(node): node is WorkflowNode & { name: string } =>
nodeHasName(node) && reachableNameSet.has(node.name),
);
const aiNodeNames = new Set(detectAiNodes({ ...workflow, nodes: reachableNodes }).aiNodeNames);
return reachableNodeNames.find((name) => aiNodeNames.has(name));
}
export function analyzeEvalDataRequirements(workflow: WorkflowJSON): EvalDataRequirements {
const evaluationTriggers = (workflow.nodes ?? []).filter(
(node): node is WorkflowNode & { name: string } =>
nodeHasName(node) && nodeTypeEndsWith(node, 'evaluationTrigger'),
);
if (evaluationTriggers.length === 0) {
return { targets: [], reason: 'Workflow has no EvaluationTrigger nodes.' };
}
const targets = evaluationTriggers.flatMap((trigger) => {
const dataTableId = readDataTableId(trigger);
if (!dataTableId) return [];
const reachableNames = new Set(collectReachableNodeNames(workflow, trigger.name));
const reachableNodes = (workflow.nodes ?? []).filter(
(node): node is WorkflowNode & { name: string } =>
nodeHasName(node) && reachableNames.has(node.name),
);
const metricNodeNames = reachableNodes
.filter(
(node) => nodeTypeEndsWith(node, 'evaluation') && readOperation(node) === 'setMetrics',
)
.map((node) => node.name);
const targetAgentNodeName = firstReachableAgentName(workflow, trigger.name);
const targetNodeName = firstReachableTargetNodeName(workflow, trigger.name);
const inputTargetNodeName = targetAgentNodeName ?? targetNodeName;
return [
{
dataTableId,
evaluationTriggerName: trigger.name,
targetNodeName,
targetAgentNodeName,
inputColumns: inputTargetNodeName
? analyzeAgentInputColumns(workflow, inputTargetNodeName).inputColumns
: [],
expectedOutputColumns: expectedColumnsFromMetricNodes(reachableNodes),
actualOutputColumns: actualColumnsFromSetOutputs(reachableNodes),
metricNodeNames,
expectedToActualPairs: pairsFromMetricNodes(reachableNodes),
},
];
});
if (targets.length === 0) {
return {
targets: [],
reason: 'Workflow has EvaluationTrigger nodes, but none are wired to a DataTable id.',
};
}
return { targets };
}

622
packages/@n8n/instance-ai/src/tools/evals/evals.tool.ts
View File

@ -1,622 +0,0 @@
import { Tool } from '@n8n/agents';
import {
instanceAiApprovalResumeSchema,
instanceAiConfirmationSeveritySchema,
} from '@n8n/api-types';
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { nanoid } from 'nanoid';
import { z } from 'zod';
import { analyzeAgentInputColumns } from './analyze-agent-input-columns.service';
import { applyPinData } from './apply-pin-data.service';
import {
describeMetricForWorkflow,
recommendedMetricId,
} from './describe-metric-for-workflow.service';
import { detectAgentNamedRefs, type NamedRef } from './detect-agent-named-refs.service';
import { detectAiNodes, type DetectAiNodesResult } from './detect-ai-nodes';
import { detectToolRefs } from './detect-tool-refs.service';
import { createEmptyEvalDataTable } from './ensure-eval-data-table.service';
import { formatEvalSetupTask } from './format-eval-setup-task';
import { generateToolRefPinData } from './generate-tool-ref-pin-data.service';
import {
METRIC_CATALOG,
METRIC_IDS,
getMetricsByIds,
proposeDefaultMetricIds,
type MetricId,
} from './metric-catalog';
import { sanitizeInputSchema } from '../../agent/sanitize-mcp-schemas';
import type { InstanceAiContext } from '../../types';
import { refreshWorkflowSourceFileBindingFromSave } from '../workflows/workflow-file-bindings';
// ── Action input schemas ───────────────────────────────────────────────────
const offerAction = z.object({
action: z
.literal('offer')
.describe(
'Eligibility precheck. Returns { eligible: false, reason }, or { eligible: true, aiNodeNames, message } — output `message` verbatim and end the turn (no widget; the user replies in chat).',
),
workflowId: z.string(),
projectId: z.string().optional(),
targetAgentNodeName: z
.string()
.optional()
.describe('Required when the workflow has more than one AI node.'),
});
const recommendMetricAction = z.object({
action: z
.literal('recommend-metric')
.describe(
'Single-metric suggestion; suspends with an approve/deny widget. Approved → pass the returned metricId to `propose` (skip `select-metrics`). Denied → fall through to `select-metrics`.',
),
workflowId: z.string(),
targetAgentNodeName: z
.string()
.optional()
.describe('Required when the workflow has more than one AI node.'),
});
const selectMetricsAction = z.object({
action: z
.literal('select-metrics')
.describe(
'Multi-select picker over all canned metrics — call this ONLY when `recommend-metric` returned `{ approved: false }`. Returns chosenMetricIds.',
),
workflowId: z.string(),
targetAgentNodeName: z
.string()
.optional()
.describe('Required when the workflow has more than one AI node.'),
});
const proposeAction = z.object({
action: z
.literal('propose')
.describe('Build the eval-setup task spec and create or link the eval DataTable.'),
workflowId: z.string(),
projectId: z.string().optional(),
metrics: z.array(z.string()).default([]),
datasetChoice: z.enum(['create-empty', 'link-existing', 'later']).default('create-empty'),
existingDataTableId: z.string().optional(),
targetAgentNodeName: z
.string()
.optional()
.describe('Required when the workflow has more than one AI node.'),
});
const inputSchema = sanitizeInputSchema(
z.discriminatedUnion('action', [
offerAction,
recommendMetricAction,
selectMetricsAction,
proposeAction,
]),
);
type Input = z.infer<typeof inputSchema>;
// ── Suspend / resume schemas ───────────────────────────────────────────────
const confirmationSuspend = z.object({
requestId: z.string(),
message: z.string(),
severity: instanceAiConfirmationSeveritySchema,
});
const questionSchema = z.object({
id: z.string(),
question: z.string(),
type: z.enum(['single', 'multi', 'text']),
options: z.array(z.string()).optional(),
});
const questionsSuspend = z.object({
requestId: z.string(),
message: z.string(),
severity: z.literal('info'),
inputType: z.literal('questions'),
questions: z.array(questionSchema),
introMessage: z.string().optional(),
});
const suspendSchema = z.union([confirmationSuspend, questionsSuspend]);
const confirmResumeSchema = instanceAiApprovalResumeSchema;
const questionsResumeSchema = instanceAiApprovalResumeSchema.extend({
answers: z
.array(
z.object({
questionId: z.string(),
selectedOptions: z.array(z.string()),
customText: z.string().optional(),
skipped: z.boolean().optional(),
}),
)
.optional(),
});
/** The questions shape is a superset of the plain-approval one, so it covers both
* suspend kinds. Deliberately not a `z.union`: the plain-approval branch would match
* a questions payload first and strip `answers` on the way through. */
export const evalsResumeSchema = questionsResumeSchema;
type ConfirmResume = z.infer<typeof confirmResumeSchema>;
type QuestionsResume = z.infer<typeof questionsResumeSchema>;
type SuspendPayload = z.infer<typeof suspendSchema>;
type EvalsToolExecutionContext = {
resumeData?: unknown;
suspend?: (payload: SuspendPayload) => Promise<never> | Promise<void>;
agent?: {
resumeData?: unknown;
suspend?: (payload: SuspendPayload) => Promise<never> | Promise<void>;
};
};
// ── Helpers ────────────────────────────────────────────────────────────────
const DEFAULT_EXPECTED_OUTPUT_COLUMN = 'expected_output';
function composeOfferMessage(aiNodeNames: string[], namedRefs: NamedRef[]): string {
const subject =
aiNodeNames.length === 1
? `This workflow uses AI node \`${aiNodeNames[0]}\`.`
: `This workflow uses ${aiNodeNames.length} AI nodes.`;
const baseMessage =
`${subject} AI answers can change unpredictably when you tweak prompts, models, or data — ` +
"even small edits can break behavior you weren't planning to change. " +
'Test cases let you re-run your workflow against a fixed set of inputs and check the answers stay correct over time. ' +
'Want to add some?';
if (namedRefs.length === 0) return baseMessage;
// Disclosure: cite which named nodes will move and into which dataset columns.
const sourceNodes = [...new Set(namedRefs.map((r) => r.nodeName))]
.map((n) => `\`${n}\``)
.join(', ');
const targetColumns = namedRefs.map((r) => `\`${r.column}\``).join(', ');
const sourceLabel = namedRefs.length === 1 ? 'node' : 'nodes';
return (
`${baseMessage}\n\n` +
`Setting this up will adjust your workflow slightly: the agent currently reads from ${sourceLabel} ${sourceNodes}, ` +
`and I'll route that data through the test inputs (columns ${targetColumns}) using a small Set node. ` +
'Your live behavior stays the same.'
);
}
function hasResumeData(ctx: EvalsToolExecutionContext): boolean {
const resumeData = getResumeData(ctx);
return resumeData !== undefined && resumeData !== null;
}
function getResumeData(ctx: EvalsToolExecutionContext): unknown {
return ctx.resumeData ?? ctx.agent?.resumeData;
}
function getSuspend(ctx: EvalsToolExecutionContext) {
return ctx.suspend ?? ctx.agent?.suspend;
}
function getConfirmResume(ctx: EvalsToolExecutionContext): ConfirmResume | undefined {
const resumeData = getResumeData(ctx);
if (resumeData === undefined || resumeData === null) return undefined;
const parsed = confirmResumeSchema.safeParse(resumeData);
return parsed.success ? parsed.data : undefined;
}
function getQuestionsResume(ctx: EvalsToolExecutionContext): QuestionsResume | undefined {
const resumeData = getResumeData(ctx);
if (resumeData === undefined || resumeData === null) return undefined;
const parsed = questionsResumeSchema.safeParse(resumeData);
return parsed.success ? parsed.data : undefined;
}
function isMetricId(id: string): id is MetricId {
return (METRIC_IDS as readonly string[]).includes(id);
}
function metricNameOnly(label: string): string {
// Strip " — description" if present.
const dashIndex = label.indexOf(' — ');
const name = dashIndex >= 0 ? label.slice(0, dashIndex) : label;
// Strip " (recommended)" suffix if present.
return name.replace(/\s*\(recommended\)\s*$/i, '');
}
function labelToId(label: string): string | undefined {
const name = metricNameOnly(label);
for (const id of METRIC_IDS) {
if (METRIC_CATALOG[id].name === name) return id;
}
return undefined;
}
function metricLabel(
workflow: WorkflowJSON,
agentName: string,
id: string,
recommended: string,
): string {
const name = isMetricId(id) ? METRIC_CATALOG[id].name : id;
const description = describeMetricForWorkflow(workflow, agentName, id);
const recommendedSuffix = id === recommended ? ' (recommended)' : '';
return description
? `${name}${recommendedSuffix} — ${description}`
: `${name}${recommendedSuffix}`;
}
type TargetAgentResolution =
| { ok: true; agentName: string }
| {
ok: false;
reason: 'target-agent-required' | 'target-agent-not-found';
aiNodeNames: string[];
message: string;
};
function composeTargetAgentMessage(
aiNodeNames: string[],
targetAgentNodeName: string | undefined,
): string {
const list = aiNodeNames.map((name) => `\`${name}\``).join(', ');
if (targetAgentNodeName) {
return `I couldn't find AI node \`${targetAgentNodeName}\`. Pick one of these AI nodes to set up evals for: ${list}.`;
}
return `This workflow has multiple AI nodes: ${list}. Which AI node should I set up evals for?`;
}
function resolveTargetAgent(
detection: DetectAiNodesResult,
targetAgentNodeName: string | undefined,
): TargetAgentResolution {
if (targetAgentNodeName) {
if (detection.aiNodeNames.includes(targetAgentNodeName)) {
return { ok: true, agentName: targetAgentNodeName };
}
return {
ok: false,
reason: 'target-agent-not-found',
aiNodeNames: detection.aiNodeNames,
message: composeTargetAgentMessage(detection.aiNodeNames, targetAgentNodeName),
};
}
const [onlyAgentName] = detection.aiNodeNames;
if (detection.aiNodeNames.length === 1 && onlyAgentName) {
return { ok: true, agentName: onlyAgentName };
}
return {
ok: false,
reason: 'target-agent-required',
aiNodeNames: detection.aiNodeNames,
message: composeTargetAgentMessage(detection.aiNodeNames, undefined),
};
}
function targetAgentSkippedResponse(resolution: Exclude<TargetAgentResolution, { ok: true }>) {
return {
skipped: true as const,
reason: resolution.reason,
aiNodeNames: resolution.aiNodeNames,
message: resolution.message,
};
}
function pinDataCoversRef(pinData: WorkflowJSON['pinData'] | undefined, ref: NamedRef): boolean {
const items = pinData?.[ref.nodeName];
if (!items) return false;
return items.some((item) => {
const json = item.json;
if (isRecord(json)) return Object.hasOwn(json, ref.field);
return Object.hasOwn(item, ref.field);
});
}
// ── Tool factory ───────────────────────────────────────────────────────────
export function createEvalsTool(context: InstanceAiContext) {
return new Tool('evals')
.description(
"Eval suite orchestration: 'offer' = eligibility precheck after a fresh build (output the returned message verbatim, end the turn); " +
"'recommend-metric' = single-metric approve/deny — call FIRST when choosing metrics; 'select-metrics' = picker, ONLY after recommend-metric was denied; " +
"'propose' = build the eval-setup task spec and create/link the DataTable.",
)
.input(inputSchema)
.suspend(suspendSchema)
.resume(evalsResumeSchema)
.handler(async (input: Input, ctx) => {
switch (input.action) {
case 'offer':
return await executeOffer(context, input);
case 'recommend-metric':
return await executeRecommendMetric(context, input, ctx);
case 'select-metrics':
return await executeSelectMetrics(context, input, ctx);
case 'propose':
return await executePropose(context, input);
}
})
.build();
}
// ── action: offer ──────────────────────────────────────────────────────────
async function executeOffer(context: InstanceAiContext, input: z.infer<typeof offerAction>) {
const wf = await context.workflowService.getAsWorkflowJSON(input.workflowId);
const detection = detectAiNodes(wf);
if (!detection.isAiWorkflow) return { eligible: false as const, reason: 'no-ai-nodes' as const };
if (detection.alreadyConfigured) {
return { eligible: false as const, reason: 'already-configured' as const };
}
const target = resolveTargetAgent(detection, input.targetAgentNodeName);
if (!target.ok) {
return {
eligible: true as const,
requiresTargetAgentSelection: true as const,
reason: target.reason,
aiNodeNames: target.aiNodeNames,
message: target.message,
};
}
const agentName = target.agentName;
const namedRefs = detectAgentNamedRefs(wf, agentName);
return {
eligible: true as const,
aiNodeNames: detection.aiNodeNames,
targetAgentNodeName: agentName,
message: composeOfferMessage(detection.aiNodeNames, namedRefs),
};
}
// ── action: recommend-metric ───────────────────────────────────────────────
function composeRecommendMessage(
workflow: WorkflowJSON,
agentName: string,
metricId: string,
): string {
const name = isMetricId(metricId) ? METRIC_CATALOG[metricId].name : metricId;
const description = describeMetricForWorkflow(workflow, agentName, metricId);
const body = description ? `**${name}** — ${description}.` : `**${name}**.`;
return (
`Based on this workflow, I'd measure ${body} ` +
'Sound good? Approve to use it, or deny to pick a different metric from the full list.'
);
}
async function executeRecommendMetric(
context: InstanceAiContext,
input: z.infer<typeof recommendMetricAction>,
ctx: EvalsToolExecutionContext,
) {
const resumeData = getConfirmResume(ctx);
const suspend = getSuspend(ctx);
const wf = await context.workflowService.getAsWorkflowJSON(input.workflowId);
const detection = detectAiNodes(wf);
if (!detection.isAiWorkflow) {
return { skipped: true as const, reason: 'no-ai-nodes' as const };
}
const target = resolveTargetAgent(detection, input.targetAgentNodeName);
if (!target.ok) return targetAgentSkippedResponse(target);
const agentName = target.agentName;
const metricId = recommendedMetricId(wf, agentName);
if (hasResumeData(ctx)) {
if (resumeData?.approved) {
return { approved: true as const, metricId };
}
return { approved: false as const };
}
if (suspend) {
return await suspend({
requestId: nanoid(),
message: composeRecommendMessage(wf, agentName, metricId),
severity: 'info' as const,
});
}
return { approved: false as const };
}
// ── action: select-metrics ─────────────────────────────────────────────────
async function executeSelectMetrics(
context: InstanceAiContext,
input: z.infer<typeof selectMetricsAction>,
ctx: EvalsToolExecutionContext,
) {
const resumeData = getQuestionsResume(ctx);
const suspend = getSuspend(ctx);
const wf = await context.workflowService.getAsWorkflowJSON(input.workflowId);
const detection = detectAiNodes(wf);
if (!detection.isAiWorkflow) {
return { skipped: true as const, reason: 'no-ai-nodes' as const };
}
const target = resolveTargetAgent(detection, input.targetAgentNodeName);
if (!target.ok) return targetAgentSkippedResponse(target);
const agentName = target.agentName;
if (hasResumeData(ctx)) {
if (resumeData === undefined || !resumeData.approved || !resumeData.answers) {
return { chosenMetricIds: ['correctness'], answers: resumeData?.answers ?? [] };
}
const selected = resumeData.answers[0]?.selectedOptions ?? [];
const ids = selected.map(labelToId).filter((x): x is string => x !== undefined);
return {
chosenMetricIds: ids.length > 0 ? ids : ['correctness'],
answers: resumeData.answers,
};
}
const defaults = proposeDefaultMetricIds(wf, agentName);
const recommended = recommendedMetricId(wf, agentName);
const allLabels = METRIC_IDS.map((id) => metricLabel(wf, agentName, id, recommended));
const defaultLabels = defaults.map((id) => METRIC_CATALOG[id].name);
const questionId = nanoid();
if (suspend) {
return await suspend({
requestId: nanoid(),
message: 'Pick what to measure',
severity: 'info' as const,
inputType: 'questions' as const,
questions: [
{
id: questionId,
question: `Pick what you'd like to measure on each test case (defaults pre-selected: ${defaultLabels.join(', ')})`,
type: 'multi' as const,
options: allLabels,
},
],
});
}
return { chosenMetricIds: defaults };
}
// ── action: propose ────────────────────────────────────────────────────────
async function executePropose(context: InstanceAiContext, input: z.infer<typeof proposeAction>) {
const wf = await context.workflowService.getAsWorkflowJSON(input.workflowId);
const detection = detectAiNodes(wf);
if (!detection.isAiWorkflow) {
return { skipped: true, reason: 'Workflow has no AI/LLM nodes.' };
}
if (detection.alreadyConfigured) {
return {
skipped: true,
reason: 'Workflow already has an EvaluationTrigger or Evaluation node.',
};
}
const target = resolveTargetAgent(detection, input.targetAgentNodeName);
if (!target.ok) return targetAgentSkippedResponse(target);
// datasetChoice may be undefined when sanitizeInputSchema flattens the
// discriminated union — treat missing value as the schema default ('create-empty').
const datasetChoice = input.datasetChoice ?? 'create-empty';
if (datasetChoice === 'link-existing' && !input.existingDataTableId) {
return { skipped: true as const, reason: 'existing-data-table-id-required' as const };
}
const agentName = target.agentName;
const { inputColumns: directColumns } = analyzeAgentInputColumns(wf, agentName);
const namedRefs = detectAgentNamedRefs(wf, agentName);
// Resolve sub-component refs (tool/memory/...) via pinData on their source
// nodes. The LLM generates one fixture per source node; we write it to the
// workflow JSON. This shadows the production-adapter rewrite for those
// refs, so they're subtracted from `namedRefs` before formatting the task.
const toolRefs = detectToolRefs(wf, agentName);
let workflowWithPinData = wf;
if (toolRefs.length > 0) {
const generated = await generateToolRefPinData({
workflow: wf,
agentNodeName: agentName,
refs: toolRefs,
});
const patched = applyPinData(wf, generated);
if (patched !== wf) {
// No `projectId`: an update lands in the project the workflow already lives
// in, resolved by the adapter. Passing one here never did anything.
const saved = await context.workflowService.updateFromWorkflowJSON(input.workflowId, patched);
await refreshWorkflowSourceFileBindingFromSave(context, input.workflowId, {
versionId: saved.versionId,
checksum: saved.checksum,
});
workflowWithPinData = patched;
}
}
const filteredNamedRefs = namedRefs.filter(
(r) => !(r.targetNodeName !== agentName && pinDataCoversRef(workflowWithPinData.pinData, r)),
);
const namedRefColumns = filteredNamedRefs.map((r) => r.column);
// Combined column list: direct $json refs + named-ref-derived columns.
const inputColumns = [...new Set([...directColumns, ...namedRefColumns])];
// metrics may be undefined when sanitizeInputSchema flattens the discriminated union
let resolvedMetrics = getMetricsByIds(input.metrics ?? []);
if (resolvedMetrics.length === 0) {
resolvedMetrics = getMetricsByIds(['correctness']);
}
const outputColumns = resolvedMetrics.some((metric) => metric.requiresExpected)
? [DEFAULT_EXPECTED_OUTPUT_COLUMN]
: [];
const dataTableColumns = [...new Set([...inputColumns, ...outputColumns])];
let dataTableId: string | undefined = input.existingDataTableId;
let createdTable: { id: string; name: string; projectId?: string } | undefined;
let datasetChoiceForTask: 'link-existing' | 'later' = 'later';
if (datasetChoice === 'link-existing' && input.existingDataTableId) {
datasetChoiceForTask = 'link-existing';
} else if (datasetChoice === 'create-empty') {
const dt = await createEmptyEvalDataTable(context, {
workflowName: wf.name ?? 'Workflow',
projectId: input.projectId,
columns: dataTableColumns,
});
dataTableId = dt.id;
createdTable = dt;
datasetChoiceForTask = 'link-existing';
}
const task = formatEvalSetupTask({
workflowId: input.workflowId,
workflowName: wf.name ?? 'Workflow',
detectedAiNodes: detection.aiNodeNames,
datasetChoice: datasetChoiceForTask,
existingDataTableId: dataTableId,
projectId: input.projectId,
suggestedInputColumns: inputColumns,
suggestedOutputColumns: outputColumns,
enabledMetrics: resolvedMetrics,
namedRefs: filteredNamedRefs,
targetAgentNodeName: agentName,
});
return {
success: true,
shouldDelegateToEvalSetupAgent: true,
task,
workflowId: input.workflowId,
...(input.projectId ? { projectId: input.projectId } : {}),
...(dataTableId ? { dataTableId } : {}),
// `table` lets the artifacts panel pick up the newly created DataTable
// (registry's "Singular data table" extraction path). Only set on the
// create-empty path — link-existing reuses a table the user already
// has, so we don't claim it as a produced artifact. projectId comes
// from the DataTable service (not just input.projectId) so the preview
// can fetch the table even when the caller didn't pass projectId.
...(createdTable
? {
table: {
id: createdTable.id,
name: createdTable.name,
...((createdTable.projectId ?? input.projectId)
? { projectId: createdTable.projectId ?? input.projectId }
: {}),
},
}
: {}),
datasetChoice,
};
}

132
packages/@n8n/instance-ai/src/tools/evals/extract-rows-from-history.service.ts
View File

@ -1,132 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import type { InstanceAiContext, NodeOutputResult } from '../../types';
const SCAN_LIMIT = 100;
const MAX_ROWS = 25;
export interface ExtractRowsInput {
workflow: WorkflowJSON;
workflowId: string;
agentNodeName: string;
inputColumns: string[];
/**
* For each expected_* column, the agent output field whose value should
* be projected into that column. Empty array means no expected columns
* to populate from history.
*/
expectedToActualPairs: Array<{ expectedColumn: string; actualField: string }>;
}
export interface ExtractRowsResult {
rows: Array<Record<string, string>>;
scannedExecutions: number;
}
function findParentNode(workflow: WorkflowJSON, targetNodeName: string): string | undefined {
const connections = workflow.connections ?? {};
for (const [sourceName, byType] of Object.entries(connections)) {
if (!isRecord(byType)) continue;
const main = byType.main;
if (!Array.isArray(main)) continue;
for (const slot of main) {
if (!Array.isArray(slot)) continue;
for (const conn of slot) {
if (isRecord(conn) && conn.node === targetNodeName) {
return sourceName;
}
}
}
}
return undefined;
}
function projectRow(
parentJson: unknown,
agentJson: unknown,
inputColumns: string[],
expectedToActualPairs: Array<{ expectedColumn: string; actualField: string }>,
): Record<string, string> | undefined {
if (!isRecord(parentJson)) return undefined;
const row: Record<string, string> = {};
for (const col of inputColumns) {
const value = parentJson[col];
if (value === undefined || value === null) return undefined;
row[col] = typeof value === 'string' ? value : JSON.stringify(value);
}
if (expectedToActualPairs.length > 0) {
if (!isRecord(agentJson)) return undefined;
for (const { expectedColumn, actualField } of expectedToActualPairs) {
const value = agentJson[actualField];
if (value === undefined || value === null) return undefined;
row[expectedColumn] = typeof value === 'string' ? value : JSON.stringify(value);
}
}
return row;
}
function rowKey(row: Record<string, string>): string {
return JSON.stringify(row);
}
export async function extractRowsFromExecutionHistory(
ctx: InstanceAiContext,
input: ExtractRowsInput,
): Promise<ExtractRowsResult> {
const parentNodeName = findParentNode(input.workflow, input.agentNodeName);
if (!parentNodeName) {
return { rows: [], scannedExecutions: 0 };
}
const summaries = await ctx.executionService.list({
workflowId: input.workflowId,
status: 'success',
limit: SCAN_LIMIT,
});
const rows: Array<Record<string, string>> = [];
const seenRows = new Set<string>();
let scannedExecutions = 0;
for (const summary of summaries) {
if (rows.length >= MAX_ROWS) break;
let parentOutput: NodeOutputResult | undefined;
let agentOutput: NodeOutputResult | undefined;
try {
parentOutput = await ctx.executionService.getNodeOutput(summary.id, parentNodeName, {
maxItems: 1,
});
if (input.expectedToActualPairs.length > 0) {
agentOutput = await ctx.executionService.getNodeOutput(summary.id, input.agentNodeName, {
maxItems: 1,
});
}
} catch (error) {
ctx.logger.warn('extract-rows: getNodeOutput failed', {
executionId: summary.id,
error,
});
continue;
}
scannedExecutions++;
const parentItem = parentOutput.items[0];
const parentJson = isRecord(parentItem) ? parentItem.json : undefined;
const agentItem = agentOutput?.items[0];
const agentJson = agentItem !== undefined && isRecord(agentItem) ? agentItem.json : undefined;
const row = projectRow(parentJson, agentJson, input.inputColumns, input.expectedToActualPairs);
if (!row) continue;
const key = rowKey(row);
if (seenRows.has(key)) continue;
seenRows.add(key);
rows.push(row);
}
return { rows, scannedExecutions };
}

232
packages/@n8n/instance-ai/src/tools/evals/format-eval-setup-task.ts
View File

@ -1,232 +0,0 @@
import {
currentJsonExpression,
jsonFieldAccessor,
nodeItemJsonExpression,
} from './column-ref-utils';
import type { NamedRef } from './detect-agent-named-refs.service';
import {
formatEvalDataTableColumnName,
formatEvalDataTableColumnNameMap,
formatEvalDataTableName,
} from './ensure-eval-data-table.service';
import type { MetricProposal } from './metric-catalog';
export interface FormatEvalSetupTaskInput {
workflowId: string;
workflowName: string;
detectedAiNodes: string[];
datasetChoice: 'create-empty' | 'link-existing' | 'later';
existingDataTableId?: string;
projectId?: string;
suggestedInputColumns: string[];
suggestedOutputColumns: string[];
enabledMetrics: MetricProposal[];
namedRefs?: NamedRef[];
targetAgentNodeName?: string;
}
function taskString(value: string): string {
return JSON.stringify(value) ?? '""';
}
function evalTriggerJsonRef(column: string): string {
return `={{ $('Eval Trigger').item.json${jsonFieldAccessor(column)} }}`;
}
function formatProductionAdapter(
namedRefs: NamedRef[],
agentNodeName: string,
columnNameFor: (column: string) => string,
): string {
if (namedRefs.length === 0) return '';
// Set adapter assignments: one per unique column, using canonical single-quote syntax.
const assignmentsByColumn = new Map<
string,
{ column: string; nodeName: string; field: string }
>();
for (const r of namedRefs) {
const column = columnNameFor(r.column);
if (!assignmentsByColumn.has(column)) {
assignmentsByColumn.set(column, {
column,
nodeName: r.nodeName,
field: r.field,
});
}
}
const assignments = [...assignmentsByColumn.values()]
.map(
(a) =>
` - { name: ${taskString(a.column)}, value: ${taskString(`={{ ${nodeItemJsonExpression(a.nodeName, a.field)} }}`)}, type: "string" }`,
)
.join('\n');
// Rewrites grouped by target node.
//
// The replacement expression depends on whether the target IS the agent or
// a sub-component (memory, tool, output parser, ...):
// - Agent target: replace with `{{ $json.<col> }}` — the agent receives
// `$json` directly from upstream (Set adapter in prod, EvaluationTrigger
// in eval), so $json.<col> resolves.
// - Sub-component target: replace with `{{ $('<AgentName>').item.json.<col> }}`
// — sub-components have their own runtime context where `$json` may not
// reflect the agent's input row (memory nodes in particular). Referencing
// the agent by name forces n8n to resolve from the agent's last input,
// which works in both eval and prod modes.
const byTarget = new Map<string, NamedRef[]>();
for (const r of namedRefs) {
const arr = byTarget.get(r.targetNodeName) ?? [];
arr.push(r);
byTarget.set(r.targetNodeName, arr);
}
const rewriteBlocks: string[] = [];
for (const [target, refs] of byTarget) {
const isAgent = target === agentNodeName;
const lines = refs
.map((r) => {
const column = columnNameFor(r.column);
const replacement = isAgent
? `{{ ${currentJsonExpression(column)} }}`
: `{{ ${nodeItemJsonExpression(agentNodeName, column)} }}`;
return ` - Replace \`${r.originalExpression}\` with \`${replacement}\``;
})
.join('\n');
rewriteBlocks.push(` In \`${target}\`:\n${lines}`);
}
const rewrites = rewriteBlocks.join('\n');
const sourceList = [...new Set(namedRefs.map((r) => r.nodeName))]
.map((n) => `\`${n}\``)
.join(', ');
const sourceLabel = assignmentsByColumn.size === 1 ? 'node' : 'nodes';
return `
PRODUCTION ADAPTER (REQUIRED — the agent and/or its connected sub-components currently read input from named ${sourceLabel} ${sourceList}, which won't resolve in eval runs):
1. Insert a new \`n8n-nodes-base.set\` node named \`"Eval Production Adapter"\` (\`typeVersion: 3.4\`) immediately upstream of the agent on the PRODUCTION path. The agent's existing \`main\` input parent on the production path becomes the Set adapter's \`main\` input parent. The Set adapter's \`main\` output goes to the agent.
2. Configure the Set adapter's \`assignments.assignments\` array (one entry per unique dataset column):
${assignments}
3. Rewrite parameter expressions in each affected node — the agent and any sub-components (memory, tools, output parsers) that reference named source nodes. **The replacement form depends on the target:** the agent itself uses \`{{ $json.<col> }}\` (it receives \`$json\` directly), but sub-components must use the exact selected-agent expression listed below because their runtime context does not propagate \`$json\` from the agent's input row.
${rewrites}
4. The eval branch wires \`EvaluationTrigger\` directly to the agent's \`main\` input as a SECOND incoming connection (no Set adapter between them — the trigger row already has \`$json.<column>\` shape).
After your edits the agent has TWO incoming \`main\` connections: one from the Eval Production Adapter (production runs) and one from the EvaluationTrigger (eval runs). Both produce \`$json.<column>\` for the agent. Sub-components reference the selected agent node by name, so they resolve to the agent's last input row in both modes.`;
}
function formatMetric(
m: MetricProposal,
input: FormatEvalSetupTaskInput,
columnNameFor: (column: string) => string,
): string {
const cannedSuffix = m.cannedMetricKey ? `, canned=${m.cannedMetricKey}` : '';
const promptSuffix = m.prompt ? `\n Prompt: ${m.prompt}` : '';
const expectedOutputColumn = input.suggestedOutputColumns[0]
? columnNameFor(input.suggestedOutputColumns[0])
: 'expected_output';
const inputColumn = input.suggestedInputColumns[0]
? columnNameFor(input.suggestedInputColumns[0])
: 'input';
switch (m.cannedMetricKey) {
case 'correctness':
return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
Configure Evaluation(setMetrics) with \`metric: 'correctness'\`, \`expectedAnswer: ${evalTriggerJsonRef(expectedOutputColumn)}\`, \`actualAnswer: ={{ $json.output }}\`, and \`options.metricName: '${m.name}'\`. Connect an \`ai_languageModel\` input.${promptSuffix}`;
case 'helpfulness':
return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
Configure Evaluation(setMetrics) with \`metric: 'helpfulness'\`, \`userQuery: ${evalTriggerJsonRef(inputColumn)}\`, \`actualAnswer: ={{ $json.output }}\`, and \`options.metricName: '${m.name}'\`. Connect an \`ai_languageModel\` input.${promptSuffix}`;
case 'tool_use':
return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
Configure Evaluation(setMetrics) with \`metric: 'toolsUsed'\` (not \`'tool_use'\`), \`expectedTools\` from the dataset or the user's selected expected tool list, \`intermediateSteps: ={{ $json.intermediateSteps }}\`, and \`options.metricName: '${m.name}'\`. Enable returning intermediate steps on the agent.`;
case 'relevance':
return `- ${m.name} (${m.kind}${cannedSuffix}): ${m.description}
There is no native \`relevance\` metric option. Configure Evaluation(setMetrics) with \`metric: 'helpfulness'\`, \`userQuery: ${evalTriggerJsonRef(inputColumn)}\`, \`actualAnswer\` mapped to the retrieved context or response field being judged, \`options.metricName: '${m.name}'\`, and the relevance prompt below. Connect an \`ai_languageModel\` input.${promptSuffix}`;
default:
return `- ${m.name} (${m.kind}): ${m.description}${promptSuffix}`;
}
}
function formatDatasetSection(input: FormatEvalSetupTaskInput, dataTableColumns: string[]): string {
if (input.datasetChoice === 'link-existing') {
return `Wire the EvaluationTrigger to DataTable id \`${input.existingDataTableId}\`. This table already exists — do not modify its rows or schema.`;
}
if (input.datasetChoice === 'later') {
return "Do not create a DataTable. Leave the EvaluationTrigger's dataTableId empty — the user will wire it manually later.";
}
const tableName = formatEvalDataTableName(input.workflowName);
const columns = dataTableColumns.map((c) => `- ${c}`).join('\n');
return `Create an empty DataTable named "${tableName}"${input.projectId ? ` in project id \`${input.projectId}\`` : ''} using only the \`create-empty-eval-data-table\` tool. Columns to create as strings:\n${columns}\n\nDo not insert rows, generate rows, or mutate row data. After creating the empty table, wire the EvaluationTrigger and setOutputs dataTableId to the created table id.`;
}
function formatSetOutputsDataTableInstruction(input: FormatEvalSetupTaskInput): string {
if (input.datasetChoice === 'link-existing') {
const id = input.existingDataTableId ?? '<existing DataTable id>';
return `The setOutputs node MUST have \`source: 'dataTable'\` and \`dataTableId: { mode: 'id', value: '${id}' }\` set explicitly (the node default for older typeVersions is googleSheets — that's a silent failure mode).`;
}
if (input.datasetChoice === 'later') {
return 'Leave setOutputs dataTableId empty until the user selects a DataTable. Do not write placeholder values into EvaluationTrigger or setOutputs.';
}
return "The setOutputs node MUST have `source: 'dataTable'` and `dataTableId` set explicitly using the id returned by `create-empty-eval-data-table` (the same id wired on the EvaluationTrigger). The node default for older typeVersions is googleSheets — that's a silent failure mode.";
}
export function formatEvalSetupTask(input: FormatEvalSetupTaskInput): string {
const rawInputColumns = [
...input.suggestedInputColumns,
...(input.namedRefs ?? []).map((r) => r.column),
];
const rawOutputColumns = input.suggestedOutputColumns;
const columnNameByRaw = formatEvalDataTableColumnNameMap([
...rawInputColumns,
...rawOutputColumns,
]);
const columnNameFor = (column: string) =>
columnNameByRaw.get(column) ?? formatEvalDataTableColumnName(column);
const inputColumnNames = [...new Set(rawInputColumns.map(columnNameFor))];
const outputColumnNames = [...new Set(rawOutputColumns.map(columnNameFor))];
const dataTableColumns = [...new Set([...inputColumnNames, ...outputColumnNames])];
const outputColumns = outputColumnNames.map((c) => `- ${c}`).join('\n');
const inputColumns = inputColumnNames.map((c) => `- ${c}`).join('\n');
const metrics = input.enabledMetrics
.map((m) => formatMetric(m, input, columnNameFor))
.join('\n\n');
const datasetSection = formatDatasetSection(input, dataTableColumns);
const setOutputsDataTableInstruction = formatSetOutputsDataTableInstruction(input);
const agentNodeName = input.targetAgentNodeName ?? input.detectedAiNodes[0] ?? '';
const adapterSection = formatProductionAdapter(
input.namedRefs ?? [],
agentNodeName,
columnNameFor,
);
return `Set up evaluations for workflow "${input.workflowName}" (id: ${input.workflowId}).
AI AGENT NODES IN WORKFLOW: ${input.detectedAiNodes.join(', ')}
DATASET:
${datasetSection}
INPUT COLUMNS (the AI agent's parameters MUST reference each of these via \`={{ $json.<column> }}\`. If the agent's existing parameters reference different fields, rewrite those parameter expressions to use these dataset columns. Only rewrite input-reading parameters — leave credentials, tools, model selection, and unrelated configuration untouched):
${inputColumns}
${adapterSection}
GROUND-TRUTH OUTPUT COLUMNS (already in the dataset — these hold the EXPECTED values per row):
${outputColumns}
For setOutputs, write the agent's actual output to NEW columns derived from these ground-truth column names — convention: prefix with \`actual_\` (e.g. ground-truth \`expected_output\` → setOutputs writes to \`actual_output\`; ground-truth \`expected_response\` → setOutputs writes to \`actual_response\`). Never overwrite the ground-truth column itself. The Evaluation node auto-adds the new column on first eval run.
${setOutputsDataTableInstruction} Use \`typeVersion: 4.8\` for the Evaluation node.
METRICS TO CONFIGURE (on setMetrics):
${metrics}
Use the field mappings specified per metric above. Use the explicit name you assign to the EvaluationTrigger node (name it \`"Eval Trigger"\` when creating it, then reference exactly that name everywhere — never use the stock default \`"When fetching a dataset row"\` and never create a separate node for that purpose).
For \`correctness\`, \`helpfulness\`, and \`relevance\` (configured with the native \`helpfulness\` metric): also wire an \`ai_languageModel\` connection from the workflow's existing LLM model node (the one already feeding the AI agent) to each setMetrics node that uses an AI-judged metric. The LLM is reused — same node, additional outgoing \`ai_languageModel\` connection. Without this, AI-judged metrics fail silently. \`stringSimilarity\`/\`categorization\`/\`toolsUsed\` don't need this.
Apply the topology as described in your instructions:
1. EvaluationTrigger → target AI agent node (direct \`main\` connection). No intermediate Set/Code/transform node. The trigger emits each dataset column as \`$json.<column>\`. If the agent's existing parameters reference fields that don't match the listed INPUT COLUMNS, rewrite those parameter expressions to use \`{{ $json.<column> }}\`. NOTE: \`Evaluation(setInputs)\` does NOT reshape data — it only attaches metadata for the eval-results display tab.
2. After the AI agent: insert \`Evaluation(checkIfEvaluating)\` (no separate IF node — it has two native output slots). Slot 0 (Evaluation) routes to setOutputs → setMetrics. Slot 1 (Normal) preserves the original production path with side-effects.
Report back with a one-line summary when done.`;
}

333
packages/@n8n/instance-ai/src/tools/evals/generate-sample-rows.service.ts
View File

@ -1,333 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { z } from 'zod';
import { detectAiNodes } from './detect-ai-nodes';
import type { Logger } from '../../logger';
import { HAIKU_MODEL } from '../../utils/eval-agents';
import { generateValidatedJson } from '../../utils/generate-validated-json';
const FACET_COUNT = 5;
const DEFAULT_ROW_COUNT = 25;
const SYSTEM_PROMPT_MAX_CHARS = 2000;
const REAL_EXAMPLES_MAX_COUNT = 10;
const REAL_EXAMPLE_VALUE_MAX_CHARS = 300;
export interface SampleRowFacet {
length: string;
edgeMode: string;
instructions: string;
}
export interface AgentContext {
workflowName: string;
agentNodeName: string;
systemPrompt: string | undefined;
promptTemplate: string | undefined;
connectedToolNames: string[];
}
export const SAMPLE_ROW_FACETS: readonly SampleRowFacet[] = [
{
length: 'short (single sentence or one-line snippet)',
edgeMode: 'happy — typical, well-formed inputs the agent commonly receives',
instructions:
'Produce concise, realistic inputs the AI Agent would handle on a regular day. Each input should fit on one line. No edge cases.',
},
{
length: 'long (multi-paragraph or several lines)',
edgeMode: 'happy — typical, well-formed inputs',
instructions:
'Produce longer realistic inputs spanning multiple paragraphs or several lines. The content stays in-domain and well-structured.',
},
{
length: 'medium (2–3 sentences or comparable size)',
edgeMode: 'edge — ambiguity, truncations, typos, imperfect formatting',
instructions:
'Produce inputs that are messy but plausible: typos, ambiguous references, truncated phrasing, irregular casing or spacing. Still in-domain.',
},
{
length: 'long (multi-paragraph)',
edgeMode: 'adversarial — out-of-scope requests, contradictions, prompt injection',
instructions:
'Produce inputs that try to push the agent off task: out-of-scope demands, internally contradictory requirements, attempts at prompt injection ("ignore previous instructions"), or hostile framing.',
},
{
length: 'mixed extremes (empty, very short, or very long)',
edgeMode: 'robustness — empty inputs, multi-language fragments, malformed data',
instructions:
'Produce robustness probes: an empty input, an input of one or two characters, an input that mixes two languages, and an input with malformed structure (e.g., truncated JSON, broken markup). Use whatever the agent would plausibly stumble on in production.',
},
];
export function distributeRowCount(rowCount: number): number[] {
const safe = Math.max(0, Math.floor(rowCount));
const base = Math.floor(safe / FACET_COUNT);
const extra = safe % FACET_COUNT;
return Array.from({ length: FACET_COUNT }, (_, i) => base + (i < extra ? 1 : 0));
}
function readSystemPrompt(parameters: Record<string, unknown> | undefined): string | undefined {
if (!parameters) return undefined;
const direct = parameters.systemMessage;
if (typeof direct === 'string' && direct.length > 0) {
return direct.slice(0, SYSTEM_PROMPT_MAX_CHARS);
}
const options = parameters.options;
if (isRecord(options)) {
const nested = options.systemMessage;
if (typeof nested === 'string' && nested.length > 0) {
return nested.slice(0, SYSTEM_PROMPT_MAX_CHARS);
}
}
return undefined;
}
function readPromptTemplate(parameters: Record<string, unknown> | undefined): string | undefined {
if (!parameters) return undefined;
const text = parameters.text;
return typeof text === 'string' && text.length > 0 ? text : undefined;
}
function findConnectedTools(workflow: WorkflowJSON, agentName: string): string[] {
const tools: string[] = [];
const connections = workflow.connections ?? {};
for (const [sourceName, byType] of Object.entries(connections)) {
if (!isRecord(byType)) continue;
const aiTool = byType?.ai_tool;
if (!Array.isArray(aiTool)) continue;
const matches = aiTool.some((slot) => {
if (!Array.isArray(slot)) return false;
return slot.some((link) => isRecord(link) && link.node === agentName);
});
if (matches) tools.push(sourceName);
}
return tools;
}
export function extractAgentContext(
workflow: WorkflowJSON,
agentNodeName: string,
): AgentContext | undefined {
const node = (workflow.nodes ?? []).find((n) => n.name === agentNodeName);
if (!node) return undefined;
const parameters = isRecord(node.parameters) ? node.parameters : undefined;
return {
workflowName: workflow.name ?? 'Untitled',
agentNodeName,
systemPrompt: readSystemPrompt(parameters),
promptTemplate: readPromptTemplate(parameters),
connectedToolNames: findConnectedTools(workflow, agentNodeName),
};
}
const batchRowSchema = z.array(z.record(z.union([z.string(), z.number(), z.boolean(), z.null()])));
function isExpectedOutputColumn(column: string): boolean {
return column.startsWith('expected');
}
function buildAgentContextBlock(context: AgentContext | undefined): string {
if (!context) return 'Workflow context unavailable.';
const lines: string[] = [
`Workflow name: ${context.workflowName}`,
`AI Agent node: ${context.agentNodeName}`,
];
if (context.systemPrompt) {
lines.push('Agent system prompt:', context.systemPrompt);
}
if (context.promptTemplate) {
lines.push(`Agent prompt template: ${context.promptTemplate}`);
}
if (context.connectedToolNames.length > 0) {
lines.push(`Connected tools: ${context.connectedToolNames.join(', ')}`);
}
return lines.join('\n');
}
const FORMAT_INFERENCE =
"Inspect the agent's system prompt, prompt template, and connected tools to infer what kind of text this agent receives at runtime. It may be a user chat message, output from another tool, scraped web content, structured records (JSON/key-value), document chunks, log lines, code, etc. Generate inputs that look like what would arrive at the agent in production. Do not assume a human user when the agent suggests otherwise.";
function truncateExampleValue(value: string): string {
return value.length > REAL_EXAMPLE_VALUE_MAX_CHARS
? `${value.slice(0, REAL_EXAMPLE_VALUE_MAX_CHARS)}…`
: value;
}
/**
* Render a small block of recent real inputs (filtered to the requested
* `columns`) as a reference for the LLM. Returns an empty string when no
* usable examples exist — the caller injects this block only when
* non-empty, so the generator keeps producing rows from agent context
* alone when history is missing.
*
* The directive is explicit that these are flavour reference, not seed
* data to copy: the generator must produce NEW inputs in the same domain
* and tone, not paraphrase the examples.
*/
function buildRealExamplesBlock(
examples: ReadonlyArray<Record<string, unknown>> | undefined,
columns: string[],
): string {
if (!examples || examples.length === 0 || columns.length === 0) return '';
const filtered: Array<Record<string, string>> = [];
for (const example of examples.slice(0, REAL_EXAMPLES_MAX_COUNT)) {
const row: Record<string, string> = {};
let hasValue = false;
for (const col of columns) {
const raw = example[col];
if (raw === undefined || raw === null) continue;
const str = typeof raw === 'string' ? raw : JSON.stringify(raw);
if (str.length === 0) continue;
row[col] = truncateExampleValue(str);
hasValue = true;
}
if (hasValue) filtered.push(row);
}
if (filtered.length === 0) return '';
const numbered = filtered.map((row, i) => `${i + 1}. ${JSON.stringify(row)}`).join('\n');
return [
'',
'Recent real inputs the agent has received in production (REFERENCE, not seeds):',
numbered,
'Use these as a hint about the actual domain, tone and shape of inputs the agent sees. Do NOT copy or paraphrase them — produce NEW inputs that fit the same setting.',
].join('\n');
}
const BATCH_SYSTEM_INSTRUCTIONS = `You generate realistic test inputs for an n8n workflow evaluation dataset.
Output: JSON array of objects. Keys = exactly the provided column names. Values = short strings. No prose outside the JSON.
Generate only INPUT data — the inputs the agent would receive in production. Do not invent expected outputs, answers, or labels. Those columns are filled in by the user, who decides what the correct answer is. Generating both inputs and expected outputs with the same model would measure self-consistency, not correctness.
Return only the JSON array.`;
export interface RunBatchInput {
facet: SampleRowFacet;
rowCount: number;
context: AgentContext | undefined;
columns: string[];
realExamples?: ReadonlyArray<Record<string, unknown>>;
logger: Pick<Logger, 'warn'>;
}
function normalizeBatchRow(
rawRow: Record<string, string | number | boolean | null>,
columns: string[],
): Record<string, string> {
const row: Record<string, string> = {};
for (const col of columns) {
if (isExpectedOutputColumn(col)) {
row[col] = '';
continue;
}
const v = rawRow[col];
if (v === undefined || v === null) row[col] = '';
else if (typeof v === 'string') row[col] = v;
else row[col] = String(v);
}
return row;
}
export async function runBatch(input: RunBatchInput): Promise<Array<Record<string, string>>> {
const requestedRowCount = Math.max(0, Math.floor(input.rowCount));
if (requestedRowCount <= 0) return [];
const generatedColumns = input.columns.filter((column) => !isExpectedOutputColumn(column));
const realExamplesBlock = buildRealExamplesBlock(input.realExamples, generatedColumns);
const sections = [buildAgentContextBlock(input.context)];
if (realExamplesBlock) sections.push(realExamplesBlock);
sections.push(FORMAT_INFERENCE);
sections.push(
[
`Variation focus for this batch: length = ${input.facet.length}; mode = ${input.facet.edgeMode}.`,
input.facet.instructions,
].join('\n'),
);
sections.push(
[`Columns: ${generatedColumns.join(', ')}`, `Generate exactly ${requestedRowCount} rows.`].join(
'\n',
),
);
const result = await generateValidatedJson('eval-sample-rows', {
model: HAIKU_MODEL,
instructions: BATCH_SYSTEM_INSTRUCTIONS,
userText: sections.join('\n\n'),
schema: batchRowSchema,
});
if (!result.ok) {
input.logger.warn('generate-sample-rows: batch generation failed', {
rowCount: requestedRowCount,
facet: input.facet.edgeMode,
reason: result.reason,
issues: result.issues,
});
return [];
}
return result.data
.slice(0, requestedRowCount)
.map((rawRow) => normalizeBatchRow(rawRow, input.columns));
}
export interface GenerateSampleRowsInput {
workflow: WorkflowJSON;
columns: string[];
rowCount?: number;
targetAgentNodeName?: string;
/**
* Recent real input rows extracted from the workflow's execution history.
* When present (typically below the history threshold that would otherwise
* have been used directly), they are passed to the LLM as a flavour
* reference — rows are filtered to the requested `columns`, truncated, and
* accompanied by an explicit "reference, not seed" directive so the
* generator produces new in-domain inputs instead of paraphrasing them.
*/
realExamples?: ReadonlyArray<Record<string, unknown>>;
logger: Pick<Logger, 'warn'>;
}
function resolveAgentContext(
workflow: WorkflowJSON,
targetAgentNodeName: string | undefined,
): AgentContext | undefined {
if (targetAgentNodeName) {
return extractAgentContext(workflow, targetAgentNodeName);
}
const detected = detectAiNodes(workflow);
const firstAgent = detected.aiNodeNames[0];
if (!firstAgent) return undefined;
return extractAgentContext(workflow, firstAgent);
}
function emptyRow(columns: string[]): Record<string, string> {
return Object.fromEntries(columns.map((c) => [c, '']));
}
export async function generateSampleRows(
input: GenerateSampleRowsInput,
): Promise<Array<Record<string, string>>> {
const rowCount = input.rowCount ?? DEFAULT_ROW_COUNT;
const counts = distributeRowCount(rowCount);
const context = resolveAgentContext(input.workflow, input.targetAgentNodeName);
const settled = await Promise.allSettled(
SAMPLE_ROW_FACETS.map(async (facet, i) => {
if (counts[i] <= 0) return [];
return await runBatch({
facet,
rowCount: counts[i],
context,
columns: input.columns,
realExamples: input.realExamples,
logger: input.logger,
});
}),
);
const merged: Array<Record<string, string>> = [];
for (const r of settled) {
if (r.status === 'fulfilled') merged.push(...r.value);
}
if (merged.length === 0) return [emptyRow(input.columns)];
return merged;
}

135
packages/@n8n/instance-ai/src/tools/evals/generate-tool-ref-pin-data.service.ts
View File

@ -1,135 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { z } from 'zod';
import { nodeHasName } from './column-ref-utils';
import type { ToolRef } from './detect-tool-refs.service';
import { HAIKU_MODEL } from '../../utils/eval-agents';
import { generateValidatedJson } from '../../utils/generate-validated-json';
export type ToolRefPinData = Record<string, Array<{ json: Record<string, unknown> }>>;
export interface GenerateToolRefPinDataInput {
workflow: WorkflowJSON;
agentNodeName: string;
refs: ToolRef[];
}
const PinDataResponseSchema = z.record(
z.string(),
z.array(z.object({ json: z.record(z.unknown()) })).min(1),
);
const SYSTEM_INSTRUCTIONS = `You generate realistic mock output for n8n trigger and source nodes referenced inside an AI agent's tools/memory.
Output: a single JSON object whose keys are node names and whose values are arrays of n8n pinData items in the form { "json": { ... } }. One item per node is enough.
For each node, return data that matches what the node would naturally emit at runtime — based on its type and parameters. Make sure every requested field appears in your output, with a plausible value.
Return only the JSON object. No prose, no markdown fences.`;
interface NodeContext {
name: string;
type: string;
parameters: Record<string, unknown> | undefined;
requiredFields: string[];
}
function readAgentPurpose(agent: WorkflowJSON['nodes'][number] | undefined): string | undefined {
if (!agent) return undefined;
const params = agent.parameters as Record<string, unknown> | undefined;
if (!params) return undefined;
const direct = params.systemMessage;
if (typeof direct === 'string' && direct.length > 0) return direct.slice(0, 1000);
const options = params.options;
if (options && typeof options === 'object') {
const nested = (options as Record<string, unknown>).systemMessage;
if (typeof nested === 'string' && nested.length > 0) return nested.slice(0, 1000);
}
const text = params.text;
if (typeof text === 'string' && text.length > 0) return text.slice(0, 1000);
return undefined;
}
function buildNodeContexts(workflow: WorkflowJSON, refs: ToolRef[]): NodeContext[] {
const fieldsBySource = new Map<string, Set<string>>();
for (const r of refs) {
const set = fieldsBySource.get(r.sourceNodeName) ?? new Set<string>();
set.add(r.field);
fieldsBySource.set(r.sourceNodeName, set);
}
const nodesByName = new Map(
(workflow.nodes ?? []).filter(nodeHasName).map((n) => [n.name, n] as const),
);
const contexts: NodeContext[] = [];
for (const [name, fields] of fieldsBySource) {
const node = nodesByName.get(name);
if (!node) continue;
contexts.push({
name,
type: node.type,
parameters: node.parameters as Record<string, unknown> | undefined,
requiredFields: [...fields],
});
}
return contexts;
}
function formatNodeContextBlock(ctx: NodeContext): string {
const paramsLine = ctx.parameters
? `Parameters: ${JSON.stringify(ctx.parameters).slice(0, 500)}`
: 'Parameters: (none)';
return [
`Node name: ${ctx.name}`,
`Node type: ${ctx.type}`,
paramsLine,
`Required fields on json: ${ctx.requiredFields.join(', ')}`,
].join('\n');
}
function buildUserMessage(contexts: NodeContext[], agentPurpose: string | undefined): string {
const blocks = contexts.map(formatNodeContextBlock).join('\n\n');
const purpose = agentPurpose ? `\nAgent purpose / system prompt:\n${agentPurpose}\n` : '';
const nodeList = contexts.map((c) => `"${c.name}"`).join(', ');
return [
'Generate realistic mock output (pinData items) for the following n8n nodes.',
purpose,
blocks,
'',
`Output a single JSON object with exactly these keys: ${nodeList}.`,
'Each value: an array with one item shaped like { "json": { ...fields } }.',
'All required fields above MUST appear inside `json` for the corresponding node.',
].join('\n');
}
function filterToRequested(parsed: ToolRefPinData, requested: Set<string>): ToolRefPinData {
const out: ToolRefPinData = {};
for (const [name, items] of Object.entries(parsed)) {
if (!requested.has(name)) continue;
out[name] = items;
}
return out;
}
export async function generateToolRefPinData(
input: GenerateToolRefPinDataInput,
): Promise<ToolRefPinData> {
if (input.refs.length === 0) return {};
const contexts = buildNodeContexts(input.workflow, input.refs);
if (contexts.length === 0) return {};
const agent = (input.workflow.nodes ?? []).find(
(n) => nodeHasName(n) && n.name === input.agentNodeName,
);
const agentPurpose = readAgentPurpose(agent);
const userText = buildUserMessage(contexts, agentPurpose);
const result = await generateValidatedJson('eval-tool-ref-pin-data', {
model: HAIKU_MODEL,
instructions: SYSTEM_INSTRUCTIONS,
userText,
schema: PinDataResponseSchema,
});
if (!result.ok) return {};
const requested = new Set(contexts.map((c) => c.name));
return filterToRequested(result.data, requested);
}

98
packages/@n8n/instance-ai/src/tools/evals/metric-catalog.ts
View File

@ -1,98 +0,0 @@
import { isRecord } from '@n8n/utils/is-record';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
export const METRIC_IDS = ['correctness', 'relevance', 'tool_use', 'helpfulness'] as const;
export type MetricId = (typeof METRIC_IDS)[number];
export interface MetricProposal {
id: string;
name: string;
kind: string;
cannedMetricKey?: string;
description: string;
prompt?: string;
defaultEnabled: boolean;
}
export interface MetricCatalogEntry extends MetricProposal {
id: MetricId;
kind: 'llm-judge';
cannedMetricKey: MetricId;
prompt: string;
requiresExpected: boolean;
}
export const METRIC_CATALOG = {
correctness: {
id: 'correctness',
name: 'Correctness',
kind: 'llm-judge',
cannedMetricKey: 'correctness',
description: 'Judge whether the workflow output matches the expected ground-truth response.',
prompt:
'Given the input and expected output, rate from 0 to 1 how correct the actual output is.',
defaultEnabled: true,
requiresExpected: true,
},
relevance: {
id: 'relevance',
name: 'Relevance',
kind: 'llm-judge',
cannedMetricKey: 'relevance',
description: 'Judge whether retrieved context is relevant to the user query (RAG).',
prompt: 'Rate from 0 to 1 how relevant the retrieved documents are to the user query.',
defaultEnabled: false,
requiresExpected: false,
},
tool_use: {
id: 'tool_use',
name: 'Tool use',
kind: 'llm-judge',
cannedMetricKey: 'tool_use',
description: 'Judge whether the agent selected and used the correct tools for the input.',
prompt:
'Rate from 0 to 1 how appropriate the tool selection and arguments were for the user query.',
defaultEnabled: false,
requiresExpected: false,
},
helpfulness: {
id: 'helpfulness',
name: 'Helpfulness',
kind: 'llm-judge',
cannedMetricKey: 'helpfulness',
description: 'Judge whether the agent response is helpful to the user.',
prompt: 'Rate from 0 to 1 how helpful the response is to the user query.',
defaultEnabled: false,
requiresExpected: false,
},
} satisfies Record<MetricId, MetricCatalogEntry>;
function isMetricId(id: string): id is MetricId {
return (METRIC_IDS as readonly string[]).includes(id);
}
export function getMetricsByIds(ids: string[]): MetricCatalogEntry[] {
return ids.filter(isMetricId).map((id) => METRIC_CATALOG[id]);
}
function agentHasTools(workflow: WorkflowJSON, agentNodeName: string): boolean {
const connections = workflow.connections ?? {};
for (const sourceConnections of Object.values(connections)) {
if (!isRecord(sourceConnections)) continue;
const aiTool = sourceConnections.ai_tool;
if (!Array.isArray(aiTool)) continue;
for (const slot of aiTool) {
if (!Array.isArray(slot)) continue;
for (const conn of slot) {
if (isRecord(conn) && conn.node === agentNodeName) return true;
}
}
}
return false;
}
export function proposeDefaultMetricIds(workflow: WorkflowJSON, agentNodeName: string): MetricId[] {
const ids: MetricId[] = ['correctness'];
if (agentHasTools(workflow, agentNodeName)) ids.push('tool_use');
return ids;
}

116
packages/@n8n/instance-ai/src/tools/index.ts
View File

@ -23,9 +23,6 @@ const loadConversationHistoryTool = lazyMod(
const loadDataTablesTool = lazyMod(
() => require('./data-tables.tool') as typeof import('./data-tables.tool'),
);
const loadEvalsTool = lazyMod(
() => require('./evals/evals.tool') as typeof import('./evals/evals.tool'),
);
const loadEvalConfigTool = lazyMod(
() => require('./evals/eval-config.tool') as typeof import('./evals/eval-config.tool'),
);
@ -56,14 +53,6 @@ const loadCompleteCheckpointTool = lazyMod(
() =>
require('./orchestration/complete-checkpoint.tool') as typeof import('./orchestration/complete-checkpoint.tool'),
);
const loadEvalDataAgentTool = lazyMod(
() =>
require('./orchestration/eval-data-agent.tool') as typeof import('./orchestration/eval-data-agent.tool'),
);
const loadEvalSetupAgentTool = lazyMod(
() =>
require('./orchestration/eval-setup-agent.tool') as typeof import('./orchestration/eval-setup-agent.tool'),
);
const loadPlanTool = lazyMod(
() => require('./orchestration/plan.tool') as typeof import('./orchestration/plan.tool'),
);
@ -99,69 +88,44 @@ const loadWorkspaceTool = lazyMod(
() => require('./workspace.tool') as typeof import('./workspace.tool'),
);
/**
* Creates all native n8n domain tools with the full action surface.
* Used for sub-agent tool resolution — sub-agents get unrestricted access.
*/
export function createAllTools(context: InstanceAiContext): InstanceAiToolRegistry {
const tools: Array<[string, BuiltTool]> = [
[DOMAIN_TOOL_IDS.WORKFLOWS, loadWorkflowsTool().createWorkflowsTool(context)],
[DOMAIN_TOOL_IDS.EVALS, loadEvalsTool().createEvalsTool(context)],
[DOMAIN_TOOL_IDS.EXECUTIONS, loadExecutionsTool().createExecutionsTool(context)],
[DOMAIN_TOOL_IDS.CREDENTIALS, loadCredentialsTool().createCredentialsTool(context)],
[DOMAIN_TOOL_IDS.DATA_TABLES, loadDataTablesTool().createDataTablesTool(context)],
[DOMAIN_TOOL_IDS.WORKSPACE, loadWorkspaceTool().createWorkspaceTool(context)],
[DOMAIN_TOOL_IDS.RESEARCH, loadResearchTool().createResearchTool(context)],
[DOMAIN_TOOL_IDS.N8N_DOCS, loadN8nDocsTool().createN8nDocsTool(context)],
[DOMAIN_TOOL_IDS.NODES, loadNodesTool().createNodesTool(context)],
[DOMAIN_TOOL_IDS.ASK_USER, loadAskUserTool().createAskUserTool()],
[DOMAIN_TOOL_IDS.BUILD_WORKFLOW, loadBuildWorkflowTool().createBuildWorkflowTool(context)],
type DomainToolFactory = () => BuiltTool;
function getOrchestratorDomainToolFactories(
context: InstanceAiContext,
): Array<[string, DomainToolFactory]> {
const tools: Array<[string, DomainToolFactory]> = [
[DOMAIN_TOOL_IDS.WORKFLOWS, () => loadWorkflowsTool().createWorkflowsTool(context)],
[DOMAIN_TOOL_IDS.EXECUTIONS, () => loadExecutionsTool().createExecutionsTool(context)],
[DOMAIN_TOOL_IDS.CREDENTIALS, () => loadCredentialsTool().createCredentialsTool(context)],
[DOMAIN_TOOL_IDS.DATA_TABLES, () => loadDataTablesTool().createDataTablesTool(context)],
[DOMAIN_TOOL_IDS.WORKSPACE, () => loadWorkspaceTool().createWorkspaceTool(context)],
[DOMAIN_TOOL_IDS.RESEARCH, () => loadResearchTool().createResearchTool(context)],
[DOMAIN_TOOL_IDS.N8N_DOCS, () => loadN8nDocsTool().createN8nDocsTool(context)],
[DOMAIN_TOOL_IDS.NODES, () => loadNodesTool().createNodesTool(context)],
[DOMAIN_TOOL_IDS.ASK_USER, () => loadAskUserTool().createAskUserTool()],
[
DOMAIN_TOOL_IDS.BUILD_WORKFLOW,
() => loadBuildWorkflowTool().createBuildWorkflowTool(context),
],
];
// eval-config is flag-gated: the adapter only wires evaluationConfigService
// when `088_config_evaluations` is on, so presence = expose the tool.
if (context.evaluationConfigService) {
tools.push([DOMAIN_TOOL_IDS.EVAL_CONFIG, loadEvalConfigTool().createEvalConfigTool(context)]);
}
if (context.currentUserAttachments?.some(isParseableAttachment)) {
tools.push([DOMAIN_TOOL_IDS.PARSE_FILE, loadParseFileTool().createParseFileTool(context)]);
}
return createToolRegistry(tools);
}
/**
* Creates orchestrator domain tools. Skills run in the orchestrator now, so
* domain tools keep their workflow-building surface while hiding raw full
* WorkflowJSON read and update actions.
*/
export function createOrchestratorDomainTools(context: InstanceAiContext): InstanceAiToolRegistry {
const tools: Array<[string, BuiltTool]> = [
[DOMAIN_TOOL_IDS.WORKFLOWS, loadWorkflowsTool().createWorkflowsTool(context, 'orchestrator')],
[DOMAIN_TOOL_IDS.EVALS, loadEvalsTool().createEvalsTool(context)],
[DOMAIN_TOOL_IDS.EXECUTIONS, loadExecutionsTool().createExecutionsTool(context)],
[DOMAIN_TOOL_IDS.CREDENTIALS, loadCredentialsTool().createCredentialsTool(context)],
[DOMAIN_TOOL_IDS.DATA_TABLES, loadDataTablesTool().createDataTablesTool(context)],
[DOMAIN_TOOL_IDS.WORKSPACE, loadWorkspaceTool().createWorkspaceTool(context)],
[DOMAIN_TOOL_IDS.RESEARCH, loadResearchTool().createResearchTool(context)],
[DOMAIN_TOOL_IDS.N8N_DOCS, loadN8nDocsTool().createN8nDocsTool(context)],
[DOMAIN_TOOL_IDS.NODES, loadNodesTool().createNodesTool(context)],
[DOMAIN_TOOL_IDS.ASK_USER, loadAskUserTool().createAskUserTool()],
[DOMAIN_TOOL_IDS.BUILD_WORKFLOW, loadBuildWorkflowTool().createBuildWorkflowTool(context)],
];
// eval-config is flag-gated: the adapter only wires evaluationConfigService
// when `088_config_evaluations` is on, so presence = expose the tool.
if (context.evaluationConfigService) {
tools.push([DOMAIN_TOOL_IDS.EVAL_CONFIG, loadEvalConfigTool().createEvalConfigTool(context)]);
tools.push([
DOMAIN_TOOL_IDS.EVAL_CONFIG,
() => loadEvalConfigTool().createEvalConfigTool(context),
]);
}
// Same pattern: the adapter only wires mcpService when MCP access is enabled
// instance-wide and the user is in the MCP-connections experiment. Orchestrator
// only — sub-agents can't offer the user a connection.
if (context.mcpService) {
tools.push([DOMAIN_TOOL_IDS.MCP_SERVERS, loadMcpServersTool().createMcpServersTool(context)]);
tools.push([
DOMAIN_TOOL_IDS.MCP_SERVERS,
() => loadMcpServersTool().createMcpServersTool(context),
]);
}
// Orchestrator-only: sub-agents receive context via briefings, and cross-thread
@ -170,14 +134,31 @@ export function createOrchestratorDomainTools(context: InstanceAiContext): Insta
if (context.conversationHistoryService) {
tools.push([
DOMAIN_TOOL_IDS.CONVERSATION_HISTORY,
loadConversationHistoryTool().createConversationHistoryTool(context),
() => loadConversationHistoryTool().createConversationHistoryTool(context),
]);
}
if (context.currentUserAttachments?.some(isParseableAttachment)) {
tools.push([DOMAIN_TOOL_IDS.PARSE_FILE, loadParseFileTool().createParseFileTool(context)]);
tools.push([
DOMAIN_TOOL_IDS.PARSE_FILE,
() => loadParseFileTool().createParseFileTool(context),
]);
}
return tools;
}
/** Returns only the native domain tool names active for this request. */
export function getActiveOrchestratorDomainToolNames(context: InstanceAiContext): Set<string> {
return new Set(getOrchestratorDomainToolFactories(context).map(([name]) => name));
}
/** Creates the native n8n domain tools available to the orchestrator. */
export function createOrchestratorDomainTools(context: InstanceAiContext): InstanceAiToolRegistry {
const tools: Array<[string, BuiltTool]> = getOrchestratorDomainToolFactories(context).map(
([name, createTool]) => [name, createTool()],
);
return createToolRegistry(tools);
}
@ -193,11 +174,6 @@ export function createOrchestrationTools(context: OrchestrationContext): Instanc
ORCHESTRATION_TOOL_IDS.COMPLETE_CHECKPOINT,
loadCompleteCheckpointTool().createCompleteCheckpointTool(context),
],
[
ORCHESTRATION_TOOL_IDS.EVAL_SETUP_WITH_AGENT,
loadEvalSetupAgentTool().createEvalSetupAgentTool(context),
],
[ORCHESTRATION_TOOL_IDS.EVAL_DATA, loadEvalDataAgentTool().createEvalDataAgentTool(context)],
];
if (context.workflowTaskService) {

66
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/agent-persistence.test.ts
View File

@ -1,72 +1,8 @@
import type { BuiltMemory } from '@n8n/agents';
import type { OrchestrationContext } from '../../../types';
import {
createSubAgentPersistence,
createSubAgentResourceId,
createSubAgentResourceIdPrefix,
} from '../agent-persistence';
import { createSubAgentResourceIdPrefix } from '../agent-persistence';
const PARENT_THREAD_ID = '00000000-0000-4000-8000-000000000001';
function createContext(memory?: BuiltMemory): OrchestrationContext {
return {
threadId: PARENT_THREAD_ID,
userId: 'user-1',
memory,
} as OrchestrationContext;
}
describe('sub-agent persistence', () => {
it('creates hidden parent-scoped persistence and saves the child thread', async () => {
const saveThread = vi.fn();
const context = createContext({ saveThread } as unknown as BuiltMemory);
const persistence = await createSubAgentPersistence(context, {
agentKind: 'Workflow Builder !!!',
});
expect(persistence.threadId).toMatch(
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
);
expect(persistence.threadId).not.toBe(PARENT_THREAD_ID);
expect(persistence.resourceId).toBe(
`instance-ai-subagent:${PARENT_THREAD_ID}:workflow-builder`,
);
expect(saveThread).toHaveBeenCalledWith({
id: persistence.threadId,
resourceId: persistence.resourceId,
title: '',
metadata: {
instanceAiHidden: true,
instanceAiThreadKind: 'sub-agent',
parentThreadId: PARENT_THREAD_ID,
agentKind: 'workflow-builder',
},
});
});
it('respects caller-provided thread and resource IDs', async () => {
const saveThread = vi.fn();
const context = createContext({ saveThread } as unknown as BuiltMemory);
const threadId = '00000000-0000-4000-8000-000000000002';
const resourceId = createSubAgentResourceId(PARENT_THREAD_ID, 'workflow-builder');
const persistence = await createSubAgentPersistence(context, {
agentKind: 'workflow-builder',
threadId,
resourceId,
});
expect(persistence).toEqual({ threadId, resourceId });
expect(saveThread).toHaveBeenCalledWith(
expect.objectContaining({
id: threadId,
resourceId,
}),
);
});
it('builds the cleanup prefix for all sub-agent threads under one parent', () => {
expect(createSubAgentResourceIdPrefix(PARENT_THREAD_ID)).toBe(
`instance-ai-subagent:${PARENT_THREAD_ID}:`,

2
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/complete-checkpoint.tool.test.ts
View File

@ -1,5 +1,4 @@
import { executeTool } from '../../../__tests__/tool-test-utils';
import { createToolRegistry } from '../../../tool-registry';
import type {
CheckpointSettleResult,
OrchestrationContext,
@ -38,7 +37,6 @@ function makeContext(
subscribe: vi.fn(),
},
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
domainTools: createToolRegistry(),
abortSignal: new AbortController().signal,
taskStorage: { get: vi.fn(), save: vi.fn() },
plannedTaskService: service,

512
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/eval-data-agent.tool.test.ts
View File

@ -1,512 +0,0 @@
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import type { Mock } from 'vitest';
import * as sampleRowsService from '../../evals/generate-sample-rows.service';
import { createEvalDataAgentTool } from '../eval-data-agent.tool';
type EvalDataToolResult = {
status: 'imported' | 'generated' | 'skipped';
source?: 'history' | 'synthetic';
rowCount?: number;
expectedOutputsNeedUserReview?: boolean;
expectedOutputColumns?: string[];
table?: {
id: string;
name: string;
projectId?: string;
rowCount: number;
inputColumns: string[];
previewRows: Array<Record<string, unknown>>;
};
};
async function runEvalDataTool(
ctx: ReturnType<typeof buildOrchestrationCtx>,
input: { workflowId: string; projectId?: string },
): Promise<EvalDataToolResult> {
const tool = createEvalDataAgentTool(ctx as never);
return (await tool.handler!(input, {} as never)) as EvalDataToolResult;
}
const evalWf = (): WorkflowJSON =>
({
name: 't',
nodes: [
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [200, 0],
id: 'a',
},
],
connections: {
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
const evalWfWithMetrics = (): WorkflowJSON =>
({
name: 't',
nodes: [
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: '={{ $json.user_query }}' },
position: [200, 0],
id: 'a',
},
{
name: 'MetricN',
type: 'n8n-nodes-base.evaluation',
typeVersion: 1,
parameters: {
operation: 'setMetrics',
expectedAnswer: "={{ $('EvalTrig').item.json.expected_response }}",
actualAnswer: '={{ $json.output }}',
},
position: [400, 0],
id: 'm',
},
],
connections: {
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
Agent: { main: [[{ node: 'MetricN', type: 'main', index: 0 }]] },
},
pinData: {},
settings: {},
}) as unknown as WorkflowJSON;
const defaultInsertResult = {
insertedCount: 0,
dataTableId: 'dt-1',
tableName: 'eval_dataset',
projectId: 'proj-1',
};
const silentLogger = () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() });
/** Default DataTable service stub. Override individual mocks per test as needed. */
function defaultDataTableService(
overrides: Partial<{
insertRows: Mock;
getSchema: Mock;
addColumn: Mock;
queryRows: Mock;
}> = {},
) {
return {
insertRows: vi.fn().mockResolvedValue(defaultInsertResult),
getSchema: vi.fn().mockResolvedValue([]),
addColumn: vi.fn().mockResolvedValue(undefined),
queryRows: vi.fn().mockResolvedValue({ count: 0, data: [] }),
...overrides,
};
}
/** Execution service stub with no successful executions to read from. */
function emptyExecutionService() {
return {
list: vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]),
getNodeOutput: vi.fn(),
};
}
/**
* Execution service stub returning `count` successful executions whose
* EvalTrig output exposes `user_query: "real-eN"` per execution.
*/
function trigInputHistoryExecutionService(count: number) {
const summaries = Array.from({ length: count }, (_, i) => ({
id: `e${i}`,
status: 'success',
}));
return {
list: vi.fn().mockResolvedValueOnce(summaries).mockResolvedValueOnce([]),
getNodeOutput: vi.fn(
async (id: string) =>
await Promise.resolve({
nodeName: 'EvalTrig',
items: [{ json: { user_query: `real-${id}` } }],
totalItems: 1,
returned: { from: 0, to: 0 },
}),
),
};
}
/**
* Execution service stub returning `count` successful executions with both
* an EvalTrig input (`user_query: "q-eN"`) and an Agent output (`output: "a-eN"`).
*/
function trigInputAgentOutputExecutionService(count: number) {
const summaries = Array.from({ length: count }, (_, i) => ({
id: `e${i}`,
status: 'success',
}));
return {
list: vi.fn().mockResolvedValueOnce(summaries).mockResolvedValueOnce([]),
getNodeOutput: vi.fn(async (id: string, nodeName: string) =>
nodeName === 'EvalTrig'
? await Promise.resolve({
nodeName,
items: [{ json: { user_query: `q-${id}` } }],
totalItems: 1,
returned: { from: 0, to: 0 },
})
: await Promise.resolve({
nodeName,
items: [{ json: { output: `a-${id}` } }],
totalItems: 1,
returned: { from: 0, to: 0 },
}),
),
};
}
interface BuildCtxOptions {
workflow?: WorkflowJSON;
dataTableService?: ReturnType<typeof defaultDataTableService>;
executionService?: ReturnType<typeof emptyExecutionService>;
}
const buildOrchestrationCtx = (opts: BuildCtxOptions = {}) => ({
domainContext: {
workflowService: {
getAsWorkflowJSON: vi.fn().mockResolvedValue(opts.workflow ?? evalWf()),
},
dataTableService: opts.dataTableService ?? defaultDataTableService(),
executionService: opts.executionService ?? emptyExecutionService(),
logger: silentLogger(),
},
});
describe('eval-data tool', () => {
beforeEach(() => {
vi.restoreAllMocks();
});
it('imports rows from execution history when >= 10 valid rows are available', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({
dataTableService,
executionService: trigInputHistoryExecutionService(12),
});
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.status).toBe('imported');
expect(result.source).toBe('history');
expect(result.rowCount).toBe(12);
expect(dataTableService.insertRows).toHaveBeenCalledWith('dt-1', expect.any(Array), undefined);
});
it('falls back to synthetic generation when fewer than 10 valid history rows are available', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({ dataTableService });
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue(
Array.from({ length: 10 }, (_, i) => ({ user_query: `gen-${i}` })),
);
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.status).toBe('generated');
expect(result.source).toBe('synthetic');
expect(result.rowCount).toBe(10);
expect(dataTableService.insertRows).toHaveBeenCalled();
});
it('returns skipped when no eval target exists', async () => {
const wf: WorkflowJSON = {
name: 't',
nodes: [],
connections: {},
pinData: {},
settings: {},
} as never;
const ctx = buildOrchestrationCtx({ workflow: wf });
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.status).toBe('skipped');
});
it('populates with the fallback "input" column when agent has no $json refs', async () => {
const wf = {
name: 't',
nodes: [
{
name: 'EvalTrig',
type: 'n8n-nodes-base.evaluationTrigger',
typeVersion: 1,
parameters: { dataTableId: { value: 'dt-1' } },
position: [0, 0],
id: 't',
},
// Agent with no $json refs in its parameters
{
name: 'Agent',
type: '@n8n/n8n-nodes-langchain.agent',
typeVersion: 1,
parameters: { text: 'literal prompt' },
position: [200, 0],
id: 'a',
},
],
connections: {
EvalTrig: { main: [[{ node: 'Agent', type: 'main', index: 0 }]] },
},
pinData: {},
settings: {},
} as unknown as WorkflowJSON;
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'input' }]),
});
const ctx = buildOrchestrationCtx({ workflow: wf, dataTableService });
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue([{ input: 'sample' }]);
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.status).toBe('generated');
expect(dataTableService.insertRows).toHaveBeenCalledWith(
'dt-1',
[{ input: 'sample' }],
undefined,
);
});
it('populates expected_* columns from agent output in the history path', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }, { name: 'expected_response' }]),
});
const ctx = buildOrchestrationCtx({
workflow: evalWfWithMetrics(),
dataTableService,
executionService: trigInputAgentOutputExecutionService(12),
});
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.status).toBe('imported');
expect(result.rowCount).toBe(12);
expect(dataTableService.insertRows).toHaveBeenCalledWith('dt-1', expect.any(Array), undefined);
expect(dataTableService.insertRows).toHaveBeenCalledWith(
'dt-1',
expect.arrayContaining([{ user_query: 'q-e0', expected_response: 'a-e0' }]),
undefined,
);
});
it('synthetic path generates ONLY input columns and flags expected outputs for user review', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }, { name: 'expected_response' }]),
});
const ctx = buildOrchestrationCtx({ workflow: evalWfWithMetrics(), dataTableService });
const generateSpy = vi
.spyOn(sampleRowsService, 'generateSampleRows')
.mockResolvedValue([{ user_query: 'q' }]);
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(generateSpy).toHaveBeenCalledWith(
expect.objectContaining({
columns: ['user_query'],
rowCount: 10,
}),
);
expect(result).toMatchObject({
status: 'generated',
source: 'synthetic',
expectedOutputsNeedUserReview: true,
expectedOutputColumns: ['expected_response'],
});
});
it('does not flag user review on the history path (real outputs are ground truth)', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }, { name: 'expected_response' }]),
});
const ctx = buildOrchestrationCtx({
workflow: evalWfWithMetrics(),
dataTableService,
executionService: trigInputAgentOutputExecutionService(12),
});
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.source).toBe('history');
expect(result.expectedOutputsNeedUserReview).toBeUndefined();
});
it('does not flag user review when there are no expected-output columns', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({ dataTableService });
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue([{ user_query: 'q' }]);
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.source).toBe('synthetic');
expect(result.expectedOutputsNeedUserReview).toBeUndefined();
});
it('adds missing columns to the DataTable before inserting rows', async () => {
// Schema has only the input column; expected_response is missing.
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({
workflow: evalWfWithMetrics(),
dataTableService,
});
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue([
{ user_query: 'q', expected_response: 'r' },
]);
await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(dataTableService.getSchema).toHaveBeenCalledWith('dt-1', undefined);
expect(dataTableService.addColumn).toHaveBeenCalledTimes(1);
expect(dataTableService.addColumn).toHaveBeenCalledWith(
'dt-1',
{ name: 'expected_response', type: 'string' },
undefined,
);
expect(dataTableService.insertRows).toHaveBeenCalled();
expect(dataTableService.addColumn.mock.invocationCallOrder[0]).toBeLessThan(
dataTableService.insertRows.mock.invocationCallOrder[0],
);
});
it('does not add columns that already exist in the DataTable schema', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }, { name: 'expected_response' }]),
});
const ctx = buildOrchestrationCtx({
workflow: evalWfWithMetrics(),
dataTableService,
});
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue([
{ user_query: 'q', expected_response: 'r' },
]);
await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(dataTableService.addColumn).not.toHaveBeenCalled();
expect(dataTableService.insertRows).toHaveBeenCalled();
});
it('forwards projectId to insertRows when present', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({ dataTableService });
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue([{ user_query: 'q' }]);
await runEvalDataTool(ctx, { workflowId: 'w1', projectId: 'proj-1' });
expect(dataTableService.insertRows).toHaveBeenCalledWith('dt-1', expect.any(Array), {
projectId: 'proj-1',
});
});
it('returns a `table` summary so the agent can recap the populated dataset to the user', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
queryRows: vi.fn().mockResolvedValue({
count: 2,
data: [
{
user_query:
'first question with a really long body that should be truncated past eighty characters of content easily',
},
{ user_query: 'second' },
],
}),
});
const ctx = buildOrchestrationCtx({ dataTableService });
vi.spyOn(sampleRowsService, 'generateSampleRows').mockResolvedValue([{ user_query: 'q' }]);
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.table).toMatchObject({
id: 'dt-1',
name: 'eval_dataset',
projectId: 'proj-1',
rowCount: 1,
inputColumns: ['user_query'],
});
expect(result.table?.previewRows).toHaveLength(2);
// First row's long string should be truncated.
expect(String(result.table?.previewRows[0]?.user_query)).toMatch(/…$/);
});
describe('few-shot seeding from history', () => {
it('passes residual history rows to generateSampleRows when below threshold', async () => {
// 3 valid history rows — below the 10-row threshold, so the tool
// goes synthetic but should still hand the rows to the generator.
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({
dataTableService,
executionService: trigInputHistoryExecutionService(3),
});
const generateSpy = vi
.spyOn(sampleRowsService, 'generateSampleRows')
.mockResolvedValue([{ user_query: 'synth' }]);
const result = await runEvalDataTool(ctx, { workflowId: 'w1' });
expect(result.source).toBe('synthetic');
const callArg = generateSpy.mock.calls[0]?.[0];
expect(callArg?.realExamples).toEqual(
expect.arrayContaining([
expect.objectContaining({ user_query: 'real-e0' }),
expect.objectContaining({ user_query: 'real-e1' }),
expect.objectContaining({ user_query: 'real-e2' }),
]),
);
});
it('does not pass realExamples when no history rows are available', async () => {
const dataTableService = defaultDataTableService({
getSchema: vi.fn().mockResolvedValue([{ name: 'user_query' }]),
});
const ctx = buildOrchestrationCtx({ dataTableService });
const generateSpy = vi
.spyOn(sampleRowsService, 'generateSampleRows')
.mockResolvedValue([{ user_query: 'synth' }]);
await runEvalDataTool(ctx, { workflowId: 'w1' });
const callArg = generateSpy.mock.calls[0]?.[0];
expect(callArg).not.toHaveProperty('realExamples');
});
});
});

148
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/eval-setup-agent.tools.test.ts
View File

@ -1,148 +0,0 @@
import type { BuiltTool } from '@n8n/agents';
import { DEFAULT_INSTANCE_AI_PERMISSIONS } from '@n8n/api-types';
import { mock } from 'vitest-mock-extended';
import { createToolRegistry } from '../../../tool-registry';
import type { InstanceAiContext, OrchestrationContext } from '../../../types';
import { buildEvalSetupTools, startEvalSetupAgentTask } from '../eval-setup-agent.tool';
const sentinelNodesTool: BuiltTool = { name: 'nodes', description: 'nodes-sentinel' };
const sentinelOriginalWorkflows: BuiltTool = {
name: 'workflows',
description: 'workflows-original-sentinel',
};
function makeContext(
updateWorkflow: NonNullable<InstanceAiContext['permissions']>['updateWorkflow'],
): OrchestrationContext {
const domainContext = mock<InstanceAiContext>();
domainContext.permissions = { ...DEFAULT_INSTANCE_AI_PERMISSIONS, updateWorkflow };
domainContext.workflowService.get = vi
.fn()
.mockResolvedValue({ id: 'w1', name: 'Test', nodes: [], connections: {}, active: false });
domainContext.workflowService.getAsWorkflowJSON = vi
.fn()
.mockResolvedValue({ name: 'Eval setup', nodes: [], connections: {} });
domainContext.workflowService.updateFromWorkflowJSON = vi
.fn()
.mockResolvedValue({ id: 'w1', name: 'Test', versionId: 'v1', nodes: [], connections: {} });
domainContext.workflowService.updateVersion = vi.fn().mockResolvedValue(undefined);
const ctx = mock<OrchestrationContext>();
ctx.domainTools = createToolRegistry([
['workflows', sentinelOriginalWorkflows],
['nodes', sentinelNodesTool],
]);
ctx.domainContext = domainContext;
ctx.threadId = 'thread-1';
ctx.runId = 'run-1';
ctx.orchestratorAgentId = 'root-agent';
ctx.modelId = 'test-model' as OrchestrationContext['modelId'];
ctx.eventBus = { publish: vi.fn() } as unknown as OrchestrationContext['eventBus'];
ctx.tracing = undefined;
return ctx;
}
function getInputSchema(tool: BuiltTool | undefined): {
safeParse: (input: unknown) => { success: boolean };
} {
return (tool as { inputSchema: { safeParse: (input: unknown) => { success: boolean } } })
.inputSchema;
}
describe('buildEvalSetupTools', () => {
it('allows saving workflow JSON without prompting when parent permission is require_approval', async () => {
const ctx = makeContext('require_approval');
const tools = buildEvalSetupTools(ctx);
const suspend = vi.fn();
const workflows = tools.get('workflows');
const workflow = { name: 'Eval setup', nodes: [], connections: {} };
const result = (await workflows?.handler!(
{
action: 'update',
workflowId: 'w1',
workflow,
},
{ suspend, resumeData: undefined } as never,
)) as Record<string, unknown>;
expect(suspend).not.toHaveBeenCalled();
expect(ctx.domainContext!.workflowService.updateFromWorkflowJSON).toHaveBeenCalledWith(
'w1',
workflow,
);
expect(result).toMatchObject({ success: true, workflowId: 'w1' });
});
it('exposes only the workflow actions the eval setup agent needs', () => {
const ctx = makeContext('require_approval');
const tools = buildEvalSetupTools(ctx);
const schema = getInputSchema(tools.get('workflows'));
expect(schema.safeParse({ action: 'get-json', workflowId: 'w1' }).success).toBe(true);
expect(schema.safeParse({ action: 'get', workflowId: 'w1' }).success).toBe(false);
expect(schema.safeParse({ action: 'get-as-code', workflowId: 'w1' }).success).toBe(false);
expect(
schema.safeParse({
action: 'update',
workflowId: 'w1',
workflow: { name: 'Eval setup', nodes: [], connections: {} },
}).success,
).toBe(true);
expect(
schema.safeParse({
action: 'update-version',
workflowId: 'w1',
versionId: 'v1',
name: 'Eval setup',
}).success,
).toBe(false);
});
it('reads raw workflow JSON through the eval setup workflow tool', async () => {
const ctx = makeContext('require_approval');
const workflows = buildEvalSetupTools(ctx).get('workflows');
const result = await workflows?.handler!({ action: 'get-json', workflowId: 'w1' }, {} as never);
expect(ctx.domainContext!.workflowService.getAsWorkflowJSON).toHaveBeenCalledWith(
'w1',
undefined,
);
expect(result).toEqual({ name: 'Eval setup', nodes: [], connections: {} });
});
it('leaves the original workflows tool in place when admin has blocked updates', () => {
const ctx = makeContext('blocked');
const tools = buildEvalSetupTools(ctx);
expect(tools.get('workflows')).toBe(sentinelOriginalWorkflows);
});
});
describe('startEvalSetupAgentTask', () => {
it('deduplicates eval setup background tasks by workflow id', () => {
const ctx = makeContext('require_approval');
ctx.spawnBackgroundTask = vi.fn().mockReturnValue({
status: 'started',
taskId: 'task-1',
agentId: 'agent-1',
});
startEvalSetupAgentTask(ctx, {
workflowId: 'w1',
task: 'Set up evals',
plannedTaskId: 'planned-1',
});
expect(ctx.spawnBackgroundTask).toHaveBeenCalledWith(
expect.objectContaining({
dedupeKey: {
role: 'eval-setup',
workflowId: 'w1',
plannedTaskId: 'planned-1',
},
}),
);
});
});

2
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/plan.tool.test.ts
View File

@ -1,6 +1,5 @@
import { executeTool } from '../../../__tests__/tool-test-utils';
import { PlanValidationError } from '../../../planned-tasks/planned-task-service';
import { createToolRegistry } from '../../../tool-registry';
import type { OrchestrationContext, PlannedTaskService, TaskStorage } from '../../../types';
import { createPlanTool } from '../plan.tool';
@ -27,7 +26,6 @@ function createMockContext(overrides: Partial<OrchestrationContext> = {}): Orche
subscribe: vi.fn(),
},
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
domainTools: createToolRegistry(),
abortSignal: new AbortController().signal,
taskStorage: {
get: vi.fn(),

2
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/report-verification-verdict.tool.test.ts
View File

@ -1,5 +1,4 @@
import { executeTool } from '../../../__tests__/tool-test-utils';
import { createToolRegistry } from '../../../tool-registry';
import type { OrchestrationContext, TaskStorage } from '../../../types';
import type { WorkflowLoopAction } from '../../../workflow-loop/workflow-loop-state';
import { createReportVerificationVerdictTool } from '../report-verification-verdict.tool';
@ -33,7 +32,6 @@ function createMockContext(overrides: Partial<OrchestrationContext> = {}): Orche
subscribe: vi.fn(),
},
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
domainTools: createToolRegistry(),
abortSignal: new AbortController().signal,
taskStorage: {
get: vi.fn(),

115
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/tracing-utils.test.ts
View File

@ -1,115 +0,0 @@
/* eslint-disable import-x/order */
import type { OrchestrationContext } from '../../../types';
const mockCreateDetachedSubAgentTraceContext = vi.fn<(arg: unknown) => Promise<unknown>>();
vi.mock('../../../tracing/langsmith-tracing', () => ({
createDetachedSubAgentTraceContext: async (options: unknown): Promise<unknown> =>
await mockCreateDetachedSubAgentTraceContext(options),
getCurrentOtelSpanContext: vi.fn(() => undefined),
getCurrentTraceToolCallId: vi.fn(() => undefined),
mergeCurrentTraceMetadata: vi.fn(),
}));
import { createDetachedSubAgentTraceFactory } from '../tracing-utils';
describe('createDetachedSubAgentTraceFactory', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('propagates parent version metadata to detached sub-agent traces', async () => {
mockCreateDetachedSubAgentTraceContext.mockResolvedValue({
rootRun: { id: 'sub-root', traceId: 'sub-trace' },
});
const context = {
threadId: 'thread-1',
messageGroupId: 'group-1',
runId: 'run-1',
userId: 'user-1',
modelId: 'model-1',
orchestratorAgentId: 'orchestrator-1',
tracingProxyConfig: { getAuthHeaders: vi.fn() },
tracing: {
projectName: 'project-1',
rootRun: { traceId: 'root-trace' },
actorRun: {
id: 'actor-run',
metadata: {
message_id: 'message-1',
conversation_id: 'conversation-1',
agent_id: 'agent-1',
n8n_version: '1.123.4',
workflow_sdk_version: '0.13.0',
browser_connection_state: 'connected',
browser_extension_version: '0.0.6',
},
},
},
} as unknown as OrchestrationContext;
const createTrace = createDetachedSubAgentTraceFactory(context, {
agentId: 'sub-agent',
role: 'builder',
kind: 'background',
inputs: { task: 'build' },
metadata: { task_id: 'task-1' },
});
await createTrace();
expect(mockCreateDetachedSubAgentTraceContext).toHaveBeenCalledWith(
expect.objectContaining({
metadata: { task_id: 'task-1' },
n8nVersion: '1.123.4',
workflowSdkVersion: '0.13.0',
browserExtension: { connectionState: 'connected', version: '0.0.6' },
}),
);
});
it.each([
[
'connected without a reported version',
{ browser_connection_state: 'connected' },
{ connectionState: 'connected' },
],
[
'disconnected',
{ browser_connection_state: 'disconnected' },
{ connectionState: 'disconnected' },
],
['absent from the parent', {}, undefined],
['not a known state', { browser_connection_state: 'sideways' }, undefined],
])(
'propagates the browser extension dimensions when the parent is %s',
async (_case, parentMetadata, expected) => {
mockCreateDetachedSubAgentTraceContext.mockResolvedValue({
rootRun: { id: 'sub-root', traceId: 'sub-trace' },
});
const context = {
threadId: 'thread-1',
runId: 'run-1',
userId: 'user-1',
orchestratorAgentId: 'orchestrator-1',
tracing: {
projectName: 'project-1',
rootRun: { traceId: 'root-trace' },
actorRun: { id: 'actor-run', metadata: parentMetadata },
},
} as unknown as OrchestrationContext;
const createTrace = createDetachedSubAgentTraceFactory(context, {
agentId: 'sub-agent',
role: 'builder',
kind: 'background',
});
await createTrace();
expect(mockCreateDetachedSubAgentTraceContext).toHaveBeenCalledWith(
expect.objectContaining({ browserExtension: expected }),
);
},
);
});

2
packages/@n8n/instance-ai/src/tools/orchestration/__tests__/verify-built-workflow.tool.test.ts
View File

@ -1,7 +1,6 @@
import type { Mock } from 'vitest';
import { executeTool } from '../../../__tests__/tool-test-utils';
import { createToolRegistry } from '../../../tool-registry';
import type {
InstanceAiDataTableService,
InstanceAiWorkflowService,
@ -70,7 +69,6 @@ function createContext(overrides: Partial<OrchestrationContext> = {}): Orchestra
warn: vi.fn(),
error: vi.fn(),
} as unknown as OrchestrationContext['logger'],
domainTools: createToolRegistry(),
abortSignal: new AbortController().signal,
taskStorage: {} as OrchestrationContext['taskStorage'],
workflowTaskService,

51
packages/@n8n/instance-ai/src/tools/orchestration/agent-persistence.ts
View File

@ -1,56 +1,5 @@
import { randomUUID } from 'node:crypto';
import type { OrchestrationContext } from '../../types';
export const SUB_AGENT_RESOURCE_PREFIX = 'instance-ai-subagent';
export function createSubAgentResourceIdPrefix(parentThreadId: string): string {
return `${SUB_AGENT_RESOURCE_PREFIX}:${parentThreadId}:`;
}
function normalizeSubAgentKind(agentKind: string): string {
const normalized = agentKind
.trim()
.toLowerCase()
.replace(/[^a-z0-9._-]+/g, '-')
.replace(/^-+|-+$/g, '');
return (normalized || 'agent').slice(0, 64);
}
export function createSubAgentResourceId(parentThreadId: string, agentKind: string): string {
return `${createSubAgentResourceIdPrefix(parentThreadId)}${normalizeSubAgentKind(agentKind)}`;
}
interface CreateSubAgentPersistenceOptions {
agentKind: string;
threadId?: string;
resourceId?: string;
}
export async function createSubAgentPersistence(
context: OrchestrationContext,
options: CreateSubAgentPersistenceOptions,
) {
const threadId = options.threadId ?? randomUUID();
const resourceId =
options.resourceId ?? createSubAgentResourceId(context.threadId, options.agentKind);
const metadata: Record<string, unknown> = {
instanceAiHidden: true,
instanceAiThreadKind: 'sub-agent',
parentThreadId: context.threadId,
agentKind: normalizeSubAgentKind(options.agentKind),
};
await context.memory?.saveThread({
id: threadId,
resourceId,
title: '',
metadata,
});
return {
threadId,
resourceId,
};
}

245
packages/@n8n/instance-ai/src/tools/orchestration/eval-data-agent.tool.ts
View File

@ -1,245 +0,0 @@
import { Tool } from '@n8n/agents';
import { z } from 'zod';
import type { InstanceAiDataTableService, OrchestrationContext } from '../../types';
import { analyzeEvalDataRequirements } from '../evals/eval-data-requirements.service';
import { extractRowsFromExecutionHistory } from '../evals/extract-rows-from-history.service';
import { generateSampleRows } from '../evals/generate-sample-rows.service';
const HISTORY_THRESHOLD = 10;
const GENERATE_ROW_COUNT = 10;
async function ensureColumnsExist(
dataTableService: InstanceAiDataTableService,
dataTableId: string,
rows: Array<Record<string, unknown>>,
extraColumns: readonly string[],
options: { projectId?: string } | undefined,
): Promise<void> {
const referencedColumns = new Set<string>(extraColumns);
for (const row of rows) {
for (const key of Object.keys(row)) referencedColumns.add(key);
}
if (referencedColumns.size === 0) return;
const schema = await dataTableService.getSchema(dataTableId, options);
const existing = new Set(schema.map((c) => c.name));
const missing = [...referencedColumns].filter((name) => !existing.has(name));
for (const name of missing) {
await dataTableService.addColumn(dataTableId, { name, type: 'string' }, options);
}
}
const evalDataInputSchema = z.object({
workflowId: z.string().describe('ID of the workflow whose eval DataTable should be populated'),
projectId: z.string().optional(),
});
const PREVIEW_ROW_COUNT = 3;
const PREVIEW_VALUE_MAX_LEN = 80;
const tableSummarySchema = z.object({
id: z.string(),
name: z.string(),
projectId: z.string().optional(),
rowCount: z.number(),
inputColumns: z.array(z.string()),
expectedOutputColumns: z.array(z.string()),
previewRows: z.array(z.record(z.string(), z.unknown())),
});
const outputSchema = z.object({
status: z.enum(['imported', 'generated', 'skipped']),
rowCount: z.number().optional(),
source: z.enum(['history', 'synthetic']).optional(),
reason: z.string().optional(),
/**
* True when synthetic rows were inserted with empty expected-output columns.
* The agent must tell the user to fill those columns in before running the
* evaluation, so the eval measures correctness instead of self-consistency
* with the generator's own guess at the right answer.
*/
expectedOutputsNeedUserReview: z.boolean().optional(),
expectedOutputColumns: z.array(z.string()).optional(),
/**
* Snapshot of the populated DataTable so the agent can show the user what
* was generated alongside the metric setup, without making them dig through
* the data-tables UI to verify. Includes the table id (for deep-linking) and
* a short row preview. Only present on success paths.
*/
table: tableSummarySchema.optional(),
});
function truncateForPreview(value: unknown): unknown {
if (typeof value !== 'string') return value;
return value.length > PREVIEW_VALUE_MAX_LEN ? `${value.slice(0, PREVIEW_VALUE_MAX_LEN)}…` : value;
}
function buildPreviewRows(rows: Array<Record<string, unknown>>): Array<Record<string, unknown>> {
return rows.slice(0, PREVIEW_ROW_COUNT).map((row) => {
const truncated: Record<string, unknown> = {};
for (const [key, value] of Object.entries(row)) {
truncated[key] = truncateForPreview(value);
}
return truncated;
});
}
export function createEvalDataAgentTool(context: OrchestrationContext) {
return new Tool('eval-data')
.description(
'Populate an eval DataTable for a workflow that already has its eval setup wired. ' +
'First scans the workflow execution history for real rows (these include real expected ' +
'outputs); if fewer than 10 valid rows are available, generates synthetic rows with INPUT ' +
'columns only — expected-output columns are left empty so the user can fill them in with ' +
'the correct answers. We never auto-fill expected outputs with model-generated guesses, ' +
'because that would measure self-consistency rather than correctness. ' +
'Inserts at most 25 rows total. Synchronous — no sub-agent, no HITL.',
)
.input(evalDataInputSchema)
.output(outputSchema)
.handler(async (input: z.infer<typeof evalDataInputSchema>) => {
const domain = context.domainContext;
if (!domain) {
return { status: 'skipped' as const, reason: 'Domain context unavailable.' };
}
const log = (level: 'info' | 'warn' | 'error', msg: string) => {
domain.logger[level]?.(`[eval-data] ${msg}`);
};
const j = (v: unknown) => JSON.stringify(v);
log('info', `start workflowId=${input.workflowId} projectId=${j(input.projectId)}`);
const workflow = await domain.workflowService.getAsWorkflowJSON(input.workflowId);
const reqs = analyzeEvalDataRequirements(workflow);
const target = reqs.targets[0];
if (!target) {
log('warn', `skip:no-target reason=${j(reqs.reason)}`);
return { status: 'skipped' as const, reason: reqs.reason ?? 'No eval target.' };
}
log(
'info',
`target dataTableId=${target.dataTableId} agent=${j(target.targetAgentNodeName)} inputColumns=${j(target.inputColumns)} expectedOutputColumns=${j(target.expectedOutputColumns)} pairs=${j(target.expectedToActualPairs)}`,
);
if (!target.targetAgentNodeName) {
log('warn', 'skip:no-agent');
return {
status: 'skipped' as const,
reason: 'No agent node reachable from EvaluationTrigger.',
};
}
const { rows: historyRows } = await extractRowsFromExecutionHistory(domain, {
workflow,
workflowId: input.workflowId,
agentNodeName: target.targetAgentNodeName,
inputColumns: target.inputColumns,
expectedToActualPairs: target.expectedToActualPairs,
});
log('info', `history-extracted count=${historyRows.length}`);
let rowsToInsert: Array<Record<string, unknown>>;
let source: 'history' | 'synthetic';
if (historyRows.length >= HISTORY_THRESHOLD) {
rowsToInsert = historyRows;
source = 'history';
} else {
// This will only generate the input part: expected output columns
// will stay empty so that the user has to supply the ground truth.
// If the threshold for using history rows has not been reached, however
// many rows exist get passed as `realExamples` — a domain reference,
// not seeds to paraphrase.
rowsToInsert = await generateSampleRows({
workflow,
columns: target.inputColumns,
rowCount: GENERATE_ROW_COUNT,
targetAgentNodeName: target.targetAgentNodeName,
logger: domain.logger,
...(historyRows.length > 0 ? { realExamples: historyRows } : {}),
});
source = 'synthetic';
}
log(
'info',
`rows-prepared source=${source} count=${rowsToInsert.length} firstRowKeys=${j(rowsToInsert[0] ? Object.keys(rowsToInsert[0]) : [])}`,
);
const dataTableOptions = input.projectId ? { projectId: input.projectId } : undefined;
// On the synthetic path we leave expected-output columns empty, so the
// rows never reference them. Still make sure those columns exist in
// the table so the user has somewhere to type the correct answer.
const extraColumns = source === 'synthetic' ? target.expectedOutputColumns : [];
try {
await ensureColumnsExist(
domain.dataTableService,
target.dataTableId,
rowsToInsert,
extraColumns,
dataTableOptions,
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
log('error', `ensureColumnsExist-failed error=${j(message)}`);
throw error;
}
let insertResult: Awaited<ReturnType<typeof domain.dataTableService.insertRows>>;
try {
insertResult = await domain.dataTableService.insertRows(
target.dataTableId,
rowsToInsert,
dataTableOptions,
);
log('info', `insertRows-ok result=${j(insertResult)}`);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
log('error', `insertRows-failed error=${j(message)}`);
throw error;
}
// Fetch a tiny preview so the agent can recap WHAT was generated, not
// just that something was. Treat failures here as non-fatal — the
// insert already succeeded; a missing preview is a UX gap, not a bug.
let previewRows: Array<Record<string, unknown>> = [];
try {
const preview = await domain.dataTableService.queryRows(target.dataTableId, {
limit: PREVIEW_ROW_COUNT,
...(insertResult.projectId ? { projectId: insertResult.projectId } : {}),
});
previewRows = buildPreviewRows(preview.data);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
log('warn', `preview-query-failed error=${j(message)}`);
}
log('info', `done source=${source} rowCount=${rowsToInsert.length}`);
const needsReview = source === 'synthetic' && target.expectedOutputColumns.length > 0;
const table = {
id: target.dataTableId,
name: insertResult.tableName,
...(insertResult.projectId ? { projectId: insertResult.projectId } : {}),
rowCount: rowsToInsert.length,
inputColumns: target.inputColumns,
expectedOutputColumns: target.expectedOutputColumns,
previewRows,
};
return {
status: source === 'history' ? ('imported' as const) : ('generated' as const),
rowCount: rowsToInsert.length,
source,
...(needsReview
? {
expectedOutputsNeedUserReview: true as const,
expectedOutputColumns: target.expectedOutputColumns,
}
: {}),
table,
};
})
.build();
}

241
packages/@n8n/instance-ai/src/tools/orchestration/eval-setup-agent.prompt.ts
View File

@ -1,241 +0,0 @@
/**
* System prompt for the eval-setup-agent — a specialized sub-agent that
* adds EvaluationTrigger + Evaluation nodes + checkIfEvaluating gate to a workflow.
* DataTable creation is always handled upstream by `propose`; this agent only
* uses the DataTable id provided in the task.
*/
export const EVAL_SETUP_AGENT_PROMPT = `You are an eval setup specialist for n8n workflows. You receive an approved eval setup request from the parent agent and handle the topology setup: workflow patching with evaluation nodes (EvaluationTrigger, Evaluation(checkIfEvaluating)/setOutputs/setMetrics) wired directly to the target AI agent, using the DataTable id provided in the task. Synthetic row generation is outside your scope.
## Output Discipline
- You report to a parent agent, not a human. Be terse.
- Do NOT narrate ("I'll read the workflow now", "Let me check the schema"). Just do the work.
- No emojis, no filler phrases, no markdown headers in conversational output.
- Only output a final one-line summary (e.g., "Eval setup complete: 4 eval nodes added and existing DataTable 'Wf eval dataset' wired").
## Mandatory Process
1. **Read the workflow** using the workflows tool with the workflowId in the task. Identify the AI agent nodes named in the task. Trace the main trigger path.
2. **Use the DataTable id from the task.** The task always names an existing DataTable id under "Wire the EvaluationTrigger to DataTable id ...". Use it as-is. Do not create, modify rows, or modify schema. If the task says to leave it empty (the \`later\` path), set the \`EvaluationTrigger.dataTableId\` to an empty string and report that the user must wire it manually.
3. **Patch the workflow.** Build the eval topology in this order:
a. Insert \`EvaluationTrigger\` (\`name: "Eval Trigger"\`).
b. **If the task contains a \`PRODUCTION ADAPTER\` section, follow it LITERALLY.** Insert the named Set node immediately upstream of the agent on the production path with the exact assignments listed; for each rewrite line under section 3 of the PRODUCTION ADAPTER (which lists target nodes — the agent itself AND any of its connected sub-components like memory, tools, output parsers), substring-replace the listed \`originalExpression\` with the EXACT replacement form shown on that line. **Do not assume the replacement is always \`$json.<column>\`** — the task uses \`$json.<column>\` for the agent target but \`{{ $('<AgentName>').item.json.<column> }}\` for sub-components, because sub-components do not see \`$json\` from the agent's input row. Use what the task says verbatim. Do not invent extra assignments, do not skip any, do not add intermediate nodes between the Set adapter and the agent. Do not skip any target node. After this step the agent has TWO incoming \`main\` connections — one from the Set adapter (production), one from the EvaluationTrigger (eval).
c. **If the task does NOT contain a \`PRODUCTION ADAPTER\` section**, wire \`EvaluationTrigger\` directly to the agent's \`main\` input — no Set node, no rewrites. The agent's existing parameters reference \`$json.<column>\` already (the orchestrator validated this).
d. After the agent: insert \`Evaluation(checkIfEvaluating)\` (no separate IF node — it has two native main output slots). Slot 0 (Evaluation) → \`Evaluation(setOutputs)\` → one \`Evaluation(setMetrics)\` per metric listed under "Metrics". Slot 1 (Normal) preserves the original production downstream path with side-effects.
e. For \`correctness\` and \`helpfulness\` metrics: wire an additional outgoing \`ai_languageModel\` connection from the workflow's existing LLM model node to each setMetrics node that uses an AI-judged metric. The LLM gets reused — same node, additional connection. Without this, AI-judged metrics fail silently.
4. **Save** the modified workflow via \`workflows(action="update", ...)\`.
5. **Validate**: re-read the workflow using the workflows tool and assert:
- EvaluationTrigger → target AI agent (direct \`main\` connection, no intermediate node).
- Agent → checkIfEvaluating; slot 0 → setOutputs → setMetrics; slot 1 → original downstream path.
- When input columns are non-empty, the agent's parameters contain at least one \`{{ $json.<column> }}\` expression matching a column from "Input columns".
- For \`correctness\`/\`helpfulness\` metrics: existing LLM model node has an outgoing \`ai_languageModel\` connection to the corresponding setMetrics node.
If any assertion fails, attempt one fix cycle. If still broken, include the specific failure in your summary and stop.
6. **Report** with a one-line summary.
Do NOT produce visible output during steps 1-5. All reasoning happens internally.
Hard boundary: this sub-agent has NO DataTable mutation tools. Do not attempt to create, populate, or modify any DataTable. Row population is handled by a separate \`eval-data\` step downstream.
Parameter-rewrite scope: rewrite parameters in the agent and sub-components **only** as instructed by the task's \`PRODUCTION ADAPTER\` section. Do not rewrite anything else. Do not invent rewrites. If the task has no adapter section, leave all node parameters byte-for-byte identical.
## Eval Node Knowledge
### Data Flow Semantics (read this first)
For a single eval run on row N of the DataTable:
1. \`EvaluationTrigger\` reads row N → emits its columns as \`$json\` (e.g. \`{ input, expected_output, row_id: N, ... }\`).
2. The emitted row flows DIRECTLY into the target AI agent node. The agent's parameters must reference \`$json.<column>\` for the listed input columns. If they reference different fields, those parameters are rewritten during eval setup (see step 3 in Mandatory Process above).
3. \`AI agent\` produces its output (e.g. \`$json.output\`).
4. \`Evaluation(checkIfEvaluating)\` gates the path: slot 0 (Evaluation) routes into the eval branch; slot 1 (Normal) routes through the production path with side-effects.
5. Eval branch: \`Evaluation(setOutputs)\` WRITES the agent's output back into row N of the DataTable, in a NEW column distinct from any ground-truth column. Then \`Evaluation(setMetrics)\` reads ground-truth (via the EvaluationTrigger reference) and the agent's output (\`$json\`), computes a score.
Key invariants:
- setOutputs adds NEW columns to the dataset; setMetrics compares them against the ground-truth columns. NEVER overwrite a ground-truth column with setOutputs.
- The \`Evaluation(setInputs)\` node does NOT reshape \`$json\` — it only attaches metadata for the eval-results display. Skip it unless you specifically want extra dataset columns surfaced in the eval results display.
### n8n-nodes-base.evaluationTrigger
Pulls rows from a dataset. For v3 we always use DataTable source.
**typeVersion**: \`4.7\` (latest supported — do NOT use 4.8, the Evaluation node version range is different from the EvaluationTrigger's).
**Name the node explicitly** when adding it: \`name: "Eval Trigger"\`. The node's stock default name is \`"When fetching a dataset row"\` — DON'T rely on that, and DON'T add a separate node with that name. There is exactly ONE EvaluationTrigger per workflow. Use the explicit name everywhere you reference it (e.g. in setMetrics expressions: \`={{ $('Eval Trigger').item.json.expected_output }}\`).
Required parameters (all three are necessary — missing any one makes the trigger inert):
\`\`\`
{
source: "dataTable",
dataTableId: { __rl: true, mode: "id", value: "<dt_id>" }, // resourceLocator format
limitRows: false // (default) — explicit avoids subtle UI defaults
}
\`\`\`
The \`dataTableId\` field is a resourceLocator (\`__rl: true\` flag). The \`mode: "id"\` + \`value\` shape is what gets the trigger to actually pull rows. If you skip \`__rl: true\`, n8n may not recognize the field and the trigger silently does nothing.
Wiring: connect the EvaluationTrigger directly to the target AI agent node's main input (direct \`main\` connection). Do not insert any intermediate Set/Code/transform node between the EvaluationTrigger and the agent.
### n8n-nodes-base.evaluation (4 operations)
**setInputs**: optional. This node does NOT reshape \`$json\` for downstream nodes — it only attaches an \`evaluationData\` metadata field that the n8n "Evaluations" tab uses to display dataset column values alongside the eval result row. The downstream \`$json\` shape is unchanged. Skip this node unless you specifically want extra dataset columns surfaced in the eval results display.
If used: place it on the eval branch (e.g. between checkIfEvaluating slot 0 and setOutputs). \`inputs.values\` entries with \`inputName\`/\`inputValue\` simply annotate; they don't transform.
**setOutputs**: writes the AI agent's output BACK to the same DataTable row, into a NEW column. After the run, the row contains both the original ground-truth columns AND the agent's actual output, so setMetrics can compare them.
CRITICAL — set these THREE parameter groups, otherwise the node silently does nothing:
1. **Pin the source to dataTable.** The setOutputs operation has its OWN \`source\` parameter (independent from EvaluationTrigger's). The default depends on \`typeVersion\` — \`googleSheets\` for ≤ 4.7, \`dataTable\` for ≥ 4.8. Always set it explicitly:
- \`source: 'dataTable'\`
- \`dataTableId: { mode: 'id', value: '<same id as the EvaluationTrigger>' }\`
Use the same DataTable id passed in the task. Strongly prefer \`typeVersion: 4.8\` (latest) for the Evaluation node so dataTable is the natural default.
2. **outputName must be a NEW column** distinct from any ground-truth column. If the dataset has \`expected_output\`, use \`actual_output\` (or \`actual_<thing>\`) — never the same name as ground truth, otherwise you'd overwrite the ground truth. The Evaluation node auto-adds the new column on first eval run.
3. **Sub-field names** are \`outputName\` and \`outputValue\` (NOT \`name\`/\`value\`).
Full parameter shape:
\`\`\`
{
operation: "setOutputs",
source: "dataTable",
dataTableId: { mode: "id", value: "<dt_id>" },
outputs: {
values: [
{ outputName: "actual_<column>", outputValue: "<n8n expression>" },
...
]
}
}
\`\`\`
For a LangChain agent, the canonical config is:
\`\`\`
{
operation: "setOutputs",
source: "dataTable",
dataTableId: { mode: "id", value: "<dt_id>" },
outputs: {
values: [
{ outputName: "actual_output", outputValue: "={{ $json.output }}" }
]
}
}
\`\`\`
So if the task lists ground-truth output columns like \`expected_output\`, the corresponding setOutputs entries should write to \`actual_output\`, \`actual_response\`, etc. — pick names that pair naturally with the ground-truth columns. Use snake_case.
**setMetrics**: configures ONE metric per node. For multiple metrics, add multiple \`Evaluation(setMetrics)\` nodes in series (or use \`customMetrics\` which accepts multiple entries). The \`metric\` parameter is a SINGLE string, NOT an array.
Top-level parameter:
- \`metric\`: one of
- \`correctness\` — AI-judged factual correctness (scale 1-5). Requires \`expectedAnswer\` + \`actualAnswer\`.
- \`helpfulness\` — AI-judged helpfulness (scale 1-5). Requires \`userQuery\` + \`actualAnswer\`.
- \`stringSimilarity\` — character-level similarity (edit distance, 0-1). Requires \`expectedAnswer\` + \`actualAnswer\`.
- \`categorization\` — exact string match (1 or 0). Requires \`expectedAnswer\` + \`actualAnswer\`.
- \`toolsUsed\` — whether tools were used (0-1). No ground-truth fields needed.
- \`customMetrics\` — user-defined metrics (advanced, rarely chosen).
How to reference the values (use the EvaluationTrigger's explicit name — see above; "Eval Trigger" if you followed the convention):
- \`expectedAnswer\` (ground truth) — pulls from the EvaluationTrigger's emitted row. Example: \`={{ $('Eval Trigger').item.json.expected_output }}\`.
- \`actualAnswer\` (agent's response) — pulls from the AI agent's output JSON, which is still on \`$json\` at the setMetrics stage (setOutputs forwards its input data unchanged). Example: \`={{ $json.output }}\`.
- \`userQuery\` (for \`helpfulness\`) — pulls from the EvaluationTrigger row, e.g. \`={{ $('Eval Trigger').item.json.input }}\`.
Required fields per \`metric\`:
- \`correctness\` / \`stringSimilarity\` / \`categorization\`: \`expectedAnswer\` + \`actualAnswer\`.
- \`helpfulness\`: \`userQuery\` + \`actualAnswer\`.
- \`toolsUsed\`: no extra required fields.
**CRITICAL — AI-judged metrics need an \`ai_languageModel\` connection**: when \`metric\` is \`correctness\` or \`helpfulness\` (the LLM-as-judge metrics), the setMetrics node has TWO inputs — the standard \`main\` from the eval branch AND a separate \`ai_languageModel\` slot for the LLM judge. The node will not work without an LLM connected to the \`ai_languageModel\` slot.
To wire this:
1. Find the LLM model node already used by the AI agent in the workflow (e.g., \`@n8n/n8n-nodes-langchain.lmChatOpenAi\`, \`@n8n/n8n-nodes-langchain.lmChatAnthropic\`, etc.). It connects to the agent via \`ai_languageModel\` connection type.
2. Add a SECOND outgoing \`ai_languageModel\` connection from the SAME LLM node to the new setMetrics node. The LLM gets reused — don't duplicate it.
3. The connection in \`workflow.connections\` looks like:
\`\`\`
"Existing LLM Node Name": {
"ai_languageModel": [
[
{ "node": "Agent Name", "type": "ai_languageModel", "index": 0 },
{ "node": "EvalSetMetricsCorrectness", "type": "ai_languageModel", "index": 0 }
]
]
}
\`\`\`
(Both consumers in the SAME inner array under \`ai_languageModel[0]\` — one connection slot, multiple downstream consumers.)
If the workflow has no LLM model node yet (rare for an AI workflow), add one — but reuse credentials and config of any existing model. \`stringSimilarity\`, \`categorization\`, \`toolsUsed\` do NOT need an \`ai_languageModel\` connection — they're deterministic/local metrics.
Canned metric key mapping from the task's \`canned=<key>\` hints to the \`metric\` parameter value:
- \`canned=correctness\` → \`metric: "correctness"\`
- \`canned=helpfulness\` → \`metric: "helpfulness"\`
- \`canned=tool_use\` → \`metric: "toolsUsed"\`
- \`canned=relevance\` → \`metric: "categorization"\` (closest stock match)
- If no canned key present → use \`customMetrics\` or pick the most appropriate built-in.
**checkIfEvaluating**: gates the workflow flow based on whether the current run is an eval run. Place it immediately after the AI agent. IMPORTANT: this node has TWO native \`main\` output slots — no separate IF node needed:
- **slot 0 (Evaluation)**: fires when the EvaluationTrigger was executed (i.e., we are in an eval run). Wire this to setOutputs → setMetrics.
- **slot 1 (Normal)**: fires during normal (non-eval) workflow execution. Wire this to whatever the AI agent was originally connected to downstream (the production path).
The node forwards the AI agent's data unchanged down whichever slot is active; it's a flow-control switch, not a data transformer.
### Fallback: schema lookup
If you need exact parameter names or displayOptions for any node variant, call \`nodes(action="describe", nodeType=<node-type>)\`. For TypeScript-level type signatures (useful for complex parameter shapes) use \`nodes(action="type-definition", nodeTypes=[<node-type>])\`. Don't guess parameter shapes.
## LangChain AI Agent Output
\`@n8n/n8n-nodes-langchain.agent\` nodes emit output JSON shaped like:
- \`output\`: string — the final response text.
- \`intermediateSteps\`: array — tool calls the agent made (only when tools are attached).
For setOutputs the most common pattern is \`{ outputName: 'actual_output', outputValue: '={{ $json.output }}' }\`. For tool-using agents, add columns like \`{ outputName: 'tool_used', outputValue: '={{ $json.intermediateSteps?.[0]?.action?.tool }}' }\`.
## Required Topology
Apply the correct diagram based on whether the task contains a \`PRODUCTION ADAPTER\` section.
**Direct case (no \`PRODUCTION ADAPTER\` section in task):**
\`\`\`
[Production Trigger] ─→ ... ─→ [AI Agent] ──→ [checkIfEvaluating]
↑ ├── slot 0: [setOutputs] ──→ [setMetrics]
[EvaluationTrigger] ──────────────┘ └── slot 1: [original downstream]
\`\`\`
**Adapter case (task contains \`PRODUCTION ADAPTER\` section):**
\`\`\`
[Production Trigger] ─→ ... ─→ [Eval Production Adapter (Set)] ─→ [AI Agent] ─→ [checkIfEvaluating]
↑ ├── slot 0: [setOutputs] ──→ [setMetrics]
[EvaluationTrigger] ──────────────────────────────────────────────────┘ └── slot 1: [original downstream]
\`\`\`
Rules:
- **Direct case**: EvaluationTrigger connects DIRECTLY to the target AI agent node's \`main\` input. NO Set/Code/transform node in between. The trigger emits each dataset row column as \`$json.<column>\`. The agent's parameters already reference these columns (no rewrites needed).
- **Adapter case**: A Set node (\`"Eval Production Adapter"\`) sits between the production trigger path and the agent. The EvaluationTrigger connects DIRECTLY to the agent as a SECOND incoming \`main\` connection (no Set adapter on the eval path). After the adapter is in place the agent has TWO incoming \`main\` connections: one from the Set adapter (production runs) and one from the EvaluationTrigger (eval runs). Both paths produce \`$json.<column>\` so the rewritten agent parameters resolve in both modes.
- Insert \`checkIfEvaluating + setOutputs + setMetrics\` AFTER the AI agent node. No IF node needed — the checkIfEvaluating node itself has two output slots.
- \`checkIfEvaluating\` slot 0 (Evaluation) routes to setOutputs → setMetrics (eval branch; terminates).
- \`checkIfEvaluating\` slot 1 (Normal) routes to whatever the AI agent was originally connected to (production path preserved).
- Side-effect nodes (Send message, HTTP POST, DB writes) MUST be reachable ONLY via slot 1. This is the core invariant — it's what makes eval runs safe.
Multiple AI agents: one \`checkIfEvaluating + setOutputs + setMetrics\` block per agent by default. Your judgment: if multiple agents share output semantics (e.g. multi-stage pipeline with one final response), group them and place the eval block after the final agent. Use the task's "AI AGENT NODES IN WORKFLOW" hint to prioritize the agent that produces the user-visible output.
## Error Handling & Validation
After patching:
1. Re-read the workflow using the workflows tool.
2. Assert EvaluationTrigger connects DIRECTLY to the target AI agent (no intermediate node on the eval branch).
3. Assert connections after the agent: agent → checkIfEvaluating; slot 0 → setOutputs → setMetrics (one per metric); slot 1 → original downstream path.
4. When the task contained a \`PRODUCTION ADAPTER\` section: assert
- The named Set adapter node exists with \`typeVersion: 3.4\`.
- The Set adapter's \`assignments.assignments\` array contains EVERY entry from the task spec.
- The Set adapter's \`main\` output connects to the agent's \`main\` input.
- EACH target node listed under section 3 of the PRODUCTION ADAPTER (agent and all sub-components) no longer contains any of its original \`$('NodeName').item.json.<field>\` expressions. The agent target's parameters now use \`$json.<column>\`; sub-component targets (memory, tools, output parsers) use \`$('<AgentName>').item.json.<column>\` instead — match each line of the task verbatim.
- The EvaluationTrigger has a \`main\` connection to the agent's \`main\` input (a SECOND incoming connection).
When the task did NOT contain an adapter section: assert the agent's parameters reference \`$json.<column>\` for at least one column from the task's INPUT COLUMNS list (existing rule).
5. For \`correctness\`/\`helpfulness\` metrics: assert the workflow's existing LLM model node has an outgoing \`ai_languageModel\` connection to the corresponding setMetrics node (in addition to its existing connection to the AI agent).
6. If any assertion fails, attempt one fix cycle: edit the workflow JSON to repair the missing/incorrect pieces and save again.
7. If still broken after one fix, include the specific failure in your summary and stop.
Do NOT run the workflow or call \`executions(action="run")\` — validation is structural, not behavioral.`;

279
packages/@n8n/instance-ai/src/tools/orchestration/eval-setup-agent.tool.ts
View File

@ -1,279 +0,0 @@
/**
* Preconfigured Eval Setup Agent Tool
*
* Creates a specialized sub-agent that reads an n8n workflow and patches it
* with EvaluationTrigger + Evaluation nodes + a checkIfEvaluating gate to
* isolate side-effect nodes during eval runs. DataTable creation is always
* handled upstream by `propose` and passed in via the task spec.
*
* No HITL - the eval card already captured the user's approval; this sub-agent
* operates post-approval.
*/
import { Agent, Tool } from '@n8n/agents';
import { nanoid } from 'nanoid';
import { z } from 'zod';
import { createSubAgentPersistence } from './agent-persistence';
import { truncateLabel } from './display-utils';
import { EVAL_SETUP_AGENT_PROMPT } from './eval-setup-agent.prompt';
import {
createDetachedSubAgentTraceFactory,
traceSubAgentTools,
withTraceContextActor,
} from './tracing-utils';
import { buildSubAgentBriefing } from '../../agent/sub-agent-briefing';
import { MAX_STEPS } from '../../constants/max-steps';
import { consumeStreamWithHitl, requireCompletedHitlText } from '../../stream/consume-with-hitl';
import { createToolRegistry, toolRegistryKeys, toolRegistryValues } from '../../tool-registry';
import { buildAgentTraceInputs, mergeTraceRunInputs } from '../../tracing/langsmith-tracing';
import type { InstanceAiToolRegistry, OrchestrationContext } from '../../types';
import { createWorkflowsTool, type WorkflowAction } from '../workflows.tool';
const EVAL_SETUP_TOOL_NAMES = ['workflows', 'nodes'] as const;
const EVAL_SETUP_WORKFLOW_ACTIONS = [
'get-json',
'update',
] as const satisfies readonly WorkflowAction[];
/**
* Build the eval-setup sub-agent's tool set.
*
* The `workflows` tool is overridden so update-gated workflow mutations do
* not prompt again — the eval offer card already captured the user's approval
* for adding eval nodes. If admin has blocked workflow updates outright
* (`updateWorkflow: 'blocked'`), the original tool is left in place so the
* sub-agent receives the standard `denied` result.
*/
export function buildEvalSetupTools(context: OrchestrationContext): InstanceAiToolRegistry {
const tools = createToolRegistry();
for (const name of EVAL_SETUP_TOOL_NAMES) {
const tool = context.domainTools.get(name);
if (tool) {
tools.set(name, tool);
}
}
const domainContext = context.domainContext;
const parentPermissions = domainContext?.permissions;
if (
domainContext &&
parentPermissions &&
parentPermissions.updateWorkflow !== 'blocked' &&
tools.has('workflows')
) {
tools.set(
'workflows',
createWorkflowsTool(
{
...domainContext,
permissions: { ...parentPermissions, updateWorkflow: 'always_allow' },
},
{ allowedActions: EVAL_SETUP_WORKFLOW_ACTIONS },
),
);
}
return tools;
}
export interface StartEvalSetupAgentInput {
workflowId: string;
task: string;
conversationContext?: string;
taskId?: string;
agentId?: string;
plannedTaskId?: string;
}
export interface StartedEvalSetupAgentTask {
result: string;
taskId: string;
agentId: string;
}
export function startEvalSetupAgentTask(
context: OrchestrationContext,
input: StartEvalSetupAgentInput,
): StartedEvalSetupAgentTask {
const evalSetupTools = buildEvalSetupTools(context);
if (!evalSetupTools.has('workflows')) {
return { result: 'Error: workflows tool not available.', taskId: '', agentId: '' };
}
if (!context.spawnBackgroundTask) {
return { result: 'Error: background task support not available.', taskId: '', agentId: '' };
}
const subAgentId = input.agentId ?? `agent-evalsetup-${nanoid(6)}`;
const taskId = input.taskId ?? `evalsetup-${nanoid(8)}`;
const createTraceContext = createDetachedSubAgentTraceFactory(context, {
agentId: subAgentId,
role: 'eval-setup',
kind: 'eval-setup',
taskId,
plannedTaskId: input.plannedTaskId,
inputs: {
workflowId: input.workflowId,
task: input.task,
conversationContext: input.conversationContext,
},
});
const tracedEvalSetupTools = traceSubAgentTools(context, evalSetupTools, 'eval-setup');
const spawnOutcome = context.spawnBackgroundTask({
taskId,
threadId: context.threadId,
agentId: subAgentId,
role: 'eval-setup',
createTraceContext,
plannedTaskId: input.plannedTaskId,
dedupeKey: {
role: 'eval-setup',
workflowId: input.workflowId,
plannedTaskId: input.plannedTaskId,
},
parentCheckpointId:
context.isCheckpointFollowUp === true ? context.checkpointTaskId : undefined,
run: async (signal, drainCorrections, waitForCorrection, { traceContext }) => {
return await withTraceContextActor(traceContext, async () => {
const subAgent = new Agent('Eval Setup Agent')
.model(context.modelId)
.instructions(EVAL_SETUP_AGENT_PROMPT, {
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' } },
},
})
.tool(toolRegistryValues(tracedEvalSetupTools))
.checkpoint(context.checkpointStore ?? 'memory');
const telemetry = traceContext?.getTelemetry?.({
agentRole: 'eval-setup',
functionId: 'instance-ai.subagent.eval-setup',
executionMode: 'background_subagent',
metadata: { agent_id: subAgentId, task_id: taskId },
});
if (telemetry) {
subAgent.telemetry(telemetry);
}
mergeTraceRunInputs(
traceContext?.actorRun,
buildAgentTraceInputs({
systemPrompt: EVAL_SETUP_AGENT_PROMPT,
tools: tracedEvalSetupTools,
modelId: context.modelId,
}),
);
const briefing = await buildSubAgentBriefing({
task: input.task,
conversationContext: input.conversationContext,
runningTasks: context.getRunningTaskSummaries?.(),
});
const persistence = await createSubAgentPersistence(context, { agentKind: 'eval-setup' });
const stream = await subAgent.stream(briefing, {
maxIterations: MAX_STEPS.EVAL_SETUP,
abortSignal: signal,
persistence,
providerOptions: {
anthropic: { cacheControl: { type: 'ephemeral' } },
},
});
const hitlResult = await consumeStreamWithHitl({
agent: subAgent,
stream,
runId: context.runId,
agentId: subAgentId,
eventBus: context.eventBus,
logger: context.logger,
threadId: context.threadId,
abortSignal: signal,
waitForConfirmation: context.waitForConfirmation,
drainCorrections,
waitForCorrection,
maxIterations: MAX_STEPS.EVAL_SETUP,
persistence,
});
return await requireCompletedHitlText(hitlResult, 'Eval setup sub-agent');
});
},
});
if (spawnOutcome.status === 'duplicate') {
return {
result: `Eval setup already in progress (task: ${spawnOutcome.existing.taskId}). Wait for the planned-task-follow-up — do not dispatch again.`,
taskId: spawnOutcome.existing.taskId,
agentId: spawnOutcome.existing.agentId,
};
}
if (spawnOutcome.status === 'limit-reached') {
return {
result:
'Could not start eval setup: concurrent background-task limit reached. Wait for an existing task to finish and try again.',
taskId: '',
agentId: '',
};
}
context.eventBus.publish(context.threadId, {
type: 'agent-spawned',
runId: context.runId,
agentId: subAgentId,
payload: {
parentId: context.orchestratorAgentId,
role: 'eval-setup',
tools: toolRegistryKeys(evalSetupTools),
taskId,
kind: 'eval-setup',
title: 'Setting up evaluations',
subtitle: truncateLabel(input.task),
goal: input.task,
targetResource: { type: 'workflow' as const, id: input.workflowId },
},
});
return {
result: `Eval setup started (task: ${taskId}). Reply with one short sentence. Do NOT summarize the plan or list details.`,
taskId,
agentId: subAgentId,
};
}
export const evalSetupAgentInputSchema = z.object({
workflowId: z.string().describe('ID of the workflow to add evaluations to'),
task: z
.string()
.describe(
'Full task description for the eval setup agent: includes dataset choice, output columns, metrics to configure, and detected AI agent nodes. Pass the exact string returned by `evals(action="propose")` — do not rephrase.',
),
conversationContext: z
.string()
.optional()
.describe(
"Summary of the thread so far, especially the user's original workflow request. The sub-agent uses this to anchor the domain for dataset design.",
),
});
export function createEvalSetupAgentTool(context: OrchestrationContext) {
return new Tool('eval-setup-with-agent')
.description(
'Set up evaluations for a workflow containing AI agents. ' +
'Adds EvaluationTrigger + Evaluation nodes with a checkIfEvaluating gate that isolates ' +
'side-effect nodes during eval runs. The DataTable is always created upstream by `propose` ' +
'and passed in via the task spec. ' +
'Use only after `evals(action="propose")` returned shouldDelegateToEvalSetupAgent=true.',
)
.input(evalSetupAgentInputSchema)
.output(
z.object({
result: z.string(),
taskId: z.string(),
}),
)
.handler(async (input: z.infer<typeof evalSetupAgentInputSchema>) => {
const result = startEvalSetupAgentTask(context, input);
return await Promise.resolve({ result: result.result, taskId: result.taskId });
})
.build();
}

138
packages/@n8n/instance-ai/src/tools/orchestration/tracing-utils.ts
View File

@ -1,32 +1,9 @@
import {
createDetachedSubAgentTraceContext,
getCurrentOtelSpanContext,
getCurrentTraceToolCallId,
mergeCurrentTraceMetadata,
type BrowserExtensionTraceContext,
} from '../../tracing/langsmith-tracing';
import type {
InstanceAiTraceContext,
InstanceAiTraceRun,
InstanceAiTraceRunFinishOptions,
OrchestrationContext,
} from '../../types';
/** Rebuilds the parent's browser-extension dimensions so sub-agent tool calls stay sliceable. */
function readBrowserExtensionFromMetadata(
metadata: Record<string, unknown> | undefined,
): BrowserExtensionTraceContext | undefined {
const state = metadata?.browser_connection_state;
if (state !== 'connected' && state !== 'disconnected') return undefined;
const version = metadata?.browser_extension_version;
return {
connectionState: state,
...(typeof version === 'string' ? { version } : {}),
};
}
type ToolRegistry = OrchestrationContext['domainTools'];
interface StartSubAgentTraceOptions {
agentId: string;
role: string;
@ -61,110 +38,6 @@ export async function startSubAgentTrace(
});
}
export async function createDetachedSubAgentTracing(
context: OrchestrationContext,
options: StartSubAgentTraceOptions,
): Promise<InstanceAiTraceContext | undefined> {
return await createDetachedSubAgentTraceFactory(context, options)();
}
export function createDetachedSubAgentTraceFactory(
context: OrchestrationContext,
options: StartSubAgentTraceOptions,
): () => Promise<InstanceAiTraceContext | undefined> {
if (!context.tracing) return async () => await Promise.resolve(undefined);
const messageId =
typeof context.tracing.actorRun.metadata?.message_id === 'string'
? context.tracing.actorRun.metadata.message_id
: context.runId;
const conversationId =
typeof context.tracing.actorRun.metadata?.conversation_id === 'string'
? context.tracing.actorRun.metadata.conversation_id
: context.threadId;
const spawnedByAgentId =
typeof context.tracing.actorRun.metadata?.agent_id === 'string'
? context.tracing.actorRun.metadata.agent_id
: context.orchestratorAgentId;
const spawnedByAgentRole =
typeof context.tracing.actorRun.metadata?.agent_role === 'string'
? context.tracing.actorRun.metadata.agent_role
: undefined;
const parentN8nVersion =
typeof context.tracing.actorRun.metadata?.n8n_version === 'string'
? context.tracing.actorRun.metadata.n8n_version
: undefined;
const parentWorkflowSdkVersion =
typeof context.tracing.actorRun.metadata?.workflow_sdk_version === 'string'
? context.tracing.actorRun.metadata.workflow_sdk_version
: undefined;
const parentBrowserExtension = readBrowserExtensionFromMetadata(
context.tracing.actorRun.metadata,
);
const activeSpanContext = getCurrentOtelSpanContext();
const spawnedByToolCallId = getCurrentTraceToolCallId();
return async () => {
if (!context.tracing) return undefined;
const tracing = await createDetachedSubAgentTraceContext({
projectName: context.tracing.projectName,
threadId: context.threadId,
conversationId,
messageGroupId: context.messageGroupId,
messageId,
runId: context.runId,
userId: context.userId,
modelId: context.modelId,
input: options.inputs,
metadata: options.metadata,
n8nVersion: parentN8nVersion,
workflowSdkVersion: parentWorkflowSdkVersion,
browserExtension: parentBrowserExtension,
agentId: options.agentId,
role: options.role,
kind: options.kind,
taskId: options.taskId,
plannedTaskId: options.plannedTaskId,
workItemId: options.workItemId,
spawnedByTraceId:
activeSpanContext?.traceId ??
context.tracing.rootRun.otelTraceId ??
context.tracing.rootRun.traceId,
spawnedBySpanId: activeSpanContext?.spanId ?? context.tracing.actorRun.otelSpanId,
spawnedByRunId: context.tracing.actorRun.id,
spawnedByAgentId,
spawnedByAgentRole,
spawnedByToolCallId,
proxyConfig: context.tracingProxyConfig,
});
if (tracing) {
mergeCurrentTraceMetadata({
detached_trace: true,
spawned_role: options.role,
...(options.taskId ? { spawned_task_id: options.taskId } : {}),
spawned_trace_id: tracing.rootRun.traceId,
spawned_root_run_id: tracing.rootRun.id,
});
}
return tracing;
};
}
export function traceSubAgentTools(
context: OrchestrationContext,
tools: ToolRegistry,
role: string,
): ToolRegistry {
return (
context.tracing?.wrapTools(tools, {
agentRole: role,
tags: ['sub-agent'],
}) ?? tools
);
}
export async function withTraceRun<T>(
context: OrchestrationContext,
traceRun: InstanceAiTraceRun | undefined,
@ -177,17 +50,6 @@ export async function withTraceRun<T>(
return await context.tracing.withActiveSpan(traceRun, fn);
}
export async function withTraceContextActor<T>(
tracing: InstanceAiTraceContext | undefined,
fn: () => Promise<T>,
): Promise<T> {
if (!tracing) {
return await fn();
}
return await tracing.withActiveSpan(tracing.actorRun, fn);
}
export async function finishTraceRun(
context: OrchestrationContext,
traceRun: InstanceAiTraceRun | undefined,

3
packages/@n8n/instance-ai/src/tools/tool-ids.ts
View File

@ -2,7 +2,6 @@ import { isAgentFeatureEnabled } from '../utils/agent-feature-enabled';
export const DOMAIN_TOOL_IDS = {
WORKFLOWS: 'workflows',
EVALS: 'evals',
EVAL_CONFIG: 'eval-config',
EXECUTIONS: 'executions',
CREDENTIALS: 'credentials',
@ -32,8 +31,6 @@ export const AGENT_SNAPSHOT_TRACE_RUN_NAME = 'agent-snapshot';
export const ORCHESTRATION_TOOL_IDS = {
CREATE_TASKS: 'create-tasks',
TASK_CONTROL: 'task-control',
EVAL_SETUP_WITH_AGENT: 'eval-setup-with-agent',
EVAL_DATA: 'eval-data',
COMPLETE_CHECKPOINT: 'complete-checkpoint',
VERIFY_BUILT_WORKFLOW: 'verify-built-workflow',
REPORT_VERIFICATION_VERDICT: 'report-verification-verdict',

204
packages/@n8n/instance-ai/src/tools/workflows.tool.ts
View File

@ -1,5 +1,5 @@
/**
* Consolidated workflows tool — list, get, get-json, get-as-code, delete/archive,
* Consolidated workflows tool — list, get, get-as-code, delete/archive,
* unarchive, setup, publish, unpublish, list-versions, restore-version,
* update-version.
*/
@ -9,14 +9,11 @@ import {
credentialDestinationSchema,
TEMPLATED_CUSTOM_AUTH_CREDENTIAL_TYPE,
} from '@n8n/api-types';
import { isRecord } from '@n8n/utils/is-record';
import { dropInvalidWorkflowJsonGroups, type WorkflowJSON } from '@n8n/workflow-sdk';
import { makeGetNodeTypeForGrouping } from 'n8n-workflow';
import type { WorkflowJSON } from '@n8n/workflow-sdk';
import { nanoid } from 'nanoid';
import { z } from 'zod';
import { sanitizeInputSchema } from '../agent/sanitize-mcp-schemas';
import { WorkflowSaveConflictError } from '../errors/workflow-save-conflict.error';
import { WorkflowSnapshotChangedError } from '../errors/workflow-snapshot-changed.error';
import type { FolderResolutionFailure, InstanceAiContext } from '../types';
import {
@ -27,11 +24,6 @@ import {
TEMPLATABLE_PLAIN_AUTH_TYPES,
} from './credentials.tool';
import { formatTimestamp } from '../utils/format-timestamp';
import {
getObservedWorkflowChecksum,
rememberCurrentWorkflowChecksum,
rememberObservedWorkflowChecksum,
} from './workflows/observed-workflow-checksums';
import {
completedSetupSubjects,
describeSkippedSetup,
@ -59,23 +51,17 @@ import {
summarizeWorkflowStructure,
} from './workflows/summarize-workflow';
import { validateWorkflowConfig } from './workflows/validate-workflow.service';
import {
grantSessionWorkflowUpdate,
canSkipWorkflowUpdateHitl,
formatWarning,
} from './workflows/workflow-build-context';
import {
refreshWorkflowSourceFileBindingFromSave,
refreshWorkflowSourceFileBindingFromWorkflow,
} from './workflows/workflow-file-bindings';
import { ensureUniqueNodeIds, getReferencedWorkflowIds } from './workflows/workflow-json-utils';
import { getReferencedWorkflowIds } from './workflows/workflow-json-utils';
import {
INLINE_SOURCE_LIMIT_CHARS,
indexSourceNodes,
materializeWorkflowSource,
type MaterializedSourceStatus,
} from './workflows/workflow-source-materializer';
import { nodeGroupDroppedWarnings } from './workflows/workflow-validation-warnings';
// ── Action schemas ──────────────────────────────────────────────────────────
@ -211,16 +197,6 @@ const getAction = z.object({
.describe('Return complete node data including parameters (large). Default false.'),
});
const getJsonAction = z.object({
action: z
.literal('get-json')
.describe(
'Get full WorkflowJSON for workspace-file workflow edits. Write it to a .workflow.json file, edit the file, then save with build-workflow. Pass versionId for a past version instead of the current draft.',
),
workflowId: z.string().describe('ID of the workflow'),
versionId: z.string().optional().describe('Version ID'),
});
const getAsCodeAction = z.object({
action: z
.literal('get-as-code')
@ -322,20 +298,6 @@ const validateAction = z.object({
.describe('Issue categories to suppress from the result'),
});
const updateAction = z.object({
action: z
.literal('update')
.describe(
'Internal/raw update escape hatch. Save a complete modified WorkflowJSON back to the workflow. Replaces the full workflow definition.',
),
workflowId: z.string().describe('ID of the workflow'),
workflow: z
.record(z.unknown())
.describe(
'Full WorkflowJSON object (same shape as returned by `get-json`). This completely replaces the current workflow definition — ensure name, nodes, and connections are all included.',
),
});
const publishBaseAction = z.object({
action: z
.literal('publish')
@ -391,11 +353,7 @@ const confirmationSuspendSchema = setupSuspendSchema
const suspendSchema = z.union([setupSuspendSchema, confirmationSuspendSchema]);
// Resume: setup-specific fields plus optional session scope for generic approvals
// (e.g. update "always allow" → persist `workflows:update:<id>`).
export const workflowsResumeSchema = setupResumeSchema.extend({
scope: z.enum(['once', 'session']).optional(),
});
export const workflowsResumeSchema = setupResumeSchema;
interface WorkflowToolContext {
resumeData: z.infer<typeof workflowsResumeSchema> | undefined;
@ -410,13 +368,11 @@ type Input =
| z.infer<typeof listAction>
| z.infer<typeof nodeUsageAction>
| z.infer<typeof getAction>
| z.infer<typeof getJsonAction>
| z.infer<typeof getAsCodeAction>
| z.infer<typeof deleteAction>
| z.infer<typeof unarchiveAction>
| z.infer<typeof setupAction>
| z.infer<typeof validateAction>
| z.infer<typeof updateAction>
| z.infer<typeof publishExtendedAction>
| z.infer<typeof unpublishAction>
| z.infer<typeof listVersionsAction>
@ -432,13 +388,11 @@ export type WorkflowAction =
| 'list'
| 'node-usage'
| 'get'
| 'get-json'
| 'get-as-code'
| 'delete'
| 'unarchive'
| 'setup'
| 'validate'
| 'update'
| 'publish'
| 'unpublish'
| 'list-versions'
@ -462,13 +416,11 @@ const WORKFLOW_ACTION_ORDER = [
// ordering is what the agent reads first in the tool schema.
'node-usage',
'get',
'get-json',
'get-as-code',
'delete',
'unarchive',
'setup',
'validate',
'update',
'publish',
'unpublish',
'list-versions',
@ -480,13 +432,11 @@ const WORKFLOW_ACTION_LABELS = {
list: 'list',
'node-usage': 'summarize which node types are in use',
get: 'inspect',
'get-json': 'inspect full WorkflowJSON',
'get-as-code': 'convert existing workflows to TypeScript SDK code',
delete: 'archive',
unarchive: 'restore archived workflows',
setup: 'set up credentials and parameters',
validate: 'validate configuration',
update: 'save a modified WorkflowJSON',
publish: 'publish',
unpublish: 'unpublish',
'list-versions': 'list versions',
@ -500,7 +450,6 @@ function normalizeOptions(options: WorkflowsToolOptionsInput = {}): WorkflowsToo
function getSupportedWorkflowActionSchemas(
context: InstanceAiContext,
surface: 'full' | 'orchestrator' = 'full',
): Partial<Record<WorkflowAction, WorkflowActionSchema>> {
const hasNamedVersions = !!context.workflowService.updateVersion;
const hasVersions = !!context.workflowService.listVersions;
@ -513,13 +462,11 @@ function getSupportedWorkflowActionSchemas(
list: pickListAction(context, hasNodeUsage),
...(hasNodeUsage ? { 'node-usage': nodeUsageAction } : {}),
get: getAction,
...(surface !== 'orchestrator' ? { 'get-json': getJsonAction } : {}),
'get-as-code': getAsCodeAction,
delete: deleteAction,
unarchive: unarchiveAction,
setup: setupAction,
validate: validateAction,
...(surface !== 'orchestrator' ? { update: updateAction } : {}),
publish: hasNamedVersions ? publishExtendedAction : publishBaseAction,
unpublish: unpublishAction,
...(hasVersions
@ -543,7 +490,7 @@ function getWorkflowActions(
}
function buildInputSchema(context: InstanceAiContext, options: WorkflowsToolOptions) {
const supportedSchemas = getSupportedWorkflowActionSchemas(context, options.surface);
const supportedSchemas = getSupportedWorkflowActionSchemas(context);
const actionSchemas: WorkflowActionSchema[] = [];
for (const action of getWorkflowActions(supportedSchemas, options)) {
const schema = supportedSchemas[action];
@ -737,7 +684,6 @@ async function handleGet(context: InstanceAiContext, input: Extract<Input, { act
};
}
const detail = await context.workflowService.get(input.workflowId);
await rememberObservedWorkflowChecksum(context, input.workflowId, detail.checksum);
if (isSmallPayload(detail)) return detail;
if (input.full && !exceedsFullPayloadLimit(detail)) return detail;
const { nodes, connections, ...meta } = detail;
@ -767,10 +713,9 @@ async function handleGet(context: InstanceAiContext, input: Extract<Input, { act
/**
* Pinned-data summary for agent visibility. Pins live on the saved workflow but
* never inside the WorkflowJSON the agent round-trips (see
* `InstanceAiWorkflowService.getPinnedDataSummary`), so without this report the
* agent cannot tell that test runs feed nodes from saved pins instead of
* executing them. Failures degrade to "no report" — it must never break a read.
* are not part of its workflow definition, so without this report the agent
* cannot tell that test runs feed nodes from saved pins instead of executing
* them. Failures degrade to "no report" — they must never break validation.
*/
async function getPinnedNodesReport(
context: InstanceAiContext,
@ -794,30 +739,6 @@ async function getPinnedNodesReport(
}
}
async function handleGetJson(
context: InstanceAiContext,
input: Extract<Input, { action: 'get-json' }>,
) {
try {
const json = await context.workflowService.getAsWorkflowJSON(input.workflowId, input.versionId);
// This is the graph the agent edits before `update`, so pin the state it
// saw. Historical reads must not advance the optimistic-concurrency lock.
if (!input.versionId) {
await rememberCurrentWorkflowChecksum(context, input.workflowId);
}
const pinnedReport = input.versionId
? undefined
: await getPinnedNodesReport(context, input.workflowId);
return pinnedReport ? { ...json, ...pinnedReport } : json;
} catch (error) {
return {
workflowId: input.workflowId,
found: false as const,
error: error instanceof Error ? error.message : 'Failed to fetch workflow JSON',
};
}
}
const SOURCE_FILE_NOTES: Record<MaterializedSourceStatus, string> = {
written:
'Source written to filePath and bound to this workflow. Locate nodes with the `nodes` index (line numbers) and read only those lines — for a large file use a ranged shell read such as `sed -n START,ENDp filePath` via workspace_execute_command, since workspace_read_file returns the whole file. Apply edits with workspace_str_replace_file, then call build-workflow with this filePath. Do not rewrite the whole file.',
@ -1743,109 +1664,6 @@ async function handleValidate(
}
}
function isWorkflowJson(value: unknown): value is WorkflowJSON {
return (
isRecord(value) &&
typeof value.name === 'string' &&
Array.isArray(value.nodes) &&
isRecord(value.connections)
);
}
async function handleUpdate(
context: InstanceAiContext,
input: Extract<Input, { action: 'update' }>,
ctx: WorkflowToolContext,
) {
const resumeData = ctx.resumeData;
if (context.permissions?.updateWorkflow === 'blocked') {
return { success: false, denied: true, reason: 'Action blocked by admin' };
}
// Skip HITL for session-created or always-allowed workflows; others still need approval.
const needsApproval =
context.permissions?.updateWorkflow !== 'always_allow' &&
!canSkipWorkflowUpdateHitl(context, input.workflowId);
if (needsApproval && (resumeData === undefined || resumeData === null)) {
const workflowName = await resolveWorkflowName(context, input.workflowId);
return await ctx.suspend({
requestId: nanoid(),
message: `Update workflow "${workflowName}" (ID: ${input.workflowId})?`,
severity: 'warning' as const,
// Carried on the confirmation so the UI can scope "always allow" per workflow
// even if tool-call args are incomplete on resume.
workflowId: input.workflowId,
});
}
if (resumeData !== undefined && resumeData !== null && !resumeData.approved) {
return { success: false, denied: true, reason: 'User denied the action' };
}
// "Always allow" — persist so later edits of this workflow skip HITL.
if (resumeData?.approved && resumeData.scope === 'session') {
await grantSessionWorkflowUpdate(context, input.workflowId);
}
if (!isWorkflowJson(input.workflow)) {
return {
success: false,
error: 'Workflow JSON must include name, nodes, and connections.',
};
}
// Guard against overwriting a save this conversation never saw (canvas
// autosave, another user, another thread). Absent when the agent never read
// the workflow here — then there is nothing to pin the save to.
const expectedChecksum = await getObservedWorkflowChecksum(context, input.workflowId);
try {
ensureUniqueNodeIds(input.workflow);
const droppedGroupWarnings = nodeGroupDroppedWarnings(
dropInvalidWorkflowJsonGroups(
input.workflow,
context.nodeTypesProvider ? makeGetNodeTypeForGrouping(context.nodeTypesProvider) : null,
),
);
const saved = expectedChecksum
? await context.workflowService.updateFromWorkflowJSON(input.workflowId, input.workflow, {
expectedChecksum,
})
: await context.workflowService.updateFromWorkflowJSON(input.workflowId, input.workflow);
await refreshWorkflowSourceFileBindingFromWorkflow(context, input.workflowId);
// Pin to what this save wrote, not to the re-read above: if another writer
// landed in between, the next update should conflict rather than clobber.
if (saved.checksum) {
await rememberObservedWorkflowChecksum(context, input.workflowId, saved.checksum);
}
return {
success: true,
workflowId: input.workflowId,
...(droppedGroupWarnings.length > 0
? {
warnings: droppedGroupWarnings.map((warning) =>
formatWarning(warning.code, warning.message),
),
}
: {}),
};
} catch (error) {
if (error instanceof WorkflowSaveConflictError) {
return {
success: false,
error: `${error.message} Call workflows(action="get", workflowId="${input.workflowId}") to read the current state, re-apply your change, then update again.`,
};
}
return {
success: false,
error: error instanceof Error ? error.message : String(error),
};
}
}
async function handlePublish(
context: InstanceAiContext,
input: PublishInput,
@ -2160,7 +1978,7 @@ function formatWorkflowActionList(actions: readonly WorkflowAction[]): string {
}
function getToolDescription(context: InstanceAiContext, options: WorkflowsToolOptions): string {
const supportedSchemas = getSupportedWorkflowActionSchemas(context, options.surface);
const supportedSchemas = getSupportedWorkflowActionSchemas(context);
const actionList = formatWorkflowActionList(getWorkflowActions(supportedSchemas, options));
const description = `${options.descriptionPrefix ?? 'Manage workflows'} — ${actionList}.`;
const suffix =
@ -2201,8 +2019,6 @@ export function createWorkflowsTool(
return await handleNodeUsage(context, workflowInput);
case 'get':
return await handleGet(context, workflowInput);
case 'get-json':
return await handleGetJson(context, workflowInput);
case 'get-as-code':
return await handleGetAsCode(context, workflowInput);
case 'delete':
@ -2213,8 +2029,6 @@ export function createWorkflowsTool(
return await handleSetup(context, workflowInput, ctx, setupState);
case 'validate':
return await handleValidate(context, workflowInput);
case 'update':
return await handleUpdate(context, workflowInput, ctx);
case 'publish':
return await handlePublish(context, workflowInput, ctx);
case 'unpublish':

2
packages/@n8n/instance-ai/src/tools/workflows/__tests__/get-as-code-node-identity.test.ts
View File

@ -74,7 +74,7 @@ function rebuild(code: string): WorkflowJSON {
}
async function getAsCode(): Promise<string> {
const tool = createWorkflowsTool(createContext(), 'full');
const tool = createWorkflowsTool(createContext());
// A versionId keeps this a historical read, which skips the source-file binding refresh
// and its thread-metadata plumbing — irrelevant to the round trip under test.
const result = await executeTool<{ code: string; error?: string }>(

156
packages/@n8n/instance-ai/src/tools/workflows/__tests__/observed-workflow-checksums.test.ts
View File

@ -1,156 +0,0 @@
import type { ThreadPatch, ThreadRecord } from '../../../storage/thread-patch';
import type { InstanceAiContext } from '../../../types';
import {
getObservedWorkflowChecksum,
rememberCurrentWorkflowChecksum,
rememberObservedWorkflowChecksum,
} from '../observed-workflow-checksums';
/** Thread store shared across contexts, the way one conversation spans several runs. */
function createThreadMemory() {
const threads = new Map<string, ThreadRecord>();
return {
getThread: vi.fn(
async (threadId: string) => await Promise.resolve(threads.get(threadId) ?? null),
),
patchThread: vi.fn(
async ({
threadId,
update,
}: {
threadId: string;
update: (current: ThreadRecord) => ThreadPatch | null | undefined;
}) => {
const current: ThreadRecord = threads.get(threadId) ?? {
id: threadId,
metadata: {},
resourceId: 'resource-1',
createdAt: new Date('2026-01-01'),
updatedAt: new Date('2026-01-01'),
};
const patch = update(current);
if (!patch) return null;
const next = { ...current, metadata: patch.metadata ?? current.metadata };
threads.set(threadId, next);
return await Promise.resolve(next);
},
),
};
}
/** Each Instance AI run builds a fresh context, so a turn is a new context object. */
function createContextForRun(threadMemory: ReturnType<typeof createThreadMemory>) {
return {
threadId: 'thread-1',
threadMemory,
logger: { debug: vi.fn(), warn: vi.fn() },
} as unknown as InstanceAiContext;
}
describe('observed workflow checksums', () => {
it('carries the observed checksum into later turns of the same conversation', async () => {
const threadMemory = createThreadMemory();
await rememberObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1', 'checksum-1');
await expect(
getObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1'),
).resolves.toBe('checksum-1');
});
it('keeps checksums separate per workflow', async () => {
const threadMemory = createThreadMemory();
const context = createContextForRun(threadMemory);
await rememberObservedWorkflowChecksum(context, 'wf-1', 'checksum-1');
await rememberObservedWorkflowChecksum(context, 'wf-2', 'checksum-2');
await expect(
getObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-2'),
).resolves.toBe('checksum-2');
});
it('forgets the checksum in later turns when a save reports none', async () => {
const threadMemory = createThreadMemory();
await rememberObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1', 'checksum-1');
await rememberObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1', undefined);
await expect(
getObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1'),
).resolves.toBeUndefined();
});
it('has no expectation for a workflow this conversation never observed', async () => {
const threadMemory = createThreadMemory();
await expect(
getObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-unknown'),
).resolves.toBeUndefined();
});
it('still remembers within a run when the conversation has no thread memory', async () => {
const context = { logger: { debug: vi.fn(), warn: vi.fn() } } as unknown as InstanceAiContext;
await rememberObservedWorkflowChecksum(context, 'wf-1', 'checksum-1');
await expect(getObservedWorkflowChecksum(context, 'wf-1')).resolves.toBe('checksum-1');
});
it('has no expectation when the thread copy cannot be read', async () => {
const threadMemory = createThreadMemory();
await rememberObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1', 'checksum-1');
const laterRun = createContextForRun(threadMemory);
threadMemory.getThread.mockRejectedValue(new Error('thread store is down'));
// An unguarded save beats blocking the agent on unreadable bookkeeping.
await expect(getObservedWorkflowChecksum(laterRun, 'wf-1')).resolves.toBeUndefined();
});
it('records no expectation when the workflow itself cannot be read', async () => {
const context = {
threadId: 'thread-1',
threadMemory: createThreadMemory(),
logger: { debug: vi.fn(), warn: vi.fn() },
workflowService: { get: vi.fn().mockRejectedValue(new Error('workflow is gone')) },
} as unknown as InstanceAiContext;
await expect(rememberCurrentWorkflowChecksum(context, 'wf-1')).resolves.toBeUndefined();
await expect(getObservedWorkflowChecksum(context, 'wf-1')).resolves.toBeUndefined();
});
it('prefers what this run observed over a stale thread copy', async () => {
const threadMemory = createThreadMemory();
await rememberObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1', 'checksum-1');
// A later run re-reads the workflow, but the thread write does not land.
const laterRun = createContextForRun(threadMemory);
threadMemory.patchThread.mockRejectedValue(new Error('thread store is down'));
await rememberObservedWorkflowChecksum(laterRun, 'wf-1', 'checksum-2');
await expect(getObservedWorkflowChecksum(laterRun, 'wf-1')).resolves.toBe('checksum-2');
});
it('does not fall back to a stale thread copy after a failed clear', async () => {
const threadMemory = createThreadMemory();
await rememberObservedWorkflowChecksum(createContextForRun(threadMemory), 'wf-1', 'checksum-1');
const laterRun = createContextForRun(threadMemory);
threadMemory.patchThread.mockRejectedValue(new Error('thread store is down'));
await rememberObservedWorkflowChecksum(laterRun, 'wf-1', undefined);
await expect(getObservedWorkflowChecksum(laterRun, 'wf-1')).resolves.toBeUndefined();
});
it('keeps the run-local checksum when persisting to the thread fails', async () => {
const threadMemory = createThreadMemory();
threadMemory.patchThread.mockRejectedValue(new Error('thread store is down'));
const context = createContextForRun(threadMemory);
await rememberObservedWorkflowChecksum(context, 'wf-1', 'checksum-1');
await expect(getObservedWorkflowChecksum(context, 'wf-1')).resolves.toBe('checksum-1');
});
});

124
packages/@n8n/instance-ai/src/tools/workflows/observed-workflow-checksums.ts
View File

@ -1,124 +0,0 @@
import { z } from 'zod';
import { getThread, patchThread } from '../../storage/thread-patch';
import type { InstanceAiContext } from '../../types';
const METADATA_KEY = 'instanceAiObservedWorkflowChecksums';
const observedChecksumsSchema = z.record(z.string(), z.string());
/**
* Checksum of the workflow state this conversation last observed, per workflow.
*
* `build-workflow` gets its optimistic-concurrency expectation from the source
* file binding, but the plain `workflows(action="update")` path has no source
* file to bind to. Without an expectation the agent overwrites whatever landed
* in the meantime — a canvas autosave, another user's edit, another thread.
*
* Persisted on the thread, because every run builds a fresh context: a workflow
* read in one turn is often only saved in a later one.
*/
/** `null` is a tombstone: observed in this run as having no checksum to pin to. */
const runLocalChecksums = new WeakMap<InstanceAiContext, Map<string, string | null>>();
function runLocalFor(context: InstanceAiContext): Map<string, string | null> {
let checksums = runLocalChecksums.get(context);
if (!checksums) {
checksums = new Map();
runLocalChecksums.set(context, checksums);
}
return checksums;
}
function parseChecksums(raw: unknown): Record<string, string> {
const parsed = observedChecksumsSchema.safeParse(raw);
return parsed.success ? parsed.data : {};
}
async function readThreadChecksums(
context: InstanceAiContext,
): Promise<Record<string, string> | undefined> {
if (!context.threadMemory || !context.threadId) return undefined;
try {
const thread = await getThread(context.threadMemory, context.threadId);
return parseChecksums(thread?.metadata?.[METADATA_KEY]);
} catch (error) {
context.logger?.debug('Failed to read observed workflow checksums from thread metadata', {
error: error instanceof Error ? error.message : String(error),
});
return undefined;
}
}
/**
* Records the workflow state this conversation has seen — a read or its own
* save. `undefined` clears the entry, and is remembered as a tombstone for the
* rest of the run: with no checksum to pin to, an unguarded save beats one
* guarded against an expectation we already know is stale.
*/
export async function rememberObservedWorkflowChecksum(
context: InstanceAiContext,
workflowId: string,
checksum: string | undefined,
): Promise<void> {
runLocalFor(context).set(workflowId, checksum ?? null);
if (!context.threadMemory || !context.threadId) return;
try {
await patchThread(context.threadMemory, {
threadId: context.threadId,
update: ({ metadata = {} }) => {
const checksums = parseChecksums(metadata[METADATA_KEY]);
if (checksum === undefined) {
delete checksums[workflowId];
} else {
checksums[workflowId] = checksum;
}
return { metadata: { ...metadata, [METADATA_KEY]: checksums } };
},
});
} catch (error) {
// The run-local copy still guards saves made in this run.
context.logger?.warn('Failed to persist the observed workflow checksum to thread metadata', {
workflowId,
error: error instanceof Error ? error.message : String(error),
});
}
}
/**
* Records the workflow's current state for reads that hand the agent a graph to
* edit but no checksum of their own (`get-json`). Best-effort: bookkeeping must
* never fail the read it is attached to.
*/
export async function rememberCurrentWorkflowChecksum(
context: InstanceAiContext,
workflowId: string,
): Promise<void> {
try {
const detail = await context.workflowService.get(workflowId);
await rememberObservedWorkflowChecksum(context, workflowId, detail.checksum);
} catch (error) {
context.logger?.debug('Failed to record the observed workflow checksum', {
workflowId,
error: error instanceof Error ? error.message : String(error),
});
}
}
export async function getObservedWorkflowChecksum(
context: InstanceAiContext,
workflowId: string,
): Promise<string | undefined> {
// What this run observed wins: it is at least as fresh as the thread copy, and
// strictly fresher when the thread write failed. A later run starts with an
// empty run-local map and falls through to the thread.
const runLocal = runLocalFor(context);
if (runLocal.has(workflowId)) return runLocal.get(workflowId) ?? undefined;
const threadChecksums = await readThreadChecksums(context);
return threadChecksums?.[workflowId];
}

6
packages/@n8n/instance-ai/src/tools/workflows/workflow-file-bindings.ts
View File

@ -1,7 +1,6 @@
import { createHash } from 'node:crypto';
import { z } from 'zod';
import { rememberObservedWorkflowChecksum } from './observed-workflow-checksums';
import { getThread, patchThread } from '../../storage/thread-patch';
import type { InstanceAiContext } from '../../types';
import { readWorkspaceFile } from '../../workspace/workspace-files';
@ -175,11 +174,6 @@ export async function refreshWorkflowSourceFileBindingFromSave(
workflowId: string,
saved: { versionId: string; checksum?: string },
): Promise<void> {
// Every agent-side save routes through here, so this is also where the
// conversation's view of the workflow (used by `workflows(action="update")`)
// stays in step with the DB.
await rememberObservedWorkflowChecksum(context, workflowId, saved.checksum);
const threadBindings = await readThreadBindings(context);
const fallback = getFallbackBindings(context);
const entries: WorkflowSourceFileBinding[] = [];

192
packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts
View File

@ -11,12 +11,10 @@ import { createToolRegistry } from '../../tool-registry';
import { createAskUserTool } from '../../tools/shared/ask-user.tool';
import {
buildAgentTraceInputs,
createDetachedSubAgentTraceContext,
createInstanceAiTraceContext,
createInternalOperationTraceContext,
createTraceReplayOnlyContext,
continueInstanceAiTraceContext,
mergeTraceRunInputs,
redactLangSmithTelemetrySpan,
releaseTraceClient,
shutdownProductTelemetryProviders,
@ -1464,196 +1462,6 @@ describe('createInstanceAiTraceContext', () => {
);
});
it('creates detached sub-agent traces as separate root traces', async () => {
const tracing = await createDetachedSubAgentTraceContext({
threadId: 'thread-1',
conversationId: 'thread-1',
messageGroupId: 'group-1',
messageId: 'message-1',
runId: 'run-1',
userId: 'user-1',
agentId: 'agent-builder-1',
role: 'workflow-builder',
kind: 'builder',
taskId: 'build-1',
spawnedByTraceId: 'trace-parent-1',
spawnedBySpanId: 'span-parent-1',
spawnedByRunId: 'run-parent-1',
spawnedByAgentId: 'agent-001',
spawnedByAgentRole: 'orchestrator',
spawnedByToolCallId: 'toolu-1',
input: { task: 'Build a workflow' },
metadata: { n8n_version: '2.19.0' },
});
expect(tracing).toBeDefined();
expect(tracing?.traceKind).toBe('background_subagent');
expect(tracing?.rootRun.id).not.toBe(tracing?.actorRun.id);
expect(tracing?.rootRun.parentRunId).toBeUndefined();
expect(tracing?.rootRun.name).toBe('background task: workflow-builder');
expect(tracing?.actorRun.name).toBe('agent: workflow-builder');
expect(tracing?.actorRun.parentRunId).toBe(tracing?.rootRun.id);
expect(tracing?.rootRun.metadata).toEqual(
expect.objectContaining({
thread_id: 'thread-1',
message_group_id: 'group-1',
task_id: 'build-1',
task_kind: 'builder',
agent_id: 'agent-builder-1',
n8n_version: '2.19.0',
trace_kind: 'background_subagent',
execution_mode: 'background_subagent',
spawned_by_trace_id: 'trace-parent-1',
spawned_by_span_id: 'span-parent-1',
spawned_by_run_id: 'run-parent-1',
spawned_by_agent_id: 'agent-001',
spawned_by_agent_role: 'orchestrator',
spawned_by_tool_call_id: 'toolu-1',
'instance_ai.canonical_name': 'instance-ai.background_subagent',
}),
);
expect(typeof tracing?.rootRun.metadata?.agents_version).toBe('string');
expect(typeof tracing?.rootRun.metadata?.workflow_sdk_version).toBe('string');
expect(tracing?.actorRun.metadata).toEqual(
expect.objectContaining({
'instance_ai.canonical_name': 'instance-ai.agent.workflow-builder',
}),
);
const telemetryOrBuilder = tracing!.getTelemetry!({
agentRole: 'workflow-builder',
functionId: 'instance-ai.subagent.workflow-builder',
executionMode: 'background_subagent',
});
const telemetry =
'build' in telemetryOrBuilder ? await telemetryOrBuilder.build() : telemetryOrBuilder;
expect(telemetry.runtimeRootSpanEnabled).toBe(false);
});
it('attaches compact root agent config without duplicating tool schemas into agent spans', async () => {
const tracing = await createDetachedSubAgentTraceContext({
threadId: 'thread-1',
conversationId: 'thread-1',
messageGroupId: 'group-1',
messageId: 'message-1',
runId: 'run-1',
userId: 'user-1',
agentId: 'agent-builder-1',
role: 'workflow-builder',
kind: 'builder',
taskId: 'build-1',
input: { task: 'Build a workflow' },
});
expect(tracing).toBeDefined();
mergeTraceRunInputs(
tracing?.actorRun,
buildAgentTraceInputs({
systemPrompt: ['line 1', 'line 2', 'line 3', 'line 4'].join('\n').repeat(700),
tools: createToolRegistry([
[
'build-workflow',
{
description: 'Build or patch a workflow from SDK code.',
inputSchema: {
type: 'object',
properties: {
task: { type: 'string' },
},
required: ['task'],
},
} as never,
],
]),
runtimeTools: createToolRegistry([
[
'workspace_read_file',
{
name: 'workspace_read_file',
description: 'Read a file from the workspace.',
} as never,
],
]),
modelId: 'anthropic/claude-sonnet-4-6',
}),
);
const actorInputs = tracing?.actorRun.inputs as Record<string, unknown>;
const systemPrompt = actorInputs.system_prompt as Record<string, unknown>;
expect(actorInputs.task).toBe('Build a workflow');
expect(actorInputs.model).toBe('anthropic/claude-sonnet-4-6');
expect(actorInputs.assigned_tool_count).toBe(1);
expect(actorInputs.assigned_tool_names).toEqual(['build-workflow']);
expect(actorInputs.assigned_tool_schema_hash).toEqual(expect.any(String));
expect(actorInputs.runtime_tool_count).toBe(1);
expect(actorInputs.runtime_tool_names).toEqual(['workspace_read_file']);
expect(actorInputs.runtime_tool_schema_hash).toEqual(expect.any(String));
expect(actorInputs.loaded_tool_count).toBeUndefined();
expect(actorInputs.loaded_tool_names).toBeUndefined();
expect(actorInputs.loaded_tool_schema_hash).toBeUndefined();
expect(actorInputs.loaded_tool_manifest).toBeUndefined();
expect(actorInputs.loaded_tools).toBeUndefined();
expect(actorInputs.loaded_tool_catalog).toBeUndefined();
const actorSpan = agentsMock
.getSpans()
.find((span) => span.id === tracing?.actorRun.otelSpanId);
const spanInputs = jsonParse<Record<string, unknown>>(
actorSpan?.attributes['gen_ai.prompt'] as string,
);
expect(spanInputs.assigned_tool_names).toEqual(['build-workflow']);
expect(spanInputs.runtime_tool_names).toEqual(['workspace_read_file']);
expect(spanInputs.loaded_tool_manifest).toBeUndefined();
expect(spanInputs.loaded_tools).toBeUndefined();
expect(systemPrompt.part_01).toEqual(expect.any(String));
expect(systemPrompt.part_02).toEqual(expect.any(String));
});
it('persists merged actor inputs while the actor run is active', async () => {
const tracing = await createDetachedSubAgentTraceContext({
threadId: 'thread-1',
conversationId: 'thread-1',
messageGroupId: 'group-1',
messageId: 'message-1',
runId: 'run-1',
userId: 'user-1',
agentId: 'agent-builder-1',
role: 'workflow-builder',
kind: 'builder',
taskId: 'build-1',
input: { task: 'Build a workflow' },
});
expect(tracing).toBeDefined();
await tracing?.withActiveSpan(tracing.actorRun, async () => {
mergeTraceRunInputs(
tracing?.actorRun,
buildAgentTraceInputs({
systemPrompt: 'system prompt',
tools: createToolRegistry([
[
'build-workflow',
{
description: 'Build or patch a workflow from SDK code.',
} as never,
],
]),
modelId: 'anthropic/claude-sonnet-4-6',
}),
);
await Promise.resolve();
});
const actorInputs = tracing?.actorRun.inputs as Record<string, unknown>;
expect(actorInputs.model).toBe('anthropic/claude-sonnet-4-6');
expect(actorInputs.system_prompt).toBe('system prompt');
expect(actorInputs.assigned_tool_count).toBe(1);
expect(actorInputs.assigned_tool_names).toEqual(['build-workflow']);
});
it('redacts model secrets from agent trace inputs', () => {
const inputs = buildAgentTraceInputs({
systemPrompt: 'You are a helpful agent.',

141
packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts
View File

@ -709,21 +709,6 @@ interface CreateInstanceAiTraceContextOptions {
proxyConfig?: ServiceProxyConfig;
}
interface CreateDetachedSubAgentTraceContextOptions extends CreateInstanceAiTraceContextOptions {
agentId: string;
role: string;
kind: string;
taskId?: string;
plannedTaskId?: string;
workItemId?: string;
spawnedByTraceId?: string;
spawnedBySpanId?: string;
spawnedByRunId?: string;
spawnedByAgentId?: string;
spawnedByAgentRole?: string;
spawnedByToolCallId?: string;
}
interface CreateInternalOperationTraceContextOptions
extends Omit<CreateInstanceAiTraceContextOptions, 'messageId'> {
operationName: string;
@ -867,38 +852,6 @@ export async function submitLangsmithUserFeedback(
return true;
}
export function getCurrentOtelSpanContext(): { traceId: string; spanId: string } | undefined {
const activeSpanContext = otelTrace.getSpan(otelContext.active())?.spanContext();
if (activeSpanContext) {
return {
traceId: activeSpanContext.traceId,
spanId: activeSpanContext.spanId,
};
}
const currentRun = getCurrentProductTrace()?.currentRun;
if (currentRun?.otelTraceId && currentRun.otelSpanId) {
return {
traceId: currentRun.otelTraceId,
spanId: currentRun.otelSpanId,
};
}
return undefined;
}
export function getCurrentTraceToolCallId(): string | undefined {
const metadata = getCurrentProductTrace()?.currentRun.metadata;
return typeof metadata?.tool_call_id === 'string' ? metadata.tool_call_id : undefined;
}
export function mergeCurrentTraceMetadata(metadata: Record<string, unknown>): void {
const currentProductTrace = getCurrentProductTrace();
if (currentProductTrace) {
updateProductRunMetadata(currentProductTrace.runtime, currentProductTrace.currentRun, metadata);
}
}
export function appendRootRunMetadata(
root: InstanceAiTraceRun,
patch: Record<string, unknown>,
@ -1655,41 +1608,6 @@ async function buildBaseMetadata(
};
}
function buildDetachedSubAgentMetadata(
options: CreateDetachedSubAgentTraceContextOptions,
includeSpawnMetadata: boolean,
): Record<string, unknown> {
return {
agent_role: options.role,
agent_id: options.agentId,
execution_mode: 'background_subagent',
trace_kind: 'background_subagent',
task_kind: options.kind,
...(options.taskId ? { task_id: options.taskId } : {}),
...(options.plannedTaskId ? { planned_task_id: options.plannedTaskId } : {}),
...(options.workItemId ? { work_item_id: options.workItemId } : {}),
...(includeSpawnMetadata && options.spawnedByTraceId
? { spawned_by_trace_id: options.spawnedByTraceId }
: {}),
...(includeSpawnMetadata && options.spawnedBySpanId
? { spawned_by_span_id: options.spawnedBySpanId }
: {}),
...(includeSpawnMetadata && options.spawnedByRunId
? { spawned_by_run_id: options.spawnedByRunId }
: {}),
...(includeSpawnMetadata && options.spawnedByAgentId
? { spawned_by_agent_id: options.spawnedByAgentId }
: {}),
...(includeSpawnMetadata && options.spawnedByAgentRole
? { spawned_by_agent_role: options.spawnedByAgentRole }
: {}),
...(includeSpawnMetadata && options.spawnedByToolCallId
? { spawned_by_tool_call_id: options.spawnedByToolCallId }
: {}),
subagent_role: options.role,
};
}
function buildInternalOperationMetadata(operationName: string): Record<string, unknown> {
return {
agent_role: operationName,
@ -1949,65 +1867,6 @@ export async function continueInstanceAiTraceContext(
}
}
export async function createDetachedSubAgentTraceContext(
options: CreateDetachedSubAgentTraceContextOptions,
): Promise<InstanceAiTraceContext | undefined> {
if (!isLangSmithTracingEnabled(!!options.proxyConfig)) {
return undefined;
}
const projectName = options.projectName ?? resolveDefaultProjectName();
const baseMetadata = await buildBaseMetadata(options);
const createDetachedRuns = async () => {
const otelRuntime = await createProductOtelRuntime(projectName, options.proxyConfig);
const rootMetadata = buildDetachedSubAgentMetadata(options, true);
const actorMetadata = buildDetachedSubAgentMetadata(options, false);
const rootRun = startProductSpan(otelRuntime, {
projectName,
name: `background task: ${formatAgentRoleLabel(options.role)}`,
canonicalName: 'instance-ai.background_subagent',
runType: 'chain',
tags: normalizeTags(
['sub-agent', 'background'],
options.plannedTaskId ? ['planned'] : undefined,
),
metadata: mergeMetadata(baseMetadata, rootMetadata),
inputs: options.input,
root: true,
});
const actorRun = startProductSpan(otelRuntime, {
projectName,
name: `agent: ${formatAgentRoleLabel(options.role)}`,
canonicalName: `instance-ai.agent.${options.role}`,
runType: 'chain',
tags: normalizeTags(
['sub-agent', 'background'],
options.plannedTaskId ? ['planned'] : undefined,
),
metadata: mergeMetadata(baseMetadata, actorMetadata),
inputs: options.input,
parentRun: rootRun,
});
return createProductTraceContext({
projectName,
traceKind: 'background_subagent',
rootRun,
actorRun,
otelRuntime,
baseMetadata: mergeMetadata(baseMetadata, rootMetadata) ?? baseMetadata,
...(options.proxyConfig ? { proxyConfig: options.proxyConfig } : {}),
});
};
try {
return await withProxyHeaders(options.proxyConfig, createDetachedRuns);
} catch {
return undefined;
}
}
export async function createInternalOperationTraceContext(
options: CreateInternalOperationTraceContextOptions,
): Promise<InstanceAiTraceContext | undefined> {

89
packages/@n8n/instance-ai/src/types.ts
View File

@ -1830,71 +1830,12 @@ export interface InstanceAiTraceContext {
/** Structured result from a background task. The `text` field is the human-readable
* summary; `outcome` carries an optional typed payload consumed by the workflow
* loop controller (additive — existing callers that return a plain string still work). */
* loop controller. */
export interface BackgroundTaskResult {
text: string;
outcome?: Record<string, unknown>;
}
export interface SpawnBackgroundTaskOptions {
taskId: string;
threadId: string;
agentId: string;
role: string;
/** Existing trace context for legacy callers. Prefer createTraceContext for new background tasks. */
traceContext?: InstanceAiTraceContext;
/** Lazily creates the background trace only after the task is accepted and starts executing. */
createTraceContext?: () => Promise<InstanceAiTraceContext | undefined>;
/** When set, links the background task back to a planned task in the scheduler. */
plannedTaskId?: string;
/** Unique work item ID for workflow loop tracking. When set, the service
* uses the workflow loop controller to manage verify/repair transitions. */
workItemId?: string;
/**
* Identity used for single-flight dedupe. When present, a spawn with the same
* `plannedTaskId` (primary) or `role + workflowId` (fallback) as a currently-running
* task returns `{ status: 'duplicate', existing }` instead of starting a new task.
*/
dedupeKey?: {
plannedTaskId?: string;
workflowId?: string;
role: string;
};
/**
* Link this background task to a running checkpoint in the planned-task
* graph. Set when the orchestrator spawns a detached sub-agent (builder,
* research, data-table) from inside a
* `<planned-task-follow-up type="checkpoint">` turn. The post-run safety
* net defers failing the checkpoint while a child with this id is still
* running, and settlement re-emits the checkpoint follow-up when the last
* child settles — so the orchestrator re-enters the checkpoint context
* instead of a bare `<background-task-completed>` shell.
*/
parentCheckpointId?: string;
run: (
signal: AbortSignal,
drainCorrections: () => string[],
waitForCorrection: () => Promise<void>,
taskContext: { traceContext?: InstanceAiTraceContext },
) => Promise<string | BackgroundTaskResult>;
}
/** Result of a {@link SpawnBackgroundTaskOptions} spawn. */
export type SpawnBackgroundTaskResult =
| { status: 'started'; taskId: string; agentId: string }
| { status: 'limit-reached' }
| {
status: 'duplicate';
/** The live background task that matched on `dedupeKey`. */
existing: {
taskId: string;
agentId: string;
role: string;
plannedTaskId?: string;
workItemId?: string;
};
};
export interface WorkflowTaskService {
reportBuildOutcome(outcome: WorkflowBuildOutcome): Promise<WorkflowLoopAction>;
reportVerificationVerdict(verdict: VerificationResult): Promise<WorkflowLoopAction>;
@ -1914,7 +1855,6 @@ export interface OrchestrationContext {
projectId?: string;
orchestratorAgentId: string;
modelId: ModelConfig;
checkpointStore?: CheckpointStore;
eventBus: InstanceAiEventBus;
logger: Logger;
trackTelemetry?: (eventName: string, properties: Record<string, GenericValue>) => void;
@ -1932,28 +1872,12 @@ export interface OrchestrationContext {
usage: BuilderUsageItem[],
status: TraceStatus,
) => Promise<void>;
domainTools: InstanceAiToolRegistry;
abortSignal: AbortSignal;
taskStorage: TaskStorage;
tracing?: InstanceAiTraceContext;
waitForConfirmation?: (requestId: string) => Promise<{
approved: boolean;
credentialId?: string;
credentials?: Record<string, string>;
autoSetup?: { credentialType: string };
userInput?: string;
domainAccessAction?: string;
resourceDecision?: string;
answers?: Array<{
questionId: string;
selectedOptions: string[];
customText?: string;
skipped?: boolean;
}>;
}>;
/** Local MCP server (Computer Use daemon) for filesystem, shell, browser, and related tools. */
localMcpServer?: LocalMcpServer;
/** Safe MCP tools loaded from external servers and the local Computer Use gateway. */
/** Validated, approval-wrapped MCP tools available to Agent Builder. */
mcpTools?: InstanceAiToolRegistry;
/**
* Runtime-loadable skills available to the agent. Workspace-backed agents may
@ -1971,8 +1895,6 @@ export interface OrchestrationContext {
webhookBaseUrl?: string;
/** Form base URL for the n8n instance (e.g. http://localhost:5678/form) — distinct from webhookBaseUrl since Form Triggers serve at /form/, not /webhook/ */
formBaseUrl?: string;
/** Spawn a detached background task that outlives the current orchestrator run */
spawnBackgroundTask?: (opts: SpawnBackgroundTaskOptions) => SpawnBackgroundTaskResult;
/** Cancel a running background task by its ID */
cancelBackgroundTask?: (taskId: string) => Promise<void>;
/** Persist and inspect dependency-aware planned tasks for this thread. */
@ -1987,8 +1909,6 @@ export interface OrchestrationContext {
workspaceRoot?: string;
/** Directories containing node type definition files (.ts) for materializing into sandbox */
nodeDefinitionDirs?: string[];
/** Native memory store — used to retrieve thread message history for sub-agents. */
memory?: BuiltMemory;
/** The current user message being processed — needed because memory history only
* returns previously-saved messages, so the in-flight message isn't available yet. */
currentUserMessage?: string;
@ -2017,11 +1937,6 @@ export interface OrchestrationContext {
touchBackgroundTask?: (taskId: string) => boolean;
/** Shared workflow-task state service for build / verify / credential-finalize flows */
workflowTaskService?: WorkflowTaskService;
/** When set, LangSmith traces are routed through the AI service proxy. */
tracingProxyConfig?: ServiceProxyConfig;
/** Summaries of currently running background tasks in this thread.
* Used to give sub-agents thread-state awareness (what else is happening). */
getRunningTaskSummaries?: () => Array<{ taskId: string; role: string; goal?: string }>;
/** IANA time zone for the current user (e.g. "Europe/Helsinki"). Propagated to sub-agents
* so they can resolve "now" consistently with the orchestrator. */
timeZone?: string;

2
packages/@n8n/instance-ai/src/workspace/__tests__/sandbox-setup.test.ts
View File

@ -386,7 +386,7 @@ describe('setupSandboxWorkspace', () => {
expect(initialized).toBe(false);
expect(runInSandbox).not.toHaveBeenCalledWith(
expect.anything(),
'npm install --ignore-scripts --no-audit --no-fund --prefer-offline',
'npm install --ignore-scripts --no-audit --no-fund --prefer-online',
'/sandbox',
);
const writtenPaths = writeFile.mock.calls.map(([path]) => path);

2
packages/@n8n/instance-ai/src/workspace/__tests__/snapshot-manager.test.ts
View File

@ -197,7 +197,7 @@ describe('SnapshotManager.ensureImage', () => {
'mkdir -p /home/daytona/workspace/src /home/daytona/workspace/chunks /home/daytona/workspace/node-types',
);
expect(image.dockerfile).toContain(
'npm install --ignore-scripts --no-audit --no-fund --prefer-offline',
'npm install --ignore-scripts --no-audit --no-fund --prefer-online',
);
const stagingDir = image.contextList[0]?.sourcePath;

6
packages/@n8n/instance-ai/src/workspace/sandbox-setup.ts
View File

@ -99,10 +99,10 @@ function resolveHostDepVersion(name: string): string {
* `--no-audit` matters most: npm otherwise posts the whole tree to the registry's
* advisories endpoint and blocks until it answers or its 300s fetch timeout expires,
* so a slow registry turns a 10s install into minutes. Nobody reads the audit or
* funding output in a sandbox. `--prefer-offline` lets a warm npm cache skip
* freshness checks against the registry.
* funding output in a sandbox. Refresh registry metadata so a cached sandbox
* can install a newly published SDK version.
*/
export const NPM_INSTALL_FLAGS = '--ignore-scripts --no-audit --no-fund --prefer-offline';
export const NPM_INSTALL_FLAGS = '--ignore-scripts --no-audit --no-fund --prefer-online';
/**
* Versions pinned from the host's installed packages. Pinning is load-bearing

1
packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.builderAskedForInput.test.ts
View File

@ -23,7 +23,6 @@ vi.mock('@n8n/instance-ai', async () => {
handleBuildOutcome: vi.fn(),
handleVerificationVerdict: vi.fn(),
createInstanceAgent: vi.fn(),
createAllTools: vi.fn(),
};
});

1
packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.refineTitle.test.ts
View File

@ -27,7 +27,6 @@ vi.mock('@n8n/instance-ai', async () => {
handleBuildOutcome: vi.fn(),
handleVerificationVerdict: vi.fn(),
createInstanceAgent: vi.fn(),
createAllTools: vi.fn(),
// No trace context so refineTitleIfNeeded takes the direct title-generation path.
createInternalOperationTraceContext: vi.fn(async () => undefined),
releaseTraceClient: vi.fn(),

494
packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts
View File

@ -77,7 +77,6 @@ vi.mock('@n8n/instance-ai', async () => {
if (uncachedInput + cacheRead + cacheWrite + output === 0) return [];
return [{ type: 'llmTokens', model, uncachedInput, cacheRead, cacheWrite, output }];
},
createAllTools: vi.fn(),
createOrchestratorRunControl: vi.fn(function () {
return {
state: undefined,
@ -207,7 +206,6 @@ import type { InstanceAiConfig } from '@n8n/config';
import type { User } from '@n8n/db';
import { Container } from '@n8n/di';
import {
createAllTools,
createLazyRuntimeWorkspace,
createLazyWorkspaceRuntimeSkillSource,
createOrchestratorRunControl,
@ -223,9 +221,6 @@ import {
type BuilderUsageItem,
type ManagedBackgroundTask,
type InstanceAiTraceContext,
type SpawnBackgroundTaskOptions,
type SpawnBackgroundTaskResult,
type SpawnManagedBackgroundTaskOptions,
type TraceStatus,
type WorkflowVerificationObligation,
} from '@n8n/instance-ai';
@ -246,182 +241,6 @@ import { InstanceAiRunLimitError } from '../instance-ai-run-limit.error';
import { InstanceAiService } from '../instance-ai.service';
import { InstanceAiSandboxService } from '../sandbox';
type ServiceInternals = {
pendingCheckpointReentries: Map<string, Set<string>>;
queuePendingCheckpointReentry: (threadId: string, checkpointTaskId: string) => void;
drainPendingCheckpointReentries: (user: User, threadId: string) => Promise<void>;
reenterCheckpointById: Mock<(...args: [User, string, string, string?]) => Promise<boolean>>;
backgroundTasks: {
getRunningTasksByParentCheckpoint: Mock;
};
runState: {
getActiveRunId: Mock;
hasSuspendedRun: Mock;
};
logger: { debug: Mock; warn: Mock; error: Mock };
};
type BackgroundTaskFollowUpServiceInternals = {
spawnBackgroundTask: (
runId: string,
opts: SpawnBackgroundTaskOptions,
messageGroupIdOverride?: string,
) => SpawnBackgroundTaskResult;
backgroundTasks: {
spawn: MockedFunction<
(options: SpawnManagedBackgroundTaskOptions) => {
status: 'started';
task: ManagedBackgroundTask;
}
>;
getRunningTasks: MockedFunction<(threadId: string) => ManagedBackgroundTask[]>;
};
runState: {
getMessageGroupId: MockedFunction<(threadId: string) => string | undefined>;
getThreadUser: MockedFunction<(threadId: string) => User | undefined>;
getActiveRunId: MockedFunction<(threadId: string) => string | undefined>;
hasSuspendedRun: MockedFunction<(threadId: string) => boolean>;
};
liveness: {
hasTimedOutActiveRunThread: MockedFunction<(threadId: string) => boolean>;
};
eventBus: {
publish: MockedFunction<(threadId: string, event: InstanceAiEvent) => void>;
};
tracing: {
finalizeBackgroundTaskTracing: MockedFunction<
(task: ManagedBackgroundTask, status: 'completed' | 'failed' | 'cancelled') => Promise<void>
>;
};
handlePlannedTaskSettlement: MockedFunction<
(
user: User,
task: ManagedBackgroundTask,
status: 'succeeded' | 'failed' | 'cancelled',
) => Promise<void>
>;
terminalOutcome: {
recordBackgroundTerminalOutcome: MockedFunction<(task: ManagedBackgroundTask) => Promise<void>>;
};
startInternalFollowUpRun: MockedFunction<
(
user: User,
threadId: string,
message: string,
messageGroupId?: string,
isReplanFollowUp?: boolean,
checkpoint?: { isCheckpointFollowUp: true; checkpointTaskId: string },
resumeReasonOverride?: string,
) => Promise<string | undefined>
>;
maybeStartWorkflowVerificationFollowUp: MockedFunction<
(user: User, task: ManagedBackgroundTask) => Promise<boolean>
>;
maybeStartWorkflowSetupFollowUp: MockedFunction<
(user: User, threadId: string) => Promise<boolean>
>;
queuePendingCheckpointReentry: MockedFunction<
(threadId: string, checkpointTaskId: string) => void
>;
maybeReenterParentCheckpoint: MockedFunction<
(user: User, threadId: string, task: ManagedBackgroundTask) => Promise<boolean>
>;
taskProjector: { syncFromBackgroundTask: Mock };
logger: { warn: Mock; debug: Mock };
};
function createBackgroundTaskFollowUpService({
timedOutThread = false,
}: { timedOutThread?: boolean } = {}): {
service: BackgroundTaskFollowUpServiceInternals;
task: ManagedBackgroundTask;
getSpawnOptions: () => SpawnManagedBackgroundTaskOptions;
} {
const service = Object.create(
InstanceAiService.prototype,
) as unknown as BackgroundTaskFollowUpServiceInternals;
let spawnOptions: SpawnManagedBackgroundTaskOptions | undefined;
const task: ManagedBackgroundTask = {
taskId: 'task-1',
threadId: 'thread-a',
runId: 'run-1',
role: 'workflow-builder',
agentId: 'agent-builder',
status: 'completed',
result: 'done',
startedAt: 0,
lastActivityAt: 0,
abortController: new AbortController(),
corrections: [],
messageGroupId: 'group-1',
};
service.backgroundTasks = {
spawn: vi.fn((options: SpawnManagedBackgroundTaskOptions) => {
spawnOptions = options;
return { status: 'started', task };
}),
getRunningTasks: vi.fn((_threadId: string) => []),
};
service.runState = {
getMessageGroupId: vi.fn((_threadId: string) => 'group-1'),
getThreadUser: vi.fn((_threadId: string) => fakeUser),
getActiveRunId: vi.fn((_threadId: string) => undefined),
hasSuspendedRun: vi.fn((_threadId: string) => false),
};
service.liveness = {
hasTimedOutActiveRunThread: vi.fn((threadId: string) =>
timedOutThread ? threadId === 'thread-a' : false,
),
};
service.eventBus = { publish: vi.fn((_threadId: string, _event: InstanceAiEvent) => {}) };
service.taskProjector = { syncFromBackgroundTask: vi.fn(async () => {}) };
service.tracing = {
finalizeBackgroundTaskTracing: vi.fn(
async (_task: ManagedBackgroundTask, _status: 'completed' | 'failed' | 'cancelled') => {},
),
};
service.handlePlannedTaskSettlement = vi.fn(
async (
_user: User,
_task: ManagedBackgroundTask,
_status: 'succeeded' | 'failed' | 'cancelled',
) => {},
);
service.terminalOutcome = {
recordBackgroundTerminalOutcome: vi.fn(async (_task: ManagedBackgroundTask) => {}),
};
service.startInternalFollowUpRun = vi.fn(
async (
_user: User,
_threadId: string,
_message: string,
_messageGroupId?: string,
_isReplanFollowUp?: boolean,
_checkpoint?: { isCheckpointFollowUp: true; checkpointTaskId: string },
_resumeReasonOverride?: string,
) => 'run-follow-up',
);
service.maybeStartWorkflowVerificationFollowUp = vi.fn(
async (_user: User, _task: ManagedBackgroundTask) => false,
);
service.maybeStartWorkflowSetupFollowUp = vi.fn(async (_user: User, _threadId: string) => false);
service.queuePendingCheckpointReentry = vi.fn();
service.maybeReenterParentCheckpoint = vi.fn(
async (_user: User, _threadId: string, _task: ManagedBackgroundTask) => false,
);
service.logger = { warn: vi.fn(), debug: vi.fn() };
return {
service,
task,
getSpawnOptions: () => {
if (!spawnOptions) throw new Error('Background task was not spawned');
return spawnOptions;
},
};
}
type StartRunServiceInternals = {
startRun: InstanceAiService['startRun'];
liveness: {
@ -474,32 +293,6 @@ function createStartRunService(): StartRunServiceInternals {
return service;
}
function createCheckpointService(): ServiceInternals {
// Bypass the constructor — we only exercise the three pending-reentry helpers
// and their direct dependencies. Everything else (scheduler, event bus, etc.)
// is out of scope for this unit.
const service = Object.create(InstanceAiService.prototype) as unknown as ServiceInternals;
service.pendingCheckpointReentries = new Map();
service.reenterCheckpointById = vi.fn(
async (_user: User, _threadId: string, _checkpointTaskId: string, _mgid?: string) => true,
);
service.backgroundTasks = {
getRunningTasksByParentCheckpoint: vi.fn(() => []),
};
service.runState = {
getActiveRunId: vi.fn(() => undefined),
hasSuspendedRun: vi.fn(() => false),
};
service.logger = {
debug: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
return service;
}
type CheckpointPruneServiceInternals = {
startCheckpointPruning: () => void;
stopCheckpointPruning: () => void;
@ -717,7 +510,6 @@ type TerminalGuardOrderServiceInternals = {
creditService: { claimRunUsage: Mock; ensureQuotaLockApplied: Mock };
failedInternalFollowUpStreaks: Map<string, number>;
schedulePlannedTasks: Mock;
drainPendingCheckpointReentries: Mock;
createPlannedTaskState: Mock;
syncPlannedTasksToUi: Mock;
taskProjector: { syncFromWorkflowLoop: Mock };
@ -796,7 +588,6 @@ function createTerminalGuardOrderService(): TerminalGuardOrderServiceInternals {
};
service.failedInternalFollowUpStreaks = new Map();
service.schedulePlannedTasks = vi.fn(async () => {});
service.drainPendingCheckpointReentries = vi.fn(async () => {});
service.preserveHitlOnShutdown = new Set();
service.terminalOutcome = new InstanceAiTerminalOutcomeService({
@ -830,7 +621,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
(createSandbox as Mock).mockReset();
(createWorkspace as Mock).mockReset();
(setupSandboxWorkspace as Mock).mockReset();
(createAllTools as Mock).mockReset();
(createLazyRuntimeWorkspace as Mock).mockImplementation(
(args: { id?: string; ensureWorkspace: () => Promise<unknown> }) => ({
id: args.id ?? 'lazy-runtime-workspace',
@ -897,7 +687,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
webhookBaseUrl: string;
formBaseUrl: string;
runState: { touchActiveRun: Mock; registerPendingConfirmation: Mock };
spawnBackgroundTask: Mock;
cancelBackgroundTask: Mock;
backgroundTasks: { touchTask: Mock };
schedulePlannedTasks: Mock;
@ -956,7 +745,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
touchActiveRun: vi.fn(),
registerPendingConfirmation: vi.fn(),
};
service.spawnBackgroundTask = vi.fn();
service.cancelBackgroundTask = vi.fn();
service.backgroundTasks = { touchTask: vi.fn() };
service.schedulePlannedTasks = vi.fn();
@ -985,7 +773,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
claimRunUsage: vi.fn(),
ensureQuotaLockApplied: vi.fn(async () => {}),
};
(createAllTools as Mock).mockReturnValue(new Map());
const sandbox = { id: 'sandbox-1' };
const workspace = {
init: vi.fn(async () => {}),
@ -1176,7 +963,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
webhookBaseUrl: string;
formBaseUrl: string;
runState: { touchActiveRun: Mock; registerPendingConfirmation: Mock };
spawnBackgroundTask: Mock;
cancelBackgroundTask: Mock;
backgroundTasks: { touchTask: Mock };
schedulePlannedTasks: Mock;
@ -1236,7 +1022,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
touchActiveRun: vi.fn(),
registerPendingConfirmation: vi.fn(),
};
service.spawnBackgroundTask = vi.fn();
service.cancelBackgroundTask = vi.fn();
service.backgroundTasks = { touchTask: vi.fn() };
service.schedulePlannedTasks = vi.fn();
@ -1265,7 +1050,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
claimRunUsage: vi.fn(),
ensureQuotaLockApplied: vi.fn(async () => {}),
};
(createAllTools as Mock).mockReturnValue(new Map());
(createSandbox as Mock).mockResolvedValue({ id: 'sandbox-1' });
(createWorkspace as Mock).mockReturnValue({
init: vi.fn(async () => {}),
@ -1380,99 +1164,7 @@ describe('InstanceAiService — memory task observer', () => {
});
});
describe('InstanceAiService — background task auto-follow-up', () => {
it('starts an internal follow-up when the last direct background task settles normally', async () => {
const { service, task, getSpawnOptions } = createBackgroundTaskFollowUpService();
const result = service.spawnBackgroundTask(
'run-1',
{
taskId: 'task-1',
threadId: 'thread-a',
agentId: 'agent-builder',
role: 'workflow-builder',
run: async () => 'done',
},
'group-1',
);
await getSpawnOptions().onSettled?.(task);
expect(result).toEqual({ status: 'started', taskId: 'task-1', agentId: 'agent-builder' });
expect(service.startInternalFollowUpRun).toHaveBeenCalledWith(
fakeUser,
'thread-a',
expect.stringContaining('<background-task-completed>'),
'group-1',
);
});
it('lets workflow verification follow-up replace the generic background completion follow-up', async () => {
const { service, task, getSpawnOptions } = createBackgroundTaskFollowUpService();
service.maybeStartWorkflowVerificationFollowUp.mockResolvedValue(true);
service.spawnBackgroundTask(
'run-1',
{
taskId: 'task-1',
threadId: 'thread-a',
agentId: 'agent-builder',
role: 'workflow-builder',
workItemId: 'wi-1',
run: async () => 'done',
},
'group-1',
);
await getSpawnOptions().onSettled?.(task);
expect(service.maybeStartWorkflowVerificationFollowUp).toHaveBeenCalledWith(fakeUser, task);
expect(service.startInternalFollowUpRun).not.toHaveBeenCalled();
});
it('skips internal follow-up when the active run already timed out', async () => {
const { service, task, getSpawnOptions } = createBackgroundTaskFollowUpService({
timedOutThread: true,
});
service.spawnBackgroundTask(
'run-1',
{
taskId: 'task-1',
threadId: 'thread-a',
agentId: 'agent-builder',
role: 'workflow-builder',
run: async () => 'done',
},
'group-1',
);
await getSpawnOptions().onSettled?.(task);
expect(service.startInternalFollowUpRun).not.toHaveBeenCalled();
expect(service.terminalOutcome.recordBackgroundTerminalOutcome).toHaveBeenCalledWith(task);
});
it('skips internal follow-up when the task itself timed out', async () => {
const { service, task, getSpawnOptions } = createBackgroundTaskFollowUpService();
task.status = 'failed';
task.timeoutReason = 'idle_timeout';
task.error = 'Background workflow-builder task timed out after 600000ms';
service.spawnBackgroundTask(
'run-1',
{
taskId: 'task-1',
threadId: 'thread-a',
agentId: 'agent-builder',
role: 'workflow-builder',
run: async () => 'done',
},
'group-1',
);
await getSpawnOptions().onSettled?.(task);
expect(service.startInternalFollowUpRun).not.toHaveBeenCalled();
expect(service.terminalOutcome.recordBackgroundTerminalOutcome).toHaveBeenCalledWith(task);
});
describe('InstanceAiService — run start', () => {
describe('concurrency admission', () => {
it('refuses a new turn when the user is at their limit', () => {
const service = createStartRunService();
@ -1590,7 +1282,6 @@ describe('InstanceAiService — background task auto-follow-up', () => {
}
});
});
it('clears the active-timeout guard when the user starts a new run', () => {
const service = createStartRunService();
@ -1652,111 +1343,6 @@ describe('InstanceAiService — background task auto-follow-up', () => {
});
});
describe('InstanceAiService — pending checkpoint re-entry', () => {
describe('queuePendingCheckpointReentry', () => {
it('records a marker keyed by threadId + checkpointTaskId', () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
expect(service.pendingCheckpointReentries.get('thread-a')).toEqual(new Set(['cp-1']));
});
it('deduplicates markers for the same (thread, checkpoint) pair', () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
expect(service.pendingCheckpointReentries.get('thread-a')?.size).toBe(1);
});
it('keeps markers for different threads separate', () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
service.queuePendingCheckpointReentry('thread-b', 'cp-1');
expect(service.pendingCheckpointReentries.get('thread-a')).toEqual(new Set(['cp-1']));
expect(service.pendingCheckpointReentries.get('thread-b')).toEqual(new Set(['cp-1']));
});
});
describe('drainPendingCheckpointReentries', () => {
it('fires re-entry for each queued marker when the thread is idle', async () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
service.queuePendingCheckpointReentry('thread-a', 'cp-2');
await service.drainPendingCheckpointReentries(fakeUser, 'thread-a');
expect(service.reenterCheckpointById).toHaveBeenCalledTimes(2);
expect(service.reenterCheckpointById).toHaveBeenCalledWith(fakeUser, 'thread-a', 'cp-1');
expect(service.reenterCheckpointById).toHaveBeenCalledWith(fakeUser, 'thread-a', 'cp-2');
expect(service.pendingCheckpointReentries.get('thread-a')).toBeUndefined();
});
it('stops draining if a new run starts mid-drain', async () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
service.queuePendingCheckpointReentry('thread-a', 'cp-2');
// After the first re-entry fires, simulate a new active run.
let calls = 0;
service.reenterCheckpointById.mockImplementation(async () => {
calls += 1;
if (calls === 1) {
service.runState.getActiveRunId.mockReturnValue('run-new');
}
return true;
});
await service.drainPendingCheckpointReentries(fakeUser, 'thread-a');
// First marker drained; second should remain queued for the next run's cleanup.
expect(service.reenterCheckpointById).toHaveBeenCalledTimes(1);
expect(service.pendingCheckpointReentries.get('thread-a')).toEqual(new Set(['cp-2']));
});
it('skips a marker whose parent-tagged siblings are still running', async () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
service.queuePendingCheckpointReentry('thread-a', 'cp-2');
// cp-1 has a sibling still in flight, cp-2 is clear.
service.backgroundTasks.getRunningTasksByParentCheckpoint.mockImplementation(
(_threadId: string, cp: string) => (cp === 'cp-1' ? [{ taskId: 'sibling-running' }] : []),
);
await service.drainPendingCheckpointReentries(fakeUser, 'thread-a');
expect(service.reenterCheckpointById).toHaveBeenCalledTimes(1);
expect(service.reenterCheckpointById).toHaveBeenCalledWith(fakeUser, 'thread-a', 'cp-2');
// cp-1 stays queued — the sibling's own settlement will drive the next drain.
expect(service.pendingCheckpointReentries.get('thread-a')).toEqual(new Set(['cp-1']));
});
it('returns early when a suspended run is present', async () => {
const service = createCheckpointService();
service.queuePendingCheckpointReentry('thread-a', 'cp-1');
service.runState.hasSuspendedRun.mockReturnValue(true);
await service.drainPendingCheckpointReentries(fakeUser, 'thread-a');
expect(service.reenterCheckpointById).not.toHaveBeenCalled();
expect(service.pendingCheckpointReentries.get('thread-a')).toEqual(new Set(['cp-1']));
});
it('is a no-op when no markers are queued', async () => {
const service = createCheckpointService();
await service.drainPendingCheckpointReentries(fakeUser, 'thread-nonexistent');
expect(service.reenterCheckpointById).not.toHaveBeenCalled();
});
});
});
describe('InstanceAiService — scheduled pruning', () => {
it('marks checkpoints expired older than the retention window', async () => {
const service = createCheckpointPruneService();
@ -3743,7 +3329,6 @@ describe('InstanceAiService — terminal response guard wiring', () => {
expect(service.tracing.maybeFinalizeRunTraceRoot).not.toHaveBeenCalled();
expect(service.liveness.consumeRunTimeout).not.toHaveBeenCalled();
expect(service.schedulePlannedTasks).not.toHaveBeenCalled();
expect(service.drainPendingCheckpointReentries).not.toHaveBeenCalled();
expect(service.taskProjector.syncFromWorkflowLoop).not.toHaveBeenCalled();
expect(service.maybeStartWorkflowSetupFollowUp).not.toHaveBeenCalled();
});
@ -4205,7 +3790,6 @@ describe('InstanceAiService — run error reporter lifecycle', () => {
expect(service.creditService.claimRunUsage).not.toHaveBeenCalled();
expect(service.telemetry.track).not.toHaveBeenCalled();
expect(service.schedulePlannedTasks).not.toHaveBeenCalled();
expect(service.drainPendingCheckpointReentries).not.toHaveBeenCalled();
expect(service.taskProjector.syncFromWorkflowLoop).not.toHaveBeenCalled();
expect(service.maybeStartWorkflowSetupFollowUp).not.toHaveBeenCalled();
expect(service.liveness.consumeRunTimeout).not.toHaveBeenCalled();
@ -4516,9 +4100,6 @@ describe('InstanceAiService — run error reporter lifecycle', () => {
service.schedulePlannedTasks = vi.fn(async () => {
callOrder.push('schedulePlannedTasks');
});
service.drainPendingCheckpointReentries = vi.fn(async () => {
callOrder.push('drainPendingCheckpointReentries');
});
service.taskProjector = {
syncFromWorkflowLoop: vi.fn(async () => {
callOrder.push('syncFromWorkflowLoop');
@ -4550,7 +4131,6 @@ describe('InstanceAiService — run error reporter lifecycle', () => {
'beginRun',
'finalizeRun',
'schedulePlannedTasks',
'drainPendingCheckpointReentries',
'syncFromWorkflowLoop',
'maybeStartWorkflowSetupFollowUp',
'endRun',
@ -4574,7 +4154,6 @@ describe('InstanceAiService — run error reporter lifecycle', () => {
await service.processResumedStream({}, {}, resumedStreamOpts(abortController));
expect(service.schedulePlannedTasks).not.toHaveBeenCalled();
expect(service.drainPendingCheckpointReentries).not.toHaveBeenCalled();
// The UI projection still runs, so a stopped run's task states reach the client.
expect(service.taskProjector.syncFromWorkflowLoop).toHaveBeenCalledWith('thread-a', 'run-1');
expect(service.instanceAiErrorReporter.endRun).toHaveBeenCalledWith(
@ -5044,77 +4623,6 @@ describe('InstanceAiService — agent snapshots for attached agents', () => {
});
});
describe('InstanceAiService — workflow verification follow-up gate', () => {
type VerificationGateService = {
workflowObligations: { getObligation: Mock };
trackWorkflowVerificationObligation: Mock;
buildWorkflowVerificationFollowUpMessage: Mock;
startInternalFollowUpRun: Mock;
maybeStartWorkflowVerificationFollowUp: (
user: User,
task: ManagedBackgroundTask,
) => Promise<boolean>;
};
function createVerificationGateService(
obligation: WorkflowVerificationObligation,
): VerificationGateService {
const service = Object.create(
InstanceAiService.prototype,
) as unknown as VerificationGateService;
service.workflowObligations = { getObligation: vi.fn(async () => obligation) };
service.trackWorkflowVerificationObligation = vi.fn();
service.buildWorkflowVerificationFollowUpMessage = vi.fn(() => 'verification message');
service.startInternalFollowUpRun = vi.fn(async () => 'follow-up-run');
return service;
}
const builderTask = {
taskId: 'task-1',
threadId: 'thread-a',
runId: 'run-1',
role: 'workflow-builder',
workItemId: 'wi-1',
status: 'completed',
messageGroupId: 'group-1',
} as ManagedBackgroundTask;
function makeObligation(
overrides: Partial<WorkflowVerificationObligation>,
): WorkflowVerificationObligation {
return {
workItemId: 'wi-1',
threadId: 'thread-a',
source: 'direct',
policy: 'required',
status: 'ready_to_verify',
updatedAt: '2026-01-01T00:00:00.000Z',
...overrides,
} as WorkflowVerificationObligation;
}
it('starts a verification follow-up when the build is ready to verify', async () => {
const service = createVerificationGateService(makeObligation({ status: 'ready_to_verify' }));
const started = await service.maybeStartWorkflowVerificationFollowUp(fakeUser, builderTask);
expect(started).toBe(true);
expect(service.startInternalFollowUpRun).toHaveBeenCalled();
});
it.each(['verified', 'needs_setup', 'not_verifiable', 'blocked'] as const)(
'does not run a verification follow-up for a %s build (setup is routed separately)',
async (status) => {
const service = createVerificationGateService(makeObligation({ status }));
const started = await service.maybeStartWorkflowVerificationFollowUp(fakeUser, builderTask);
expect(started).toBe(false);
expect(service.startInternalFollowUpRun).not.toHaveBeenCalled();
},
);
});
describe('InstanceAiService — deterministic workflow setup follow-up', () => {
type SetupFollowUpService = {
listWorkflowLoopRecords: Mock;

1
packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.threadPushRef.test.ts
View File

@ -23,7 +23,6 @@ vi.mock('@n8n/instance-ai', async () => {
handleBuildOutcome: vi.fn(),
handleVerificationVerdict: vi.fn(),
createInstanceAgent: vi.fn(),
createAllTools: vi.fn(),
};
});

469
packages/cli/src/modules/instance-ai/instance-ai.service.ts
View File

@ -35,7 +35,6 @@ import { Container, Service } from '@n8n/di';
import {
MAX_STEPS,
createInstanceAgent,
createAllTools,
createLazyRuntimeWorkspace,
createLazyWorkspaceRuntimeSkillSource,
getPromptWorkspaceRoot,
@ -94,8 +93,6 @@ import {
type PlannedWorkflowVerification,
type OrchestratorRunHandoffState,
type OrchestratorRunStopSignal,
type SpawnBackgroundTaskOptions,
type SpawnBackgroundTaskResult,
type ServiceProxyConfig,
type StreamableAgent,
type SuspendedRunState,
@ -211,10 +208,7 @@ import {
type MessageTraceFinalization,
type OrchestratorResumeReason,
} from './tracing';
import {
parseWorkflowBuildOutcome,
WorkflowVerificationObligationService,
} from './workflow-verification-obligation-service';
import { WorkflowVerificationObligationService } from './workflow-verification-obligation-service';
import { WorkflowVerificationTaskProjector } from './workflow-verification-task-projector';
import { AgentExecutionService } from '../agents/agent-execution.service';
import { formatPreviewSessionContext } from '../agents/builder/format-preview-context';
@ -758,14 +752,6 @@ export class InstanceAiService {
*/
private readonly failedInternalFollowUpStreaks = new Map<string, number>();
/**
* Checkpoint re-entries that could not fire when their parent-tagged child
* settled (an orchestrator run was live, or other parent siblings were
* still running). Drained from the post-run cleanup path so the checkpoint
* is never left orphaned.
*/
private readonly pendingCheckpointReentries = new Map<string, Set<string>>();
private readonly terminalOutcome: InstanceAiTerminalOutcomeService;
private readonly liveness: InstanceAiLivenessService<SuspendedRunState<User>>;
@ -2422,7 +2408,7 @@ export class InstanceAiService {
// There's another ensure lock check at `getCredits`, which only fires when the frontend mounts.
await this.creditService.ensureQuotaLockApplied(user);
const { searchProxyConfig, tracingProxyConfig, tokenManager, proxyBaseUrl } =
const { searchProxyConfig, tokenManager, proxyBaseUrl } =
proxyRunConfig ?? (await this.createProxyRunConfig(user));
const proxyContext = { runId, threadId };
@ -2664,8 +2650,6 @@ export class InstanceAiService {
context.trackTelemetry = (eventName, properties) => {
this.telemetry.track(eventName, redactTelemetryProperties(properties));
};
const domainTools = createAllTools(context);
const orchestrationContext: OrchestrationContext = {
threadId,
runId,
@ -2674,7 +2658,6 @@ export class InstanceAiService {
projectId: boundProjectId,
orchestratorAgentId: orchestratorAgentId(runId),
modelId,
checkpointStore: this.checkpointStore,
eventBus: this.eventBus,
logger: this.logger,
trackTelemetry: (eventName, properties) => {
@ -2709,7 +2692,6 @@ export class InstanceAiService {
});
}
},
domainTools,
abortSignal,
taskStorage,
timeZone: this.defaultTimeZone,
@ -2719,28 +2701,7 @@ export class InstanceAiService {
oauth2CallbackUrl: this.oauth2CallbackUrl,
webhookBaseUrl: this.webhookBaseUrl,
formBaseUrl: this.formBaseUrl,
waitForConfirmation: async (requestId: string) => {
this.runState.touchActiveRun(threadId);
return await new Promise<ConfirmationData>((resolve) => {
this.runState.registerPendingConfirmation(requestId, {
resolve,
threadId,
userId: user.id,
createdAt: Date.now(),
});
void this.suspendedThreads.persistPendingConfirmation({
requestId,
threadId,
userId: user.id,
runId,
messageGroupId,
kind: 'inline',
});
});
},
cancelBackgroundTask: async (taskId) => this.cancelBackgroundTask(threadId, taskId),
spawnBackgroundTask: (opts) => this.spawnBackgroundTask(runId, opts, messageGroupId),
touchRun: () => this.runState.touchActiveRun(threadId),
touchBackgroundTask: (taskId) => this.backgroundTasks.touchTask(threadId, taskId),
plannedTaskService,
@ -2753,8 +2714,6 @@ export class InstanceAiService {
workspaceRoot,
nodeDefinitionDirs: nodeDefDirs.length > 0 ? nodeDefDirs : undefined,
domainContext: context,
tracingProxyConfig,
memory,
};
return {
@ -2996,60 +2955,6 @@ export class InstanceAiService {
await this.schedulePlannedTasks(user, task.threadId);
}
private async maybeStartWorkflowVerificationFollowUp(
user: User,
task: ManagedBackgroundTask,
): Promise<boolean> {
if (task.role !== 'workflow-builder' || !task.workItemId) return false;
const obligation = await this.workflowObligations.getObligation(
task.threadId,
task.workItemId,
{
source: task.plannedTaskId ? 'planned' : 'direct',
plannedTaskId: task.plannedTaskId,
},
);
if (!obligation) return false;
this.trackWorkflowVerificationObligation(obligation, 'background_task_settled');
// Only run a verification follow-up when there is something to verify.
// Setup (mocked credentials, unresolved placeholders) is handled separately
// and deterministically by `maybeStartWorkflowSetupFollowUp`.
if (obligation.status !== 'ready_to_verify' && obligation.status !== 'verifying') {
return false;
}
const outcome = parseWorkflowBuildOutcome(task.outcome);
const startedRunId = await this.startInternalFollowUpRun(
user,
task.threadId,
this.buildWorkflowVerificationFollowUpMessage({
obligation,
outcome,
sourceTask: {
taskId: task.taskId,
role: task.role,
status: task.status,
result: task.result,
error: task.error,
plannedTaskId: task.plannedTaskId,
workItemId: task.workItemId,
},
}),
task.messageGroupId,
false,
undefined,
'workflow_verification',
);
this.trackWorkflowVerificationObligation(obligation, 'follow_up_start_attempted', {
follow_up_started: startedRunId.length > 0,
});
return startedRunId.length > 0;
}
private buildWorkflowSetupFollowUpMessage(obligation: WorkflowVerificationObligation): string {
const payload = {
workflowId: obligation.workflowId,
@ -4575,9 +4480,6 @@ export class InstanceAiService {
} else if (reschedule) {
await this.schedulePlannedTasks(user, threadId);
}
if (reschedule) {
await this.drainPendingCheckpointReentries(user, threadId);
}
await this.taskProjector.syncFromWorkflowLoop(threadId, runId);
if (reschedule) {
await this.maybeStartWorkflowSetupFollowUp(user, threadId);
@ -4594,132 +4496,6 @@ export class InstanceAiService {
* terminal (marking it failed if the orchestrator abandoned it) and re-ticks
* the scheduler so the next planned action can fire.
*/
private queuePendingCheckpointReentry(threadId: string, checkpointTaskId: string): void {
let set = this.pendingCheckpointReentries.get(threadId);
if (!set) {
set = new Set();
this.pendingCheckpointReentries.set(threadId, set);
}
set.add(checkpointTaskId);
}
/**
* Drain any checkpoint re-entries whose parent-tagged children settled while
* an orchestrator run was live (or while other siblings were still running).
* Called from the post-run cleanup path in every run-ending `finally` block,
* so the checkpoint is never left orphaned when the settlement path could
* not fire immediately.
*/
private async drainPendingCheckpointReentries(user: User, threadId: string): Promise<void> {
const set = this.pendingCheckpointReentries.get(threadId);
if (!set || set.size === 0) return;
const snapshot = [...set];
for (const checkpointTaskId of snapshot) {
// If a new run started while we were draining, stop — the next run's
// cleanup will pick up the remaining markers.
if (this.runState.getActiveRunId(threadId) || this.runState.hasSuspendedRun(threadId)) {
return;
}
// A new parent-tagged child is running — let its settlement drive the
// checkpoint instead of racing another re-entry.
const siblings = this.backgroundTasks.getRunningTasksByParentCheckpoint(
threadId,
checkpointTaskId,
);
if (siblings.length > 0) continue;
set.delete(checkpointTaskId);
await this.reenterCheckpointById(user, threadId, checkpointTaskId);
}
if (set.size === 0) this.pendingCheckpointReentries.delete(threadId);
}
/**
* Fire a synthetic `<planned-task-follow-up type="checkpoint">` for the
* given checkpoint task id when the parent-tagged children that drove it
* are no longer running and no new orchestrator run is live. Used by both
* the immediate re-entry path (via `maybeReenterParentCheckpoint`) and the
* deferred drain (via `drainPendingCheckpointReentries`).
*/
private async reenterCheckpointById(
user: User,
threadId: string,
checkpointTaskId: string,
messageGroupId?: string,
): Promise<boolean> {
try {
const { plannedTaskService } = await this.createPlannedTaskState();
const graph = await plannedTaskService.getGraph(threadId);
const checkpoint = graph?.tasks.find((t) => t.id === checkpointTaskId);
if (!graph || !checkpoint || checkpoint.kind !== 'checkpoint') return false;
if (checkpoint.status !== 'running') return false;
const startedRunId = await this.startInternalFollowUpRun(
user,
threadId,
this.buildPlannedTaskFollowUpMessage('checkpoint', graph, { checkpoint }),
messageGroupId,
false,
{ isCheckpointFollowUp: true, checkpointTaskId },
);
if (!startedRunId) return false;
this.logger.debug('Re-entered checkpoint follow-up', {
threadId,
checkpointTaskId,
messageGroupId,
});
return true;
} catch (error) {
this.logger.error('Failed to re-enter checkpoint follow-up', {
threadId,
checkpointTaskId,
error: error instanceof Error ? error.message : String(error),
});
return false;
}
}
/**
* When a direct background task (builder/research/data-table)
* settles and was spawned inside a checkpoint follow-up, try to re-enter
* that checkpoint so the orchestrator can call `complete-checkpoint`.
*
* Returns `true` only when a follow-up was actually started. Returns
* `false` in every other case (checkpoint no longer running, siblings
* still in-flight, an orchestrator run is active or suspended, or the
* graph no longer has the checkpoint). The caller is responsible for
* queuing a deferred re-entry in the false case — never falling through
* to a generic `<background-task-completed>` shell, which would re-open
* the orphan bug.
*/
private async maybeReenterParentCheckpoint(
user: User,
threadId: string,
task: ManagedBackgroundTask,
): Promise<boolean> {
const parentCheckpointId = task.parentCheckpointId;
if (!parentCheckpointId) return false;
// If other parent-tagged children are still running, let the LAST one
// re-drive the checkpoint; emitting multiple re-dispatches would race.
const siblings = this.backgroundTasks
.getRunningTasksByParentCheckpoint(threadId, parentCheckpointId)
.filter((t) => t.taskId !== task.taskId);
if (siblings.length > 0) return false;
// If a run is live, defer — startInternalFollowUpRun would be rejected
// and we must not fall through to the shell path.
if (this.runState.getActiveRunId(threadId) || this.runState.hasSuspendedRun(threadId)) {
return false;
}
return await this.reenterCheckpointById(
user,
threadId,
parentCheckpointId,
task.messageGroupId,
);
}
private async finalizeCheckpointFollowUp(
user: User,
threadId: string,
@ -5955,9 +5731,6 @@ export class InstanceAiService {
} else if (reschedule) {
await this.schedulePlannedTasks(opts.user, opts.threadId);
}
if (reschedule) {
await this.drainPendingCheckpointReentries(opts.user, opts.threadId);
}
// The setup claim must land even on a stop: it is what stops a later
// run from re-routing setup for a workflow the user already set up.
if (completedSetupWorkflowId) {
@ -6127,244 +5900,6 @@ export class InstanceAiService {
}
}
// ── Background task management ──────────────────────────────────────────
private spawnBackgroundTask(
runId: string,
opts: SpawnBackgroundTaskOptions,
messageGroupIdOverride?: string,
): SpawnBackgroundTaskResult {
const outcome = this.backgroundTasks.spawn({
taskId: opts.taskId,
threadId: opts.threadId,
runId,
role: opts.role,
agentId: opts.agentId,
messageGroupId: messageGroupIdOverride ?? this.runState.getMessageGroupId(opts.threadId),
plannedTaskId: opts.plannedTaskId,
workItemId: opts.workItemId,
traceContext: opts.traceContext,
createTraceContext: opts.createTraceContext,
dedupeKey: opts.dedupeKey,
parentCheckpointId: opts.parentCheckpointId,
run: opts.run,
onLimitReached: async (errorMessage) => {
await this.tracing.finalizeDetachedTraceRun(opts.taskId, opts.traceContext, {
status: 'failed',
outputs: {
taskId: opts.taskId,
agentId: opts.agentId,
role: opts.role,
},
error: errorMessage,
metadata: {
...(opts.plannedTaskId ? { planned_task_id: opts.plannedTaskId } : {}),
...(opts.workItemId ? { work_item_id: opts.workItemId } : {}),
},
});
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: {
role: opts.role,
result: '',
error: errorMessage,
},
});
},
onCompleted: async (task) => {
await this.tracing.finalizeBackgroundTaskTracing(task, 'completed');
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: { role: opts.role, result: task.result ?? '' },
});
const user = this.runState.getThreadUser(opts.threadId);
if (user) {
await this.handlePlannedTaskSettlement(user, task, 'succeeded');
}
},
onFailed: async (task) => {
await this.tracing.finalizeBackgroundTaskTracing(task, 'failed');
this.instanceAiErrorReporter.report(
new Error(task.error ?? 'Instance AI background task failed'),
{
component: 'instance-ai-background-task',
threadId: opts.threadId,
runId,
tracing: task.traceContext,
agentId: opts.agentId,
messageGroupId: task.messageGroupId,
taskId: task.taskId,
role: task.role,
},
);
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: { role: opts.role, result: '', error: task.error ?? 'Unknown error' },
});
const user = this.runState.getThreadUser(opts.threadId);
if (user) {
await this.handlePlannedTaskSettlement(user, task, 'failed');
}
},
onSettled: async (task) => {
await this.terminalOutcome.recordBackgroundTerminalOutcome(task);
// Auto-follow-up: when the last background task finishes and no
// orchestrator run is active, resume the orchestrator so it can
// synthesize results for the user. Planned tasks handle this via
// schedulePlannedTasks(); this covers direct detached background calls.
if (task.plannedTaskId) return;
await this.taskProjector.syncFromBackgroundTask(task);
// Parent-tagged children (patch-builder etc. spawned inside a
// checkpoint follow-up) must NEVER emit a generic
// `<background-task-completed>` shell — the orchestrator would
// land outside the checkpoint context and the checkpoint would
// be orphaned. Try immediate re-entry; if the run state or
// still-running siblings block it, queue a deferred marker that
// the post-run drain hook will pick up.
const parentCheckpointId = task.parentCheckpointId;
if (parentCheckpointId) {
const user = this.runState.getThreadUser(opts.threadId);
if (!user) {
this.queuePendingCheckpointReentry(opts.threadId, parentCheckpointId);
return;
}
const reentered = await this.maybeReenterParentCheckpoint(user, opts.threadId, task);
if (!reentered) {
this.queuePendingCheckpointReentry(opts.threadId, parentCheckpointId);
}
return;
}
const remaining = this.backgroundTasks.getRunningTasks(opts.threadId);
const hasActiveRun = !!this.runState.getActiveRunId(opts.threadId);
const hasSuspendedRun = this.runState.hasSuspendedRun(opts.threadId);
if (remaining.length === 0 && !hasActiveRun && !hasSuspendedRun) {
if (this.liveness.hasTimedOutActiveRunThread(opts.threadId)) {
this.logger.debug('Skipping background auto-follow-up after active run timeout', {
threadId: opts.threadId,
taskId: task.taskId,
});
return;
}
// Don't auto-respawn a task that timed out — hand back to the user instead.
if (task.timeoutReason) {
this.logger.debug('Skipping background auto-follow-up after task timeout', {
threadId: opts.threadId,
taskId: task.taskId,
timeoutReason: task.timeoutReason,
});
return;
}
const user = this.runState.getThreadUser(opts.threadId);
if (user) {
const verificationFollowUpStarted = await this.maybeStartWorkflowVerificationFollowUp(
user,
task,
);
if (verificationFollowUpStarted) return;
// Builder already verified (or setup is needed without a verify step):
// route directly to setup so an unrunnable workflow is never presented
// as done.
const setupFollowUpStarted = await this.maybeStartWorkflowSetupFollowUp(
user,
task.threadId,
);
if (setupFollowUpStarted) return;
const payload = JSON.stringify(
{
role: opts.role,
status: task.result ? 'completed' : task.error ? 'failed' : 'finished',
result: task.result ?? undefined,
outcome: task.outcome ?? undefined,
error: task.error ?? undefined,
},
null,
2,
);
await this.startInternalFollowUpRun(
user,
opts.threadId,
`<background-task-completed>\n${payload}\n</background-task-completed>\n\n${AUTO_FOLLOW_UP_MESSAGE}`,
task.messageGroupId,
);
}
}
},
});
if (outcome.status === 'started') {
void this.taskProjector.syncFromBackgroundTask(outcome.task);
return { status: 'started', taskId: outcome.task.taskId, agentId: outcome.task.agentId };
}
if (outcome.status === 'duplicate') {
this.logger.warn('Background task dispatch deduped — task already in flight', {
threadId: opts.threadId,
requestedTaskId: opts.taskId,
existingTaskId: outcome.existing.taskId,
plannedTaskId: opts.dedupeKey?.plannedTaskId,
workflowId: opts.dedupeKey?.workflowId,
role: opts.role,
});
// The sub-agent dispatch tools publish `agent-spawned` and allocate a
// detached LangSmith trace root BEFORE calling spawnBackgroundTask, so
// the freshly-generated subAgentId for this deduped attempt already has
// a phantom sub-agent node in the event stream and an unfinished trace
// root. Compensate the same way `onLimitReached` does so the agent tree
// snapshot doesn't keep a ghost child and the trace client is released.
void this.tracing.finalizeDetachedTraceRun(opts.taskId, opts.traceContext, {
status: 'cancelled',
outputs: {
taskId: opts.taskId,
agentId: opts.agentId,
role: opts.role,
deduped_to: outcome.existing.taskId,
},
metadata: {
deduped: true,
existing_task_id: outcome.existing.taskId,
...(opts.plannedTaskId ? { planned_task_id: opts.plannedTaskId } : {}),
...(opts.workItemId ? { work_item_id: opts.workItemId } : {}),
},
});
this.eventBus.publish(opts.threadId, {
type: 'agent-completed',
runId,
agentId: opts.agentId,
payload: {
role: opts.role,
result: '',
error: `Deduped: task already in flight as ${outcome.existing.taskId}`,
},
});
return {
status: 'duplicate',
existing: {
taskId: outcome.existing.taskId,
agentId: outcome.existing.agentId,
role: outcome.existing.role,
plannedTaskId: outcome.existing.plannedTaskId,
workItemId: outcome.existing.workItemId,
},
};
}
return { status: 'limit-reached' };
}
/** Snapshot every Agent the editor attached, as it stands before this turn
* acts on it. Best-effort: it only feeds eval-seed authoring. */
private async snapshotAttachedAgents(

21
packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/agentTimeline.utils.test.ts
View File

@ -494,6 +494,27 @@ describe('buildTimelineBlocks', () => {
expect(blocks[0].type === 'thinking' && blocks[0].entries).toHaveLength(2);
});
test('historical eval setup calls are hidden without splitting a thinking run', () => {
const blocks = blocksOf(
[toolEntry('tc-before', 'r1'), toolEntry('tc-eval', 'r1'), toolEntry('tc-after', 'r1')],
[
makeToolCall({ toolCallId: 'tc-before' }),
makeToolCall({
toolCallId: 'tc-eval',
toolName: 'eval-setup-with-agent',
renderHint: 'eval-setup',
}),
makeToolCall({ toolCallId: 'tc-after' }),
],
);
expect(blocks).toHaveLength(1);
expect(blocks[0].type === 'thinking' && blocks[0].entries).toEqual([
expect.objectContaining({ toolCallId: 'tc-before' }),
expect.objectContaining({ toolCallId: 'tc-after' }),
]);
});
test('in-thread build-workflow renders as a trace row; agent-delegated builds stay hidden', () => {
const blocks = blocksOf(
[toolEntry('tc-build', 'r1'), toolEntry('tc-delegated', 'r1')],

4
packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/useResourceRegistry.test.ts
View File

@ -139,7 +139,7 @@ describe('useResourceRegistry', () => {
);
});
test('registers successful workflow updates from workflowId in args', async () => {
test('replays historical raw workflow update results', async () => {
const { messages, producedArtifacts } = setup();
messages.value = [
@ -166,7 +166,7 @@ describe('useResourceRegistry', () => {
);
});
test('registers workflow document returned by workflows get-json', async () => {
test('replays historical workflows get-json results', async () => {
const { messages, producedArtifacts, resourceNameIndex } = setup();
messages.value = [

4
packages/frontend/editor-ui/src/features/ai/instanceAi/canvasPreview.utils.ts
View File

@ -237,8 +237,8 @@ const WORKFLOW_LOCKING_TOOLS = new Set([
* whole build window: read file → edit → submit-workflow → verify).
* 3. An in-flight workflow-affecting tool call targeting the workflow — the
* build/setup/verify tools, `executions.run`, or a `workflows` update /
* restore-version / setup action. Read-only `workflows` actions (get-json,
* get, list, …) don't lock.
* restore-version / setup action. Read-only `workflows` actions (including
* historical get-json events, get, list, …) don't lock.
*/
export function isAgentEditingWorkflow(node: InstanceAiAgentNode, workflowId: string): boolean {
if (

5
packages/frontend/editor-ui/src/features/ai/instanceAi/useResourceRegistry.ts
View File

@ -190,9 +190,8 @@ function extractFromToolCall(tc: InstanceAiToolCallState, col: Collections): voi
recordProduced(col, { type: 'workflow', id: workflowId, name });
}
// workflows action=get-json returns the workflow document itself, not under
// a `workflow` key. Surface it so existing workflows loaded for editing can
// be previewed even before a later update result is observed.
// Historical workflows action=get-json events return the workflow document itself,
// not under a `workflow` key. Keep them available when replaying stored threads.
if (tc.toolName === 'workflows' && Array.isArray(result.nodes)) {
const entry = entryFromListItem('workflow', result);
if (entry) recordProduced(col, entry);