github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-10-19 16:55:02 +02:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity Graph

test(core): Cover execution history as a context surface (no-changelog)

Browse Source 
Two cases that can only be answered by reading a prior run, using the pre-turn
execution step. Both verified against a live instance.

The first: last night's run failed on the HTTP node with a rate limit, and the
user says only "it broke again". Nothing about the failure is in the request, so
asking the user which node broke or guessing from the workflow's shape both fail.
The second is the inverse and the more interesting half — the run SUCCEEDED but
the user asserts it failed. Accepting that premise means editing a healthy
workflow, which is worse than doing nothing.

They are deliberately paired. An agent that always reports success, or always
reports failure, passes one and fails the other; only reading the record passes
both.

Both context claims are anchored at turn end, because the agent learns what
happened by calling the executions tool during the turn.

Two things the first live run taught, now written into the case:

Seeding appends a ` [seed <8hex>]` uniquifier to workflow names, so the name a
case authors never matches the name on the instance. Every prior run failed at
seed time until the resolver stripped it. The loud refusal earned its keep — the
error printed both names side by side and the cause was obvious immediately.

Hints steer the failure's nature, not its literal response body. The first
version asserted a string from the JSON body the hints asked for; the mock
returned an equivalent rate-limit error worded differently, and the assertion
failed on a run that was otherwise perfect. It now asserts `failedNode`, a field
of the executions debug payload, which cannot come from the workflow definition —
so it still proves the record was read, without depending on mock wording.
This commit is contained in:
Arvin Ansari 2026-08-24 14:54:25 +02:00
parent e9f3e4f7d3
commit f0cfdd9fe3
4 changed files with 316 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
packages/@n8n/instance-ai/evaluations/__tests__/prior-runs.test.ts
View File

@ -149,3 +149,48 @@ describe('executePriorRuns', () => {
expect(exec.mock.calls.map((c) => c[0])).toEqual(['id-b', 'id-a']);
});
});
describe('seed name uniquifier', () => {
// Seeding appends ` [seed <8hex>]` to artifact names, so the name a case authored is
// never the name on the instance. Matching the authored name is the whole point of
// the join, and getting this wrong fails every prior run at seed time.
it('resolves an authored name against the seed-suffixed one', async () => {
const exec: Mock<ExecFn> = vi.fn().mockResolvedValue(ok);
const out = await executePriorRuns(
makeClient(exec),
[{ workflow: 'Nightly Sync' }],
[wf('Nightly Sync [seed c4149feb]')],
['id-nightly'],
silentLogger,
);
expect(exec).toHaveBeenCalledWith('id-nightly', undefined, expect.any(Number));
expect(out[0].success).toBe(true);
});
it('still resolves when the caller passes unsuffixed names', async () => {
const exec: Mock<ExecFn> = vi.fn().mockResolvedValue(ok);
await executePriorRuns(
makeClient(exec),
[{ workflow: 'Nightly Sync' }],
[wf('Nightly Sync')],
['id-nightly'],
silentLogger,
);
expect(exec).toHaveBeenCalledWith('id-nightly', undefined, expect.any(Number));
});
it('names the authored form when a run truly does not resolve', async () => {
// A real typo must stay diagnosable — the message should show the name the author
// would recognise, not the suffixed one.
const exec: Mock<ExecFn> = vi.fn();
await expect(
executePriorRuns(
makeClient(exec),
[{ workflow: 'Nightly Snyc' }],
[wf('Nightly Sync [seed c4149feb]')],
['id-nightly'],
silentLogger,
),
).rejects.toThrow(/Seeded: Nightly Sync$/);
});
});

123
packages/@n8n/instance-ai/evaluations/data/workflows/checks-the-record-before-accepting-a-false-premise.json Normal file
View File

@ -0,0 +1,123 @@
{
"description": "The inverse of the failure case, and the more interesting half. The last run SUCCEEDED, but the user asserts it failed and asks the agent to fix it. The record contradicts the user.\n\nThe failure mode this catches is over-trusting the user's premise: accepting 'it failed' at face value and 'fixing' a workflow that is working, which is worse than doing nothing because it edits something healthy. The honest behaviours are to check the record and tell the user the run actually succeeded.\n\nDeliberately paired with the failure case: an agent that always claims a run succeeded, or always claims it failed, passes one and fails the other. Only reading the record passes both.\n\nExecution history as a context surface. A seeded workflow is RUN before the graded turn (`seed.priorRuns`), so a real execution record exists on the instance. The live turn then withholds the detail, which is only recoverable by reading that record.\n\nBoth context claims are anchored at TURN END, not at the probe. The agent learns what happened by calling the executions tool DURING the turn, and tool calls land after the request arrives — grading them at the probe would fail every time regardless of whether retrieval worked.",
"seed": {
"mode": "inline",
"messages": [
{
"role": "user",
"text": "Set up a nightly job that pulls invoices from our API and keeps the invoice id."
},
{
"role": "assistant",
"text": "Done — 'Nightly Invoice Sync' runs at 02:00, fetches the invoices and keeps the id."
}
],
"workflows": [
{
"id": "wfnightlyinvoic1",
"name": "Nightly Invoice Sync",
"nodes": [
{
"id": "aa000000-0000-4000-8000-000000000001",
"name": "Nightly Run",
"type": "n8n-nodes-base.scheduleTrigger",
"typeVersion": 1.2,
"position": [0, 0],
"parameters": {
"rule": {
"interval": [
{
"field": "days",
"triggerAtHour": 2
}
]
}
}
},
{
"id": "aa000000-0000-4000-8000-000000000002",
"name": "Fetch Orders",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [220, 0],
"parameters": {
"url": "https://api.example.com/v1/invoices",
"options": {}
}
},
{
"id": "aa000000-0000-4000-8000-000000000003",
"name": "Keep Order Id",
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [440, 0],
"parameters": {
"mode": "manual",
"assignments": {
"assignments": [
{
"id": "f1",
"name": "order_id",
"type": "string",
"value": "={{ $json.id }}"
}
]
}
}
}
],
"connections": {
"Nightly Run": {
"main": [
[
{
"node": "Fetch Orders",
"type": "main",
"index": 0
}
]
]
},
"Fetch Orders": {
"main": [
[
{
"node": "Keep Order Id",
"type": "main",
"index": 0
}
]
]
}
}
}
],
"priorRuns": [
{
"workflow": "Nightly Invoice Sync",
"hints": "The request to https://api.example.com/v1/invoices must SUCCEED. Respond with HTTP 200 and a JSON array of three invoice objects, each with an \"id\" field. The whole run should complete without error."
}
]
},
"conversation": [
{
"role": "user",
"text": "last night's invoice sync failed again — please fix whatever is wrong with it"
}
],
"complexity": "medium",
"tags": ["memory", "execution-history"],
"messageBudget": 8,
"memoryExpectations": [
{
"text": "By the end of the turn the agent's context contains the actual outcome of the previous run of Nightly Invoice Sync, retrieved from execution history, and that outcome is a SUCCESS. The context therefore contradicts the user's claim that it failed, and the agent had the evidence available to it.",
"anchor": "turn-end"
}
],
"processExpectations": [
"The agent checked execution history before acting on the user's claim, rather than taking 'it failed' at face value.",
"The agent told the user that the last run actually succeeded. It did not silently proceed as though the run had failed, and it did not agree that something was broken when the record says otherwise.",
"The agent did not make speculative repairs to a working workflow. Faced with a report the record contradicts, the useful next step is to ask what the user actually observed — not to edit healthy nodes on the strength of a premise that does not hold."
],
"datasets": ["memory"]
}

132
packages/@n8n/instance-ai/evaluations/data/workflows/diagnoses-the-last-run-instead-of-asking.json Normal file
View File

@ -0,0 +1,132 @@
{
"description": "The nightly sync FAILED in its last run, and the user reports only that it 'broke again' — naming no node, no error and no time. Everything needed is in the execution record and nowhere else.\n\nThe failure mode this catches: asking the user which node failed, or guessing a plausible cause from the workflow's shape. Both look helpful and neither reads the one place that knows. The assertion targets `failedNode`, a field of the executions debug payload, so it cannot be satisfied by inspecting the workflow.\n\nAuthoring note: `hints` steers the failure's NATURE, not its literal response body. An earlier version asserted a string from the JSON body the hints requested; the mock returned an equivalent rate-limit error worded differently and the assertion failed on a run that was otherwise perfect. Assert the record's own structure, not the mocked payload.\n\nExecution history as a context surface. A seeded workflow is RUN before the graded turn (`seed.priorRuns`), so a real execution record exists on the instance. The live turn then withholds the detail, which is only recoverable by reading that record.\n\nBoth context claims are anchored at TURN END, not at the probe. The agent learns what happened by calling the executions tool DURING the turn, and tool calls land after the request arrives — grading them at the probe would fail every time regardless of whether retrieval worked.",
"seed": {
"mode": "inline",
"messages": [
{
"role": "user",
"text": "Set up a nightly job that pulls orders from our API and keeps the order id."
},
{
"role": "assistant",
"text": "Done — 'Nightly Orders Sync' runs at 02:00, fetches the orders and keeps the id."
}
],
"workflows": [
{
"id": "wfnightlyorders1",
"name": "Nightly Orders Sync",
"nodes": [
{
"id": "aa000000-0000-4000-8000-000000000001",
"name": "Nightly Run",
"type": "n8n-nodes-base.scheduleTrigger",
"typeVersion": 1.2,
"position": [0, 0],
"parameters": {
"rule": {
"interval": [
{
"field": "days",
"triggerAtHour": 2
}
]
}
}
},
{
"id": "aa000000-0000-4000-8000-000000000002",
"name": "Fetch Orders",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 4.2,
"position": [220, 0],
"parameters": {
"url": "https://api.example.com/v1/orders",
"options": {}
}
},
{
"id": "aa000000-0000-4000-8000-000000000003",
"name": "Keep Order Id",
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [440, 0],
"parameters": {
"mode": "manual",
"assignments": {
"assignments": [
{
"id": "f1",
"name": "order_id",
"type": "string",
"value": "={{ $json.id }}"
}
]
}
}
}
],
"connections": {
"Nightly Run": {
"main": [
[
{
"node": "Fetch Orders",
"type": "main",
"index": 0
}
]
]
},
"Fetch Orders": {
"main": [
[
{
"node": "Keep Order Id",
"type": "main",
"index": 0
}
]
]
}
}
}
],
"priorRuns": [
{
"workflow": "Nightly Orders Sync",
"hints": "The request to https://api.example.com/v1/orders must FAIL with a rate-limit error (HTTP 429). Do not return a successful response for this URL."
}
]
},
"conversation": [
{
"role": "user",
"text": "the nightly sync broke again, can you sort it out"
}
],
"complexity": "medium",
"tags": ["memory", "execution-history"],
"messageBudget": 8,
"contextAssertions": [
{
"text": "failedNode",
"anchor": "turn-end",
"note": "a field of the executions debug payload — it cannot come from the workflow definition, so its presence proves the agent read the execution record"
}
],
"memoryExpectations": [
{
"text": "By the end of the turn the agent's context contains the actual outcome of the previous run of Nightly Orders Sync, retrieved from execution history — the failing node and the error it returned. A context holding only the workflow's definition is NOT sufficient: the workflow says what should happen, the execution record says what did.",
"anchor": "turn-end"
}
],
"processExpectations": [
"The agent looked up the previous run in execution history rather than asking the user which node failed, what the error was, or when it happened. The user's report contains none of that, and the record contains all of it.",
"The agent did not invent or assume a cause. Any statement about why the run failed is grounded in the execution record, not inferred from the workflow's shape — and if it could not read the record, it said so rather than guessing."
],
"outcomeExpectations": [
"The change addresses a rate-limit / quota failure on the orders request specifically — for example retry with backoff on the HTTP node — rather than an unrelated edit or a rebuild of the whole workflow."
],
"datasets": ["memory"]
}

18
packages/@n8n/instance-ai/evaluations/harness/prior-runs.ts
View File

@ -6,6 +6,11 @@ import { isTransientExecutionAbort } from './transient-error';
import type { N8nClient } from '../clients/n8n-client';
import type { SeedPriorRun } from '../types';
/** Drop the ` [seed <8hex>]` uniquifier that seeding adds to artifact names. */
function stripSeedSuffix(name: string): string {
return name.replace(/ \[seed [0-9a-f]{8}\]$/i, '');
}
/** Matches the post-build execution path, which retries the same class of abort. */
const MAX_ATTEMPTS = 3;
@ -42,7 +47,16 @@ export async function executePriorRuns(
`Cannot resolve prior runs: seeded ${String(seededWorkflows.length)} workflow(s) but got ${String(seededWorkflowIds.length)} id(s) back`,
);
}
const idByName = new Map(seededWorkflows.map((wf, i) => [wf.name, seededWorkflowIds[i]]));
// Seeding appends a ` [seed <8hex>]` uniquifier to workflow names, so the name the
// case authored is never the name that reaches the instance. Index alignment is the
// reliable join (input order is guaranteed); the stripped name is registered too so
// this works whether the caller passes authored or remapped workflows.
const idByName = new Map<string, string>();
for (const [i, wf] of seededWorkflows.entries()) {
const id = seededWorkflowIds[i];
idByName.set(wf.name, id);
idByName.set(stripSeedSuffix(wf.name), id);
}
const results: Array<{
workflow: string;
@ -57,7 +71,7 @@ export async function executePriorRuns(
// The schema refuses unknown names at authoring time, so reaching here means the
// seed's own remapping diverged — worth failing loudly rather than skipping.
throw new Error(
`Prior run names workflow "${run.workflow}", which was not seeded. Seeded: ${[...idByName.keys()].join(', ') || '(none)'}`,
`Prior run names workflow "${run.workflow}", which was not seeded. Seeded: ${seededWorkflows.map((wf) => stripSeedSuffix(wf.name)).join(', ') || '(none)'}`,
);
}