diff --git a/.agents/skills/create-instance-ai-eval/SKILL.md b/.agents/skills/create-instance-ai-eval/SKILL.md index 56fc0c6dea5..c5bb3b760e2 100644 --- a/.agents/skills/create-instance-ai-eval/SKILL.md +++ b/.agents/skills/create-instance-ai-eval/SKILL.md @@ -29,11 +29,12 @@ exhaustive field reference; this skill is the opinionated *how*. > `--source langtracer`. You still write the JSON file — it's just the input to > the push, not a committed artifact. > -> **Exception — seeded cases.** The case-write API has no `seed` field, so seeded -> cases are never pushed. A `seed.mode: "replay"` case is a local throwaway (don't -> commit it either — it dies when its trace is pruned); a `seed.mode: "inline"` -> case isn't transient and has no suite home, so it's the one sanctioned -> exception — it lives as committed JSON. See [`case-shapes.md`](case-shapes.md). +> **Seeded cases.** An `inline` seed pushes with the case — the case-write API +> stores it verbatim, so the suite is its home like any other case. Only a +> `seed.mode: "replay"` case is refused (listed under `skipped:`): it's +> reconstructed from a LangSmith trace at run time, so it dies when that trace is +> pruned and has no durable home. Don't commit a replay case either — derive a +> synthetic case from it. See [`case-shapes.md`](case-shapes.md). ## Set the autonomy level first @@ -206,9 +207,9 @@ calibration you hand the driver the thread link + login to review the real build [Push to a lang-tracer suite](#push-to-a-lang-tracer-suite)); the suite is the case's home, not the repo. Leave the `data/workflows/*.json` file uncommitted (or delete it once it's in the suite). Committing new case JSONs into the repo - is no longer the approach. (Exception: seeded cases can't be pushed — an - `inline`-seeded case stays committed JSON, a `replay` case is - a local throwaway; see [`case-shapes.md`](case-shapes.md).) For a sourced case, + is no longer the approach. (An `inline` seed pushes with the case; only a + `replay` case is refused — it's a local throwaway; see + [`case-shapes.md`](case-shapes.md).) For a sourced case, finish by **linking it to its source thread/finding** over the MCP — see [Link the pushed case to its source](#link-the-pushed-case-to-its-source-provenance-step--always-do-this). @@ -667,12 +668,12 @@ npx dotenvx run -f .env.eval -- pnpm eval:langtracer-push --suite baseline --cha lang-tracer #48), so scenario edits re-push like any other field. A lang-tracer deployment predating that change silently ignores the key; if a pushed scenario edit doesn't land, update the scenario in the lang-tracer UI. -- **Seeded cases can't be pushed:** the case-write API has no `seed` field, so the - push lists any seeded case under `skipped:` and it never reaches the suite. A - `replay` case shouldn't be committed either — it dies when its trace is pruned - or deleted — so derive a durable synthetic case as the artifact instead. An - `inline`-seeded case isn't transient and has no suite home, so it's the one - exception to "don't commit the JSON" — it lives as a committed artifact. +- **An `inline` seed pushes with the case:** the case-write API stores it + verbatim, so a seeded case lives in a suite like any other. Only a `replay` + case is refused — the push lists it under `skipped:`, because it's + reconstructed from a LangSmith trace at run time and dies when that trace is + pruned or deleted. Don't commit a replay case either; derive a durable + synthetic case as the artifact instead. ### Link the pushed case to its source (provenance step — always do this) diff --git a/.agents/skills/create-instance-ai-eval/case-shapes.md b/.agents/skills/create-instance-ai-eval/case-shapes.md index 91809f75ae7..a57510f5fc9 100644 --- a/.agents/skills/create-instance-ai-eval/case-shapes.md +++ b/.agents/skills/create-instance-ai-eval/case-shapes.md @@ -443,13 +443,12 @@ which user turn goes live. seeds with **no workflow to inspect**. Reproduce the target workflow yourself (a synthetic case whose `executionScenarios` precondition builds the stand-in), or grade the live turn with `processExpectations` only. -- **Can't be pushed to a lang-tracer suite either.** The case-write API has no - `seed` field, so `eval:langtracer-push` silently lists any seeded case under - `skipped:`. Combined with the don't-commit rule above, a `replay` case has **no - durable home by design** — the durable artifact is always the synthetic case you - derive from it. (An `inline` seed carries no thread dependency and can't be - pushed either, so — unlike a normal case — it's the one exception to the skill's - "push, don't commit the JSON" rule: it lives as a committed artifact.) +- **Can't be pushed to a lang-tracer suite either.** `eval:langtracer-push` + refuses a `replay` case and lists it under `skipped:` — a suite is a durable + home and this seed isn't. Combined with the don't-commit rule above, a `replay` + case has **no durable home by design** — the durable artifact is always the + synthetic case you derive from it. (An `inline` seed has no such problem: it + pushes with the case and lives in the suite like any other.) ### `mode: "inline"` — durable synthetic fixture @@ -487,10 +486,73 @@ write a message as `{role, text}` and the schema expands it to a full envelope: `text` also takes an array of lines (newline-joined), like a `conversation` turn. The expansion stamps `createdAt` itself — ascending, in the past — so a shorthand message can't order *after* the live turn. Shorthand and full envelopes mix freely -in one array; a full envelope keeps its authored `createdAt` — **unless any message -in the array is in the FUTURE**, in which case the whole sequence is restamped onto -ascending pre-live slots (a future stamp would sort a seeded turn after the live -turn, and moving only that one entry would reorder it against the array the -transcript is graded from). A near-miss (say +in one array; a full envelope keeps its authored `createdAt` — **unless the authored +stamps don't already ascend and sit in the past**, in which case the whole sequence +is restamped onto ascending pre-live slots. A future stamp would sort a seeded turn +after the live turn, and a non-ascending sequence (a shorthand turn appended after +later-stamped envelopes, say) would present the history in an order the graded +transcript never had; restamping only the offending entry would reorder it against +the array the transcript is graded from. A near-miss (say `text: 123`) is deliberately **not** expanded — it fails at load instead of becoming a message the transcript builder would silently drop. + +#### Which opening shape — the agent is handed the workflow, or it has to find it + +Two real conversations look the same in a case file but test different things, and +picking wrong makes the case harder than reality. + +**Handed it.** The user is looking at a workflow and opens the assistant: "why is +this failing?", "add error handling". They never name it — the editor sends the +workflow along as a resource reference and the agent resolves it by **id**. Declare +that with `attach` on the opening turn: + +```json +"conversation": [ + { "role": "user", "text": "why is this failing?", "attach": { "workflow": "wKk3RmT9xQ2bVn7L" } } +], +"seed": { + "mode": "inline", + "messages": [ … ], + "workflows": [ { "id": "wKk3RmT9xQ2bVn7L", "name": "Batch loop", … } ] +} +``` + +The id is the one the **seed declares**; the harness swaps in the per-run id, so you +track nothing. Only the opening turn may carry `attach` (an attachment is a hand-off, +not something a user re-sends), and it must name a workflow the inline seed declares — +both are refused at load rather than ignored. + +**The opening often has no text at all** — the user opens the assistant on a workflow +and waits for it to speak first. Keep `"text": ""` when that's what happened; it's the +faithful shape, and openings with no user text jumped from 1% to 31% of the corpus when +the editor hand-off shipped, so it is not an edge case. Note that an empty text is +valid *only* alongside `attach`: the chat API rejects a message that is empty with +nothing attached ("Either message or attachments must be provided"), so the two stand +or fall together. + +**Has to find it.** The user refers to the workflow in words: by name ("the Wait node +in *Generate leads* failed") or loosely ("the batch image workflow"). No `attach` — +finding it *is* part of what the case tests. This also works when the seeded history +already shows the agent building it, since the id is in its own record. + +Get this wrong in the "handed it" direction — omit `attach` on a conversation that +really had one — and the agent has to guess from prose that deliberately names +nothing. It will list workflows and pick, or ask which one, and you will score a +clarification failure the real user never hit. + +#### Before you ship a seeded case — three checks + +1. **The defect still bites.** A seed whose workflow isn't broken any more makes the + case a silent no-op that passes forever. +2. **Run it once with the seed removed. It must fail.** Copy the case, delete `seed`, + run both. If the no-seed copy also passes, the seed isn't carrying the test. Name + the copy so it doesn't share a `--filter` substring with the real case + (`control-noseed-` works; `-noseed` would match both). +3. **The workflow's skeleton is untouched** if you scrubbed it from a real one: node + types, versions and connection topology byte-identical, only string leaves moved. + +Don't grade a seeded case on cost, turn count, `messageBudget`, or "fixed it in one +build". A seeded thread starts with an empty sandbox, so the agent re-reads the +workflow from the database and re-derives SDK source a real resumed session would +still have on disk. The bias is *harder* than reality, so those numbers read worse +for a reason that has nothing to do with the builder. diff --git a/.agents/skills/create-instance-ai-eval/running-evals.md b/.agents/skills/create-instance-ai-eval/running-evals.md index b4d60033acc..144b79f6bfd 100644 --- a/.agents/skills/create-instance-ai-eval/running-evals.md +++ b/.agents/skills/create-instance-ai-eval/running-evals.md @@ -69,7 +69,7 @@ cleaned up by the next run rather than contaminating it. | Source | When to use it | |---|---| -| **`disk`** (default) | **Preferred for local development** — authoring and calibrating the case in front of you: drop the JSON into `data/workflows/`, `--filter` it, iterate. Also the only home of the `agents` tier and the seeded carve-out cases; since the corpus migration the directory holds only those, not the full suite. | +| **`disk`** (default) | **Preferred for local development** — authoring and calibrating the case in front of you: drop the JSON into `data/workflows/`, `--filter` it, iterate. Also the only home of the `agents` tier and of a `replay`-seeded case (reconstructed from a trace at run time, so no suite can hold it); since the corpus migration the directory holds only those, not the full suite. | | **`langtracer`** (`--source langtracer --suite baseline`) | Bigger runs (the full corpus or a whole tier), re-running specific cases that already live in the suite, and CI — which always runs this way. Needs `LANGTRACER_URL`/`LANGTRACER_API_KEY` in your env. | ## Configuration & secrets diff --git a/.agents/skills/create-instance-ai-eval/sourcing-cases.md b/.agents/skills/create-instance-ai-eval/sourcing-cases.md index 19e69825243..140a601b391 100644 --- a/.agents/skills/create-instance-ai-eval/sourcing-cases.md +++ b/.agents/skills/create-instance-ai-eval/sourcing-cases.md @@ -62,12 +62,38 @@ connected.) voice. 5. **Push it to a curated suite** (don't commit the JSON) with `eval:langtracer-push` — see - [Push to a lang-tracer suite](SKILL.md#push-to-a-lang-tracer-suite). Exception: - seeded cases (any `seed` mode) can't be pushed — the case-write API has no - `seed` field, so the push lists them under - `skipped:`. And a `replay` case shouldn't be committed either — it dies when - its trace is pruned or deleted — so it has no durable home; that's exactly why - step 4 turns the confirmed failure into a durable synthetic case. + [Push to a lang-tracer suite](SKILL.md#push-to-a-lang-tracer-suite). An `inline` + seed rides along with the case. Exception: a `replay` case is refused and listed + under `skipped:` — it's reconstructed from a trace at run time, so it dies when + that trace is pruned and has no durable home; that's exactly why step 4 turns the + confirmed failure into a durable synthetic case. + +## Was the workflow handed over? (sourcing an `attach` opening) + +Whether the opening turn carries `attach` is a **fact about the thread**, not a +judgement call — and guessing wrong makes the case harder than reality. Read it two +ways, depending on when the thread was imported: + +- **Recorded.** Turn 0 of `get_conversation` carries + `resourceAttachments: [{ "type": "workflow", "id": "…" }]`. That's the editor + hand-off verbatim. Only kind and ids are stored, never the workflow's name. +- **Inferred**, for threads imported before n8n traced it. The tell is an opening turn + whose `userMessage` is **empty** — the editor's context block is stripped before the + trace, so a hand-off where the user typed nothing leaves a blank record. Corroborate + with an early `workflows[get]` on a workflow the user never named. + +Then use the id to recover the workflow itself: find the tool call whose *input* +carries that `workflowId` (usually `workflows[get]`, `full: true`) — its **output** is +the workflow, nodes and connections and parameters, and that is what you scrub into +`seed.workflows`. The id alone is only a join key; it addresses the user's own +instance, so it is worthless by itself. + +Two things to expect: + +- **The agent may never have read it.** Then you have an id and no content, and the + workflow is yours to write — keep the topology plausible for the complaint. +- **The opening often has no text at all.** Keep it that way; see the empty-opening + note in [`case-shapes.md`](case-shapes.md). ## Scrubbing a real workflow into a synthetic seed diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index f44be8ac9c3..6dc641c6ea4 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -181,7 +181,7 @@ A case can belong to multiple groupings — e.g. PR-tier cases declare `"dataset **LangTracer is the source of truth for the workflow-eval corpus** — the `baseline` suite holds the cases, and CI pulls it on every run (see `.github/workflows/test-evals-instance-ai.yml`). The two `--source` modes split the work: -- **`disk` (the default) — the preferred mode for local development.** Reads `data/workflows/` and `data/agents/`. Use it while authoring and calibrating a case: drop the JSON in, `--filter` it, iterate. It is also the only home of the seeded carve-out cases (the case-write API can't represent them yet). Everything else — including the agents-team cases (suite `agents`: agent-artifact + intent-resolution) — lives in LangTracer, so disk mode is about the case in front of you, not the full suite. +- **`disk` (the default) — the preferred mode for local development.** Reads `data/workflows/` and `data/agents/`. Use it while authoring and calibrating a case: drop the JSON in, `--filter` it, iterate. It is also the only home of a `replay`-seeded case — that seed is reconstructed from a LangSmith trace at run time, so no suite can be its durable home. Everything else — including the agents-team cases (suite `agents`: agent-artifact + intent-resolution) — lives in LangTracer, so disk mode is about the case in front of you, not the full suite. - **`langtracer` — for bigger runs, already-pushed cases, and CI.** Pulls a suite from [LangTracer](https://github.com/n8n-io/lang-tracer)'s REST API (`GET /api/v1/suites/:id/export`), validated through the same `EvalTestCaseSchema`. Reach for it locally when you want the real corpus (a full or tier run) or to re-run a specific case that already lives in the suite; CI always runs this way. Set these in `.env.local`: @@ -414,7 +414,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai \ For runs that need to leave the n8n repo (for example, driving the build from a separate Claude project where you have skills configured), three flags decouple the script from its default assumptions: -- `--workflow-dir ` — read test-case JSONs from a different directory (the default `evaluations/data/workflows/` now holds only the seeded carve-out cases; use `--source langtracer --suite ` for the real corpus). When set, the script no longer needs `git rev-parse` to find the repo. +- `--workflow-dir ` — read test-case JSONs from a different directory (the default `evaluations/data/workflows/` is an authoring dir, not the corpus; use `--source langtracer --suite ` for that). When set, the script no longer needs `git rev-parse` to find the repo. - `--build-cwd ` — set the working directory the `claude` subprocess spawns from. Affects which `~/.claude.json` `projects` entry (and which skills) Claude loads. - `--project-id ` — instructs the model to pass `projectId` to `create_workflow_from_code` so workflows land in a specific n8n project instead of the user's personal one. @@ -673,7 +673,7 @@ To record an isolated cohort without touching the shared dataset or baseline — ## Adding test cases -The corpus lives in **LangTracer** — suite `baseline` is what CI runs. Author a case as a local JSON file in `evaluations/data/workflows/` (disk mode picks it up, no registration step), calibrate it against a real build, then push it to the suite with `pnpm eval:langtracer-push --suite baseline ` and delete the local file rather than committing it. Seeded cases (the `seed` slot) are the exception — the case-write API can't represent them yet, so they stay as committed JSON. Every case is validated against `harness/schema.ts`. +The corpus lives in **LangTracer** — suite `baseline` is what CI runs. Author a case as a local JSON file in `evaluations/data/workflows/` (disk mode picks it up, no registration step), calibrate it against a real build, then push it to the suite with `pnpm eval:langtracer-push --suite baseline ` and delete the local file rather than committing it. An `inline` seed pushes with the case; a `replay`-seeded case is refused (it's reconstructed from a LangSmith trace at run time, so a suite can't be its home). Every case is validated against `harness/schema.ts`. > The essentials are below. For the full authoring guide — picking a case archetype, sizing assertions so a wrong build fails, multi-turn director scripts, seeding vs synthetic, and calibrating against a real build — follow the [`create-instance-ai-eval` skill](../../../../.agents/skills/create-instance-ai-eval/SKILL.md) (with [`case-shapes.md`](../../../../.agents/skills/create-instance-ai-eval/case-shapes.md) and [`running-evals.md`](../../../../.agents/skills/create-instance-ai-eval/running-evals.md)). @@ -837,16 +837,52 @@ message as `{role, text}` and the schema expands it into a full envelope for you The expansion stamps `createdAt` itself — ascending and in the past — so a shorthand message can't accidentally order *after* the live turn. Shorthand and full envelopes can be mixed in one `messages` array; a full envelope keeps its authored `createdAt` -— **unless any message in the array is stamped in the FUTURE**, in which case the -whole sequence is restamped onto the same ascending pre-live slots. A future stamp -would sort a seeded turn after the live turn, and clamping only the offending entry -would reorder it relative to the array the transcript is graded from. +— **unless the authored stamps don't already ascend and sit in the past**, in which +case the whole sequence is restamped onto the same ascending pre-live slots. A future +stamp would sort a seeded turn after the live turn, and a non-ascending sequence (a +shorthand turn appended after later-stamped envelopes, say) would present the history +in an order the graded transcript never had; restamping only the offending entry would +reorder it relative to the array the transcript is graded from. The slots are fixed, +never derived from the current time, so re-pushing an unchanged case is a no-op. A near-miss (say `text: 123`) is deliberately **not** expanded: it falls through to the envelope rules above and fails at load, rather than becoming a message the transcript builder would silently drop. The seed lives **in the case body** rather than in a sibling file, so it travels with the case whatever the source — a JSON on disk, a suite pulled with `--source langtracer`, or a case body handed to a dispatcher. (There used to be a `seedFile` path pointing at a sibling JSON. Only the disk loader could resolve it, so a case delivered any other way lost its seed; the key is gone and a case still carrying it fails at load.) +#### Handing the agent the workflow (`attach`) + +When a real user opens the assistant with a workflow in front of them — "why is this +failing?" — the editor sends that workflow as a resource reference and the agent +resolves it by **id**, never by name. Declare it on the opening turn: + +```json +"conversation": [ + { "role": "user", "text": "why is this failing?", "attach": { "workflow": "wKk3RmT9xQ2bVn7L" } } +] +``` + +The id is the one the seed declares; the harness substitutes the per-run remapped id, +so the attachment always points at the workflow that actually exists. Only the opening +turn may carry it, and it must name a workflow the inline seed declares — both are +refused at case load rather than silently ignored. + +Omit it when the user refers to the workflow in words instead ("the batch image +workflow") — finding it is then part of what the case tests. Getting this backwards +makes a case harder than reality: the agent has to guess from prose that deliberately +names nothing, and a clarification the real user never saw scores as a failure. + +> **Pushing an `attach` case needs lang-tracer [#119](https://github.com/n8n-io/lang-tracer/pull/119) deployed.** +> Carrying `attach` through import and case-write is that PR's job; a deployment +> predating it stores the turn without the key, so the case would come back from +> the suite as a hand-off with neither text nor attachment — a quietly different +> test. **You don't have to remember this:** the push re-reads the suite export +> after every write and fails if the server didn't store what it was sent, naming +> the fields it dropped. The same check covers a `seed` against a deployment +> predating [#113](https://github.com/n8n-io/lang-tracer/pull/113). Until the +> server is upgraded, keep such a case on disk (`--source disk`). +> Round-trip coverage: `langtracer-to-exported.test.ts`. + #### How restore works (all paths) At build time the seed is restored right after the credential pin: seeded workflows are recreated under **fresh ids and a per-restore unique name** (`… [seed <8 hex>]`) with node credentials stripped, and the message log is written verbatim. Both are remapped through the history, so parallel iterations never share a workflow row *or* a name — a seeded case's live turn names its workflow the way a user would, so a same-named copy is one the agent can ground on instead, and the judge would then grade a different workflow than the agent edited. Any leftover carrying the seed suffix with the same base name is deleted before the restore; workflows without the suffix (real ones, and anything the agent built) are never touched. Restore failures fail the build — a seeded case cannot meaningfully run unseeded. Seeded turns join the transcript marked as *seeded prior context*, visible to the expectations judge and prompt-aware checks but distinguishable from live behaviour. @@ -963,7 +999,7 @@ evaluations/ ├── checklist/ # LLM verification with retry ├── credentials/ # Test credential seeding ├── data/agents/ # authoring dir for intent-resolution cases (the corpus lives in LangTracer suite `agents`) -├── data/workflows/ # seeded carve-out case JSONs (the corpus lives in LangTracer) +├── data/workflows/ # authoring dir for case JSONs (the corpus lives in LangTracer) ├── data/subagent/ # workflow-build compatibility fixture JSON files ├── data/pairwise/ # Local pairwise fixture (small smoke set) ├── harness/ # Runners: buildWorkflow + executeScenario (e2e), in-memory event bus (discovery) diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts index da25812c013..e71297d0f92 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts @@ -2,6 +2,7 @@ import { vi } from 'vitest'; import type { N8nClient } from '../clients/n8n-client'; import { buildWorkflow } from '../harness/build-workflow'; +import { recordUserTurn } from '../harness/chat-loop'; import type { ConversationSeed } from '../harness/conversation-seed'; import type { EvalLogger } from '../harness/logger'; @@ -14,6 +15,25 @@ vi.mock('../harness/chat-loop', () => ({ recordUserTurn: vi.fn(), })); +// The proxy's real constructor builds an LLM agent; only the script it is handed +// matters here, so capture that and stub the rest (runMultiTurnConversation is +// mocked above, so no other method is reached). +const { proxyScripts } = vi.hoisted(() => ({ + proxyScripts: [] as Array>, +})); + +vi.mock('../utils/user-proxy', () => ({ + UserProxyLlm: class { + constructor(config: { conversation: Array<{ text: string }> }) { + proxyScripts.push(config.conversation); + } + respondToConfirmation = vi.fn().mockResolvedValue({ approve: true }); + ingestEvents = vi.fn(); + decideFollowUp = vi.fn().mockResolvedValue({ kind: 'done' }); + getDecisionStats = vi.fn().mockReturnValue({}); + }, +})); + vi.mock('../outcome/workflow-discovery', () => ({ buildAgentOutcome: vi.fn().mockResolvedValue({ workflowsCreated: [{ id: 'built-wf-1', name: 'Built', nodeCount: 3, active: false }], @@ -251,7 +271,6 @@ describe('buildWorkflow with an inline seed', () => { preRunWorkflowIds: new Set(['leftover-1', 'leftover-2', 'leftover-3']), seed: { mode: 'inline' as const, ...inlineSeed() }, }); - expect(deleteWorkflow.mock.calls.map((call) => String(call[0]))).toEqual([ 'leftover-1', 'leftover-2', @@ -260,6 +279,135 @@ describe('buildWorkflow with an inline seed', () => { expect(build.success).toBe(true); }); + // The shape a real user creates by opening the assistant with a workflow in front + // of them: the product hands the agent a resource reference, so it resolves by id + // and never hunts by name. + it('sends the attached seed workflow with the opening message, using the REMAPPED id', async () => { + const sendMessage = vi.fn().mockResolvedValue({ runId: 'run-1' }); + const restoreThread = vi + .fn() + .mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] }); + await buildWorkflow({ + client: makeClient(restoreThread, { sendMessage }), + ...baseConfig, + conversation: [ + { role: 'user' as const, text: 'why is this failing?', attach: { workflow: SEED_WF_ID } }, + ], + seed: { mode: 'inline' as const, ...inlineSeed() }, + }); + + const [, , attachments] = sendMessage.mock.calls[0] as [ + string, + string, + Array<{ type: string; id: string; name: string }> | undefined, + ]; + expect(attachments).toHaveLength(1); + expect(attachments?.[0].type).toBe('workflow'); + // The authored id is rewritten per run, so sending it verbatim would point the + // agent at a workflow that doesn't exist on this instance. + expect(attachments?.[0].id).not.toBe(SEED_WF_ID); + const [, , workflows] = restoreThread.mock.calls[0] as [ + string, + unknown, + Array<{ id: string; name: string }>, + ]; + expect(attachments?.[0].id).toBe(workflows[0].id); + expect(attachments?.[0].name).toBe(workflows[0].name); + }); + + // The API carries the attachment out of band, so the graded transcript would show a + // faithful hand-off (`text: ''` + attach) as a bare empty message: an anomaly to the + // judge, and an EMPTY prompt for the prompt-aware checks (userTurnsAsText drops + // empty strings). The recorded turn names it instead. + it('names the attached workflow in the RECORDED turn, so judges can see the hand-off', async () => { + const sendMessage = vi.fn().mockResolvedValue({ runId: 'run-1' }); + const restoreThread = vi + .fn() + .mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] }); + vi.mocked(recordUserTurn).mockClear(); + + await buildWorkflow({ + client: makeClient(restoreThread, { sendMessage }), + ...baseConfig, + // No typed text — exactly the shape the docs promote. + conversation: [{ role: 'user' as const, text: '', attach: { workflow: SEED_WF_ID } }], + seed: { mode: 'inline' as const, ...inlineSeed() }, + }); + + const restoredWorkflows = ( + restoreThread.mock.calls[0] as [string, unknown, Array<{ id: string; name: string }>] + )[2]; + const [, recordedText] = vi.mocked(recordUserTurn).mock.calls[0]; + const [, sentText] = sendMessage.mock.calls[0] as [string, string, unknown]; + + // The RESTORED (per-run) name, not the authored one — that's what exists on the + // instance and what the judge will see referenced. + expect(recordedText).toBe(`[attached workflow: ${restoredWorkflows[0].name}]`); + // The agent still gets the user's real (empty) text plus the attachment itself. + expect(sentText).toBe(''); + }); + + // The proxy renders its script and running transcript from `text` alone, so a + // hand-off case with follow-ups would otherwise audit plans and decide follow-ups + // against a blank opening turn that never mentions the workflow. + it('names the attached workflow in the script the user proxy reads', async () => { + const restoreThread = vi + .fn() + .mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] }); + proxyScripts.length = 0; + + await buildWorkflow({ + client: makeClient(restoreThread), + ...baseConfig, + conversation: [ + { role: 'user' as const, text: '', attach: { workflow: SEED_WF_ID } }, + { role: 'user' as const, text: 'now add error handling' }, + ], + seed: { mode: 'inline' as const, ...inlineSeed() }, + }); + + const restoredWorkflows = ( + restoreThread.mock.calls[0] as [string, unknown, Array<{ id: string; name: string }>] + )[2]; + expect(proxyScripts[0][0].text).toBe(`[attached workflow: ${restoredWorkflows[0].name}]`); + // Later turns are the author's own text, untouched. + expect(proxyScripts[0][1].text).toBe('now add error handling'); + }); + + it('fails loudly when the attached seed workflow is missing from the restore', async () => { + // The schema refuses an `attach` the seed does not declare, so a miss here means + // the restore/remap lost it. Running on would silently downgrade the case to a + // find-it test and grade the wrong thing. + const restoreThread = vi + .fn() + .mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] }); + + const result = await buildWorkflow({ + client: makeClient(restoreThread, { sendMessage: vi.fn().mockResolvedValue({ runId: 'r' }) }), + ...baseConfig, + conversation: [ + { role: 'user' as const, text: 'why?', attach: { workflow: 'never-declared' } }, + ], + seed: { mode: 'inline' as const, ...inlineSeed() }, + }); + + expect(result.success).toBe(false); + expect(result.error).toMatch(/attaches seeded workflow "never-declared"/); + }); + + it('sends no attachments when the opening turn declares none', async () => { + const sendMessage = vi.fn().mockResolvedValue({ runId: 'run-1' }); + await buildWorkflow({ + client: makeClient( + vi.fn().mockResolvedValue({ restored: 1, workflowIds: [], dataTableIds: [] }), + { sendMessage }, + ), + ...baseConfig, + seed: { mode: 'inline' as const, ...inlineSeed() }, + }); + expect(sendMessage.mock.calls[0][2]).toBeUndefined(); + }); + it('does not restore anything for a case with no seed', async () => { const restoreThread = vi .fn() diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts index f345dfd895f..dad26e77ad0 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts @@ -213,6 +213,32 @@ describe('remapSeedWorkflowIds', () => { expect(serialized).toContain(`/workflow/${newId}`); }); + it('gives each workflow its own fresh id when one id is a prefix of another', () => { + // Rewriting the short id first would eat the long one's prefix, leaving it with + // a derived id no later pass matches — so both must come back as clean nanoids. + const seed = makeSeed(); + seed.workflows = [ + { id: 'abcdefgh', name: 'Short', nodes: [], connections: {} }, + { id: 'abcdefgh12', name: 'Long', nodes: [], connections: {} }, + ]; + seed.messages = [ + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2026-01-01T00:00:00.000Z', + content: [{ type: 'text', text: 'compare /workflow/abcdefgh and /workflow/abcdefgh12' }], + }, + ]; + + const remapped = remapSeedWorkflowIds(seed); + const [short, long] = remapped.workflows.map((workflow) => workflow.id); + + expect(short).toMatch(/^[0-9A-Za-z]{16}$/); + expect(long).toMatch(/^[0-9A-Za-z]{16}$/); + expect(JSON.stringify(remapped.messages)).toContain(`/workflow/${short} and /workflow/${long}`); + }); + it('returns the seed untouched when there are no workflows', () => { const seed: ConversationSeed = { messages: [ diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-text.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-text.test.ts index 1e60bed8dcf..1137f7cfc69 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-text.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-text.test.ts @@ -1,6 +1,8 @@ +import type { CaseSeed } from '../harness/schema'; import type { ConversationTurn, TranscriptTurn } from '../types'; import { agentTurnsAsText, + caseDisplayPrompt, conversationUserTurnsAsText, lastAgentText, perTurnToolCallCounts, @@ -88,6 +90,73 @@ describe('conversationUserTurnsAsText', () => { const conversation: ConversationTurn[] = [{ role: 'assistant', text: 'hello' }]; expect(conversationUserTurnsAsText(conversation)).toBe(''); }); + + // The editor hands the agent a resource reference, not text, so the faithful + // hand-off is `text: '' + attach`. Filtered as empty, it would hand the + // prompt-aware checks (fulfills-user-request) an EMPTY prompt. + it('names an attached workflow so a text-less hand-off is not dropped', () => { + const conversation: ConversationTurn[] = [ + { role: 'user', text: '', attach: { workflow: 'Batch loop' } }, + ]; + expect(conversationUserTurnsAsText(conversation)).toBe('[attached workflow: Batch loop]'); + }); + + it('keeps both the attachment and the text when the user typed something', () => { + const conversation: ConversationTurn[] = [ + { role: 'user', text: 'why is this failing?', attach: { workflow: 'Batch loop' } }, + ]; + expect(conversationUserTurnsAsText(conversation)).toBe( + '[attached workflow: Batch loop] why is this failing?', + ); + }); + + // `attach.workflow` is an id; the id means nothing to a prompt-aware check, so + // the note carries the name the seed declares for it (what the live path shows). + it('names the attachment by the seed workflow name, not its id', () => { + const conversation: ConversationTurn[] = [ + { role: 'user', text: '', attach: { workflow: 'wKk3RmT9xQ2bVn7L' } }, + ]; + expect(conversationUserTurnsAsText(conversation, seedDeclaring('Batch loop'))).toBe( + '[attached workflow: Batch loop]', + ); + }); +}); + +/** An inline seed declaring one workflow under the id the tests attach. */ +function seedDeclaring(name: string): CaseSeed { + return { + mode: 'inline', + messages: [ + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2020-01-01T00:00:00.000Z', + content: [{ type: 'text', text: 'earlier' }], + }, + ], + workflows: [{ id: 'wKk3RmT9xQ2bVn7L', name, nodes: [], connections: {} }], + dataTables: [], + }; +} + +describe('caseDisplayPrompt', () => { + it('uses the first authored turn', () => { + expect(caseDisplayPrompt({ conversation: [{ role: 'user', text: 'build a webhook' }] })).toBe( + 'build a webhook', + ); + }); + + // Without this the report labels, the comparison table and `Running case: ""` + // all come out empty for the faithful hand-off shape. + it('names the attachment when the opening turn carries no text', () => { + expect( + caseDisplayPrompt({ + conversation: [{ role: 'user', text: '', attach: { workflow: 'wKk3RmT9xQ2bVn7L' } }], + seed: seedDeclaring('Batch loop'), + }), + ).toBe('[attached workflow: Batch loop]'); + }); }); describe('transcriptAsText', () => { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts index 4e29b2b6dab..f8c69c96f02 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts @@ -215,6 +215,56 @@ describe('EvalTestCaseSchema', () => { expect(inlineSeedOf(parsed).messages[0].createdAt).toBe(authored); }); + // The restamp lands on fixed slots, never `Date.now()`-derived: `createdAt` is + // part of the projection `planPush` compares, so a now-based rewrite would read + // as an edit on every parse and re-PATCH the case forever. + it('restamps a future createdAt to the same value on every parse', () => { + const caseJson = { + ...validFixture(), + seed: { + mode: 'inline', + messages: [ + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2099-01-01T00:00:00.000Z', + content: [{ type: 'text', text: 'build it' }], + }, + ], + }, + }; + const first = inlineSeedOf(EvalTestCaseSchema.parse(caseJson)).messages[0].createdAt; + const second = inlineSeedOf(EvalTestCaseSchema.parse(caseJson)).messages[0].createdAt; + expect(first).toBe(second); + expect(Date.parse(String(first))).toBeLessThan(Date.now()); + }); + + // A shorthand turn appended after full envelopes stamps at the fixed epoch, + // which is BEFORE their authored stamps — the store would present it first while + // `transcriptPrefixFromSeed` still grades array order. + it('restamps a mixed seed whose authored timestamps do not ascend', () => { + const parsed = EvalTestCaseSchema.parse({ + ...validFixture(), + seed: { + mode: 'inline', + messages: [ + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'build it' }], + }, + { role: 'assistant', text: 'Done.' }, + ], + }, + }); + const at = inlineSeedOf(parsed).messages.map((m) => Date.parse(String(m.createdAt))); + expect(at[0]).toBeLessThan(at[1]); + expect(at[1]).toBeLessThan(Date.now()); + }); + // Both arms are strict, so a seed mixing them fails instead of having the // wrong-arm field stripped — which would run the case unseeded and grade it // as a build from scratch. @@ -325,6 +375,163 @@ describe('EvalTestCaseSchema', () => { ).toThrow(/full envelope[\s\S]*shorthand/); }); + it('accepts an attach on the opening turn naming a seeded workflow', () => { + const parsed = EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [ + { role: 'user', text: 'why is this failing?', attach: { workflow: 'wf12345678' } }, + ], + seed: { + mode: 'inline', + messages: [{ role: 'user', text: 'build it' }], + workflows: [{ id: 'wf12345678', name: 'Batch loop', nodes: [], connections: {} }], + }, + }); + expect(parsed.conversation?.[0].attach).toEqual({ workflow: 'wf12345678' }); + }); + + // An attachment models the user opening the assistant with a workflow already in + // front of them, so the turn it rides has to BE the user's. An assistant-first + // opener carrying one would be graded against a transcript that never happened. + it('rejects an attach on an assistant opening turn', () => { + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [ + { role: 'assistant', text: 'here is what I built', attach: { workflow: 'wf12345678' } }, + { role: 'user', text: 'why is this failing?' }, + ], + seed: { + mode: 'inline', + messages: [{ role: 'user', text: 'build it' }], + workflows: [{ id: 'wf12345678', name: 'Batch loop', nodes: [], connections: {} }], + }, + }), + ).toThrow(/attach.*user|user.*attach/i); + }); + + it('rejects an attach on a later turn — an attachment is a hand-off', () => { + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [ + { role: 'user', text: 'why is this failing?' }, + { role: 'user', text: 'and now this', attach: { workflow: 'wf12345678' } }, + ], + seed: { + mode: 'inline', + messages: [{ role: 'user', text: 'build it' }], + workflows: [{ id: 'wf12345678', name: 'Batch loop', nodes: [], connections: {} }], + }, + }), + ).toThrow(/only the first conversation turn may carry .attach./); + }); + + it('rejects an attach naming a workflow the seed does not declare', () => { + // A dangling reference hands the agent nothing and reads as a builder failure. + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [{ role: 'user', text: 'why?', attach: { workflow: 'not-in-the-seed' } }], + seed: { + mode: 'inline', + messages: [{ role: 'user', text: 'build it' }], + workflows: [{ id: 'wf12345678', name: 'Batch loop', nodes: [], connections: {} }], + }, + }), + ).toThrow(/must be the id of a workflow the inline seed declares/); + }); + + it('rejects an empty opening turn that carries no attach', () => { + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [{ role: 'user', text: '' }], + }), + ).toThrow(/opening turn with empty text must carry .attach./); + }); + + it('rejects an empty LATER turn too — the chat API 400s on it just the same', () => { + // The guard used to cover only turn 0, so this reached the API mid-run and + // surfaced as what reads like an infrastructure fault. + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [ + { role: 'user', text: 'build a thing' }, + { role: 'user', text: ' ' }, + ], + }), + ).toThrow(/a conversation turn needs text/); + }); + + it('leaves an empty ASSISTANT turn alone — script data, never posted to chat', () => { + // The guard exists for the chat API's 400. Assistant turns are the proxy's + // script, so an empty one has nothing to do with that rule. + const parsed = EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [ + { role: 'user', text: 'build a thing' }, + { role: 'assistant', text: '' }, + ], + }); + expect(parsed.conversation?.[1].text).toBe(''); + }); + + it('does not tell a replay author to add an attach they cannot use', () => { + // On a replay case conversation[0] CONTINUES the trace's live turn, and `attach` + // needs an inline seed to point at — so the opening-turn advice is a dead end. + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [{ role: 'user', text: '' }], + seed: { mode: 'replay', threadId: 'thread-1' }, + }), + ).toThrow(/a conversation turn needs text/); + }); + + it('rejects a seed declaring two workflows with the same id', () => { + // Restore index-aligns authored ids with remapped ones and rewrites references + // by id, so a duplicate would silently resolve to the wrong workflow. + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + seed: { + mode: 'inline', + messages: [{ role: 'user', text: 'build it' }], + workflows: [ + { id: 'wf12345678', name: 'First', nodes: [], connections: {} }, + { id: 'wf12345678', name: 'Second', nodes: [], connections: {} }, + ], + }, + }), + ).toThrow(/seed workflow ids must be unique/); + }); + + it('accepts an empty opening turn when a seeded workflow is attached', () => { + // The faithful hand-off shape: opened on a workflow, nothing typed. + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [{ role: 'user', text: '', attach: { workflow: 'wf12345678' } }], + seed: { + mode: 'inline', + messages: [{ role: 'user', text: 'build it' }], + workflows: [{ id: 'wf12345678', name: 'Batch loop', nodes: [], connections: {} }], + }, + }), + ).not.toThrow(); + }); + + it('rejects an attach on a case with no inline seed at all', () => { + expect(() => + EvalTestCaseSchema.parse({ + ...validFixture(), + conversation: [{ role: 'user', text: 'why?', attach: { workflow: 'wf12345678' } }], + }), + ).toThrow(/must be the id of a workflow the inline seed declares/); + }); + it('accepts a replay seed with no conversation (live turn from the trace)', () => { const { conversation: _omit, ...rest } = validFixture(); const parsed = EvalTestCaseSchema.parse({ diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts index d7a7dfdb4c2..21ea4263708 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts @@ -1,5 +1,5 @@ import type { WorkflowTestCaseWithFile } from '../data/workflows'; -import { planPush, toUpdatePatch } from '../langtracer/push'; +import { comparableDiff, planPush, toUpdatePatch } from '../langtracer/push'; import type { LangTracerCreateCaseBody } from '../langtracer/to-exported'; function item(fileSlug: string, overrides: Record = {}): WorkflowTestCaseWithFile { @@ -15,6 +15,25 @@ function item(fileSlug: string, overrides: Record = {}): Workfl } as WorkflowTestCaseWithFile; } +/** The authored durable seed, in the shape both the disk case and the export carry. */ +function inlineSeed(overrides: Record = {}) { + return { + mode: 'inline' as const, + messages: [ + { + id: 'm1', + type: 'llm', + role: 'assistant' as const, + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'built it' }], + }, + ], + workflows: [{ id: 'wKk3RmT9xQ2bVn7L', name: 'Batch loop', nodes: [], connections: {} }], + dataTables: [], + ...overrides, + }; +} + /** A disk-shape exported body (what `GET /suites/:id/export` returns per case). */ function body(overrides: Record = {}): Record { return { @@ -54,14 +73,165 @@ describe('planPush', () => { expect(plan.unchanged).toEqual([]); }); - it('skips a case that uses an unsupported seeding mode', () => { + it('skips a replay-seeded case — its trace expires, so it has no suite home', () => { const plan = planPush([item('c', { seed: { mode: 'replay', threadId: 't' } })], {}, {}); expect(plan.skipped).toHaveLength(1); expect(plan.skipped[0].fileSlug).toBe('c'); - expect(plan.skipped[0].reason).toMatch(/seed \(mode: replay\)/); + expect(plan.skipped[0].reason).toMatch(/replay seed/); expect(plan.toCreate).toEqual([]); }); + it('PUSHES an inline-seeded case — the durable kind is exactly what suites are for', () => { + const plan = planPush([item('repair-it', { seed: inlineSeed() })], {}, {}); + expect(plan.skipped).toEqual([]); + expect(plan.toCreate).toHaveLength(1); + }); + + // `seed` joined COMPARED_KEYS for these three: while it was excluded, every + // seed-only edit was classified `unchanged` and silently never pushed. + it('treats a seed-only addition to a hosted case as an update', () => { + const plan = planPush([item('c', { seed: inlineSeed() })], { 'c.json': body() }, { c: 5 }); + expect(plan.toUpdate.map((u) => u.id)).toEqual([5]); + expect(plan.unchanged).toEqual([]); + }); + + it('treats a seed EDIT on a hosted case as an update', () => { + const plan = planPush( + [item('c', { seed: inlineSeed({ dataTables: [{ name: 'Orders', columns: [] }] }) })], + { 'c.json': body({ seed: inlineSeed() }) }, + { c: 5 }, + ); + expect(plan.toUpdate.map((u) => u.id)).toEqual([5]); + expect(plan.unchanged).toEqual([]); + }); + + it('treats a seed REMOVAL as an update, so the patch can clear the stored one', () => { + const plan = planPush([item('c')], { 'c.json': body({ seed: inlineSeed() }) }, { c: 5 }); + expect(plan.toUpdate.map((u) => u.id)).toEqual([5]); + expect(plan.unchanged).toEqual([]); + }); + + // Shorthand is the shape case-shapes.md promotes, and expansion mints a fresh `id` + // on every parse. Comparing ids would classify the case `toUpdate` on EVERY push + // forever, leaving `--dry-run` permanently dirty. The identical-envelope test + // can't catch it, because there both sides share one envelope. + it('converges a shorthand-authored seed against its expanded stored export', () => { + // Shorthand mints a fresh `id` per parse; its `createdAt` is deterministic, so + // the two sides differ only in the id. + const disk = inlineSeed({ + messages: [ + { + id: 'freshly-minted-uuid', + createdAt: '2020-01-01T00:00:00.000Z', + role: 'assistant', + type: 'llm', + content: [{ type: 'text', text: 'built it' }], + }, + ], + }); + const stored = inlineSeed({ + messages: [ + { + id: 'stored-uuid', + createdAt: '2020-01-01T00:00:00.000Z', + role: 'assistant', + type: 'llm', + content: [{ type: 'text', text: 'built it' }], + }, + ], + }); + + const plan = planPush( + [item('c', { seed: disk })], + { 'c.json': body({ seed: stored }) }, + { c: 5 }, + ); + + expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); + expect(plan.toUpdate).toEqual([]); + }); + + it('still detects a real seed edit under a differing id', () => { + const disk = inlineSeed({ + messages: [ + { + id: 'a', + createdAt: '2026-06-29T09:00:00.000Z', + role: 'assistant', + type: 'llm', + content: [{ type: 'text', text: 'DIFFERENT text' }], + }, + ], + }); + const stored = inlineSeed({ + messages: [ + { + id: 'b', + createdAt: '2026-06-29T09:00:00.000Z', + role: 'assistant', + type: 'llm', + content: [{ type: 'text', text: 'built it' }], + }, + ], + }); + + const plan = planPush( + [item('c', { seed: disk })], + { 'c.json': body({ seed: stored }) }, + { c: 5 }, + ); + + expect(plan.toUpdate.map((u) => u.id)).toEqual([5]); + expect(plan.unchanged).toEqual([]); + }); + + // cubic's P2: `createdAt` drives restore ordering, so an authored envelope's + // timestamp edit changes what the agent sees and MUST reach the suite. Dropping + // it from the comparison alongside `id` would have hidden that. + it('detects an authored createdAt edit, which reorders the restored history', () => { + const disk = inlineSeed({ + messages: [ + { + id: 'same-id', + createdAt: '2026-07-01T12:00:00.000Z', + role: 'assistant', + type: 'llm', + content: [{ type: 'text', text: 'built it' }], + }, + ], + }); + const stored = inlineSeed({ + messages: [ + { + id: 'same-id', + createdAt: '2026-06-29T09:00:00.000Z', + role: 'assistant', + type: 'llm', + content: [{ type: 'text', text: 'built it' }], + }, + ], + }); + + const plan = planPush( + [item('c', { seed: disk })], + { 'c.json': body({ seed: stored }) }, + { c: 5 }, + ); + + expect(plan.toUpdate.map((u) => u.id)).toEqual([5]); + expect(plan.unchanged).toEqual([]); + }); + + it('still reports an identically-seeded case as unchanged', () => { + const plan = planPush( + [item('c', { seed: inlineSeed() })], + { 'c.json': body({ seed: inlineSeed() }) }, + { c: 5 }, + ); + expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); + expect(plan.toUpdate).toEqual([]); + }); + it('treats a scenario-only difference as an update (PATCH reconciles scenarios by name)', () => { const plan = planPush( [ @@ -227,4 +397,56 @@ describe('toUpdatePatch', () => { const patch = toUpdatePatch(createBody({ scenarios })); expect(patch.scenarios).toEqual(scenarios); }); + + it('sends an explicit null seed when the case has none, so a PATCH clears a stored one', () => { + // lang-tracer treats an omitted `seed` as a no-op, so dropping the seed from a + // disk case could never un-seed the hosted case without this. + const patch = toUpdatePatch(createBody()); + expect(patch.seed).toBeNull(); + }); + + it('keeps the seed when the case still has one', () => { + const seed = inlineSeed(); + const patch = toUpdatePatch(createBody({ seed })); + expect(patch.seed).toEqual(seed); + }); +}); + +// The post-write check the push runs after every create/update. A lang-tracer +// deployment predating a field's support ignores the key and still answers 200 +// (`seed` needs #113, `attach` needs #119), so only re-reading the export can tell +// you the suite holds what you authored. +describe('comparableDiff (post-write verification)', () => { + it('reports nothing when the server stored everything', () => { + expect( + comparableDiff(body({ seed: inlineSeed() }), item('c', { seed: inlineSeed() }).testCase), + ).toEqual([]); + }); + + it('names `seed` when a pre-#113 server dropped it', () => { + const written = item('c', { seed: inlineSeed() }).testCase; + expect(comparableDiff(body(), written)).toEqual(['seed']); + }); + + it('names `conversation` when a pre-#119 server dropped `attach`', () => { + const handoff = [{ role: 'user', text: '', attach: { workflow: 'wKk3RmT9xQ2bVn7L' } }]; + const written = item('c', { conversation: handoff, seed: inlineSeed() }).testCase; + // What such a server gives back: the turn, minus the attachment. + const stored = body({ conversation: [{ role: 'user', text: '' }], seed: inlineSeed() }); + + expect(comparableDiff(stored, written)).toEqual(['conversation']); + }); + + it('names every dropped field, not just the first', () => { + const handoff = [{ role: 'user', text: '', attach: { workflow: 'wKk3RmT9xQ2bVn7L' } }]; + const written = item('c', { conversation: handoff, seed: inlineSeed() }).testCase; + const stored = body({ conversation: [{ role: 'user', text: '' }] }); + + expect(comparableDiff(stored, written).sort()).toEqual(['conversation', 'seed']); + }); + + it('treats a case missing from the export as everything dropped', () => { + const written = item('c', { seed: inlineSeed() }).testCase; + expect(comparableDiff(undefined, written)).toContain('seed'); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts index 5d428141c96..206b09ddc47 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts @@ -1,4 +1,5 @@ -import type { EvalTestCaseInput } from '../harness/schema'; +import { EvalTestCaseSchema, type EvalTestCaseInput } from '../harness/schema'; +import { normalizeExportedCase } from '../langtracer/normalize'; import { diskCaseToLangTracerCreate, unsupportedPushReason } from '../langtracer/to-exported'; /** A minimal schema-parsed disk case (conversation text already collapsed to a string). */ @@ -110,15 +111,12 @@ describe('unsupportedPushReason', () => { expect(unsupportedPushReason(diskCase())).toBeNull(); }); - // Keyed off the discriminant, so a new mode is flagged without editing the - // mapper — the point of the one-slot union. - it('flags a replay seed as unsupported, naming the mode', () => { + it('flags a replay seed as unsupported — its trace expires, so it has no suite home', () => { const reason = unsupportedPushReason(diskCase({ seed: { mode: 'replay', threadId: 't' } })); - expect(reason).toMatch(/seed/); - expect(reason).toMatch(/replay/); + expect(reason).toMatch(/replay seed/); }); - it('flags an inline seed as unsupported, naming the mode', () => { + it('ALLOWS an inline seed — a durable fixture the write API stores verbatim', () => { const reason = unsupportedPushReason( diskCase({ seed: { @@ -137,7 +135,117 @@ describe('unsupportedPushReason', () => { }, }), ); - expect(reason).toMatch(/seed/); - expect(reason).toMatch(/inline/); + expect(reason).toBeNull(); + }); + + it('carries the inline seed into the create body verbatim', () => { + const seed = { + mode: 'inline' as const, + messages: [ + { + id: 'm1', + type: 'llm', + role: 'assistant' as const, + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'built it' }], + }, + ], + workflows: [{ id: 'wKk3RmT9xQ2bVn7L', name: 'Batch loop', nodes: [], connections: {} }], + dataTables: [], + }; + const body = diskCaseToLangTracerCreate(diskCase({ seed }), 'repair-it', { + suiteId: 1, + setKind: 'regression', + synthetic: true, + }); + expect(body.seed).toEqual(seed); + }); + + it('omits the seed key entirely for an unseeded case', () => { + const body = diskCaseToLangTracerCreate(diskCase(), 'plain', { + suiteId: 1, + setKind: 'regression', + synthetic: true, + }); + expect('seed' in body).toBe(false); + }); +}); + +// A suite is only a safe home for a hand-off case if `attach` survives the whole +// loop — push, store, export, reparse. It needs lang-tracer #119 deployed to carry +// the key; the last case here is what a pre-#119 deployment gives back, and it must +// fail loudly rather than run as a quietly different (find-it) test. +describe('attach round-trip: write → export → reparse', () => { + const WORKFLOW_ID = 'wKk3RmT9xQ2bVn7L'; + + function handoffCase(): EvalTestCaseInput { + return diskCase({ + // The faithful editor hand-off: no typed text, a workflow attached. + conversation: [{ role: 'user', text: '', attach: { workflow: WORKFLOW_ID } }], + seed: { + mode: 'inline', + messages: [ + { + id: 'm1', + type: 'llm', + role: 'assistant', + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'built it' }], + }, + ], + workflows: [{ id: WORKFLOW_ID, name: 'Batch loop', nodes: [], connections: {} }], + dataTables: [], + }, + } as Partial); + } + + /** What `GET /suites/:id/export` returns: disk shape plus export-only keys. */ + function exportedFrom(conversation: unknown, seed: unknown) { + return { + id: 42, + name: 'handoff', + suiteId: 1, + createdAt: '2026-08-04T00:00:00.000Z', + conversation, + seed, + complexity: 'simple', + tags: ['build'], + datasets: ['full'], + processExpectations: ['acknowledges the workflow it was handed'], + }; + } + + it('carries attach into the create body', () => { + const body = diskCaseToLangTracerCreate(handoffCase(), 'handoff', { + suiteId: 1, + setKind: 'regression', + synthetic: true, + }); + + expect(body.conversation).toEqual([ + { role: 'user', text: '', attach: { workflow: WORKFLOW_ID } }, + ]); + }); + + it('reparses from the export with the attachment intact', () => { + const body = diskCaseToLangTracerCreate(handoffCase(), 'handoff', { + suiteId: 1, + setKind: 'regression', + synthetic: true, + }); + + const parsed = EvalTestCaseSchema.parse( + normalizeExportedCase(exportedFrom(body.conversation, body.seed)), + ); + + expect(parsed.conversation?.[0].attach).toEqual({ workflow: WORKFLOW_ID }); + }); + + it('fails at load when the deployment stripped attach, instead of running as a find-it case', () => { + const stripped = exportedFrom([{ role: 'user', text: '' }], handoffCase().seed); + + expect(() => EvalTestCaseSchema.parse(normalizeExportedCase(stripped))).toThrow( + /opening turn with empty text must carry/, + ); }); }); diff --git a/packages/@n8n/instance-ai/evaluations/build-expectations/select.ts b/packages/@n8n/instance-ai/evaluations/build-expectations/select.ts index 7c720455a62..bfd399f7c0f 100644 --- a/packages/@n8n/instance-ai/evaluations/build-expectations/select.ts +++ b/packages/@n8n/instance-ai/evaluations/build-expectations/select.ts @@ -9,7 +9,10 @@ const NO_AGENT_OUTPUT_REASON = 'not judged — the build produced no agent output, so there was nothing to grade'; export interface SelectAuthorExpectationsArgs { - testCase: Pick; + testCase: Pick< + WorkflowTestCase, + 'processExpectations' | 'outcomeExpectations' | 'conversation' | 'seed' + >; /** Captured build transcript, if any. Empty/absent for prebuilt/MCP builds. */ transcript: TranscriptTurn[] | undefined; buildSucceeded: boolean; @@ -64,7 +67,12 @@ export function selectAuthorExpectations(args: SelectAuthorExpectationsArgs): { const transcript: TranscriptTurn[] = hasAgentOutput ? args.transcript! - : [{ userMessage: conversationUserTurnsAsText(testCase.conversation), steps: [] }]; + : [ + { + userMessage: conversationUserTurnsAsText(testCase.conversation, testCase.seed), + steps: [], + }, + ]; // A failed build that produced nothing at all: record every expectation as // ungraded rather than handing the judge an empty conversation to describe. diff --git a/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts b/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts index 49210879b13..a99efad1b19 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts @@ -11,10 +11,10 @@ import { execFileSync } from 'node:child_process'; import { basename } from 'node:path'; import { loadAgentEvalTestCasesWithFiles } from '../data/agents'; -import { loadWorkflowTestCasesWithFiles } from '../data/workflows'; +import { loadWorkflowTestCasesWithFiles, type WorkflowTestCaseWithFile } from '../data/workflows'; import { LangTracerClient } from '../langtracer/client'; import { resolveLangTracerConfig } from '../langtracer/config'; -import { planPush, toUpdatePatch } from '../langtracer/push'; +import { comparableDiff, planPush, toUpdatePatch } from '../langtracer/push'; import { diskCaseToLangTracerCreate } from '../langtracer/to-exported'; interface CliArgs { @@ -252,11 +252,52 @@ async function main() { console.log(` ~ updated ${item.fileSlug} (#${String(id)}, rev ${String(res.revision)})`); } + await verifyWrites(client, suite.id, [...plan.toCreate, ...plan.toUpdate.map((u) => u.item)]); + console.log( `\nDone: ${String(plan.toCreate.length)} created, ${String(plan.toUpdate.length)} updated, ${String(plan.unchanged.length)} unchanged, ${String(plan.skipped.length)} skipped.`, ); } +/** + * Re-read the suite and confirm the server stored what we sent. + * + * A lang-tracer deployment predating a field's support ignores that key and still + * answers 200 — `seed` before #113, `attach` before #119. Without this the push + * reports success while the suite holds a quietly different case: a seeded case + * that will run unseeded, or a hand-off that became a find-it test. Both are + * deploy-ordering hazards no local check can catch, so ask the server. + */ +async function verifyWrites( + client: LangTracerClient, + suiteId: number, + written: WorkflowTestCaseWithFile[], +): Promise { + if (written.length === 0) return; + + const after = await client.exportSuite(suiteId); + const dropped = written + .map((item) => ({ + fileSlug: item.fileSlug, + keys: comparableDiff(after.files[`${item.fileSlug}.json`], item.testCase), + })) + .filter((result) => result.keys.length > 0); + + if (dropped.length === 0) { + console.log(` verified ${String(written.length)} case(s) round-trip intact`); + return; + } + + for (const result of dropped) { + console.error(` ! ${result.fileSlug}: server did not store ${result.keys.join(', ')}`); + } + throw new Error( + `${String(dropped.length)} case(s) did not round-trip: the fields above were sent but are absent from the suite export. ` + + 'A lang-tracer deployment can silently ignore a key it predates (`seed` needs #113, `attach` needs #119) — ' + + 'upgrade it, then re-push. The cases in the suite are NOT what you authored until you do.', + ); +} + function printBucket(label: string, slugs: string[]): void { const pad = label.padEnd(9); console.log(` ${pad} ${String(slugs.length)}${slugs.length ? ` (${slugs.join(', ')})` : ''}`); diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 298c09666bd..c1cab975414 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -16,6 +16,7 @@ import type { InstanceAiThreadStatusResponse, InstanceAiEvalSeedDataTable, InstanceAiEvalSeedWorkflow, + InstanceAiWorkflowAttachment, AgentJsonConfig, AgentSkill, EvaluationConfigDto, @@ -212,12 +213,20 @@ export class N8nClient { /** * Send a chat message to the instance-ai agent. - * POST /rest/instance-ai/chat/:threadId body: { message } + * POST /rest/instance-ai/chat/:threadId body: { message, attachments? } + * + * `attachments` are resource references the agent resolves with its tools — the + * same channel the editor uses when a user opens the assistant with a workflow + * in front of them, so the agent is handed it by id instead of hunting by name. */ - async sendMessage(threadId: string, message: string): Promise<{ runId: string }> { + async sendMessage( + threadId: string, + message: string, + attachments?: InstanceAiWorkflowAttachment[], + ): Promise<{ runId: string }> { const result = await this.fetch(`/rest/instance-ai/chat/${threadId}`, { method: 'POST', - body: { message }, + body: attachments && attachments.length > 0 ? { message, attachments } : { message }, }); return result as { runId: string }; } diff --git a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts index cdc18087406..7694280c5b8 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts @@ -7,7 +7,7 @@ // execution and cleanup. // --------------------------------------------------------------------------- -import type { InstanceAiConfirmRequest } from '@n8n/api-types'; +import type { InstanceAiConfirmRequest, InstanceAiWorkflowAttachment } from '@n8n/api-types'; import crypto from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; @@ -59,6 +59,7 @@ import type { } from '../types'; import { agentTurnsAsText, + attachedWorkflowNote, failedBuildsPerTurn, lastAgentText, userTurnsAsText, @@ -98,6 +99,10 @@ interface MultiTurnDriverConfig { /** Appended to the FIRST sent message only (pre-seeded-table hint); the * recorded turn and the proxy's conversation keep the clean prompt. */ openingMessageSuffix?: string; + /** What the RECORDED opening turn says, when it must differ from what's sent — + * an out-of-band attachment has to be named in the transcript the judge reads. + * Defaults to the sent text. */ + recordedOpeningMessage?: string; /** Ids already allowlisted for this thread (from pre-run `createDeclaredCredentials` * seeding) — wires `UserProxyLlm.credentialCreation` so `manual` can create a * real credential when a setup card shows zero existing candidates. Omitted @@ -108,15 +113,25 @@ interface MultiTurnDriverConfig { /** Shared with `createDeclaredCredentials`'s pre-run seeding — see * `CredentialCreationConfig.nameCounts`. */ credentialNameCounts?: Map; + /** Resource references sent with the FIRST message only — an attachment is a + * hand-off, not something a user re-sends every turn. */ + openingAttachments?: InstanceAiWorkflowAttachment[]; } async function driveMultiTurnConversation( config: MultiTurnDriverConfig, ): Promise { const openingMessage = config.conversation[0]?.text ?? ''; + const recordedOpeningMessage = config.recordedOpeningMessage ?? openingMessage; + // The proxy renders both its script and its running transcript from `text` alone, + // so it needs the recorded opening too — otherwise it audits every plan and + // follow-up against a blank turn that never mentions the workflow. + const proxyConversation = config.conversation.map((turn, index) => + index === 0 ? { ...turn, text: recordedOpeningMessage } : turn, + ); const proxy = new UserProxyLlm({ - conversation: config.conversation, + conversation: proxyConversation, messageBudget: config.messageBudget, logger: config.logger, ...(config.allowlistedCredentialIds !== undefined @@ -143,10 +158,11 @@ async function driveMultiTurnConversation( return decision; }; - recordUserTurn(config.events, openingMessage); + recordUserTurn(config.events, recordedOpeningMessage); await config.client.sendMessage( config.threadId, openingMessage + (config.openingMessageSuffix ?? ''), + config.openingAttachments, ); await runMultiTurnConversation({ @@ -318,6 +334,9 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise the workflow as actually restored (fresh id and + // name). Lets an authored `attach` reference survive the per-run remap. + let seedWorkflowsBySeedId = new Map(); try { const buildStart = Date.now(); @@ -416,6 +435,12 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise [workflow.id, remapped.workflows[index]]), + ); await evictLeftoverSeedWorkflows( client, remapped, @@ -504,6 +529,34 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise new Set(workflows.map((workflow) => workflow.id)).size === workflows.length, + { message: 'seed workflow ids must be unique — references resolve by id' }, + ), /** Data tables the history references, recreated (and id-rewritten) on restore. */ dataTables: z.array(SeedDataTableSchema).default([]), }); @@ -153,17 +163,27 @@ function isShorthandTurn( ); } +/** Base for every timestamp this module stamps. A CONSTANT, not `Date.now()`: the + * same case must yield the same messages on every parse, or the push diff (which + * compares `seed`) reads a fresh timestamp as an edit and re-PATCHes the case on + * every run forever. Only ordering depends on these values, and fixed past slots + * order exactly as well. */ +const SEED_EPOCH_MS = Date.parse('2020-01-01T00:00:00.000Z'); + +/** The slot for the message at `index` — ascending, and in the past, so array + * order survives a store that presents messages by `createdAt`. */ +const seedStampAt = (index: number) => new Date(SEED_EPOCH_MS + index * 1000).toISOString(); + /** * Expand `{role, text}` shorthand messages into native llm envelopes; anything * else passes through for the envelope schema to validate. * * Array-level rather than per-message because the stamped timestamps ascend by - * position — slightly in the past, so seeded history always orders before the - * live turn and a shorthand author cannot get the ordering wrong. A full - * envelope keeps its own authored `createdAt`. + * position, so seeded history always orders before the live turn and a shorthand + * author cannot get the ordering wrong. A full envelope keeps its own authored + * `createdAt` — see `normalizeSeedTimestamps` for when that is overridden. */ export function expandSeedMessageShorthand(messages: unknown[]): unknown[] { - const base = Date.now() - (messages.length + 1) * 1000; return messages.map((message, index) => { if (!isShorthandTurn(message)) return message; return { @@ -178,48 +198,47 @@ export function expandSeedMessageShorthand(messages: unknown[]): unknown[] { text: Array.isArray(message.text) ? message.text.join('\n') : message.text, }, ], - createdAt: new Date(base + index * 1000).toISOString(), + createdAt: seedStampAt(index), }; }); } /** - * Pull an inline seed's timestamps back into the past when the author put any of - * them in the future. + * Restamp an inline seed's timestamps when the authored ones don't present the + * messages the way the array orders them: ascending, and entirely before the live + * turn. Only a full envelope can get this wrong (the shorthand stamps its own + * slots), and either failure makes the agent see its own history out of order. * - * The shorthand stamps its own ascending pre-live timestamps, so it can't get - * this wrong; a full envelope keeps what it was authored with, and a future - * stamp sorts the seeded turn AFTER the live turn — the agent then sees its own - * history out of order, and the judge grades a transcript that never happened. + * Restamps the WHOLE sequence, not just the offending entry: a per-message fix + * reorders relative to the array (`[future A, past B]` moves only A, leaving the + * store presenting B then A while the transcript still grades array order). + * Authored timestamps therefore survive only when already ascending and past. * - * Restamps the WHOLE sequence, not just the offending entry. A per-message clamp - * reorders relative to the array: `[future A, past B]` leaves B alone and moves A - * to ~now, so the store presents B then A while `transcriptPrefixFromSeed` still - * grades array order. Array order is the authority, so the rewrite reproduces it - * on the same ascending slots the shorthand uses. Authored timestamps are - * therefore only preserved when every one of them is already in the past. - * - * Inline (hand-authored) seeds only — a `replay` seed is reconstructed from a - * real trace and never reaches this schema, so no real timestamp can be moved. + * Inline seeds only — a `replay` seed is reconstructed from a real trace and never + * reaches this schema, so no real timestamp can be moved. */ -export function clampFutureSeedTimestamps(messages: unknown[]): unknown[] { - const now = Date.now(); +export function normalizeSeedTimestamps(messages: unknown[]): unknown[] { const stampOf = (message: unknown): number | undefined => { if (!isRecord(message) || typeof message.createdAt !== 'string') return undefined; const at = Date.parse(message.createdAt); // Unparseable is the envelope schema's error to report, not ours to paper over. return Number.isNaN(at) ? undefined : at; }; - const anyFuture = messages.some((message) => (stampOf(message) ?? -Infinity) >= now); - if (!anyFuture) return messages; - const base = now - (messages.length + 1) * 1000; + const now = Date.now(); + let previous = -Infinity; + const presentsInArrayOrder = messages.every((message) => { + const at = stampOf(message); + if (at === undefined) return true; + if (at >= now || at <= previous) return false; + previous = at; + return true; + }); + if (presentsInArrayOrder) return messages; + return messages.map((message, index) => { - if (stampOf(message) === undefined) return message; - return { - ...(message as Record), - createdAt: new Date(base + index * 1000).toISOString(), - }; + if (!isRecord(message) || stampOf(message) === undefined) return message; + return { ...message, createdAt: seedStampAt(index) }; }); } @@ -305,7 +324,11 @@ export function remapSeedWorkflowIds(seed: ConversationSeed): ConversationSeed { const originalIds = new Set(seed.workflows.map((workflow) => workflow.id)); let serialized = JSON.stringify({ messages: seed.messages, workflows: seed.workflows }); - for (const workflow of seed.workflows) { + // Longest id first, for the same reason the name pass below sorts: if one id were a + // prefix of another ("abcdefgh" / "abcdefgh12"), rewriting the short one first would + // eat the long one's prefix and leave it with a derived id no later pass matches. + const byLongestId = [...seed.workflows].sort((a, b) => b.id.length - a.id.length); + for (const workflow of byLongestId) { // Workflow ids are long random tokens; a short id would risk rewriting // unrelated substrings, so refuse instead of corrupting the seed. if (workflow.id.length < 8) { diff --git a/packages/@n8n/instance-ai/evaluations/harness/schema.ts b/packages/@n8n/instance-ai/evaluations/harness/schema.ts index abfecc2f30b..02f6c729b40 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/schema.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/schema.ts @@ -4,8 +4,8 @@ import { z } from 'zod'; import { ConversationSeedSchema, SeedMessageSchema, - clampFutureSeedTimestamps, expandSeedMessageShorthand, + normalizeSeedTimestamps, } from './conversation-seed'; import { SUPPORTED_CREDENTIAL_TYPES } from '../credentials/seeder'; @@ -33,6 +33,17 @@ export const conversationTurnTextSchema = z export const ConversationTurnSchema = z.object({ role: z.enum(['user', 'assistant']), text: conversationTurnTextSchema, + /** Hand the agent a seeded workflow with this turn, the way the editor does when + * a user opens the assistant with a workflow in front of them — without it the + * eval agent has to guess which workflow prose like "why is this failing?" + * means, and we score a clarification failure the real user never hit. + * + * `workflow` is the id as the seed declares it; the harness swaps in the + * per-run remapped id. Opening turn only (refined below). */ + attach: z + .object({ workflow: z.string().min(1) }) + .strict() + .optional(), }); const ExecutionScenarioSchema = z.object({ @@ -52,10 +63,10 @@ const ExecutionScenarioSchema = z.object({ /** Prior messages for an inline seed. Accepts a full envelope or the `{role, text}` * shorthand, expanded BEFORE validation so the envelope rules apply to the * expansion and error paths stay per-message (`seed.messages.2.createdAt`). - * Future `createdAt` values are pulled back after expansion, so seeded history - * can never sort after the live turn. */ + * Timestamps are normalized after expansion, so seeded history always presents + * in array order and never sorts after the live turn. */ const inlineSeedMessagesSchema = z.preprocess( - (raw) => (Array.isArray(raw) ? clampFutureSeedTimestamps(expandSeedMessageShorthand(raw)) : raw), + (raw) => (Array.isArray(raw) ? normalizeSeedTimestamps(expandSeedMessageShorthand(raw)) : raw), z.array(SeedMessageSchema).min(1), ); @@ -188,6 +199,51 @@ export const EvalTestCaseSchema = evalTestCaseObjectSchema message: 'a case needs a conversation, or a seed with mode: replay (which supplies the live turn from the trace)', }) + // Rejected rather than ignored on a later turn, so a misplaced one can't silently + // do nothing. + .refine((c) => (c.conversation ?? []).slice(1).every((turn) => turn.attach === undefined), { + message: 'only the first conversation turn may carry `attach` — an attachment is a hand-off', + }) + // Grading an assistant turn that carries one would score a transcript that could + // not have happened. + .refine((c) => c.conversation?.[0]?.attach === undefined || c.conversation[0].role === 'user', { + message: + 'only a `user` turn may carry `attach` — the attachment is the hand-off that opens the conversation', + }) + // A dangling attachment would hand the agent a reference to nothing, which reads + // as a builder failure. Only an inline seed declares workflows to point at. + .refine( + (c) => { + const attached = c.conversation?.[0]?.attach?.workflow; + if (attached === undefined) return true; + const declared = c.seed?.mode === 'inline' ? c.seed.workflows : []; + return declared.some((workflow) => workflow.id === attached); + }, + { + message: + '`attach.workflow` must be the id of a workflow the inline seed declares — otherwise the attachment points at nothing', + }, + ) + // The chat API refuses a message that is empty with nothing attached, so catch it + // at load rather than mid-run as a 400 that reads like an infrastructure fault. + // Every user turn, not just the opening; assistant turns are proxy script data + // and never posted. Only a non-replay opening may substitute `attach` for text — + // a replay case has no inline seed for it to point at. + .superRefine((c, ctx) => { + const isReplay = c.seed?.mode === 'replay'; + (c.conversation ?? []).forEach((turn, index) => { + if (turn.role === 'assistant' || turn.text.trim().length > 0) return; + const openingMayAttach = index === 0 && !isReplay; + if (openingMayAttach && turn.attach !== undefined) return; + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['conversation', index, 'text'], + message: openingMayAttach + ? 'an opening turn with empty text must carry `attach` — the chat API rejects a message that is empty with nothing attached' + : 'a conversation turn needs text — the chat API rejects an empty message, and only a non-replay opening turn may substitute `attach`', + }); + }); + }) .superRefine((c, ctx) => { // Note: this message avoids double quotes — ZodError.message is a JSON.stringify of // the issue list, which would otherwise backslash-escape them and break substring/regex diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/client.ts b/packages/@n8n/instance-ai/evaluations/langtracer/client.ts index 8391faccda3..2e4e4fef401 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/client.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/client.ts @@ -37,8 +37,12 @@ export type LangTracerCaseRef = z.infer; * by name on PATCH (upsert + delete missing); a server predating lang-tracer #48 * strips the key silently, leaving the old scenarios in place. */ export type LangTracerUpdateCaseBody = Partial< - Omit ->; + Omit +> & { + /** Explicit `null` CLEARS a stored seed. An omitted key is a server-side no-op, + * so a disk case that drops its seed needs the null to take effect. */ + seed?: LangTracerCreateCaseBody['seed'] | null; +}; export class LangTracerClient { constructor(private readonly config: LangTracerConfig) {} diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/push.ts b/packages/@n8n/instance-ai/evaluations/langtracer/push.ts index 67ebbafc89e..eaa0bcd9e2b 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/push.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/push.ts @@ -35,19 +35,24 @@ const COMPARED_KEYS = [ // Round-trips faithfully: PATCH /cases/:id reconciles scenario rows by name // (lang-tracer #48) and the export emits them back in disk shape. 'executionScenarios', + // Stored at `metadata.seed` and emitted back by the export (lang-tracer #113). + // Compared so a seed-only edit isn't misread as unchanged and left unpushed. + 'seed', ] as const; /** Drop the create-only fields, leaving the patchable set (`scenarios` included — * `PATCH /cases/:id` reconciles them by name since lang-tracer #48). An absent - * `scenarios` is sent as an explicit `[]`: a partial PATCH leaves missing keys - * untouched, so omitting it would keep the server's old scenario rows alive - * forever after a disk case drops its `executionScenarios`. */ + * `scenarios` is sent as an explicit `[]` and an absent `seed` as an explicit + * `null`: a partial PATCH leaves missing keys untouched, so omitting them would + * keep the server's old scenario rows / stored seed alive forever after a disk + * case drops its `executionScenarios` / `seed`. Both defaults sit before the + * spread, so a case that still has them overrides. */ export function toUpdatePatch({ suiteId, synthetic, ...patch }: LangTracerCreateCaseBody): LangTracerUpdateCaseBody { - return { scenarios: [], ...patch }; + return { scenarios: [], seed: null, ...patch }; } /** `existingBodies`: `.json` → exported (disk-shape) body from `GET /suites/:id/export`. @@ -86,9 +91,16 @@ export function planPush( /** Compare only the PATCH-able disk fields of an exported body against a disk case, * after folding lang-tracer's export-only keys / legacy `buildExpectations`. */ function sameComparableFields(existingRaw: unknown, diskTestCase: unknown): boolean { + return comparableDiff(existingRaw, diskTestCase).length === 0; +} + +/** Which comparable fields differ between an exported body and a disk case. Same + * rules as the create/update/unchanged split, so a post-write check can reuse them + * to name exactly what a server failed to store. */ +export function comparableDiff(existingRaw: unknown, diskTestCase: unknown): string[] { const existing = projectComparable(normalizeExportedCase(existingRaw)); const disk = projectComparable(diskTestCase); - return canonicalize(existing) === canonicalize(disk); + return COMPARED_KEYS.filter((key) => canonicalize(existing[key]) !== canonicalize(disk[key])); } function projectComparable(src: unknown): Record { @@ -116,11 +128,38 @@ function projectComparable(src: unknown): Record { out[key] = datasets; continue; } + // Shorthand expansion stamps a fresh `randomUUID()` per parse, so comparing + // message ids can never converge — the case would re-PATCH on every push + // forever. Ids carry no meaning to a case, so they're dropped here. + // `createdAt` is NOT dropped: restore ordering depends on it, so an authored + // envelope's timestamp edit must still register. Shorthand's own timestamps + // are deterministic (see SHORTHAND_SEED_EPOCH_MS), so they converge anyway. + if (key === 'seed') { + out[key] = seedWithoutMessageIds(value); + continue; + } out[key] = value; } return out; } +/** Drop message `id`s from a seed before comparing: shorthand expansion mints a + * new one per parse, so keeping them would make a shorthand-authored case differ + * from its stored export forever. Everything else the author wrote — role, + * content, `createdAt`, workflows, data tables — still compares. */ +function seedWithoutMessageIds(value: unknown): unknown { + if (value === null || typeof value !== 'object') return value; + const seed: Record = { ...(value as Record) }; + const messages: unknown = seed.messages; + if (!Array.isArray(messages)) return seed; + seed.messages = (messages as unknown[]).map((message) => { + if (message === null || typeof message !== 'object') return message; + const { id, ...rest } = message as Record; + return rest; + }); + return seed; +} + /** Stable JSON with sorted object keys, so field/scenario ordering never affects equality. */ function canonicalize(value: unknown): string { return JSON.stringify(sortKeysDeep(value)); diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts index 296dd0bac84..31fb1e222d1 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts @@ -2,7 +2,7 @@ // lang-tracer REST `POST /api/v1/cases` (create_test_case) expects. Split from the // network call so the disk→API key-renaming contract is unit-testable without a server. -import type { EvalTestCaseInput } from '../harness/schema'; +import type { CaseSeed, EvalTestCaseInput } from '../harness/schema'; /** One scenario in the create-case payload (`executionScenarios` renamed to `scenarios`). */ export interface LangTracerScenario { @@ -21,7 +21,14 @@ export interface LangTracerCreateCaseBody { synthetic: boolean; suiteId: number; description?: string; - conversation?: Array<{ role: 'user' | 'assistant'; text: string }>; + /** `attach` is declared, not just tolerated: the turn shape is the push contract, + * and leaving it off let a hand-off case type-check while losing its attachment. + * Carrying it end-to-end needs lang-tracer #119 deployed. */ + conversation?: Array<{ + role: 'user' | 'assistant'; + text: string; + attach?: { workflow: string }; + }>; evalComplexity: 'simple' | 'medium' | 'complex'; evalTags: string[]; evalTriggerType?: string; @@ -31,6 +38,10 @@ export interface LangTracerCreateCaseBody { datasets?: string[]; messageBudget?: number; credentials?: Array<{ type: string; name?: string }>; + /** Inline seed, forwarded verbatim — lang-tracer stores it at `metadata.seed`. + * Only the authored arm: a replay seed is derived from a source thread by + * promote/scrub over there, so pushing one would fabricate provenance. */ + seed?: Extract; } export interface ToLangTracerOptions { @@ -39,12 +50,30 @@ export interface ToLangTracerOptions { synthetic: boolean; } -/** The case-write API has no `seed` field, so no seeded case can be pushed yet — - * keyed off the discriminant so a new mode needs no edit here. Returns a +/** Seeding modes the case-write API can't take. An INLINE seed is pushable — it's a + * durable fixture, and the API stores it verbatim. A REPLAY seed isn't: it points at + * a LangSmith trace that expires, lang-tracer derives it from a source thread it + * already holds, and such a case is barred from suites anyway. Returns a * human-readable reason, else null. */ export function unsupportedPushReason(testCase: EvalTestCaseInput): string | null { - if (!testCase.seed) return null; - return `uses seed (mode: ${testCase.seed.mode}) — not supported by the case-write API`; + const seed = testCase.seed; + switch (seed?.mode) { + case undefined: + case 'inline': + return null; + case 'replay': + return ( + 'uses a replay seed — reconstructed from a LangSmith trace at run time, so it has no ' + + 'durable home in a suite. Derive a synthetic case from it instead.' + ); + default: { + // A new arm must decide its own push-ability here. Approving by default + // would push the case while `diskCaseToLangTracerCreate` forwards only + // `inline` — landing it in the suite stripped of its seed. + const unhandled: never = seed; + throw new Error(`Unhandled seed mode: ${JSON.stringify(unhandled)}`); + } + } } /** Map a schema-parsed disk case to a create-case body. `conversation.text` is already @@ -78,6 +107,8 @@ export function diskCaseToLangTracerCreate( if (testCase.datasets !== undefined) body.datasets = testCase.datasets; if (testCase.messageBudget !== undefined) body.messageBudget = testCase.messageBudget; if (testCase.credentials !== undefined) body.credentials = testCase.credentials; + // Replay never reaches here — `unsupportedPushReason` skips those cases upstream. + if (testCase.seed?.mode === 'inline') body.seed = testCase.seed; return body; } diff --git a/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts b/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts index fe1fb81fc2e..b2a3b386135 100644 --- a/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts +++ b/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts @@ -438,7 +438,7 @@ export function createBuildOrchestrator(deps: BuildOrchestratorDeps): BuildOrche if (build.success && !build.workflowChecks) { build.workflowChecks = await runWorkflowChecks({ workflow: build.workflowJsons[0], - prompt: conversationUserTurnsAsText(entry.conversation ?? []), + prompt: conversationUserTurnsAsText(entry.conversation ?? [], entry.seed), agentText: undefined, logger, }); @@ -467,10 +467,13 @@ export function createBuildOrchestrator(deps: BuildOrchestratorDeps): BuildOrche // No transcript in prebuilt mode, but the authored conversation still // carries the user's request — feed it so prompt-aware checks (e.g. // fulfills_user_request) grade against real intent instead of "". - const conversation = testCaseByFileSlug.get(fileSlug)?.conversation ?? []; + const prebuiltCase = testCaseByFileSlug.get(fileSlug); build.workflowChecks = await runWorkflowChecks({ workflow: build.workflowJsons[0], - prompt: conversationUserTurnsAsText(conversation), + prompt: conversationUserTurnsAsText( + prebuiltCase?.conversation ?? [], + prebuiltCase?.seed, + ), agentText: undefined, logger, }); diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 6f64c779012..8b68eb63c90 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -198,6 +198,11 @@ export interface ExecutionScenario { export interface ConversationTurn { role: 'user' | 'assistant'; text: string; + /** Hand the agent a seeded workflow with this turn (opening turn only), the way + * the editor does when a user opens the assistant with a workflow in front of + * them. `workflow` is the id as the seed declares it; the harness swaps in the + * per-run remapped id. See `ConversationTurnSchema`. */ + attach?: { workflow: string }; } export interface TestCaseCredential { diff --git a/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts b/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts index bac506c81bf..d2f1f60b882 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts @@ -3,10 +3,39 @@ import { isRecord } from '@n8n/utils/is-record'; import type { CaseSeed } from '../harness/schema'; import type { ConversationTurn, ToolInteraction, TranscriptStep, TranscriptTurn } from '../types'; +/** Render a turn's out-of-band workflow attachment for a transcript/prompt, e.g. + * `[attached workflow: Batch loop]`, or '' when it has none. The editor hands the + * agent a resource reference rather than text, so without this the faithful + * hand-off shape (`text: ""` + `attach`) reaches judges and prompt-aware checks + * as an empty message. + * + * `label` is the workflow's NAME — the restored one where the harness knows it + * (the live path), else the name the seed declares for that id + * (`attachedWorkflowLabel`). An id would mean nothing to a prompt-aware check or + * to a human reading the report. */ +export function attachedWorkflowNote(label: string | undefined): string { + return label ? `[attached workflow: ${label}]` : ''; +} + +/** The name a seed declares for an attached workflow id. The authored-conversation + * path has only the id, and the seed is where that id gets its name; falls back + * to the id when the seed can't resolve it, so the hand-off stays visible. */ +function attachedWorkflowLabel( + turn: ConversationTurn | undefined, + seed: CaseSeed | undefined, +): string | undefined { + const id = turn?.attach?.workflow; + if (id === undefined) return undefined; + const declared = seed?.mode === 'inline' ? seed.workflows.find((w) => w.id === id) : undefined; + return declared?.name ?? id; +} + /** * Human-readable prompt label for a test case. Authored cases use their first * turn; a `replay` seed carries no authored conversation, so fall back to the * live (non-seeded) user turn captured in the transcript, then to the thread id. + * A text-less hand-off (`text: ""` + `attach`) has no prompt text at all, so it + * falls back to naming the attachment — otherwise it labels as '' in every report. */ export function caseDisplayPrompt( testCase: { conversation?: ConversationTurn[]; seed?: CaseSeed }, @@ -17,7 +46,8 @@ export function caseDisplayPrompt( const liveTurn = transcript?.find((t) => !t.seeded && t.userMessage)?.userMessage; if (liveTurn) return liveTurn; const { seed } = testCase; - return seed?.mode === 'replay' ? `[seeded] thread ${seed.threadId.slice(0, 8)}` : ''; + if (seed?.mode === 'replay') return `[seeded] thread ${seed.threadId.slice(0, 8)}`; + return attachedWorkflowNote(attachedWorkflowLabel(testCase.conversation?.[0], seed)); } /** @@ -42,12 +72,20 @@ export function userTurnsAsText(transcript: TranscriptTurn[]): string { * * Accepts `undefined` because `testCase.conversation` is optional (a `replay`-seeded * case carries none) and callers pass it straight through — no conversation → ''. + * `seed` resolves an attachment's id to its declared name. */ -export function conversationUserTurnsAsText(conversation: ConversationTurn[] | undefined): string { +export function conversationUserTurnsAsText( + conversation: ConversationTurn[] | undefined, + seed?: CaseSeed, +): string { if (!conversation) return ''; const turns = conversation .filter((t) => t.role === 'user') - .map((t) => t.text) + // Name an attachment, so a text-less hand-off isn't filtered out below and + // handed to the prompt-aware checks as an empty prompt. + .map((t) => + [attachedWorkflowNote(attachedWorkflowLabel(t, seed)), t.text].filter(Boolean).join(' '), + ) .filter((text) => text.length > 0); if (turns.length === 0) return '';