From fe11536df10deae832bce11942872ae8cc3fe5fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Braulio=20Gonz=C3=A1lez=20Valido?= Date: Tue, 23 Jun 2026 18:30:30 +0100 Subject: [PATCH] feat(ai-builder): Add conversation pre-seeding to workflow evals (no-changelog) (#32196) Co-authored-by: Claude Fable 5 --- packages/@n8n/api-types/src/index.ts | 3 + .../src/schemas/instance-ai.schema.ts | 40 ++ .../@n8n/instance-ai/evaluations/README.md | 69 +++ .../__tests__/conversation-seed.test.ts | 294 ++++++++++++ .../__tests__/data-workflows-schema.test.ts | 56 +++ .../__tests__/langsmith-seed.test.ts | 440 ++++++++++++++++++ .../__tests__/workflow-report.test.ts | 23 + .../@n8n/instance-ai/evaluations/cli/index.ts | 28 +- .../evaluations/clients/n8n-client.ts | 34 ++ .../evaluations/comparison/format.ts | 7 +- .../evaluations/data/workflows/index.ts | 19 +- .../evaluations/data/workflows/schema.ts | 29 +- .../evaluations/harness/conversation-seed.ts | 247 ++++++++++ .../evaluations/harness/langsmith-seed.ts | 422 +++++++++++++++++ .../harness/parse-seed-workflow.ts | 24 + .../instance-ai/evaluations/harness/runner.ts | 141 +++++- .../outcome/transcript-from-events.ts | 10 +- .../evaluations/report/workflow-report.ts | 9 +- .../@n8n/instance-ai/evaluations/types.ts | 26 +- .../evaluations/utils/conversation-text.ts | 25 +- .../instance-ai-memory.service.test.ts | 83 ++++ .../__tests__/instance-ai.controller.test.ts | 110 +++++ .../__tests__/thread-restore.service.test.ts | 186 ++++++++ .../eval/thread-restore.service.ts | 185 ++++++++ .../instance-ai/instance-ai-memory.service.ts | 52 +++ .../instance-ai/instance-ai.controller.ts | 61 +++ 26 files changed, 2573 insertions(+), 50 deletions(-) create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts create mode 100644 packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts create mode 100644 packages/@n8n/instance-ai/evaluations/harness/parse-seed-workflow.ts create mode 100644 packages/cli/src/modules/instance-ai/eval/__tests__/thread-restore.service.test.ts create mode 100644 packages/cli/src/modules/instance-ai/eval/thread-restore.service.ts diff --git a/packages/@n8n/api-types/src/index.ts b/packages/@n8n/api-types/src/index.ts index efb3d429673..5ba2d6312b5 100644 --- a/packages/@n8n/api-types/src/index.ts +++ b/packages/@n8n/api-types/src/index.ts @@ -317,6 +317,7 @@ export { InstanceAiEvalExecutionRequest, InstanceAiEvalCredentialAllowlistRequest, INSTANCE_AI_MEMORY_TASK_WAIT_TIMEOUT_MS, + InstanceAiEvalRestoreThreadRequest, instanceAiGatewayKeySchema, InstanceAiGatewayEventsQuery, InstanceAiEventsQuery, @@ -419,6 +420,8 @@ export type { InstanceAiEvalMockedCredential, InstanceAiEvalRewrittenCredential, InstanceAiEvalExecutionResult, + InstanceAiEvalSeedWorkflow, + InstanceAiEvalSeedDataTable, } from './schemas/instance-ai.schema'; export type { diff --git a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts index adebef56839..298015b7b10 100644 --- a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts +++ b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts @@ -1324,3 +1324,43 @@ export class InstanceAiEvalCredentialAllowlistRequest extends Z.class({ */ credentialIds: z.array(z.string().min(1)).max(50), }) {} + +/** A workflow a conversation seed references, recreated at its given id so the + * seeded history resolves. Content is opaque here; the server validates it. */ +const instanceAiEvalSeedWorkflowSchema = z.object({ + id: z.string().min(1).max(64), + name: z.string().min(1).max(255), + nodes: z.array(z.record(z.unknown())).max(500), + connections: z.record(z.unknown()), +}); + +export type InstanceAiEvalSeedWorkflow = z.infer; + +/** A data table a seed references. Recreated on restore (its id is server- + * generated, so the seed workflows' references are rewritten to the new id). + * Schema only — no rows (the table just needs to exist; rows are the trace's + * highest-PII payload and are never sent here). */ +const instanceAiEvalSeedDataTableSchema = z.object({ + id: z.string().min(1).max(64), + name: z.string().min(1).max(128), + columns: z + .array( + z.object({ + name: z.string().min(1).max(128), + type: z.enum(['string', 'number', 'boolean', 'date']), + }), + ) + .max(50), +}); + +export type InstanceAiEvalSeedDataTable = z.infer; + +export class InstanceAiEvalRestoreThreadRequest extends Z.class({ + threadId: z.string().uuid(), + /** Native agent message log (ISO `createdAt`), stored verbatim. */ + messages: z.array(z.record(z.unknown())).min(1).max(1000), + /** Data tables the workflows reference; recreated first so ids can be rewritten. */ + dataTables: z.array(instanceAiEvalSeedDataTableSchema).max(20).optional(), + /** Workflows the history references; recreated (node credentials stripped). */ + workflows: z.array(instanceAiEvalSeedWorkflowSchema).max(50).optional(), +}) {} diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index bbdd012045e..e3fa38acbc8 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -592,6 +592,75 @@ Declared credentials are created for real (placeholder token; set the matching ` Each type needs a data template in `credentials/seeder.ts`; declaring an unknown type fails the build with a pointer there. +### Seeded cases (conversation pre-seeding) + +A seeded case starts **mid-conversation**: prior history is restored into the build thread before the live turn, so the eval drives only the turn under test. Use it to replicate a real misbehaviour — restore the conversation up to the moment it went wrong, re-drive that turn, and assert what should happen instead. + +Pick the lightest path that fits: + +| Situation | Path | +|---|---| +| Reproduce a real conversation (the common case) | `seedThread` — fetch + reconstruct its LangSmith trace at run time; nothing committed | +| Prelude is just "what was discussed" (no tool calls, no workflows) | `priorConversation` — prose turns, authored inline | +| A synthetic/sanitized fixture you want durable | `seedFile` — a committed seed JSON (no real conversation data) | +| Shallow 2–3 turn prelude where the agent's live replies matter | Neither — a plain multi-turn `conversation` script re-drives it live | + +#### `seedThread` — reproduce a real conversation (no repo content) + +The case carries only a **thread id**. At run time the harness pulls that thread's runs from LangSmith, reconstructs the message log (user/assistant text + resolved tool-call blocks, deduped across suspend/resume), and splits at the **last user message**: everything before it is restored as the seed, that last message is sent live. The seed workflow is compiled from the build/patch tool's captured SDK code **as of the seed boundary**, so it matches what the live turn first saw. + +```json +"seedThread": { "threadId": "", "project": "instance-ai" } +``` + +No `conversation` field needed — the live turn comes from the trace. `project` is optional (defaults to `instance-ai`). No conversation content lands in the repo — only the opaque thread id. + +**What's restored.** The workflows the seed references are recreated pinned to their ids (node credentials are stripped — the eval credential pin owns the credential view, so a pre-attached id would bypass it). Data tables those workflows reference are recreated **schema-only** — columns and a remapped id, **no rows**: an empty table is all a data-table node needs to resolve, and a real conversation's rows are the highest-PII payload, so they're never reconstructed, sent, or inserted (the same row values are also redacted out of the restored message history). The row content stays in the source trace and never reaches the eval instance. + +**Continuing past the live turn.** Add a `conversation` to keep driving *after* the trace's last message is replayed — the effective conversation becomes `[, ...conversation]`, so the live turn is sent for real and your authored turns become proxy-driven follow-ups (multi-turn). Use it to push a reproduced conversation further (e.g. "now also add error handling", or pressure-test the next decision): + +```json +"seedThread": { "threadId": "…", "project": "instance-ai" }, +"conversation": [ + { "role": "assistant", "text": "Updated the schedule to every 30 minutes." }, + { "role": "user", "text": "Now also send a copy to #ops." } +] +``` + +(The first authored turn is typically the expected assistant reply as proxy reference; subsequent `user` turns are sent as follow-ups. Omit `conversation` to just send the live turn and stop.) + +**Cross-workspace, zero config (e.g. prod traces, staging eval).** A source thread can live in a different LangSmith **workspace** than the eval writes to. You don't declare which, and there are no extra env vars — the harness enumerates the workspaces your `LANGSMITH_API_KEY` can access and finds the one holding the thread (the workspace is selected per request via the `x-tenant-id` header; a personal access token typically spans staging/prod/feature). Reads use the ambient key; the eval still writes its own traces/datasets to its own workspace, so **nothing is ever written to the source workspace**. The resolved workspace is logged (`[Prod/instance-ai]`). + +`seedThread.project` overrides the source project name (default `instance-ai`); the same name is searched in every workspace, so if prod and staging share it you need nothing. ⚠️ Reconstructing a prod conversation pulls its content (incl. any PII) into the staging eval instance, the model, the staging traces and local report artifacts — handle per the source-side redaction decision and team data policy before using real customer threads. + +> **Transient.** LangSmith base-tier traces retain ~14 days, so a `seedThread` case is runnable only while its trace lives. Keep these out of CI datasets (tag them `["seeded"]`, not `full`/`pr`) until durable seed snapshots land; the resolver fails loudly when a trace has aged out. Durable snapshotting (e.g. materialising the reconstructed seed into a private LangSmith dataset on first resolve) is a planned follow-up. + +To find the thread id, open the conversation's trace in LangSmith (or read it from the instance the conversation happened on) and copy its `thread_id`. + +#### `priorConversation` — prose prelude + +```json +"priorConversation": [ + { "role": "user", "text": "We agreed: digests go to #growth, daily at 9am." }, + { "role": "assistant", "text": "Noted — #growth, daily at 9am." } +] +``` + +Paired with a normal `conversation` for the live turn. Plain text only — no tool calls, no restored workflows. + +#### `seedFile` — durable synthetic fixture + +For a **synthetic, sanitized** fixture you want pinned in git (never a real user's conversation): hand-author a `data/workflows/seeds/.seed.json` (schema in `harness/conversation-seed.ts` — `messages` + optional `workflows`) and point `seedFile` at it. Real conversations belong in `seedThread`, which keeps their content out of the repo entirely. Paired with a normal `conversation` for the live turn. + +#### How restore works (all paths) + +At build time the seed is restored right after the credential pin: seeded workflows are recreated under **fresh ids** (every reference in the history is remapped, so parallel iterations never share a workflow row) with node credentials stripped, and the message log is written verbatim. Restore failures fail the build — a seeded case cannot meaningfully run unseeded. Seeded turns join the transcript marked as *seeded prior context*, visible to the expectations judge and prompt-aware checks but distinguishable from live behaviour. + +Rules of thumb: + +- **A seeded case is only worth shipping with `buildExpectations` that detect the misbehaviour recurring** — without them it passes vacuously. Sanity-check by running the case once with the seed removed: it should fail. +- `seedThread`, `priorConversation` and `seedFile` are mutually exclusive; all order strictly before the live turn. `seedThread` provides its own live turn (omit `conversation`); the other two pair with `conversation`. + ## Failure categories When a scenario fails, the verifier categorizes the root cause: diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts new file mode 100644 index 00000000000..da25947efed --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts @@ -0,0 +1,294 @@ +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + loadConversationSeed, + remapSeedWorkflowIds, + seedFromProse, + transcriptPrefixFromSeed, + type ConversationSeed, +} from '../harness/conversation-seed'; + +const WF_ID = 'AbCdEf1234567890'; + +function makeSeed(): ConversationSeed { + return { + source: { kind: 'thread-export', threadId: 'thread-1' }, + messages: [ + { + id: 'msg-user', + type: 'llm', + role: 'user', + content: [{ type: 'text', text: 'Send a daily digest to #cosmic-otter-alerts' }], + createdAt: '2026-01-01T00:00:00.000Z', + }, + { + id: 'msg-assistant', + type: 'llm', + role: 'assistant', + content: [ + { type: 'text', text: 'Built it.' }, + { + type: 'tool-call', + toolCallId: 'tc-1', + toolName: 'build-workflow', + state: 'resolved', + input: { workflowId: WF_ID }, + output: { success: true, workflowId: WF_ID, url: `/workflow/${WF_ID}` }, + }, + ], + createdAt: '2026-01-01T00:00:01.000Z', + }, + ], + workflows: [{ id: WF_ID, name: 'Daily digest', nodes: [], connections: {} }], + }; +} + +describe('loadConversationSeed', () => { + const dir = mkdtempSync(join(tmpdir(), 'conversation-seed-')); + + it('loads and validates a seed file', () => { + const path = join(dir, 'valid.seed.json'); + writeFileSync(path, JSON.stringify(makeSeed())); + + const seed = loadConversationSeed(path); + expect(seed.messages).toHaveLength(2); + expect(seed.workflows[0].id).toBe(WF_ID); + }); + + it('defaults workflows to an empty array', () => { + const path = join(dir, 'no-workflows.seed.json'); + writeFileSync(path, JSON.stringify({ messages: [{ id: 'm1' }] })); + + expect(loadConversationSeed(path).workflows).toEqual([]); + }); + + it('rejects a seed without messages, naming the file', () => { + const path = join(dir, 'empty.seed.json'); + writeFileSync(path, JSON.stringify({ messages: [] })); + + expect(() => loadConversationSeed(path)).toThrow(/Invalid conversation seed .*empty\.seed/); + }); + + it('rejects a missing file with a readable error', () => { + expect(() => loadConversationSeed(join(dir, 'nope.seed.json'))).toThrow( + /Failed to read conversation seed/, + ); + }); +}); + +describe('seedFromProse', () => { + it('converts turns to llm text messages with ascending past timestamps', () => { + const seed = seedFromProse([ + { role: 'user', text: 'Digest to #cosmic-otter-alerts please' }, + { role: 'assistant', text: 'Done — daily at 9am.' }, + ]); + + expect(seed.workflows).toEqual([]); + expect(seed.messages).toHaveLength(2); + const [first, second] = seed.messages; + expect(first).toMatchObject({ + type: 'llm', + role: 'user', + content: [{ type: 'text', text: 'Digest to #cosmic-otter-alerts please' }], + }); + expect(first.id).not.toBe(second.id); + + const t0 = new Date(String(first.createdAt)).getTime(); + const t1 = new Date(String(second.createdAt)).getTime(); + expect(t1).toBeGreaterThan(t0); + expect(t1).toBeLessThan(Date.now()); + }); +}); + +describe('remapSeedWorkflowIds', () => { + it('rewrites the workflow id and every reference to it across the seed', () => { + const remapped = remapSeedWorkflowIds(makeSeed()); + + const newId = remapped.workflows[0].id; + expect(newId).not.toBe(WF_ID); + expect(newId).toMatch(/^[0-9A-Za-z]{16}$/); + + const serialized = JSON.stringify(remapped); + expect(serialized).not.toContain(WF_ID); + // Tool-call input, output and canvas URL all moved to the fresh id. + expect(serialized).toContain(`/workflow/${newId}`); + }); + + it('returns the seed untouched when there are no workflows', () => { + const seed = seedFromProse([{ role: 'user', text: 'hi' }]); + expect(remapSeedWorkflowIds(seed)).toBe(seed); + }); + + it('generates distinct ids per call so parallel iterations never collide', () => { + const a = remapSeedWorkflowIds(makeSeed()).workflows[0].id; + const b = remapSeedWorkflowIds(makeSeed()).workflows[0].id; + expect(a).not.toBe(b); + }); + + it('refuses to remap a dangerously short workflow id', () => { + const seed = makeSeed(); + seed.workflows[0].id = 'abc'; + expect(() => remapSeedWorkflowIds(seed)).toThrow(/too short to remap/); + }); +}); + +describe('transcriptPrefixFromSeed', () => { + it('renders user text, assistant narration and tool calls as seeded turns', () => { + const turns = transcriptPrefixFromSeed(makeSeed().messages); + + expect(turns).toHaveLength(1); + expect(turns[0].seeded).toBe(true); + expect(turns[0].userMessage).toBe('Send a daily digest to #cosmic-otter-alerts'); + expect(turns[0].steps).toEqual([ + { kind: 'agent-text', text: 'Built it.' }, + { + kind: 'tool-call', + toolName: 'build-workflow', + args: { workflowId: WF_ID }, + result: { success: true, workflowId: WF_ID, url: `/workflow/${WF_ID}` }, + }, + ]); + }); + + it('renders a seeded ask-user block as an ask-user step with the chosen answers', () => { + const turns = transcriptPrefixFromSeed([ + { + id: 'a1', + type: 'llm', + role: 'assistant', + content: [ + { + type: 'tool-call', + toolName: 'ask-user', + state: 'resolved', + input: { + introMessage: 'A couple of questions', + questions: [{ id: 'q1', question: 'Which channel?', options: ['#growth', '#ops'] }], + }, + // Resume block carries the user's answers in its output. + output: { + answered: true, + answers: [{ questionId: 'q1', selectedOptions: ['#growth'], skipped: false }], + }, + }, + ], + createdAt: '2026-01-01T00:00:00Z', + }, + ]); + expect(turns[0].steps).toEqual([ + { + kind: 'ask-user', + questions: [{ id: 'q1', question: 'Which channel?', options: ['#growth', '#ops'] }], + answers: [ + { questionId: 'q1', selectedOptions: ['#growth'], customText: undefined, skipped: false }, + ], + }, + ]); + }); + + it('renders a seeded setup-card block as a setup-card step from output.payload.setupRequests', () => { + const turns = transcriptPrefixFromSeed([ + { + id: 'a1', + type: 'llm', + role: 'assistant', + content: [ + { + type: 'tool-call', + toolName: 'workflows[setup]', + state: 'resolved', + input: { action: 'setup', workflowId: 'wf1' }, + output: { + payload: { + requestId: 'req1', + setupRequests: [{ node: { name: 'Slack' }, credentialType: 'slackApi' }], + }, + }, + }, + ], + createdAt: '2026-01-01T00:00:00Z', + }, + ]); + expect(turns[0].steps).toEqual([ + { + kind: 'setup-card', + requests: [{ nodeName: 'Slack', credentialType: 'slackApi', params: undefined }], + outcome: 'pending', + }, + ]); + }); + + it('renders a seeded setup outcome (completedNodes/skippedNodes) as a setup-wizard step', () => { + const turns = transcriptPrefixFromSeed([ + { + id: 'a1', + type: 'llm', + role: 'assistant', + content: [ + { + type: 'tool-call', + toolName: 'workflows[setup]', + state: 'resolved', + input: { action: 'setup', workflowId: 'wf1' }, + output: { + success: true, + completedNodes: [{ nodeName: 'Schedule', parametersSet: ['rule'] }], + skippedNodes: [{ nodeName: 'Slack', credentialType: 'slackApi' }], + }, + }, + ], + createdAt: '2026-01-01T00:00:00Z', + }, + ]); + expect(turns[0].steps).toEqual([ + { + kind: 'setup-wizard', + completedNodes: [{ nodeName: 'Schedule', parametersSet: ['rule'] }], + skippedNodes: [{ nodeName: 'Slack', credentialType: 'slackApi' }], + reason: undefined, + }, + ]); + }); + + it('renders a seeded create-tasks block as a plan step', () => { + const turns = transcriptPrefixFromSeed([ + { + id: 'a1', + type: 'llm', + role: 'assistant', + content: [ + { + type: 'tool-call', + toolName: 'create-tasks', + state: 'resolved', + input: { tasks: [{ title: 'Add trigger', description: 'schedule' }] }, + output: {}, + }, + ], + createdAt: '2026-01-01T00:00:00Z', + }, + ]); + expect(turns[0].steps).toEqual([ + { kind: 'plan', tasks: [{ title: 'Add trigger', description: 'schedule' }] }, + ]); + }); + + it('skips custom messages and tolerates a history starting with an assistant turn', () => { + const turns = transcriptPrefixFromSeed([ + { id: 'c1', type: 'custom', data: { widget: 'card' }, createdAt: '2026-01-01T00:00:00Z' }, + { + id: 'a1', + type: 'llm', + role: 'assistant', + content: [{ type: 'text', text: 'Picking up where we left off.' }], + createdAt: '2026-01-01T00:00:01Z', + }, + ]); + + expect(turns).toHaveLength(1); + expect(turns[0].userMessage).toBeUndefined(); + expect(turns[0].steps).toEqual([{ kind: 'agent-text', text: 'Picking up where we left off.' }]); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts index b604099d342..7607c710b82 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts @@ -56,6 +56,62 @@ describe('WorkflowTestCaseSchema', () => { ).toThrow(); }); + it('accepts a prose priorConversation prelude', () => { + const parsed = WorkflowTestCaseSchema.parse({ + ...validFixture(), + priorConversation: [{ role: 'user', text: 'We already agreed on #cosmic-otter-alerts' }], + }); + expect(parsed.priorConversation).toHaveLength(1); + }); + + it('rejects seedFile combined with priorConversation', () => { + expect(() => + WorkflowTestCaseSchema.parse({ + ...validFixture(), + seedFile: 'seeds/some-thread.seed.json', + priorConversation: [{ role: 'user', text: 'prelude' }], + }), + ).toThrow(/mutually exclusive/); + }); + + it('accepts a seedThread case with no conversation (live turn from the trace)', () => { + const { conversation: _omit, ...rest } = validFixture(); + const parsed = WorkflowTestCaseSchema.parse({ + ...rest, + seedThread: { threadId: 'example-thread-id' }, + }); + expect(parsed.seedThread?.threadId).toBe('example-thread-id'); + expect(parsed.conversation).toBeUndefined(); + }); + + it('accepts seedThread WITH a conversation (continuation after the live turn)', () => { + const parsed = WorkflowTestCaseSchema.parse({ + ...validFixture(), + seedThread: { threadId: 't1' }, + conversation: [{ role: 'user', text: 'now also add error handling' }], + }); + expect(parsed.seedThread?.threadId).toBe('t1'); + expect(parsed.conversation).toHaveLength(1); + }); + + it('rejects seedThread combined with another seeding mode', () => { + const { conversation: _omit, ...rest } = validFixture(); + expect(() => + WorkflowTestCaseSchema.parse({ + ...rest, + seedThread: { threadId: 't1' }, + seedFile: 'seeds/x.seed.json', + }), + ).toThrow(/mutually exclusive/); + }); + + it('rejects a non-seedThread case that omits conversation', () => { + const { conversation: _omit, ...rest } = validFixture(); + expect(() => WorkflowTestCaseSchema.parse(rest)).toThrow( + /needs a conversation, or a seedThread/, + ); + }); + it('accepts the optional triggerType field', () => { const parsed = WorkflowTestCaseSchema.parse({ ...validFixture(), triggerType: 'webhook' }); expect(parsed.triggerType).toBe('webhook'); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts new file mode 100644 index 00000000000..b9ad900ddf1 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts @@ -0,0 +1,440 @@ +import type { Client } from 'langsmith'; +import { vi } from 'vitest'; + +// Stub the SDK parser so the reconstructor test doesn't depend on the real +// workflow-builder; we assert the build run is selected + compiled, not parsing. +vi.mock('../harness/parse-seed-workflow', () => ({ + parseSeedWorkflowCode: (code: string) => ({ + workflow: { + name: 'Compiled WF', + nodes: [{ id: 'n1', name: 'Schedule', type: 'n8n-nodes-base.scheduleTrigger', __code: code }], + connections: {}, + }, + }), +})); + +import { reconstructSeedFromThread } from '../harness/langsmith-seed'; + +// Discovery test doubles return fixed, already-resolved workspace lists. +/* eslint-disable @typescript-eslint/promise-function-async */ + +interface FakeRun { + id: string; + run_type: 'chain' | 'tool'; + name: string; + start_time: string; + parent_run_id?: string; + trace_id?: string; + inputs?: Record; + outputs?: Record; + extra?: { metadata?: Record }; +} + +/** Minimal stand-in for the LangSmith Client: yields a fixed run list. */ +function fakeClient(runs: FakeRun[]) { + return { + // `for await` accepts a sync iterable, so a plain generator is enough. + *listRuns(): Generator { + for (const run of runs) yield run; + }, + } as unknown as Client; +} + +const t = (s: number) => `2026-06-12T08:00:${String(s).padStart(2, '0')}.000Z`; + +function turn(id: string, sec: number, message: string): FakeRun { + return { id, run_type: 'chain', name: 'turn', start_time: t(sec), inputs: { message } }; +} + +describe('reconstructSeedFromThread', () => { + it('splits at the last user turn: seed = before, liveTurn = last', async () => { + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build Otter Digest, daily 9am'), outputs: { response: 'Built it.' } }, + turn('r2', 30, 'Change the schedule to every 30 minutes instead.'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + + expect(result.liveTurn).toBe('Change the schedule to every 30 minutes instead.'); + // Seed holds turn 1's user message + assistant response; the live turn is excluded. + const roles = result.seed.messages.map((m) => m.role); + expect(roles).toEqual(['user', 'assistant']); + expect(result.seed.messages[0].content).toEqual([ + { type: 'text', text: 'Build Otter Digest, daily 9am' }, + ]); + }); + + it('rebuilds resolved tool-call blocks and compiles the seed workflow at the boundary', async () => { + const buildTool: FakeRun = { + id: 'tool1', + run_type: 'tool', + name: 'build-workflow', + start_time: t(5), + inputs: { code: 'workflow().addNode(...)' }, + outputs: { success: true, workflowId: 'WF-ORIGINAL-123', workflowName: 'Otter Digest' }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Done.' } }, + buildTool, + turn('r2', 30, 'Now change the schedule'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + + const assistant = result.seed.messages.find((m) => m.role === 'assistant')!; + const toolBlock = (assistant.content as Array>).find( + (b) => b.type === 'tool-call', + ); + expect(toolBlock).toMatchObject({ toolName: 'build-workflow', state: 'resolved' }); + + expect(result.seed.workflows).toHaveLength(1); + expect(result.seed.workflows[0]).toMatchObject({ id: 'WF-ORIGINAL-123', name: 'Otter Digest' }); + }); + + it('reconstructs data tables (schema only — never rows) created before the boundary', async () => { + const createTable: FakeRun = { + id: 'dt-create', + run_type: 'tool', + name: 'data-tables[create]', + start_time: t(4), + inputs: { action: 'create' }, + outputs: { + table: { + id: 's8srkfMDKYIAjEHR', + name: 'Size Up Coffee FAQs', + columns: [ + { id: 'c1', name: 'keywords', type: 'string' }, + { id: 'c2', name: 'is_active', type: 'boolean' }, + ], + }, + }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + // An insert-rows run carrying real (PII) row content must be ignored — we + // reconstruct the schema only, never the rows. + const insertRows: FakeRun = { + id: 'dt-insert', + run_type: 'tool', + name: 'data-tables[insert-rows]', + start_time: t(5), + inputs: { + action: 'insert-rows', + dataTableId: 's8srkfMDKYIAjEHR', + rows: [{ keywords: 'price', is_active: true }], + }, + outputs: { insertedCount: 1 }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build a FAQ bot'), outputs: { response: 'Done.' } }, + createTable, + insertRows, + turn('r2', 30, 'Now change something'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + + expect(result.seed.dataTables).toEqual([ + { + id: 's8srkfMDKYIAjEHR', + name: 'Size Up Coffee FAQs', + columns: [ + { name: 'keywords', type: 'string' }, + { name: 'is_active', type: 'boolean' }, + ], + }, + ]); + + // The insert-rows tool-call survives in the seeded message history, but its + // row values must be redacted there too — the messages are written to the + // eval instance and shown to the judge. + const assistant = result.seed.messages.find((m) => m.role === 'assistant')!; + const insertBlock = (assistant.content as Array>).find( + (b) => b.toolName === 'data-tables[insert-rows]', + )!; + expect((insertBlock.input as Record).rows).toBe('<1 row(s) omitted>'); + // Non-row fields are preserved. + expect((insertBlock.input as Record).dataTableId).toBe('s8srkfMDKYIAjEHR'); + }); + + it('takes the latest successful build per workflow id before the boundary', async () => { + const earlyBuild: FakeRun = { + id: 'tool1', + run_type: 'tool', + name: 'build-workflow', + start_time: t(5), + inputs: { code: 'v1' }, + outputs: { success: true, workflowId: 'WF1' }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const lateBuild: FakeRun = { + id: 'tool2', + run_type: 'tool', + name: 'patch-workflow', + start_time: t(8), + inputs: { code: 'v2' }, + outputs: { success: true, workflowId: 'WF1' }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const runs: FakeRun[] = [ + turn('r1', 1, 'Build'), + earlyBuild, + lateBuild, + turn('r2', 30, 'Change'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + + expect(result.seed.workflows).toHaveLength(1); + // v2 (the later patch) wins — its code reaches the compiled node. + expect(result.seed.workflows[0].nodes[0]).toMatchObject({ __code: 'v2' }); + }); + + it('ignores builds at or after the boundary (the dropped live response)', async () => { + const postBoundaryBuild: FakeRun = { + id: 'tool9', + run_type: 'tool', + name: 'build-workflow', + start_time: t(31), + inputs: { code: 'post' }, + outputs: { success: true, workflowId: 'WF-LIVE' }, + extra: { metadata: { langsmith_root_run_id: 'r2' } }, + }; + const runs: FakeRun[] = [turn('r1', 1, 'Build'), turn('r2', 30, 'Change'), postBoundaryBuild]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + expect(result.seed.workflows).toHaveLength(0); + }); + + it('throws when the trace has fewer than two user turns', async () => { + const runs: FakeRun[] = [turn('r1', 1, 'Only one user turn')]; + await expect(reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs))).rejects.toThrow( + /need ≥2 to seed/, + ); + }); + + it('throws a retention-aware error when no runs are found', async () => { + await expect(reconstructSeedFromThread({ threadId: 'gone' }, fakeClient([]))).rejects.toThrow( + /aged out/, + ); + }); + + it('throws when a renamed build tool produced a workflow we did not recognize', async () => { + // SDK code in + workflowId out + success = unmistakably a build, but the + // tool name isn't in the known set → drift detector fires. + const renamedBuild: FakeRun = { + id: 'tool1', + run_type: 'tool', + name: 'compose-workflow', // not in WORKFLOW_BUILD_TOOLS + start_time: t(5), + inputs: { code: 'workflow()...' }, + outputs: { success: true, workflowId: 'WF1' }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Done.' } }, + renamedBuild, + turn('r2', 30, 'Change'), + ]; + await expect(reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs))).rejects.toThrow( + /likely renamed/, + ); + }); + + it('does not false-positive on a read-only tool that returns a workflowId', async () => { + // get-workflow returns a workflowId but takes no `code` — not build-like, + // so the drift detector stays quiet and the seed simply has no workflow. + const readOnly: FakeRun = { + id: 'tool1', + run_type: 'tool', + name: 'get-workflow', + start_time: t(5), + inputs: { workflowId: 'WF1' }, + outputs: { success: true, workflowId: 'WF1' }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Look at it'), outputs: { response: 'Here.' } }, + readOnly, + turn('r2', 30, 'Change'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + expect(result.seed.workflows).toHaveLength(0); + }); + + it('skips internal resume turns when finding the split point', async () => { + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Done.' } }, + turn('r2', 20, ''), // internal, not a user turn + turn('r3', 30, 'Real follow-up'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + expect(result.liveTurn).toBe('Real follow-up'); + }); + + it('collapses an ask-user suspend+resume pair to one block carrying the answer', async () => { + const questions = [{ id: 'q1', question: 'Which channel?', options: ['#a', '#b'] }]; + const suspend: FakeRun = { + id: 'tool-suspend', + run_type: 'tool', + name: 'ask-user', + start_time: t(3), + inputs: { questions }, + // Suspend: a re-statement of the pending request, no answer. + outputs: { payload: { inputType: 'questions', requestId: 'req1', questions } }, + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const resume: FakeRun = { + id: 'tool-resume', + run_type: 'tool', + name: 'ask-user', + start_time: t(6), + inputs: { questions }, + outputs: { answered: true, answers: [{ questionId: 'q1', selectedOptions: ['#a'] }] }, + extra: { metadata: { langsmith_root_run_id: 'r1', pending_tool_call_id: 'toolu_x' } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Asking…' } }, + suspend, + resume, + turn('r2', 30, 'live turn'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + + const askBlocks = result.seed.messages + .filter((m) => Array.isArray(m.content)) + .flatMap((m) => m.content as Array>) + .filter((b) => b.type === 'tool-call' && b.toolName === 'ask-user'); + expect(askBlocks).toHaveLength(1); // suspend dropped, resume kept — no duplication + expect(askBlocks[0]).toMatchObject({ + output: { answers: [{ questionId: 'q1', selectedOptions: ['#a'] }] }, + }); + }); + + it('collapses a setup-card suspend+resume pair to one block', async () => { + const card = { requestId: 'req1', setupRequests: [{ node: { name: 'Slack' } }] }; + const suspend: FakeRun = { + id: 'tool-setup-suspend', + run_type: 'tool', + name: 'workflows[setup]', + start_time: t(3), + inputs: { action: 'setup', workflowId: 'wf1' }, + outputs: { payload: card }, // HITL request envelope, no pending id → suspend half + extra: { metadata: { langsmith_root_run_id: 'r1' } }, + }; + const resume: FakeRun = { + id: 'tool-setup-resume', + run_type: 'tool', + name: 'workflows[setup]', + start_time: t(6), + inputs: { action: 'setup', workflowId: 'wf1' }, + outputs: { payload: card }, + extra: { metadata: { langsmith_root_run_id: 'r1', pending_tool_call_id: 'toolu_s' } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Setting up…' } }, + suspend, + resume, + turn('r2', 30, 'live turn'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + const setupBlocks = result.seed.messages + .filter((m) => Array.isArray(m.content)) + .flatMap((m) => m.content as Array>) + .filter((b) => b.type === 'tool-call' && b.toolName === 'workflows[setup]'); + expect(setupBlocks).toHaveLength(1); // suspend dropped, resume kept + }); + + it('keeps the answer when a suspend shares the toolCallId and comes after the resume', async () => { + const questions = [{ id: 'q1', question: 'Which channel?', options: ['#a', '#b'] }]; + const tcid = 'toolu_shared'; + const resume: FakeRun = { + id: 'tool-resume', + run_type: 'tool', + name: 'ask-user', + start_time: t(4), + inputs: { questions }, + outputs: { answered: true, answers: [{ questionId: 'q1', selectedOptions: ['#a'] }] }, + extra: { metadata: { langsmith_root_run_id: 'r1', pending_tool_call_id: tcid } }, + }; + // A later suspend re-statement with the SAME toolCallId must not overwrite + // the resume's answer in the resolved-output map. + const lateSuspend: FakeRun = { + id: 'tool-suspend-late', + run_type: 'tool', + name: 'ask-user', + start_time: t(7), + inputs: { questions }, + outputs: { payload: { inputType: 'questions', requestId: 'req1', questions } }, + extra: { metadata: { langsmith_root_run_id: 'r1', pending_tool_call_id: tcid } }, + }; + const runs: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Asking…' } }, + resume, + lateSuspend, + turn('r2', 30, 'live turn'), + ]; + const result = await reconstructSeedFromThread({ threadId: 'th1' }, fakeClient(runs)); + const askBlocks = result.seed.messages + .filter((m) => Array.isArray(m.content)) + .flatMap((m) => m.content as Array>) + .filter((b) => b.type === 'tool-call' && b.toolName === 'ask-user'); + expect(askBlocks).toHaveLength(1); + expect(askBlocks[0]).toMatchObject({ + output: { answers: [{ questionId: 'q1', selectedOptions: ['#a'] }] }, + }); + }); +}); + +describe('reconstructSeedFromThread — workspace auto-discovery', () => { + const seedableRuns: FakeRun[] = [ + { ...turn('r1', 1, 'Build it'), outputs: { response: 'Built.' } }, + turn('r2', 30, 'Change the schedule'), + ]; + + const twoWorkspaces = [ + { id: 'staging-id', name: 'Staging' }, + { id: 'prod-id', name: 'Prod' }, + ]; + + it('finds the thread in whichever workspace holds it and tags the source', async () => { + const result = await reconstructSeedFromThread({ threadId: 'th1' }, undefined, { + listWorkspaces: () => Promise.resolve(twoWorkspaces), + // Only Prod has the thread; Staging is empty. + clientForWorkspace: (id: string) => fakeClient(id === 'prod-id' ? seedableRuns : []), + ambientClient: () => fakeClient([]), + }); + expect(result.sourceWorkspace).toBe('Prod'); + expect(result.liveTurn).toBe('Change the schedule'); + }); + + it('falls back to the ambient client when no workspaces can be listed', async () => { + const result = await reconstructSeedFromThread({ threadId: 'th1' }, undefined, { + listWorkspaces: () => Promise.resolve([]), + clientForWorkspace: () => fakeClient([]), + ambientClient: () => fakeClient(seedableRuns), + }); + expect(result.liveTurn).toBe('Change the schedule'); + expect(result.sourceWorkspace).toBeUndefined(); + }); + + it('throws listing the workspaces tried when the thread is in none', async () => { + await expect( + reconstructSeedFromThread({ threadId: 'gone' }, undefined, { + listWorkspaces: () => Promise.resolve(twoWorkspaces), + clientForWorkspace: () => fakeClient([]), + ambientClient: () => fakeClient([]), + }), + ).rejects.toThrow(/not found in project .* across 2 workspace\(s\): Staging, Prod/); + }); + + it('propagates a found-but-not-seedable error instead of trying the next workspace', async () => { + const oneTurn: FakeRun[] = [turn('r1', 1, 'Only one user turn')]; + await expect( + reconstructSeedFromThread({ threadId: 'th1' }, undefined, { + // Staging HAS the thread but it isn't seedable (<2 turns) — must not + // be masked by trying Prod. + listWorkspaces: () => Promise.resolve(twoWorkspaces), + clientForWorkspace: (id: string) => + fakeClient(id === 'staging-id' ? oneTurn : seedableRuns), + ambientClient: () => fakeClient([]), + }), + ).rejects.toThrow(/need ≥2 to seed/); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/workflow-report.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/workflow-report.test.ts index 9012c649de5..850d76a6d35 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/workflow-report.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/workflow-report.test.ts @@ -132,4 +132,27 @@ describe('transcript rendering', () => { const html = generateWorkflowReport([result]); expect(html).toContain('workflow-builder'); }); + + it('surfaces a skipped ask-user answer so it is not mistaken for unanswered', () => { + const result: WorkflowTestCaseResult = { + testCase: TEST_CASE, + workflowBuildSuccess: true, + executionScenarioResults: [], + transcript: [ + { + userMessage: 'Build it', + steps: [ + { + kind: 'ask-user', + questions: [{ id: 'q1', question: 'Which channel?', options: ['#a', '#b'] }], + answers: [{ questionId: 'q1', selectedOptions: [], skipped: true }], + }, + ], + }, + ], + }; + const html = generateWorkflowReport([result]); + expect(html).toContain('👤 (skipped)'); + expect(html).toContain('ask-user (with answers)'); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index 81ecd2d0a47..0dc613c24d4 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -74,6 +74,7 @@ import type { WorkflowTestCase, WorkflowTestCaseResult, } from '../types'; +import { caseDisplayPrompt } from '../utils/conversation-text'; // n8n degrades above ~4 concurrent builds. const MAX_CONCURRENT_BUILDS = 4; @@ -279,7 +280,15 @@ async function runWithLangSmith(config: RunConfig): Promise<{ // inflightKeys) plus the lane's traced LangSmith wrappers. `runner` is // the underlying Lane (n8n client, credential state) — named distinctly so // it doesn't shadow the iteration variable `lane` in lanes.map(). - type BuildArgs = Pick; + type BuildArgs = Pick< + WorkflowTestCase, + | 'conversation' + | 'messageBudget' + | 'credentials' + | 'seedFile' + | 'priorConversation' + | 'seedThread' + >; interface LaneState { runner: Lane; activeBuilds: number; @@ -307,6 +316,9 @@ async function runWithLangSmith(config: RunConfig): Promise<{ conversation: buildArgs.conversation, messageBudget: buildArgs.messageBudget, credentials: buildArgs.credentials, + seedFile: buildArgs.seedFile, + priorConversation: buildArgs.priorConversation, + seedThread: buildArgs.seedThread, createdCredentialIds: lane.createdCredentialIds, timeoutMs: args.timeoutMs, preRunWorkflowIds: lane.preRunWorkflowIds, @@ -407,6 +419,9 @@ async function runWithLangSmith(config: RunConfig): Promise<{ conversation: entry.conversation, messageBudget: entry.messageBudget, credentials: entry.credentials, + seedFile: entry.seedFile, + priorConversation: entry.priorConversation, + seedThread: entry.seedThread, }); const buildDurationMs = Date.now() - start; buildDurations.set(key, buildDurationMs); @@ -478,7 +493,9 @@ async function runWithLangSmith(config: RunConfig): Promise<{ passed: false, score: 0, reasoning: `Build failed: ${build.error ?? 'unknown'}`, - failureCategory: 'build_failure', + // Seeding failures are a harness setup problem, not an agent build + // failure — keep them out of the agent's build_failure bucket. + failureCategory: build.seedingFailed ? 'framework_issue' : 'build_failure', execErrors: build.error ? [build.error] : [], buildDurationMs, execDurationMs: 0, @@ -975,6 +992,9 @@ function flattenRunsForReport(evaluation: MultiRunEvaluation): WorkflowTestCaseR } return evaluation.testCases.flatMap((tc) => tc.runs.map((run, iter) => { + // seedThread cases carry no authored conversation (the live turn comes + // from the trace) — nothing to relabel. + if (!run.testCase.conversation?.length) return run; const [opening, ...rest] = run.testCase.conversation; return { ...run, @@ -1117,7 +1137,7 @@ function writeEvalResults( comparisonStatus: outcome?.kind ?? 'not_attempted', comparisonError: outcome?.kind === 'fetch_failed' ? outcome.error : undefined, testCases: testCases.map((tc) => ({ - name: tc.testCase.conversation[0].text.slice(0, 70), + name: caseDisplayPrompt(tc.testCase, tc.runs[0]?.transcript).slice(0, 70), testCaseFile: slugByTestCase?.get(tc.testCase), buildSuccessCount: tc.buildSuccessCount, totalRuns, @@ -1261,7 +1281,7 @@ function bucketFromEvaluation( const fileSlug = slugByTestCase.get(tc.testCase); if (!fileSlug) { throw new Error( - `bucketFromEvaluation: no fileSlug for test case "${tc.testCase.conversation[0].text.slice(0, 60)}"`, + `bucketFromEvaluation: no fileSlug for test case "${caseDisplayPrompt(tc.testCase, tc.runs[0]?.transcript).slice(0, 60)}"`, ); } const total = tc.runs.length; diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 72471af6154..160a03fdcec 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -13,9 +13,23 @@ import type { InstanceAiRunDebugResponse, InstanceAiThreadDebugRunsResponse, InstanceAiThreadStatusResponse, + InstanceAiEvalSeedDataTable, + InstanceAiEvalSeedWorkflow, } from '@n8n/api-types'; import { z } from 'zod'; +// -- Conversation seeding response shapes ------------------------------------- + +const RestoreThreadEnvelope = z.object({ + data: z.object({ + ok: z.literal(true), + threadId: z.string(), + restored: z.number(), + workflowIds: z.array(z.string()), + dataTableIds: z.array(z.string()).default([]), + }), +}); + // --------------------------------------------------------------------------- // Computer-use gateway response shapes (Zod-validated to keep the client // honest about API drift instead of trusting `as` casts) @@ -490,6 +504,26 @@ export class N8nClient { }); } + /** + * Seed an existing thread with a previously exported conversation: the + * referenced workflows are recreated (node credentials stripped server-side) + * and the native message log is written verbatim, so the thread continues + * as if the conversation really happened. + * POST /rest/instance-ai/eval/restore-thread + */ + async restoreThread( + threadId: string, + messages: Array>, + workflows: InstanceAiEvalSeedWorkflow[], + dataTables: InstanceAiEvalSeedDataTable[] = [], + ): Promise<{ restored: number; workflowIds: string[]; dataTableIds: string[] }> { + const result = await this.fetch('/rest/instance-ai/eval/restore-thread', { + method: 'POST', + body: { threadId, messages, workflows, dataTables }, + }); + return RestoreThreadEnvelope.parse(result).data; + } + // -- Data tables --------------------------------------------------------- /** diff --git a/packages/@n8n/instance-ai/evaluations/comparison/format.ts b/packages/@n8n/instance-ai/evaluations/comparison/format.ts index 46be12b9a85..4241a649855 100644 --- a/packages/@n8n/instance-ai/evaluations/comparison/format.ts +++ b/packages/@n8n/instance-ai/evaluations/comparison/format.ts @@ -34,6 +34,7 @@ import type { WorkflowTestCase, WorkflowTestCaseResult, } from '../types'; +import { caseDisplayPrompt } from '../utils/conversation-text'; interface FormatOptions { /** Optional commit SHA for the terminal heading. Truncated to 8 chars. */ @@ -446,7 +447,7 @@ function renderPerTestCaseDetails( lines.push(''); const renderName = (tc: TestCaseAggregation): string => { const slug = slugByTestCase?.get(tc.testCase); - return slug ? `\`${slug}\`` : `\`${tc.testCase.conversation[0].text.slice(0, 70)}\``; + return slug ? `\`${slug}\`` : `\`${caseDisplayPrompt(tc.testCase).slice(0, 70)}\``; }; if (totalRuns > 1) { lines.push(`| Workflow | Built | pass@${totalRuns} | pass^${totalRuns} |`); @@ -568,7 +569,7 @@ function renderFailureDetails( for (const { tc, fileSlug, scenarioName, failedRuns } of failed) { const slug = fileSlug ? `${fileSlug}/${scenarioName}` - : `${tc.testCase.conversation[0].text.slice(0, 50).trim()} / ${scenarioName}`; + : `${caseDisplayPrompt(tc.testCase).slice(0, 50).trim()} / ${scenarioName}`; lines.push(`**\`${slug}\`** — ${failedRuns.length} failed`); for (const fr of failedRuns) { const tag = fr.category ? ` [${fr.category}]` : ''; @@ -878,7 +879,7 @@ function formatTerminalPerTestCase( const nameOf = (tc: TestCaseAggregation, max: number): string => { const slug = slugByTestCase?.get(tc.testCase); - return slug ?? tc.testCase.conversation[0].text.slice(0, max); + return slug ?? caseDisplayPrompt(tc.testCase).slice(0, max); }; if (totalRuns > 1) { diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts b/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts index 2fc5ffe3e10..fb2c83ee4b6 100644 --- a/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/index.ts @@ -1,7 +1,8 @@ import { readFileSync, readdirSync } from 'fs'; -import { basename, join } from 'path'; +import { basename, dirname, join, resolve } from 'path'; import { WorkflowTestCaseSchema } from './schema'; +import { loadConversationSeed } from '../../harness/conversation-seed'; import type { WorkflowTestCase } from '../../types'; export interface WorkflowTestCaseWithFile { @@ -30,7 +31,21 @@ function parseTestCaseFile(filePath: string): WorkflowTestCase { throw new Error(`Invalid test case ${filePath}:\n${issues}`); } - return parsed.data as WorkflowTestCase; + const testCase = parsed.data as WorkflowTestCase; + if (testCase.seedFile) { + // Resolve relative to the case file and validate now, so an authoring + // typo fails at load time instead of per-build as an agent failure. + const resolved = resolve(dirname(filePath), testCase.seedFile); + try { + loadConversationSeed(resolved); + } catch (error) { + throw new Error( + `Invalid test case ${filePath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + testCase.seedFile = resolved; + } + return testCase; } /** Split a comma-separated CLI value into a normalized list of substring tokens. */ diff --git a/packages/@n8n/instance-ai/evaluations/data/workflows/schema.ts b/packages/@n8n/instance-ai/evaluations/data/workflows/schema.ts index 6254c847ce2..d10e722fc27 100644 --- a/packages/@n8n/instance-ai/evaluations/data/workflows/schema.ts +++ b/packages/@n8n/instance-ai/evaluations/data/workflows/schema.ts @@ -20,10 +20,12 @@ const ExecutionScenarioSchema = z.object({ requires: z.string().optional(), }); -export const WorkflowTestCaseSchema = z.object({ +const workflowTestCaseObjectSchema = z.object({ /** Optional human-readable note on what this case is testing (esp. for behaviour cases). */ description: z.string().optional(), - conversation: z.array(ConversationTurnSchema).min(1), + // Optional only because `seedThread` derives the live turn from the trace; + // a refine() below requires it for every other case. + conversation: z.array(ConversationTurnSchema).min(1).optional(), complexity: z.enum(['simple', 'medium', 'complex']), tags: z.array(z.string()), triggerType: z.enum(['manual', 'webhook', 'schedule', 'form']).optional(), @@ -51,6 +53,18 @@ export const WorkflowTestCaseSchema = z.object({ }), ) .optional(), + /** Synthetic seed file (relative path), resolved + validated at case load. + * Synthetic fixtures only; real conversations use `seedThread`. */ + seedFile: z.string().min(1).optional(), + /** Prose turns seeded as plain-text history (no tool calls / workflows). */ + priorConversation: z.array(ConversationTurnSchema).min(1).optional(), + /** Reproduce a real conversation from its LangSmith trace at run time (seed = + * before the last user message, live = that message). Commits only the thread + * id; workspace auto-discovered. Supplies the live turn, so `conversation` is + * optional (continues after it). */ + seedThread: z + .object({ threadId: z.string().min(1), project: z.string().min(1).optional() }) + .optional(), /** * Logical groupings this case belongs to (e.g. `['pr', 'full']`). Used by * the eval CLI's `--tier` flag and propagated to LangSmith as example @@ -60,4 +74,15 @@ export const WorkflowTestCaseSchema = z.object({ datasets: z.array(z.string()).min(1).default(['full']), }); +// At most one seeding mode, and a source for the live turn. +export const WorkflowTestCaseSchema = workflowTestCaseObjectSchema + .refine((c) => [c.seedFile, c.priorConversation, c.seedThread].filter(Boolean).length <= 1, { + message: + 'seedFile, priorConversation and seedThread are mutually exclusive — pick one seeding mode', + }) + .refine((c) => c.seedThread !== undefined || c.conversation !== undefined, { + message: + 'a case needs a conversation, or a seedThread (which supplies the live turn from the trace)', + }); + export type WorkflowTestCaseInput = z.infer; diff --git a/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts b/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts new file mode 100644 index 00000000000..a7c7862d19c --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts @@ -0,0 +1,247 @@ +// Conversation seeding for eval builds — backs the `seedFile` (synthetic) and +// `priorConversation` (prose) paths. Real conversations use `seedThread` +// (reconstructed from a LangSmith trace; see langsmith-seed.ts). + +import { generateNanoId, isRecord } from '@n8n/utils'; +import { jsonParse } from 'n8n-workflow'; +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { z } from 'zod'; + +import { + extractAskUserAnswers, + extractAskUserQuestions, + extractPlanTasks, + extractSetupCardRequests, + extractSetupWizardOutcome, +} from '../outcome/transcript-from-events'; +import type { ConversationTurn, TranscriptStep, TranscriptTurn } from '../types'; + +// --------------------------------------------------------------------------- +// Seed file schema +// --------------------------------------------------------------------------- + +const SeedWorkflowSchema = z.object({ + id: z.string().min(1), + name: z.string().min(1), + nodes: z.array(z.record(z.unknown())), + connections: z.record(z.unknown()), +}); + +const SeedDataTableSchema = z.object({ + /** The table's id as it appears in the trace — the value baked into the + * seed workflow's data-table node. Rewritten to the recreated table's id on + * restore (the server generates a fresh id; it can't be pinned). */ + id: z.string().min(1), + name: z.string().min(1), + columns: z.array( + z.object({ name: z.string().min(1), type: z.enum(['string', 'number', 'boolean', 'date']) }), + ), + // Schema only — rows are intentionally not seeded (the table exists empty, + // which is all the workflow node needs). Real rows are the highest-PII part + // of a trace and are kept out of the eval instance entirely. +}); + +export const ConversationSeedSchema = z.object({ + /** Provenance (thread id, instance, export time) — informational only. */ + source: z.record(z.unknown()).optional(), + /** Native agent message log (user/assistant turns with resolved tool-call blocks). */ + messages: z.array(z.record(z.unknown())).min(1), + /** Workflows the history references, recreated on restore. */ + workflows: z.array(SeedWorkflowSchema).default([]), + /** Data tables the history references, recreated (and id-rewritten) on restore. */ + dataTables: z.array(SeedDataTableSchema).default([]), +}); + +export type ConversationSeed = z.infer; + +export function loadConversationSeed(filePath: string): ConversationSeed { + let raw: unknown; + try { + raw = JSON.parse(readFileSync(filePath, 'utf-8')); + } catch (error) { + throw new Error( + `Failed to read conversation seed ${filePath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const parsed = ConversationSeedSchema.safeParse(raw); + if (!parsed.success) { + const issues = parsed.error.issues + .map((i) => ` - ${i.path.join('.') || '(root)'}: ${i.message}`) + .join('\n'); + throw new Error(`Invalid conversation seed ${filePath}:\n${issues}`); + } + return parsed.data; +} + +// --------------------------------------------------------------------------- +// Prose prior turns → seed messages +// --------------------------------------------------------------------------- + +/** Convert authored prose turns into native llm messages, stamped slightly in + * the past (ascending) so they order before the live turn. */ +export function seedFromProse(turns: ConversationTurn[]): ConversationSeed { + const base = Date.now() - (turns.length + 1) * 1000; + return { + messages: turns.map((turn, index) => ({ + id: randomUUID(), + type: 'llm', + role: turn.role, + content: [{ type: 'text', text: turn.text }], + createdAt: new Date(base + index * 1000).toISOString(), + })), + workflows: [], + dataTables: [], + }; +} + +// --------------------------------------------------------------------------- +// Workflow id remapping +// --------------------------------------------------------------------------- + +/** Give every seeded workflow a fresh id, rewriting all references across the + * seed — so parallel iterations don't share (and clobber) one workflow row. */ +export function remapSeedWorkflowIds(seed: ConversationSeed): ConversationSeed { + if (seed.workflows.length === 0) return seed; + + const originalIds = new Set(seed.workflows.map((workflow) => workflow.id)); + let serialized = JSON.stringify({ messages: seed.messages, workflows: seed.workflows }); + for (const workflow of seed.workflows) { + // Workflow ids are long random tokens; a short id would risk rewriting + // unrelated substrings, so refuse instead of corrupting the seed. + if (workflow.id.length < 8) { + throw new Error( + `Seed workflow id "${workflow.id}" is too short to remap safely (need ≥8 chars)`, + ); + } + // Keep the fresh id disjoint from every original id so this sequential + // replace can't rewrite a not-yet-processed workflow's id. + let newId = generateNanoId(); + while (originalIds.has(newId)) newId = generateNanoId(); + serialized = serialized.replaceAll(workflow.id, newId); + } + + const remapped = ConversationSeedSchema.parse(jsonParse(serialized)); + // Data table ids are remapped server-side on restore (id is generated, not + // pinnable), so carry them through untouched here. + return { ...remapped, source: seed.source, dataTables: seed.dataTables }; +} + +// Transcript prefix — seeded history rendered for the judge/checks. Turns carry +// `seeded: true` so consumers can tell restored context from evaluated behaviour. + +function textOf(blocks: unknown[]): string { + return blocks + .flatMap((block) => + isRecord(block) && block.type === 'text' && typeof block.text === 'string' + ? [block.text] + : [], + ) + .join('\n'); +} + +export function transcriptPrefixFromSeed( + messages: Array>, +): TranscriptTurn[] { + const turns: TranscriptTurn[] = []; + const lastTurn = () => turns[turns.length - 1]; + + for (const message of messages) { + if (message.type === 'custom' || !Array.isArray(message.content)) continue; + + if (message.role === 'user') { + turns.push({ userMessage: textOf(message.content), steps: [], seeded: true }); + continue; + } + if (message.role !== 'assistant') continue; + + if (turns.length === 0) turns.push({ steps: [], seeded: true }); + const steps: TranscriptStep[] = lastTurn().steps; + const text = textOf(message.content); + if (text) steps.push({ kind: 'agent-text', text }); + for (const block of message.content) { + if (!isRecord(block) || block.type !== 'tool-call') continue; + steps.push(toTranscriptStep(block)); + } + } + + return turns; +} + +/** A seeded tool-call block, normalized for the interpreters below. */ +interface SeedToolCall { + toolName: string; + input?: Record; + output?: Record; +} + +/** Maps a seeded tool-call to its special transcript step (ask-user/plan/setup), + * mirroring the live SSE transcript. Returns null to fall through. */ +type SeedStepInterpreter = (call: SeedToolCall) => TranscriptStep | null; + +const interpretAskUser: SeedStepInterpreter = (call) => { + const questions = call.input?.questions; + if (call.toolName !== 'ask-user' || !Array.isArray(questions)) return null; + const parsed = extractAskUserQuestions(questions); + if (parsed.length === 0) return null; + // The kept (resume) block carries the user's answers in its output. + const answers = Array.isArray(call.output?.answers) + ? extractAskUserAnswers(call.output.answers) + : undefined; + return { kind: 'ask-user', questions: parsed, answers }; +}; + +const interpretPlan: SeedStepInterpreter = (call) => { + const tasks = call.input?.tasks; + if (call.toolName !== 'create-tasks' || !Array.isArray(tasks)) return null; + const parsed = extractPlanTasks(tasks); + return parsed.length > 0 ? { kind: 'plan', tasks: parsed } : null; +}; + +// The applied setup outcome: which nodes were configured / skipped (same +// rendering as the live `workflows` result). +const interpretSetupWizard: SeedStepInterpreter = (call) => { + const { output } = call; + if (!output || !(Array.isArray(output.completedNodes) || Array.isArray(output.skippedNodes))) { + return null; + } + return extractSetupWizardOutcome(output); +}; + +// The setup-card prompt: its asks live in output.payload.setupRequests. The fill +// outcome isn't in the trace (only SSE proxy responses capture it), so it +// renders as 'pending'. +const interpretSetupCard: SeedStepInterpreter = (call) => { + const payload = isRecord(call.output?.payload) ? call.output.payload : undefined; + const setupRequests = payload?.setupRequests; + if (!Array.isArray(setupRequests)) return null; + const requests = extractSetupCardRequests(setupRequests); + return requests.length > 0 ? { kind: 'setup-card', requests, outcome: 'pending' } : null; +}; + +const SEED_STEP_INTERPRETERS: SeedStepInterpreter[] = [ + interpretAskUser, + interpretPlan, + interpretSetupWizard, + interpretSetupCard, +]; + +/** Map a seeded tool-call block to a transcript step (special interpreters above, + * else a generic tool-call). */ +function toTranscriptStep(block: Record): TranscriptStep { + const call: SeedToolCall = { + toolName: typeof block.toolName === 'string' ? block.toolName : 'unknown-tool', + input: isRecord(block.input) ? block.input : undefined, + output: isRecord(block.output) ? block.output : undefined, + }; + for (const interpret of SEED_STEP_INTERPRETERS) { + const step = interpret(call); + if (step) return step; + } + return { + kind: 'tool-call', + toolName: call.toolName, + args: call.input, + result: 'output' in block ? block.output : undefined, + }; +} diff --git a/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts b/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts new file mode 100644 index 00000000000..8095171489b --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts @@ -0,0 +1,422 @@ +// Reconstruct a conversation seed from a LangSmith trace at run time: pull the +// thread's runs, rebuild the message log, split at the last user turn (before = +// seed, last = live), and compile the seed workflow from the build tool's +// captured SDK code at the boundary. Transient: traces retain ~14 days. + +import { isRecord } from '@n8n/utils'; +import { Client } from 'langsmith'; +import type { Run } from 'langsmith/schemas'; + +import type { ConversationSeed } from './conversation-seed'; +import { parseSeedWorkflowCode } from './parse-seed-workflow'; + +/** Default project that instance-ai conversations are traced to (same name in + * every workspace). Override per case with `seedThread.project` if it differs. */ +const DEFAULT_SOURCE_PROJECT = 'instance-ai'; + +const WORKFLOW_BUILD_TOOLS = new Set(['build-workflow', 'patch-workflow', 'submit-workflow']); + +// The source thread may live in a different workspace than the eval writes to +// (e.g. seed from prod, trace to staging). A PAT spans workspaces, so we +// enumerate them and find the one holding the thread; reads are read-only. + +/** Ambient LangSmith host + key (the eval's own), used to enumerate workspaces. */ +function ambientLangSmithConfig(): { apiUrl: string; apiKey: string } { + const raw = + process.env.LANGSMITH_ENDPOINT ?? + process.env.LANGCHAIN_ENDPOINT ?? + 'https://api.smith.langchain.com'; + const host = raw.replace(/\/$/, ''); + const apiUrl = host.endsWith('/api/v1') ? host : `${host}/api/v1`; + const apiKey = process.env.LANGSMITH_API_KEY ?? process.env.LANGCHAIN_API_KEY ?? ''; + return { apiUrl, apiKey }; +} + +/** Workspaces the ambient key can access. Returns [] on any failure so the + * caller falls back to the key's default workspace. */ +async function listAccessibleWorkspaces(): Promise> { + const { apiUrl, apiKey } = ambientLangSmithConfig(); + if (!apiKey) return []; + try { + const res = await fetch(`${apiUrl}/workspaces`, { headers: { 'x-api-key': apiKey } }); + if (!res.ok) return []; + const data: unknown = await res.json(); + if (!Array.isArray(data)) return []; + return data.flatMap((entry) => { + if (typeof entry !== 'object' || entry === null) return []; + const record = entry as Record; + const id = asString(record.id); + const name = asString(record.display_name) ?? asString(record.name) ?? id; + return id ? [{ id, name: name ?? id }] : []; + }); + } catch { + return []; + } +} + +/** Seams for unit-testing workspace discovery without the network. */ +export interface SeedDiscoveryDeps { + listWorkspaces: () => Promise>; + clientForWorkspace: (workspaceId: string) => Client; + ambientClient: () => Client; +} + +const realDiscoveryDeps: SeedDiscoveryDeps = { + listWorkspaces: listAccessibleWorkspaces, + clientForWorkspace: (workspaceId) => new Client({ workspaceId }), + ambientClient: () => new Client(), +}; + +/** Thrown when a thread has no runs in the queried (workspace, project) — used + * as control flow during discovery to advance to the next workspace. */ +class ThreadNotInWorkspaceError extends Error {} + +export interface SeedThreadRef { + threadId: string; + /** Override the LangSmith project to read the source trace from. */ + project?: string; +} + +export interface ReconstructedSeed { + seed: ConversationSeed; + /** The thread's last genuine user message — sent live, not seeded. */ + liveTurn: string; + /** Provenance, for logging. */ + runCount: number; + sourceProject: string; + /** Workspace the thread was found in (when auto-discovered). */ + sourceWorkspace?: string; +} + +function metadata(run: Run): Record { + return ((run.extra ?? {}) as { metadata?: Record }).metadata ?? {}; +} + +function asString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +/** A genuine user turn: a 'turn' root with free-text `inputs.message`. System + * control inputs (e.g. ``) are angle-bracket tags — + * matched as a leading ``, not any `<`, so real messages like "<3" stay. */ +function userMessageOf(run: Run): string | undefined { + const message = asString(run.inputs?.message); + if (run.name !== 'turn' || !message || /^<[a-z][\w-]*>/i.test(message)) return undefined; + return message; +} + +/** LangSmith redacts the `credentials:` key in captured SDK code; restore the + * opener so the code parses (secret values were never captured anyway). */ +function unredactCode(code: string): string { + return code.replace(/\[REDACTED\]/g, 'credentials: {'); +} + +/** A HITL suspend artifact (the pending request, no result) — dropped in favour + * of its resume run, which holds both request and answer. */ +function isSuspendArtifact(output: unknown): boolean { + if (!isRecord(output)) return false; + if (output.deferred === true) return true; + return isRecord(output.payload) && typeof output.payload.inputType === 'string'; +} + +/** A HITL request envelope: `{ payload: { requestId, … } }` — emitted by both the + * suspend and resume halves of ask-user / setup-card. Used (with the absence of + * a pending id) to identify the suspend half to drop. */ +function isHitlRequestEnvelope(output: unknown): boolean { + return ( + isRecord(output) && isRecord(output.payload) && typeof output.payload.requestId === 'string' + ); +} + +/** Top-level fields that carry data-table row values across the data-table + * tools (insert/upsert/update take `rows`; reads return `data`). */ +const DATA_TABLE_ROW_FIELDS = new Set(['rows', 'data']); + +/** Replace data-table row arrays with an omission marker, keeping the rest of + * the payload — so seeded history carries no real (PII) row values. */ +function redactDataTableRowPayload(value: unknown): Record { + if (!isRecord(value)) return {}; + const out: Record = {}; + for (const [key, val] of Object.entries(value)) { + out[key] = + DATA_TABLE_ROW_FIELDS.has(key) && Array.isArray(val) ? `<${val.length} row(s) omitted>` : val; + } + return out; +} + +interface TextBlock { + type: 'text'; + text: string; +} +interface ToolCallBlock { + type: 'tool-call'; + toolCallId: string; + toolName: string; + state: 'resolved'; + input: unknown; + output: unknown; +} + +/** + * Reconstruct a thread's seed + live turn. The workspace holding the thread is + * auto-discovered (the user only supplies a thread id); pass an explicit + * `client` to bypass discovery (tests). + * + * Throws when the thread isn't found in any accessible workspace (trace aged out + * or wrong project), or when it has fewer than two user turns. + */ +export async function reconstructSeedFromThread( + ref: SeedThreadRef, + client?: Client, + deps: SeedDiscoveryDeps = realDiscoveryDeps, +): Promise { + const project = ref.project ?? DEFAULT_SOURCE_PROJECT; + // An explicit client (tests) reads that one source; otherwise auto-discover. + if (client) return await reconstructWithClient(ref, client, project); + return await discoverAndReconstruct(ref, project, deps); +} + +/** Find the workspace holding the thread and reconstruct from it. Falls back to + * the key's default workspace when workspaces can't be enumerated. */ +async function discoverAndReconstruct( + ref: SeedThreadRef, + project: string, + deps: SeedDiscoveryDeps, +): Promise { + const workspaces = await deps.listWorkspaces(); + if (workspaces.length === 0) { + return await reconstructWithClient(ref, deps.ambientClient(), project); + } + const tried: string[] = []; + for (const workspace of workspaces) { + tried.push(workspace.name); + try { + const result = await reconstructWithClient( + ref, + deps.clientForWorkspace(workspace.id), + project, + ); + return { ...result, sourceWorkspace: workspace.name }; + } catch (error) { + // Not in this workspace → try the next; anything else (found but not + // seedable, drift) is a real problem and propagates. + if (error instanceof ThreadNotInWorkspaceError) continue; + throw error; + } + } + throw new Error( + `Thread ${ref.threadId} not found in project "${project}" across ${String(workspaces.length)} workspace(s): ${tried.join(', ')}. The trace may have aged out (~14-day base retention), or the project name differs (set seedThread.project).`, + ); +} + +/** Pull a thread's runs from one client/project and rebuild the seed + live turn. */ +async function reconstructWithClient( + ref: SeedThreadRef, + client: Client, + sourceProject: string, +): Promise { + const runs: Run[] = []; + for await (const run of client.listRuns({ + projectName: sourceProject, + filter: `and(eq(metadata_key, "thread_id"), eq(metadata_value, "${ref.threadId}"))`, + })) { + runs.push(run); + } + if (runs.length === 0) { + // Recognised by discovery to advance to the next workspace; the message + // still reads well if it surfaces directly (explicit-client path). + throw new ThreadNotInWorkspaceError( + `No runs for thread ${ref.threadId} in LangSmith project "${sourceProject}" — the trace may have aged out (~14-day base retention) or the project name is wrong.`, + ); + } + + const byStartTime = (a: Run, b: Run) => + new Date(a.start_time).getTime() - new Date(b.start_time).getTime(); + const rootRuns = runs.filter((r) => r.run_type === 'chain' && !r.parent_run_id).sort(byStartTime); + const toolRuns = runs.filter((r) => r.run_type === 'tool').sort(byStartTime); + + // Split point: the last genuine user turn is sent live; everything strictly + // before it is the seed. + const userTurns = rootRuns.filter((r) => userMessageOf(r) !== undefined); + if (userTurns.length < 2) { + throw new Error( + `Thread ${ref.threadId} has ${userTurns.length} user turn(s) — need ≥2 to seed (one prior + one live). Use a plain conversation case instead.`, + ); + } + const liveTurnRun = userTurns[userTurns.length - 1]; + const boundaryMs = new Date(liveTurnRun.start_time).getTime(); + const liveTurn = userMessageOf(liveTurnRun)!; + + const messages = buildSeedMessages(rootRuns, toolRuns, boundaryMs); + if (messages.length === 0) { + throw new Error( + `Thread ${ref.threadId} reconstructed to zero seed messages before the live turn — the trace shape may have drifted (expected root runs named 'turn' with inputs.message / outputs.response).`, + ); + } + const workflows = buildSeedWorkflows(toolRuns, boundaryMs, ref.threadId); + const dataTables = buildSeedDataTables(toolRuns, boundaryMs); + + return { + seed: { + source: { kind: 'langsmith', threadId: ref.threadId, sourceProject }, + messages, + workflows, + dataTables, + }, + liveTurn, + runCount: runs.length, + sourceProject, + }; +} + +/** Rebuild the native message log for every run before the seed boundary. */ +function buildSeedMessages( + rootRuns: Run[], + toolRuns: Run[], + boundaryMs: number, +): Array> { + const toolsByRoot = new Map(); + for (const tool of toolRuns) { + const rootId = asString(metadata(tool).langsmith_root_run_id) ?? tool.trace_id ?? ''; + const list = toolsByRoot.get(rootId) ?? []; + list.push(tool); + toolsByRoot.set(rootId, list); + } + + const emittedToolCallIds = new Set(); + const messages: Array> = []; + + for (const root of rootRuns) { + if (new Date(root.start_time).getTime() >= boundaryMs) break; + + const userText = userMessageOf(root); + if (userText) { + messages.push({ + id: `${root.id}-user`, + role: 'user', + type: 'llm', + createdAt: new Date(root.start_time).toISOString(), + content: [{ type: 'text', text: userText }], + }); + } + + const content: Array = []; + const responseText = asString(root.outputs?.response); + if (responseText) content.push({ type: 'text', text: responseText }); + + for (const tool of toolsByRoot.get(root.id) ?? []) { + // HITL tools split into a suspend run + a later resume run (no shared id): + // drop the suspend, keep the resume which holds request + answer. + if (isSuspendArtifact(tool.outputs)) continue; + const pendingId = asString(metadata(tool).pending_tool_call_id); + // Setup-card suspend half (request envelope, no pending id) — its resume is kept. + if (!pendingId && isHitlRequestEnvelope(tool.outputs)) continue; + const toolCallId = pendingId ?? tool.id; + if (emittedToolCallIds.has(toolCallId)) continue; + emittedToolCallIds.add(toolCallId); + // Redact data-table row payloads: seeded messages are written to the eval + // instance + shown to the judge, so real (PII) rows must not ride along. + const isDataTable = tool.name.startsWith('data-tables'); + content.push({ + type: 'tool-call', + toolCallId, + toolName: tool.name, + state: 'resolved', + input: isDataTable ? redactDataTableRowPayload(tool.inputs) : (tool.inputs ?? {}), + output: isDataTable ? redactDataTableRowPayload(tool.outputs) : (tool.outputs ?? {}), + }); + } + + if (content.length > 0) { + messages.push({ + id: `${root.id}-assistant`, + role: 'assistant', + type: 'llm', + // +1ms so the assistant reply orders after its user turn. + createdAt: new Date(new Date(root.start_time).getTime() + 1).toISOString(), + content, + }); + } + } + + return messages; +} + +/** Compile the seed's workflows from the build tool's captured SDK code — latest + * successful build per workflow id before the boundary. */ +function buildSeedWorkflows( + toolRuns: Run[], + boundaryMs: number, + threadId: string, +): ConversationSeed['workflows'] { + const latestBuildByWorkflowId = new Map(); + // Name-independent build signal (code in + workflowId out): detects a renamed + // build tool instead of silently dropping the workflow. + let sawBuildLikeRun = false; + for (const tool of toolRuns) { + if (new Date(tool.start_time).getTime() >= boundaryMs) continue; + const out = (tool.outputs ?? {}) as Record; + const workflowId = asString(out.workflowId); + const buildLike = out.success === true && !!workflowId && !!asString(tool.inputs?.code); + if (buildLike) sawBuildLikeRun = true; + if (!WORKFLOW_BUILD_TOOLS.has(tool.name)) continue; + if (!buildLike || !workflowId) continue; + // Sorted ascending, so a later run overwrites — last successful build wins. + latestBuildByWorkflowId.set(workflowId, tool); + } + + if (latestBuildByWorkflowId.size === 0 && sawBuildLikeRun) { + throw new Error( + `Thread ${threadId}: a tool run built a workflow (SDK code in, workflowId out) but its name isn't in the known build set {${[...WORKFLOW_BUILD_TOOLS].join(', ')}} — the build tool was likely renamed; update WORKFLOW_BUILD_TOOLS.`, + ); + } + + const workflows: ConversationSeed['workflows'] = []; + for (const [workflowId, build] of latestBuildByWorkflowId) { + const code = unredactCode(asString(build.inputs?.code) ?? ''); + const parsed = parseSeedWorkflowCode(code); + const out = (build.outputs ?? {}) as Record; + workflows.push({ + id: workflowId, + name: asString(out.workflowName) ?? parsed.workflow.name ?? 'workflow', + nodes: (parsed.workflow.nodes ?? []) as Array>, + connections: (parsed.workflow.connections ?? {}) as Record, + }); + } + + return workflows; +} + +type DataTableColumnType = 'string' | 'number' | 'boolean' | 'date'; +const DATA_TABLE_COLUMN_TYPES = new Set(['string', 'number', 'boolean', 'date']); +function isDataTableColumnType(value: string): value is DataTableColumnType { + return DATA_TABLE_COLUMN_TYPES.has(value); +} + +/** Reconstruct the seed's data tables, schema only — enough for a restored + * workflow's data-table node to resolve. Rows are deliberately not pulled + * (highest-PII payload). Detected by shape (`create` returns `table.id` + + * `table.columns`), so a renamed tool still reconstructs. */ +function buildSeedDataTables(toolRuns: Run[], boundaryMs: number): ConversationSeed['dataTables'] { + const created = new Map(); + + for (const tool of toolRuns) { + if (new Date(tool.start_time).getTime() >= boundaryMs) continue; + const out = isRecord(tool.outputs) ? tool.outputs : {}; + + // A `create`: output carries the new table's id, name and columns. + const table = isRecord(out.table) ? out.table : undefined; + const tableId = table ? asString(table.id) : undefined; + if (!table || !tableId || !Array.isArray(table.columns)) continue; + const columns = table.columns.flatMap((col) => { + if (!isRecord(col)) return []; + const name = asString(col.name); + const type = asString(col.type); + if (!name || !type || !isDataTableColumnType(type)) return []; + return [{ name, type }]; + }); + created.set(tableId, { id: tableId, name: asString(table.name) ?? 'data table', columns }); + } + + return [...created.values()]; +} diff --git a/packages/@n8n/instance-ai/evaluations/harness/parse-seed-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/parse-seed-workflow.ts new file mode 100644 index 00000000000..30240941159 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/harness/parse-seed-workflow.ts @@ -0,0 +1,24 @@ +// Compile build-tool SDK code into workflow JSON for a seed. Uses +// @n8n/workflow-sdk directly (resolves to built dist) rather than +// @n8n/ai-workflow-builder's handler, whose `@/`-aliased source doesn't resolve +// under the eval harness's tsx runtime. Seeding only needs the JSON. + +import { parseWorkflowCodeToBuilder } from '@n8n/workflow-sdk'; +import type { WorkflowJSON } from '@n8n/workflow-sdk'; + +/** Matches any import statement — SDK functions are globals when parsed. */ +const IMPORT_REGEX = /^\s*import\s+(?:[\s\S]*?from\s+)?['"]([^'"]+)['"];?\s*$/gm; + +function stripImportStatements(code: string): string { + return code + .replace(IMPORT_REGEX, '') + .replace(/^\s*\n/, '') + .trim(); +} + +export function parseSeedWorkflowCode(code: string): { workflow: WorkflowJSON } { + const builder = parseWorkflowCodeToBuilder(stripImportStatements(code)); + // Deterministic node ids across re-parses, matching how the builder persists. + builder.regenerateNodeIds(); + return { workflow: builder.toJSON({ tidyUp: true }) }; +} diff --git a/packages/@n8n/instance-ai/evaluations/harness/runner.ts b/packages/@n8n/instance-ai/evaluations/harness/runner.ts index 9a9f47f1f0f..8f15a53dffe 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/runner.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/runner.ts @@ -21,6 +21,14 @@ import { recordUserTurn, type ConfirmationStrategy, } from './chat-loop'; +import { + loadConversationSeed, + remapSeedWorkflowIds, + seedFromProse, + transcriptPrefixFromSeed, + type ConversationSeed, +} from './conversation-seed'; +import { reconstructSeedFromThread, type SeedThreadRef } from './langsmith-seed'; import { type EvalLogger } from './logger'; import { fetchPrebuiltBuild } from './prebuilt-workflows'; import { buildWorkflowContextBlock } from './workflow-context'; @@ -89,7 +97,7 @@ function slugifyArtifactSegment(value: string, fallback: string): string { } function deriveTestCaseArtifactName(testCase: WorkflowTestCase): string { - return slugifyArtifactSegment(testCase.conversation[0]?.text ?? '', 'workflow'); + return slugifyArtifactSegment(testCase.conversation?.[0]?.text ?? '', 'workflow'); } async function writeScenarioVerificationSnapshot(input: { @@ -189,6 +197,9 @@ export async function runWorkflowTestCase( conversation: testCase.conversation, messageBudget: testCase.messageBudget, credentials: testCase.credentials, + seedFile: testCase.seedFile, + priorConversation: testCase.priorConversation, + seedThread: testCase.seedThread, createdCredentialIds: config.createdCredentialIds, timeoutMs, preRunWorkflowIds: config.preRunWorkflowIds, @@ -385,23 +396,30 @@ export interface BuildResult { workflowChecks?: CheckOutcome[]; /** False when the backend lacks the credential-pin endpoint and the build ran unpinned. */ credentialViewPinned?: boolean; + /** True when the build failed while setting up the conversation seed (trace + * gone, reconstruction drift, restore failed) — a harness/framework problem, + * not an agent build failure. Routed to `framework_issue`. */ + seedingFailed?: boolean; } export interface BuildWorkflowConfig { client: N8nClient; - /** - * Hand-authored conversation. ≥1 turn, first turn must be `user`. - * - * - One user turn, no assistant turns → auto-approve all confirmations. - * - Anything else → UserProxyLlm engages. - */ - conversation: ConversationTurn[]; + /** Hand-authored conversation (≥1 turn, first `user`; one user turn → + * auto-approve, more → proxy). Optional when `seedThread` derives the live turn. */ + conversation?: ConversationTurn[]; /** Max follow-up messages the proxy will send. Ignored in auto-approve mode. */ messageBudget?: number; /** Credentials this build should see (created for real, view pinned to them). */ credentials?: TestCaseCredential[]; /** Run-level registry the created credential IDs are added to for cleanup. */ createdCredentialIds?: Set; + /** Synthetic seed file (path) restored before the live message. */ + seedFile?: string; + /** Prose turns seeded as plain-text history. */ + priorConversation?: ConversationTurn[]; + /** Reproduce a real conversation from its LangSmith trace (seed = before the + * last user message, live = that message). */ + seedThread?: SeedThreadRef; timeoutMs?: number; preRunWorkflowIds: Set; claimedWorkflowIds: Set; @@ -431,8 +449,7 @@ function isMultiTurnConversation(conversation: ConversationTurn[]): boolean { * executeScenario(). Call cleanupBuild() when done. */ export async function buildWorkflow(config: BuildWorkflowConfig): Promise { - const { client, conversation, logger } = config; - const openingMessage = conversation[0]?.text ?? ''; + const { client, logger } = config; const threadId = crypto.randomUUID(); const startTime = Date.now(); const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; @@ -443,9 +460,51 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise(); const followUpMessages: string[] = []; let credentialViewPinned = true; + let restoredWorkflowIds: string[] = []; + let restoredDataTableIds: string[] = []; + let seededTranscript: TranscriptTurn[] = []; + let seedingFailed = false; try { const buildStart = Date.now(); + + // `seedThread` derives both seed and live turn from a trace; otherwise the + // seed (if any) is a file/prose prelude and the conversation is authored. + let seed: ConversationSeed | undefined; + let conversation = config.conversation ?? []; + try { + if (config.seedThread) { + const reconstructed = await reconstructSeedFromThread(config.seedThread); + seed = reconstructed.seed; + // The trace's last user message is the live opening; any authored + // `conversation` continues from there (proxy-driven follow-ups). + conversation = [ + { role: 'user', text: reconstructed.liveTurn }, + ...(config.conversation ?? []), + ]; + const contSuffix = + (config.conversation?.length ?? 0) > 0 + ? ` + ${String(config.conversation!.length)} continuation turn(s)` + : ''; + const wsLabel = reconstructed.sourceWorkspace + ? `${reconstructed.sourceWorkspace}/${reconstructed.sourceProject}` + : reconstructed.sourceProject; + logger.info( + ` Reconstructed seed from thread ${config.seedThread.threadId}: ${String(reconstructed.runCount)} runs → ${String(seed.messages.length)} message(s), ${String(seed.workflows.length)} workflow(s)${contSuffix} [${wsLabel}]${config.laneTag ?? ''}`, + ); + } else if (config.seedFile) { + seed = loadConversationSeed(config.seedFile); + } else if (config.priorConversation && config.priorConversation.length > 0) { + seed = seedFromProse(config.priorConversation); + } + } catch (error: unknown) { + // A seed that can't be resolved is a harness/framework problem, not an + // agent build failure — tag it and fail before spending a live turn. + seedingFailed = true; + throw new Error(`Seeding failed: ${error instanceof Error ? error.message : String(error)}`); + } + + const openingMessage = conversation[0]?.text ?? ''; const isMultiTurn = isMultiTurnConversation(conversation); logger.info( ` Building workflow${isMultiTurn ? ' [multi-turn]' : ''}: "${truncate(openingMessage, 60)}"${config.laneTag ?? ''}`, @@ -480,6 +539,35 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise 0 + ? `, ${String(restoredDataTableIds.length)} data table(s)` + : ''; + logger.info( + ` Seeded ${String(restoreResult.restored)} prior message(s), ${String(restoredWorkflowIds.length)} workflow(s)${dtSuffix}${config.laneTag ?? ''}`, + ); + } catch (error: unknown) { + seedingFailed = true; + throw new Error( + `Seeding failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + const ssePromise = startSseConnection(client, threadId, events, abortController.signal).catch( () => {}, ); @@ -520,12 +608,15 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise {}); const conversationMetrics = buildConversationMetrics(events); - const transcript = buildTranscriptFromEvents({ - events, - openingMessage, - followUpMessages, - proxyResponses, - }); + const transcript = [ + ...seededTranscript, + ...buildTranscriptFromEvents({ + events, + openingMessage, + followUpMessages, + proxyResponses, + }), + ]; let threadMessages; try { @@ -536,7 +627,11 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise wf.id), - createdDataTableIds: outcome.dataTablesCreated, + createdDataTableIds: [...outcome.dataTablesCreated, ...restoredDataTableIds], conversationMetrics, events, threadId, @@ -638,12 +734,13 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise): ToolInteraction | null { +export function extractSetupWizardOutcome(result: Record): ToolInteraction | null { const completed = Array.isArray(result.completedNodes) ? extractCompletedNodes(result.completedNodes) : []; @@ -291,7 +291,7 @@ function interpretSetupCard( return { kind: 'setup-card', requests, outcome, filled }; } -function extractSetupCardRequests(raw: unknown[]): SetupCardRequest[] { +export function extractSetupCardRequests(raw: unknown[]): SetupCardRequest[] { const requests: SetupCardRequest[] = []; for (const item of raw) { if (!isRecord(item)) continue; @@ -398,7 +398,7 @@ function inferFeedback(response: InstanceAiConfirmRequest | undefined): string | return undefined; } -function extractPlanTasks(raw: unknown[]): PlanTask[] { +export function extractPlanTasks(raw: unknown[]): PlanTask[] { const tasks: PlanTask[] = []; for (const item of raw) { if (!isRecord(item)) continue; @@ -409,7 +409,7 @@ function extractPlanTasks(raw: unknown[]): PlanTask[] { return tasks; } -function extractAskUserQuestions(raw: unknown[]): AskUserQuestion[] { +export function extractAskUserQuestions(raw: unknown[]): AskUserQuestion[] { const questions: AskUserQuestion[] = []; for (const item of raw) { if (!isRecord(item)) continue; @@ -423,7 +423,7 @@ function extractAskUserQuestions(raw: unknown[]): AskUserQuestion[] { return questions; } -function extractAskUserAnswers(raw: unknown): AskUserAnswer[] { +export function extractAskUserAnswers(raw: unknown): AskUserAnswer[] { if (!Array.isArray(raw)) return []; const answers: AskUserAnswer[] = []; for (const item of raw) { diff --git a/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts b/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts index 88f4c942853..e886aa7eb35 100644 --- a/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts +++ b/packages/@n8n/instance-ai/evaluations/report/workflow-report.ts @@ -23,6 +23,7 @@ import type { TurnCounter, WorkflowTestCaseResult, } from '../types'; +import { caseDisplayPrompt } from '../utils/conversation-text'; // --------------------------------------------------------------------------- // Helpers @@ -287,7 +288,7 @@ function renderStageReview(stage: StageReview): string { } function firstPromptText(result: WorkflowTestCaseResult): string { - return result.testCase.conversation[0]?.text ?? ''; + return caseDisplayPrompt(result.testCase, result.transcript); } function promptReview(result: WorkflowTestCaseResult, sr: ExecutionScenarioResult): StageReview { @@ -816,7 +817,9 @@ function renderInteraction(interaction: ToolInteraction): string | null { const answerByQId = new Map(); for (const a of interaction.answers ?? []) { const selected = a.selectedOptions.join(', '); - const text = [selected, a.customText].filter(Boolean).join(' — '); + // A skipped answer is a real response — surface it so it's not mistaken + // for an unanswered question (mirrors the judge-text transcript). + const text = a.skipped ? '(skipped)' : [selected, a.customText].filter(Boolean).join(' — '); if (text) answerByQId.set(a.questionId, text); } const lines = interaction.questions @@ -1167,7 +1170,7 @@ function renderTestCase(result: WorkflowTestCaseResult, tcIndex: number): string ? `${String(passCount)}/${String(totalCount)}` : ''; - const prompt = result.testCase.conversation[0].text; + const prompt = caseDisplayPrompt(result.testCase, result.transcript); const truncatedPrompt = prompt.length > 100 ? prompt.slice(0, 100) + '...' : prompt; // Header label = the source-file slug (the same identifier the PR comment // uses), falling back to the description then the prompt. The full prompt diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 72d84bc228a..8a457cd9d40 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -185,14 +185,12 @@ export interface WorkflowTestCase { /** Optional human-readable note on what this case is testing (esp. for behaviour cases). */ description?: string; /** - * Hand-authored conversation that drives the build. Must have ≥1 turn, - * and the first turn must be `user`. - * - * - One user turn, no assistant turns → auto-approve mode (single-prompt build). - * - Anything else → multi-turn UserProxyLlm engages (answers clarifications, - * sends follow-ups consuming `messageBudget`). + * Hand-authored conversation that drives the build (≥1 turn, first `user`). + * One user turn → auto-approve single-prompt build; more → multi-turn proxy. + * Required unless `seedThread` is set, in which case it's optional and + * continues after the trace's live turn (`[, ...conversation]`). */ - conversation: ConversationTurn[]; + conversation?: ConversationTurn[]; complexity: 'simple' | 'medium' | 'complex'; tags: string[]; triggerType?: 'manual' | 'webhook' | 'schedule' | 'form'; @@ -209,6 +207,17 @@ export interface WorkflowTestCase { * field build with an empty view (everything mocks). */ credentials?: TestCaseCredential[]; + /** Synthetic seed file (messages + workflows) restored before the live turn. + * Synthetic fixtures only; mutually exclusive with the other seeds. */ + seedFile?: string; + /** Prose turns seeded as plain-text history (no tool calls/workflows). + * Mutually exclusive with `seedFile`. */ + priorConversation?: ConversationTurn[]; + /** Reproduce a real conversation from its LangSmith trace at run time: restore + * up to the last user message, send that live. Commits only the thread id + * (workspace auto-discovered; `project` overrides the source project). + * Supplies the live turn, so `conversation` is optional. Transient (~14d). */ + seedThread?: { threadId: string; project?: string }; /** Logical groupings this case belongs to (e.g. `['pr', 'full']`). Defaults to `['full']`. */ datasets: string[]; } @@ -271,6 +280,9 @@ export interface TranscriptTurn { userMessage?: string; /** Agent narration and tool interactions, interleaved in the order they occurred. */ steps: TranscriptStep[]; + /** True for turns restored from a conversation seed — context that predates + * the evaluated run, as opposed to behaviour captured live. */ + seeded?: boolean; } /** One ordered step within a turn: a slice of agent narration or a tool interaction. */ diff --git a/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts b/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts index bf5b3e4008f..bce153215d0 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/conversation-text.ts @@ -1,4 +1,20 @@ -import type { ToolInteraction, TranscriptStep, TranscriptTurn } from '../types'; +import type { ConversationTurn, ToolInteraction, TranscriptStep, TranscriptTurn } from '../types'; + +/** + * Human-readable prompt label for a test case. Authored cases use their first + * turn; seedThread cases carry no authored conversation, so fall back to the + * live (non-seeded) user turn captured in the transcript, then to the thread id. + */ +export function caseDisplayPrompt( + testCase: { conversation?: ConversationTurn[]; seedThread?: { threadId: string } }, + transcript?: TranscriptTurn[], +): string { + const authored = testCase.conversation?.[0]?.text; + if (authored) return authored; + const liveTurn = transcript?.find((t) => !t.seeded && t.userMessage)?.userMessage; + if (liveTurn) return liveTurn; + return testCase.seedThread ? `[seeded] thread ${testCase.seedThread.threadId.slice(0, 8)}` : ''; +} /** * User-side turns from a captured transcript, flattened as a text block for @@ -18,7 +34,12 @@ export function userTurnsAsText(transcript: TranscriptTurn[]): string { export function transcriptAsText(transcript: TranscriptTurn[]): string { return transcript .map((turn, i) => { - const lines: string[] = [`### Turn ${String(i + 1)}`]; + // Seeded turns are restored prior context — they predate the evaluated + // run, and judges must not score them as live behaviour. + const seededSuffix = turn.seeded + ? ' (seeded prior context — predates the evaluated run)' + : ''; + const lines: string[] = [`### Turn ${String(i + 1)}${seededSuffix}`]; if (turn.userMessage) lines.push(`User: ${turn.userMessage}`); for (const step of turn.steps) { const line = describeStep(step); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-memory.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-memory.service.test.ts index f074be99a99..f09acb0d1f2 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-memory.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-memory.service.test.ts @@ -10,6 +10,7 @@ const mockDeleteThreadsByResourceIdPrefix = jest.fn(); const mockListThreads = jest.fn(); const mockSaveThreadWithProject = jest.fn(); const mockGetThreadProjectId = jest.fn(); +const mockSaveMessages = jest.fn(); const mockAgentMemory = { listMessages: mockListMessages, getThread: mockGetThread, @@ -19,6 +20,7 @@ const mockAgentMemory = { listThreads: mockListThreads, saveThreadWithProject: mockSaveThreadWithProject, getThreadProjectId: mockGetThreadProjectId, + saveMessages: mockSaveMessages, }; // Mock GlobalConfig @@ -342,6 +344,87 @@ describe('InstanceAiMemoryService.ensureThread', () => { }); }); +describe('InstanceAiMemoryService.restoreThreadMessages', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('preserves message ids and content verbatim, coercing createdAt back to a Date', async () => { + const service = createService(); + + const result = await service.restoreThreadMessages('user-1', 'thread-1', [ + { + id: 'msg-user', + type: 'llm', + role: 'user', + content: [{ type: 'text', text: 'Send a daily digest to #cosmic-otter-alerts' }], + createdAt: '2026-01-01T00:00:00.000Z', + }, + { + id: 'msg-assistant', + type: 'llm', + role: 'assistant', + content: [ + { type: 'text', text: 'Built it.' }, + { + type: 'tool-call', + toolCallId: 'tc-1', + toolName: 'build-workflow', + state: 'resolved', + input: { code: '…' }, + output: { success: true, workflowId: 'wf-1' }, + }, + ], + createdAt: '2026-01-01T00:00:01.000Z', + }, + ]); + + expect(mockSaveMessages).toHaveBeenCalledTimes(1); + const args = mockSaveMessages.mock.calls[0][0]; + expect(args.threadId).toBe('thread-1'); + expect(args.resourceId).toBe('user-1'); + expect(args.messages).toHaveLength(2); + // Verbatim restore: same ids and content blocks, createdAt as ascending Dates. + expect(args.messages[0].id).toBe('msg-user'); + expect(args.messages[1].content[1].toolCallId).toBe('tc-1'); + expect(args.messages[0].createdAt).toEqual(new Date('2026-01-01T00:00:00.000Z')); + expect(args.messages[1].createdAt).toEqual(new Date('2026-01-01T00:00:01.000Z')); + expect(result).toEqual({ restored: 2 }); + }); + + it('accepts custom messages (no role, data payload)', async () => { + const service = createService(); + + await service.restoreThreadMessages('user-1', 'thread-1', [ + { + id: 'msg-custom', + type: 'custom', + data: { widget: 'setup-card' }, + createdAt: '2026-01-01T00:00:00.000Z', + }, + ]); + + expect(mockSaveMessages.mock.calls[0][0].messages[0].data).toEqual({ widget: 'setup-card' }); + }); + + it.each([ + ['missing id', { role: 'user', content: [], createdAt: '2026-01-01T00:00:00.000Z' }], + ['unparseable createdAt', { id: 'm', role: 'user', content: [], createdAt: 'not-a-date' }], + ['missing content', { id: 'm', role: 'user', createdAt: '2026-01-01T00:00:00.000Z' }], + ['custom without data', { id: 'm', type: 'custom', createdAt: '2026-01-01T00:00:00.000Z' }], + ])( + 'rejects a structurally invalid message (%s) without writing anything', + async (_label, bad) => { + const service = createService(); + + await expect(service.restoreThreadMessages('user-1', 'thread-1', [bad])).rejects.toThrow( + 'Seed message at index 0', + ); + expect(mockSaveMessages).not.toHaveBeenCalled(); + }, + ); +}); + describe('InstanceAiMemoryService.deleteThread', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts index fecc6ccfd73..8533678697b 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts @@ -29,6 +29,7 @@ jest.mock('../eval/execution.service', () => ({ import type { InstanceAiAdminSettingsUpdateRequest, InstanceAiEvalCredentialAllowlistRequest, + InstanceAiEvalRestoreThreadRequest, InstanceAiSendMessageRequest, InstanceAiCorrectTaskRequest, InstanceAiConfirmRequest, @@ -51,6 +52,7 @@ import type { Scope } from '@n8n/permissions'; import type { Request, Response } from 'express'; import { mock } from 'jest-mock-extended'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ConflictError } from '@/errors/response-errors/conflict.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; @@ -61,6 +63,7 @@ import type { UrlService } from '@/services/url.service'; import type { EvalExecutionService } from '../eval/execution.service'; import { EvalThreadCredentialAllowlistService } from '../eval/thread-credential-allowlist.service'; +import type { EvalThreadRestoreService } from '../eval/thread-restore.service'; import type { InProcessEventBus } from '../event-bus/in-process-event-bus'; import type { LocalGateway } from '../filesystem/local-gateway'; import type { InstanceAiGatewayService } from '../instance-ai-gateway.service'; @@ -103,6 +106,7 @@ describe('InstanceAiController', () => { const projectService = mock(); const evalCredentialAllowlists = new EvalThreadCredentialAllowlistService(); + const evalThreadRestore = mock(); const controller = new InstanceAiController( instanceAiService, @@ -111,6 +115,7 @@ describe('InstanceAiController', () => { settingsService, mock(), evalCredentialAllowlists, + evalThreadRestore, eventBus, moduleRegistry, push, @@ -537,6 +542,111 @@ describe('InstanceAiController', () => { }); }); + describe('restoreEvalThread', () => { + const seedMessages = [ + { id: 'm1', type: 'llm', role: 'user', content: [], createdAt: '2026-01-01T00:00:00.000Z' }, + ]; + const seedWorkflow = { id: 'wf-1', name: 'Seeded', nodes: [], connections: {} }; + const payload = { + threadId: THREAD_ID, + messages: seedMessages, + workflows: [seedWorkflow], + } as InstanceAiEvalRestoreThreadRequest; + + it('should require instanceAi:eval scope', () => { + expect(scopeOf('restoreEvalThread')).toEqual({ scope: 'instanceAi:eval', globalOnly: true }); + }); + + it('should recreate referenced workflows in the thread project, then seed messages', async () => { + memoryService.checkThreadOwnership.mockResolvedValue('owned'); + memoryService.getThreadProjectId.mockResolvedValue('project-1'); + memoryService.restoreThreadMessages.mockResolvedValue({ restored: 1 }); + evalThreadRestore.restoreDataTables.mockResolvedValue(new Map()); + + const result = await controller.restoreEvalThread(req, res, payload); + + expect(evalThreadRestore.restoreWorkflows).toHaveBeenCalledWith( + [seedWorkflow], + 'project-1', + expect.any(Map), + ); + expect(memoryService.restoreThreadMessages).toHaveBeenCalledWith( + USER_ID, + THREAD_ID, + seedMessages, + ); + expect(result).toEqual({ + ok: true, + threadId: THREAD_ID, + restored: 1, + workflowIds: ['wf-1'], + dataTableIds: [], + }); + }); + + it('should recreate data tables first and pass their id map to workflow restore', async () => { + memoryService.checkThreadOwnership.mockResolvedValue('owned'); + memoryService.getThreadProjectId.mockResolvedValue('project-1'); + memoryService.restoreThreadMessages.mockResolvedValue({ restored: 1 }); + const idMap = new Map([['dt-old-1234', 'dt-new']]); + evalThreadRestore.restoreDataTables.mockResolvedValue(idMap); + + const dataTable = { + id: 'dt-old-1234', + name: 'FAQs', + columns: [{ name: 'a', type: 'string' as const }], + }; + const result = await controller.restoreEvalThread(req, res, { + ...payload, + dataTables: [dataTable], + } as InstanceAiEvalRestoreThreadRequest); + + expect(evalThreadRestore.restoreDataTables).toHaveBeenCalledWith([dataTable], 'project-1'); + expect(evalThreadRestore.restoreWorkflows).toHaveBeenCalledWith( + [seedWorkflow], + 'project-1', + idMap, + ); + expect(result).toMatchObject({ dataTableIds: ['dt-new'] }); + }); + + it('should roll back created workflows and data tables when a later step fails', async () => { + memoryService.checkThreadOwnership.mockResolvedValue('owned'); + memoryService.getThreadProjectId.mockResolvedValue('project-1'); + evalThreadRestore.restoreDataTables.mockResolvedValue(new Map([['dt-old-1234', 'dt-new']])); + evalThreadRestore.restoreWorkflows.mockResolvedValue(['wf-1']); + memoryService.restoreThreadMessages.mockRejectedValue(new Error('boom')); + + await expect(controller.restoreEvalThread(req, res, payload)).rejects.toThrow('boom'); + + expect(evalThreadRestore.deleteWorkflows).toHaveBeenCalledWith(['wf-1']); + expect(evalThreadRestore.deleteDataTables).toHaveBeenCalledWith(['dt-new'], 'project-1'); + }); + + it('should reject a thread that does not exist', async () => { + memoryService.checkThreadOwnership.mockResolvedValue('not_found'); + + await expect(controller.restoreEvalThread(req, res, payload)).rejects.toThrow(NotFoundError); + expect(evalThreadRestore.restoreWorkflows).not.toHaveBeenCalled(); + }); + + it("should reject another user's thread", async () => { + memoryService.checkThreadOwnership.mockResolvedValue('other_user'); + + await expect(controller.restoreEvalThread(req, res, payload)).rejects.toThrow(ForbiddenError); + }); + + it('should reject a thread without a project binding', async () => { + memoryService.checkThreadOwnership.mockResolvedValue('owned'); + memoryService.getThreadProjectId.mockResolvedValue(undefined); + + await expect(controller.restoreEvalThread(req, res, payload)).rejects.toThrow( + BadRequestError, + ); + expect(evalThreadRestore.restoreWorkflows).not.toHaveBeenCalled(); + }); + }); + describe('cancelTask', () => { it('should require instanceAi:message scope', () => { expect(scopeOf('cancelTask')).toEqual({ scope: 'instanceAi:message', globalOnly: true }); diff --git a/packages/cli/src/modules/instance-ai/eval/__tests__/thread-restore.service.test.ts b/packages/cli/src/modules/instance-ai/eval/__tests__/thread-restore.service.test.ts new file mode 100644 index 00000000000..3a9df7c0c64 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/__tests__/thread-restore.service.test.ts @@ -0,0 +1,186 @@ +import type { Project, SharedWorkflowRepository, WorkflowRepository } from '@n8n/db'; +import { mock } from 'jest-mock-extended'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import type { DataTable } from '@/modules/data-table/data-table.entity'; +import type { DataTableService } from '@/modules/data-table/data-table.service'; + +import { EvalThreadRestoreService } from '../thread-restore.service'; + +function makeNode(overrides: Record = {}): Record { + return { + id: 'node-1', + name: 'Slack', + type: 'n8n-nodes-base.slack', + typeVersion: 2.2, + position: [100, 200], + parameters: { channel: '#cosmic-otter-alerts' }, + ...overrides, + }; +} + +describe('EvalThreadRestoreService', () => { + const workflowRepo = mock(); + const sharedWorkflowRepo = mock(); + const dataTableService = mock(); + const service = new EvalThreadRestoreService(workflowRepo, sharedWorkflowRepo, dataTableService); + + beforeEach(() => { + jest.clearAllMocks(); + workflowRepo.create.mockImplementation((entity) => entity as never); + sharedWorkflowRepo.getWorkflowOwningProject.mockResolvedValue(undefined); + }); + + it('recreates the workflow pinned to its seeded id and grants project ownership', async () => { + await service.restoreWorkflows( + [{ id: 'wf-original', name: 'Daily digest', nodes: [makeNode()], connections: {} }], + 'project-1', + ); + + expect(workflowRepo.save).toHaveBeenCalledTimes(1); + const saved = workflowRepo.create.mock.calls[0][0]; + expect(saved).toMatchObject({ + id: 'wf-original', + name: 'Daily digest', + active: false, + }); + expect(saved.versionId).toEqual(expect.any(String)); + expect(sharedWorkflowRepo.makeOwner).toHaveBeenCalledWith(['wf-original'], 'project-1'); + }); + + it('strips pre-attached node credentials so they cannot bypass the credential pin', async () => { + const node = makeNode({ + credentials: { slackApi: { id: 'cred-from-source-instance', name: 'Slack' } }, + }); + + await service.restoreWorkflows( + [{ id: 'wf-1', name: 'wf', nodes: [node], connections: {} }], + 'project-1', + ); + + const saved = workflowRepo.create.mock.calls[0][0]; + expect(saved.nodes).toHaveLength(1); + expect(saved.nodes?.[0]).not.toHaveProperty('credentials'); + expect(saved.nodes?.[0]).toMatchObject({ name: 'Slack', parameters: expect.any(Object) }); + }); + + it('does not re-grant ownership when the workflow already exists in this project', async () => { + sharedWorkflowRepo.getWorkflowOwningProject.mockResolvedValue({ id: 'project-1' } as Project); + + const created = await service.restoreWorkflows( + [{ id: 'wf-1', name: 'wf', nodes: [makeNode()], connections: {} }], + 'project-1', + ); + + expect(workflowRepo.save).toHaveBeenCalledTimes(1); + expect(sharedWorkflowRepo.makeOwner).not.toHaveBeenCalled(); + expect(created).toEqual([]); // not newly created + }); + + it('refuses to overwrite a workflow owned by another project', async () => { + sharedWorkflowRepo.getWorkflowOwningProject.mockResolvedValue({ + id: 'other-project', + } as Project); + + await expect( + service.restoreWorkflows( + [{ id: 'wf-1', name: 'wf', nodes: [makeNode()], connections: {} }], + 'project-1', + ), + ).rejects.toThrow(BadRequestError); + expect(workflowRepo.save).not.toHaveBeenCalled(); + }); + + it('rejects a structurally invalid node without writing anything', async () => { + await expect( + service.restoreWorkflows( + [{ id: 'wf-1', name: 'wf', nodes: [{ name: 'no type' }], connections: {} }], + 'project-1', + ), + ).rejects.toThrow(BadRequestError); + expect(workflowRepo.save).not.toHaveBeenCalled(); + }); + + describe('data tables', () => { + it('recreates each table (schema only) under a unique name and maps the id', async () => { + dataTableService.createDataTable.mockResolvedValue(mock({ id: 'dt-new' })); + + const idMap = await service.restoreDataTables( + [ + { + id: 'dt-old-1234', + name: 'Size Up Coffee FAQs', + columns: [ + { name: 'keywords', type: 'string' }, + { name: 'is_active', type: 'boolean' }, + ], + }, + ], + 'project-1', + ); + + expect(idMap.get('dt-old-1234')).toBe('dt-new'); + expect([...idMap.values()]).toEqual(['dt-new']); + + const [projectId, dto] = dataTableService.createDataTable.mock.calls[0]; + expect(projectId).toBe('project-1'); + // Original name is kept, with a unique suffix appended to dodge the + // per-project unique-name constraint across parallel iterations. + expect(dto.name).toMatch(/^Size Up Coffee FAQs \[seed [0-9a-f]{8}\]$/); + expect(dto.columns).toEqual([ + { name: 'keywords', type: 'string' }, + { name: 'is_active', type: 'boolean' }, + ]); + // Rows are never seeded — the table is recreated empty to keep trace + // PII out of the eval instance. + expect(dataTableService.insertRows).not.toHaveBeenCalled(); + }); + + it('rejects a too-short table id without creating anything (unsafe to string-replace)', async () => { + await expect( + service.restoreDataTables( + [{ id: 'short', name: 'T', columns: [{ name: 'a', type: 'string' }] }], + 'project-1', + ), + ).rejects.toThrow(BadRequestError); + expect(dataTableService.createDataTable).not.toHaveBeenCalled(); + }); + + it('rolls back already-created tables when a later table fails', async () => { + dataTableService.createDataTable + .mockResolvedValueOnce(mock({ id: 'dt-new-1' })) + .mockRejectedValueOnce(new Error('name conflict')); + + await expect( + service.restoreDataTables( + [ + { id: 'dt-old-1111', name: 'A', columns: [{ name: 'a', type: 'string' }] }, + { id: 'dt-old-2222', name: 'B', columns: [{ name: 'b', type: 'string' }] }, + ], + 'project-1', + ), + ).rejects.toThrow('name conflict'); + + // The first table was created, so it must be deleted on rollback. + expect(dataTableService.deleteDataTable).toHaveBeenCalledWith('dt-new-1', 'project-1'); + }); + + it('rewrites seed data-table ids in workflow nodes to the recreated ids', async () => { + const node = makeNode({ + type: 'n8n-nodes-base.dataTable', + parameters: { dataTableId: { __rl: true, mode: 'id', value: 'dt-old' } }, + }); + + await service.restoreWorkflows( + [{ id: 'wf-1', name: 'wf', nodes: [node], connections: {} }], + 'project-1', + new Map([['dt-old', 'dt-new']]), + ); + + const saved = workflowRepo.create.mock.calls[0][0]; + expect(saved.nodes?.[0]?.parameters).toEqual({ + dataTableId: { __rl: true, mode: 'id', value: 'dt-new' }, + }); + }); + }); +}); diff --git a/packages/cli/src/modules/instance-ai/eval/thread-restore.service.ts b/packages/cli/src/modules/instance-ai/eval/thread-restore.service.ts new file mode 100644 index 00000000000..287eb078638 --- /dev/null +++ b/packages/cli/src/modules/instance-ai/eval/thread-restore.service.ts @@ -0,0 +1,185 @@ +import type { InstanceAiEvalSeedDataTable, InstanceAiEvalSeedWorkflow } from '@n8n/api-types'; +import { SharedWorkflowRepository, WorkflowRepository } from '@n8n/db'; +import { Service } from '@n8n/di'; +import { jsonParse, type IConnections, type INode } from 'n8n-workflow'; +import { randomUUID } from 'node:crypto'; + +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; +import { DataTableService } from '@/modules/data-table/data-table.service'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isWorkflowNode(value: unknown): value is INode { + if (!isRecord(value)) return false; + return ( + typeof value.id === 'string' && + typeof value.name === 'string' && + typeof value.type === 'string' && + typeof value.typeVersion === 'number' && + Array.isArray(value.position) && + value.position.length === 2 && + value.position.every((coordinate) => typeof coordinate === 'number') && + isRecord(value.parameters) + ); +} + +function isConnections(value: unknown): value is IConnections { + return isRecord(value); +} + +/** Recreates the data tables and workflows a conversation seed references, so a + * restored message history's ids resolve. Used by the eval restore endpoint. */ +@Service() +export class EvalThreadRestoreService { + constructor( + private readonly workflowRepo: WorkflowRepository, + private readonly sharedWorkflowRepo: SharedWorkflowRepository, + private readonly dataTableService: DataTableService, + ) {} + + /** + * Recreate each seed data table (schema only — no rows) and map its seed id to + * the freshly created one. Tables are created under a uniquified name (names + * are unique per project; the id, which the workflow references, is what + * matters). Rolls back tables already created if a later one fails. + */ + async restoreDataTables( + dataTables: InstanceAiEvalSeedDataTable[], + projectId: string, + ): Promise> { + const idMap = new Map(); + try { + for (const table of dataTables) { + // Short ids would risk corrupting unrelated substrings in the + // whole-document id remap below; refuse them. + if (table.id.length < 8) { + throw new BadRequestError( + `Seed data table id "${table.id}" is too short to remap safely (need ≥8 chars)`, + ); + } + const suffix = ` [seed ${randomUUID().slice(0, 8)}]`; + const name = `${table.name.slice(0, 128 - suffix.length)}${suffix}`; + const created = await this.dataTableService.createDataTable(projectId, { + name, + columns: table.columns, + }); + idMap.set(table.id, created.id); + } + } catch (error) { + await this.deleteDataTables([...idMap.values()], projectId); + throw error; + } + return idMap; + } + + /** Best-effort delete (rollback of a failed restore). */ + async deleteDataTables(dataTableIds: string[], projectId: string): Promise { + for (const id of dataTableIds) { + try { + await this.dataTableService.deleteDataTable(id, projectId); + } catch { + // best-effort + } + } + } + + /** Recreate the seed workflows; returns the ids actually created (newly), and + * rolls them back if a later one fails. */ + async restoreWorkflows( + workflows: InstanceAiEvalSeedWorkflow[], + projectId: string, + dataTableIdMap: Map = new Map(), + ): Promise { + const created: string[] = []; + try { + for (const workflow of workflows) { + if (await this.createWorkflowPinnedToId(workflow, projectId, dataTableIdMap)) { + created.push(workflow.id); + } + } + } catch (error) { + await this.deleteWorkflows(created); + throw error; + } + return created; + } + + /** Best-effort delete (rollback of a failed restore). */ + async deleteWorkflows(workflowIds: string[]): Promise { + for (const id of workflowIds) { + try { + await this.workflowRepo.delete({ id }); + } catch { + // best-effort + } + } + } + + /** + * Insert a workflow at its seeded id (the BeforeInsert hook only generates an + * id when unset) and make the project its owner. Node credentials are stripped + * (the eval credential pin owns the credential view). Data-table references are + * rewritten to the recreated tables' ids. Returns true if newly created. + */ + private async createWorkflowPinnedToId( + workflow: InstanceAiEvalSeedWorkflow, + projectId: string, + dataTableIdMap: Map, + ): Promise { + const remapDataTableIds = (value: unknown): unknown => { + if (dataTableIdMap.size === 0) return value; + let serialized = JSON.stringify(value); + for (const [oldId, newId] of dataTableIdMap) { + serialized = serialized.replaceAll(oldId, newId); + } + return jsonParse(serialized); + }; + + const nodes: INode[] = workflow.nodes.map((node, index) => { + if (!isWorkflowNode(node)) { + throw new BadRequestError( + `Seed workflow ${workflow.id} node at index ${index} is not a valid workflow node`, + ); + } + const { credentials: _stripped, ...rest } = node; + const remapped = remapDataTableIds(rest); + if (!isWorkflowNode(remapped)) { + throw new BadRequestError( + `Seed workflow ${workflow.id} node at index ${index} became invalid after data-table id remap`, + ); + } + return remapped; + }); + + const connections = remapDataTableIds(workflow.connections); + if (!isConnections(connections)) { + throw new BadRequestError(`Seed workflow ${workflow.id} connections must be an object`); + } + + // Never overwrite a workflow owned by a different project (id collision). + const owningProject = await this.sharedWorkflowRepo.getWorkflowOwningProject(workflow.id); + if (owningProject && owningProject.id !== projectId) { + throw new BadRequestError( + `Seed workflow id ${workflow.id} already exists in another project; refusing to overwrite`, + ); + } + + await this.workflowRepo.save( + this.workflowRepo.create({ + id: workflow.id, + name: workflow.name, + nodes, + connections, + active: false, + versionId: randomUUID(), + }), + ); + if (!owningProject) { + await this.sharedWorkflowRepo.makeOwner([workflow.id], projectId); + return true; + } + return false; + } +} diff --git a/packages/cli/src/modules/instance-ai/instance-ai-memory.service.ts b/packages/cli/src/modules/instance-ai/instance-ai-memory.service.ts index 74dd7353690..777fdbcc777 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai-memory.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai-memory.service.ts @@ -18,6 +18,7 @@ import { import { DbSnapshotStorage } from './storage/db-snapshot-storage'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { NotFoundError } from '@/errors/response-errors/not-found.error'; import { @@ -38,6 +39,29 @@ function isAgentMessageLike(value: unknown): value is AgentDbMessage { ); } +function isRestorableMessage( + value: Record & { createdAt: Date }, +): value is AgentDbMessage & Record { + if (typeof value.id !== 'string' || value.id.length === 0) return false; + if (value.type === 'custom') return typeof value.data === 'object' && value.data !== null; + return typeof value.role === 'string' && Array.isArray(value.content); +} + +/** Coerce a wire-format seed message (ISO `createdAt`) into a persistable + * AgentDbMessage, or undefined if it fails the structural contract. */ +function toRestorableMessage(value: Record): AgentDbMessage | undefined { + const rawCreatedAt = value.createdAt; + const createdAt = + rawCreatedAt instanceof Date + ? rawCreatedAt + : typeof rawCreatedAt === 'string' + ? new Date(rawCreatedAt) + : undefined; + if (!createdAt || Number.isNaN(createdAt.getTime())) return undefined; + const candidate = { ...value, createdAt }; + return isRestorableMessage(candidate) ? candidate : undefined; +} + function messageCreatedAtMs(message: AgentDbMessage): number { const at = message.createdAt; if (at instanceof Date) return at.getTime(); @@ -119,6 +143,34 @@ export class InstanceAiMemoryService { }; } + /** Eval-only: seed a thread with a native message log (id/role/content/createdAt + * preserved verbatim) so the runtime continues as if it really happened. The + * thread must exist; referenced artifacts are recreated by the caller. */ + async restoreThreadMessages( + userId: string, + threadId: string, + messages: Array>, + ): Promise<{ restored: number }> { + const restorable: AgentDbMessage[] = []; + for (const [index, raw] of messages.entries()) { + const message = toRestorableMessage(raw); + if (!message) { + throw new BadRequestError( + `Seed message at index ${index} is not a valid agent message (id, createdAt, and role+content or type:custom+data are required)`, + ); + } + restorable.push(message); + } + + await this.agentMemory.saveMessages({ threadId, resourceId: userId, messages: restorable }); + return { restored: restorable.length }; + } + + /** Project a thread is bound to (undefined for legacy unbound threads). */ + async getThreadProjectId(threadId: string): Promise { + return (await this.agentMemory.getThreadProjectId(threadId)) ?? undefined; + } + async getThreadMessages( _userId: string, threadId: string, diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index 199dc90a9ed..b6047446201 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -15,6 +15,7 @@ import { InstanceAiUserPreferencesUpdateRequest, InstanceAiEvalExecutionRequest, InstanceAiEvalCredentialAllowlistRequest, + InstanceAiEvalRestoreThreadRequest, } from '@n8n/api-types'; import type { InstanceAiAgentNode } from '@n8n/api-types'; import { ModuleRegistry } from '@n8n/backend-common'; @@ -40,6 +41,7 @@ import type { NextFunction, Request, Response } from 'express'; import { randomUUID, timingSafeEqual } from 'node:crypto'; import { EvalExecutionService } from './eval/execution.service'; import { EvalThreadCredentialAllowlistService } from './eval/thread-credential-allowlist.service'; +import { EvalThreadRestoreService } from './eval/thread-restore.service'; import { InProcessEventBus } from './event-bus/in-process-event-bus'; import { InstanceAiGatewayService } from './instance-ai-gateway.service'; import { InstanceAiMemoryService } from './instance-ai-memory.service'; @@ -100,6 +102,7 @@ export class InstanceAiController { private readonly settingsService: InstanceAiSettingsService, private readonly evalExecutionService: EvalExecutionService, private readonly evalCredentialAllowlists: EvalThreadCredentialAllowlistService, + private readonly evalThreadRestore: EvalThreadRestoreService, private readonly eventBus: InProcessEventBus, private readonly moduleRegistry: ModuleRegistry, private readonly push: Push, @@ -700,6 +703,64 @@ export class InstanceAiController { return { ok: true }; } + /** + * Seed an existing (owned) thread with a previously exported conversation: + * recreate the workflow artifacts the history references (node credentials + * stripped — see `EvalThreadRestoreService`), then write the native message + * log verbatim. The thread then continues as if the conversation really + * happened, so an eval can drive the next turn live. + */ + @Post('/eval/restore-thread') + @GlobalScope('instanceAi:eval') + async restoreEvalThread( + req: AuthenticatedRequest, + _res: Response, + @Body payload: InstanceAiEvalRestoreThreadRequest, + ) { + this.requireInstanceAiEnabled(); + await this.assertThreadAccess(req.user.id, payload.threadId); + const projectId = await this.memoryService.getThreadProjectId(payload.threadId); + if (!projectId) { + throw new BadRequestError('Thread is not bound to a project'); + } + + const workflows = payload.workflows ?? []; + // Data tables first: the workflows reference them, and their ids are + // rewritten to the recreated tables' ids during workflow restore. + const idMap = await this.evalThreadRestore.restoreDataTables( + payload.dataTables ?? [], + projectId, + ); + const dataTableIds = [...idMap.values()]; + // Roll back everything we created if a later step fails, so a partial + // restore doesn't leak workflows/tables into the shared eval project. + let restored: number; + let createdWorkflowIds: string[] = []; + try { + createdWorkflowIds = await this.evalThreadRestore.restoreWorkflows( + workflows, + projectId, + idMap, + ); + ({ restored } = await this.memoryService.restoreThreadMessages( + req.user.id, + payload.threadId, + payload.messages, + )); + } catch (error) { + await this.evalThreadRestore.deleteWorkflows(createdWorkflowIds); + await this.evalThreadRestore.deleteDataTables(dataTableIds, projectId); + throw error; + } + return { + ok: true, + threadId: payload.threadId, + restored, + workflowIds: workflows.map((workflow) => workflow.id), + dataTableIds, + }; + } + // ── Gateway endpoints (daemon ↔ server) ────────────────────────────────── @Post('/gateway/create-link')