From b6ce05d7e24a6fa534bc4585a4e5924976370790 Mon Sep 17 00:00:00 2001 From: "n8n-assistant[bot]" <100856346+n8n-assistant[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:24:19 +0000 Subject: [PATCH] feat(core): Add progressive workflow building (no-changelog) (backport to release-candidate/2.39.x) (#38354) Co-authored-by: oleg --- .../create-instance-ai-eval/case-shapes.md | 1 + ...oaded-skill-drives-a-domain-tool-call.json | 93 +++---- ...ollows-its-instructions-with-generate.json | 55 ++-- .../loads-and-follows-a-skill-in-stream.json | 61 ++--- packages/@n8n/agents/src/index.ts | 2 + .../runtime/__tests__/active-skills.test.ts | 194 ++++++++++++++ .../runtime/__tests__/agent-runtime.test.ts | 134 +++++++++- .../agents/src/runtime/loop/agent-runtime.ts | 21 +- .../agents/src/runtime/memory/memory-store.ts | 19 ++ .../agents/src/runtime/model/message-list.ts | 4 + .../src/runtime/skills/active-skills.ts | 150 +++++++++++ .../agents/src/runtime/tools/tool-adapter.ts | 2 + .../src/runtime/tools/tool-call-executor.ts | 3 + packages/@n8n/agents/src/sdk/agent.ts | 1 + .../skills/__tests__/runtime-skills.test.ts | 1 + packages/@n8n/agents/src/skills/index.ts | 2 + packages/@n8n/agents/src/skills/tools.ts | 39 ++- packages/@n8n/agents/src/skills/types.ts | 12 + .../agents/src/types/runtime/message-list.ts | 2 + packages/@n8n/agents/src/types/sdk/memory.ts | 3 + packages/@n8n/agents/src/types/sdk/tool.ts | 5 + packages/@n8n/api-types/src/index.ts | 6 + .../__tests__/instance-ai.schema.test.ts | 17 ++ .../src/schemas/instance-ai.schema.ts | 26 ++ .../@n8n/instance-ai/docs/prompt-profiles.md | 111 ++++++++ .../evaluations/__tests__/build-mode.test.ts | 89 +++++++ .../build-workflow-conversation-seed.test.ts | 63 ++++- .../__tests__/event-parser.test.ts | 46 ++++ .../__tests__/langtracer-to-exported.test.ts | 12 + .../evaluations/__tests__/mcp-builder.test.ts | 16 ++ .../__tests__/scenario-seed-tables.test.ts | 52 +++- .../__tests__/user-execution.test.ts | 238 ++++++++++++++++++ .../evaluations/__tests__/user-proxy.test.ts | 67 ++++- .../evaluations/cli/mcp-builder.ts | 7 + .../clients/__tests__/n8n-client.test.ts | 45 ++++ .../evaluations/clients/n8n-client.ts | 39 ++- .../evaluations/harness/build-mode.ts | 33 +++ .../evaluations/harness/build-workflow.ts | 59 ++++- .../evaluations/harness/chat-loop.ts | 78 +++++- .../instance-ai/evaluations/harness/schema.ts | 4 + .../evaluations/harness/seed-tables.ts | 7 +- .../evaluations/langtracer/to-exported.ts | 9 + .../evaluations/outcome/event-parser.ts | 49 ++-- .../evaluations/run/build-orchestrator.ts | 6 + .../evaluations/run/case-selection.ts | 5 + .../evaluations/run/eval-session.ts | 3 + .../@n8n/instance-ai/evaluations/types.ts | 7 + .../evaluations/utils/user-proxy/agent.ts | 23 +- .../evaluations/utils/user-proxy/index.ts | 30 ++- .../evaluations/utils/user-proxy/prompts.ts | 6 + .../evaluations/utils/user-proxy/tools.ts | 27 +- .../@n8n/instance-ai/scripts/print-prompts.ts | 53 +++- .../@n8n/instance-ai/skills/planning/SKILL.md | 3 + .../skills/progressive-building/SKILL.md | 112 +++++++++ .../agent/__tests__/instance-agent.test.ts | 55 +++- .../instance-ai/src/agent/instance-agent.ts | 65 +++-- packages/@n8n/instance-ai/src/index.ts | 20 ++ .../prompts/__tests__/prompt-profiles.test.ts | 175 +++++++++++++ .../src/prompts/prompt-profiles.ts | 102 ++++++++ .../instance-ai/src/prompts/skill-variants.ts | 99 ++++++++ .../__tests__/run-state-registry.test.ts | 49 +++- .../src/runtime/run-state-registry.ts | 47 +++- .../__tests__/progressive-building.test.ts | 101 ++++++++ .../skills/__tests__/runtime-skills.test.ts | 19 +- .../instance-ai/src/skills/runtime-skills.ts | 28 +++ .../src/tools/__tests__/index.test.ts | 13 +- .../__tests__/build-workflow.tool.test.ts | 112 +++++++-- .../tools/workflows/build-workflow.tool.ts | 53 ++-- .../__tests__/langsmith-tracing.test.ts | 31 +++ .../src/tracing/langsmith-tracing.ts | 44 +++- .../instance-ai/src/tracing/trace-payloads.ts | 10 + packages/@n8n/instance-ai/src/types.ts | 6 + ...ance-ai-temporary-workflow.service.test.ts | 1 + ...stance-ai-terminal-outcome.service.test.ts | 8 + .../instance-ai.adapter.service.test.ts | 8 + .../__tests__/instance-ai.controller.test.ts | 35 +++ .../instance-ai.service.build-mode.test.ts | 133 ++++++++++ .../__tests__/instance-ai.service.test.ts | 173 +++++++++++-- .../__tests__/run-debug-gating.test.ts | 4 +- .../instance-ai-terminal-outcome.service.ts | 3 + .../instance-ai.adapter.service.ts | 7 + .../instance-ai/instance-ai.controller.ts | 2 + .../instance-ai/instance-ai.service.ts | 144 +++++++++-- .../__tests__/typeorm-agent-memory.test.ts | 40 +++ .../storage/typeorm-agent-memory.ts | 42 ++++ .../src/app/constants/experiments.ts | 10 +- .../__tests__/agentTimeline.utils.test.ts | 47 ++++ .../ai/instanceAi/agentTimeline.utils.ts | 2 + 88 files changed, 3502 insertions(+), 338 deletions(-) create mode 100644 packages/@n8n/agents/src/runtime/__tests__/active-skills.test.ts create mode 100644 packages/@n8n/agents/src/runtime/skills/active-skills.ts create mode 100644 packages/@n8n/instance-ai/docs/prompt-profiles.md create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/build-mode.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/user-execution.test.ts create mode 100644 packages/@n8n/instance-ai/evaluations/harness/build-mode.ts create mode 100644 packages/@n8n/instance-ai/skills/progressive-building/SKILL.md create mode 100644 packages/@n8n/instance-ai/src/prompts/__tests__/prompt-profiles.test.ts create mode 100644 packages/@n8n/instance-ai/src/prompts/prompt-profiles.ts create mode 100644 packages/@n8n/instance-ai/src/prompts/skill-variants.ts create mode 100644 packages/@n8n/instance-ai/src/skills/__tests__/progressive-building.test.ts create mode 100644 packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.build-mode.test.ts diff --git a/.agents/skills/create-instance-ai-eval/case-shapes.md b/.agents/skills/create-instance-ai-eval/case-shapes.md index 25bf7c4f22e..4dbfdd251e4 100644 --- a/.agents/skills/create-instance-ai-eval/case-shapes.md +++ b/.agents/skills/create-instance-ai-eval/case-shapes.md @@ -103,6 +103,7 @@ recognises this vocabulary: | Dismiss a setup card / skip a value | `[When the setup card asks for the API base URL, dismiss it — the user hasn't decided yet.]` (proxy dismisses via `approve_or_reject(false)`) | | Reject a plan that misses a requirement | `[When the agent shows its plan, reject it unless it sorts descending by count.]` | | Iterate change-by-change, in order | `[Send each change below in order, waiting for the build after each; keep bundled changes in one message.]` | +| Run the workflow themselves and report back | `[When the agent asks you to test the workflow, run it yourself, then tell it you ran it.]` Set `allowUserExecution: true` on the case. The proxy then sets `runWorkflowId` to the intended saved workflow. The harness uses normal execution without mocks. This action supports credential-free manual or schedule workflows without pinned data. | The `text` of a turn may be an **array of strings** (joined with newlines) so a long director note stays readable in JSON. A note governs only what it covers; diff --git a/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loaded-skill-drives-a-domain-tool-call.json b/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loaded-skill-drives-a-domain-tool-call.json index 7942bc0ad8f..7bc9463d6b5 100644 --- a/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loaded-skill-drives-a-domain-tool-call.json +++ b/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loaded-skill-drives-a-domain-tool-call.json @@ -67,30 +67,29 @@ } }, "status": 200, - "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_0153mr7MZ5RaibNuJF3t1kkn\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_017u8Xd5bungZRiNvzXXLg7p\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"engineering-status\"},\"caller\":{\"type\":\"direct\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1167,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", + "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqshPA7QoqLUtLRhcA\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Krds9KGve8gJNTNa9jn6N7\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"engineering-status\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1045,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", - "anthropic-ratelimit-input-tokens-remaining": "10000000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:45:04Z", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:45:04Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:45:03Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-tokens-limit": "12000000", - "anthropic-ratelimit-tokens-remaining": "12000000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:45:04Z", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:04Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d54c1fb0e589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f485dd8bc7b-PRG", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Mon, 06 Jul 2026 08:45:04 GMT", - "request-id": "req_011CckWYjo6QtopbKeU9fVgX", + "date": "Thu, 10 Sep 2026 07:12:04 GMT", + "request-id": "req_011CeuLqs6fxp1AkSuMSRWYb", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-4575c6c2498118f4235cb15c53994d86-dd185342703b474e-01", + "traceresponse": "00-77c37e49f4c62f7623d8cc66c4f3a26a-6bf7325e3f3f69d3-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, @@ -106,7 +105,7 @@ "system": [ { "type": "text", - "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies." + "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually.\n\n" } ], "messages": [ @@ -124,7 +123,7 @@ "content": [ { "type": "tool_use", - "id": "toolu_017u8Xd5bungZRiNvzXXLg7p", + "id": "toolu_01Krds9KGve8gJNTNa9jn6N7", "name": "load_skill", "input": { "skillId": "engineering-status" @@ -140,13 +139,8 @@ "content": [ { "type": "tool_result", - "tool_use_id": "toolu_017u8Xd5bungZRiNvzXXLg7p", - "content": [ - { - "type": "text", - "text": "[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually." - } - ] + "tool_use_id": "toolu_01Krds9KGve8gJNTNa9jn6N7", + "content": "{\"skillId\":\"engineering-status\",\"active\":true}" } ] } @@ -195,30 +189,29 @@ } }, "status": 200, - "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01AncHteAYY9SRacTTHzKv6w\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Now I'll format the status token as instructed:\"},{\"type\":\"tool_use\",\"id\":\"toolu_01LaHkuQAUWKJMdaMRfDEgPM\",\"name\":\"format_status\",\"input\":{\"status\":\"waiting_on_vendor_info\"},\"caller\":{\"type\":\"direct\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1320,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":71,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", + "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqxFjGLPD3QE1zh7Ef\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Now I'll format the status token:\"},{\"type\":\"tool_use\",\"id\":\"toolu_01CKRsNPiekmX9TWf3ooQdHU\",\"name\":\"format_status\",\"input\":{\"status\":\"waiting_on_vendor_info\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1244,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":68,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:45:05Z", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:45:05Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:45:04Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:45:05Z", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:05Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d54c878bc589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f4f38babc7b-PRG", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Mon, 06 Jul 2026 08:45:05 GMT", - "request-id": "req_011CckWYpHUBbu1v83kyVSuv", + "date": "Thu, 10 Sep 2026 07:12:06 GMT", + "request-id": "req_011CeuLqwpw3fPBk6xt6nrrm", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-5d29afad8d6483cabba40e5acf4a5fbf-0d9b0424157724b8-01", + "traceresponse": "00-d17795404368021a8e502b8755cc4b35-c750063766d5eb87-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, @@ -234,7 +227,7 @@ "system": [ { "type": "text", - "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies." + "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually.\n\n" } ], "messages": [ @@ -252,7 +245,7 @@ "content": [ { "type": "tool_use", - "id": "toolu_017u8Xd5bungZRiNvzXXLg7p", + "id": "toolu_01Krds9KGve8gJNTNa9jn6N7", "name": "load_skill", "input": { "skillId": "engineering-status" @@ -268,13 +261,8 @@ "content": [ { "type": "tool_result", - "tool_use_id": "toolu_017u8Xd5bungZRiNvzXXLg7p", - "content": [ - { - "type": "text", - "text": "[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually." - } - ] + "tool_use_id": "toolu_01Krds9KGve8gJNTNa9jn6N7", + "content": "{\"skillId\":\"engineering-status\",\"active\":true}" } ] }, @@ -283,11 +271,11 @@ "content": [ { "type": "text", - "text": "Now I'll format the status token as instructed:" + "text": "Now I'll format the status token:" }, { "type": "tool_use", - "id": "toolu_01LaHkuQAUWKJMdaMRfDEgPM", + "id": "toolu_01CKRsNPiekmX9TWf3ooQdHU", "name": "format_status", "input": { "status": "waiting_on_vendor_info" @@ -303,7 +291,7 @@ "content": [ { "type": "tool_result", - "tool_use_id": "toolu_01LaHkuQAUWKJMdaMRfDEgPM", + "tool_use_id": "toolu_01CKRsNPiekmX9TWf3ooQdHU", "content": "{\"formatted\":\"STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO\"}" } ] @@ -353,30 +341,29 @@ } }, "status": 200, - "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_018vDNTGTPYvQft63MaduXM2\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The status token has been formatted as requested:\\n\\n**STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO**\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1425,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":33,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", + "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLr3AQ92UTrYgeFRerd\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The status token has been formatted successfully:\\n\\n**STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO**\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1346,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":32,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:45:06Z", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:45:06Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:45:06Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:45:06Z", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:06Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d54cf88d1589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f561a95bc7b-PRG", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Mon, 06 Jul 2026 08:45:06 GMT", - "request-id": "req_011CckWYu6wR6tBum89WmZrE", + "date": "Thu, 10 Sep 2026 07:12:07 GMT", + "request-id": "req_011CeuLr2Whz42pkAm7AzGhs", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-0b21690b3bc7148f30f536a4baea9082-3a53115aa7a77a17-01", + "traceresponse": "00-80319e2d05f81fe007810b9d0f60ed01-c7956b0a3a60554e-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, diff --git a/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-a-matching-skill-and-follows-its-instructions-with-generate.json b/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-a-matching-skill-and-follows-its-instructions-with-generate.json index f8b6601bbb6..4382b1f71e1 100644 --- a/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-a-matching-skill-and-follows-its-instructions-with-generate.json +++ b/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-a-matching-skill-and-follows-its-instructions-with-generate.json @@ -51,30 +51,29 @@ } }, "status": 200, - "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_017Ydh5amLQEJMZi2SL2BFZp\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01L1gQY84Qqkbb2Vj91eQHay\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"summary-marker\"},\"caller\":{\"type\":\"direct\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1079,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", + "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqj5DrEFutdjSD8ETs\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Eq5aFLJhGriLQF4UBg7tyT\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"summary-marker\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":957,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", - "anthropic-ratelimit-input-tokens-remaining": "10000000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:44:58Z", + "anthropic-ratelimit-input-tokens-remaining": "9999000", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:44:58Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:44:57Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-tokens-limit": "12000000", - "anthropic-ratelimit-tokens-remaining": "12000000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:44:58Z", + "anthropic-ratelimit-tokens-remaining": "11999000", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:02Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d549adc3e589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f3bd9d2bc7b-PRG", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Mon, 06 Jul 2026 08:44:58 GMT", - "request-id": "req_011CckWYH41USJ75EenpGCQY", + "date": "Thu, 10 Sep 2026 07:12:02 GMT", + "request-id": "req_011CeuLqiWV7tDxhVGbXmda8", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-97b48f09aca0aecc15e1ed8c9fe8651d-dd31bf983e86dc71-01", + "traceresponse": "00-7cf7d7c3cfa9378a22214da3c62a1ea0-1ad2b8e7d42a1569-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, @@ -90,7 +89,7 @@ "system": [ { "type": "text", - "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies." + "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged.\n\n" } ], "messages": [ @@ -108,7 +107,7 @@ "content": [ { "type": "tool_use", - "id": "toolu_01L1gQY84Qqkbb2Vj91eQHay", + "id": "toolu_01Eq5aFLJhGriLQF4UBg7tyT", "name": "load_skill", "input": { "skillId": "summary-marker" @@ -124,13 +123,8 @@ "content": [ { "type": "tool_result", - "tool_use_id": "toolu_01L1gQY84Qqkbb2Vj91eQHay", - "content": [ - { - "type": "text", - "text": "[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged." - } - ] + "tool_use_id": "toolu_01Eq5aFLJhGriLQF4UBg7tyT", + "content": "{\"skillId\":\"summary-marker\",\"active\":true}" } ] } @@ -163,30 +157,29 @@ } }, "status": 200, - "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_01MLCY35L5o8Yag5erP4BPCM\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"**Summary:**\\n\\nn8n agents maintain workflow context efficiently by keeping it compact. This allows for streamlined execution and better memory management within automation workflows, reducing overhead while preserving essential information needed for agent operations.\\n\\nSKILL_MARKER_ALPHA\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1215,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":55,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", + "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqo1c45tLcGPFM5dEA\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"**Summary:** n8n agents maintain efficient workflow operations by keeping context compact, reducing overhead and improving performance.\\n\\nSKILL_MARKER_ALPHA\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1139,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":34,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:44:59Z", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:03Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:45:00Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:03Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:44:58Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:44:59Z", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:03Z", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d54a14884589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f417db9bc7b-PRG", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "application/json", - "date": "Mon, 06 Jul 2026 08:45:00 GMT", - "request-id": "req_011CckWYMTgCwuyKTBSpSK89", + "date": "Thu, 10 Sep 2026 07:12:03 GMT", + "request-id": "req_011CeuLqnRP3vkdg9opi7gNC", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-2c3af9579d58ddd9db43455722b54b60-d02bea40ae7237b6-01", + "traceresponse": "00-86beb3374d817d1d427aabf5c903ca34-f5a5fa52f1cb2789-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, diff --git a/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-and-follows-a-skill-in-stream.json b/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-and-follows-a-skill-in-stream.json index 1d834e83e65..481a5c37b3f 100644 --- a/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-and-follows-a-skill-in-stream.json +++ b/packages/@n8n/agents/src/__tests__/integration/cassettes/runtime-skills/loads-and-follows-a-skill-in-stream.json @@ -53,34 +53,32 @@ }, "status": 200, "response": [ - "1f8b0800000000000013cd55cb6edb3010bcfb2b8c3dcb8094564dca6b50f4851445d25b1d101b7163d3a248950fd786a17f2f6859b1e447da4b9beac699a5766756bba22569cfc615398733e2cea3f523811ed978037e5d13301890907467601ba88c20050c0a8541d0648eb20c93d7937c72915ee4599a66908014f11d6ec6d3ecc65fb99b578bdbfcfdd5dab80fd75fc2cf2cbb83e4201324608d8a003a279d471dd316467bd21ed8f7fb049c3735b784ce68603a28b5831cfd08a40b1a80823c4ae53a2c74d54b5d07cfbd29493b60597af92681028b39f1c2127a69341f86a41d6f09c539aebb1b33503da78a2c2a9e57c7f17b369b1fb24d0226f83e945f26e0c82e6541dc4bb2c0205a23d08ae8b27e241b95f3191960a08de7b844a9f0411134cd78ff34a311b56ddf59ca1f9429cad3cd3f11b2cd2668d56aeef351f2ee9e3746f1e0a8fb00e239f034fb5abf7bfbed512fe6b3dbebcf2bfb290fe5dd6a050968ace23d655070574aa5b669eae0816d9ae8ad5251f35302212d15bea76c2fab967a76a0630c118473d205298fcf4bdf860ca4b7c8bea2b6830b67f453708dd64b545b101840f36c03fe51159b294483a1ff4dbc744deaa398021b4f5b8b5eb81817aa0aed7a52a1fd3f3a56929d4203cdf9c935f5ef06d7d4bd228e9adeedf893820624f4040cb6707fe4ff7017377f71191f6dcfc34eee7f6bc7e6f539180cc92ff4b205fb31070000" + "1f8b0800000000000013c5554d8bdb3010bde757189d1348c2ba21be9585d2427ae852e8a12942b1a6896259f2ea23ec12f4df3b72e2f823d96d2f253e18346fa479ef69d0c00194cb9212ac655ba0d631e3469c39962547e25e2b2019e98164dcac497624a5e620312597cc7398ec9828fce461924ee6d3793a9b4e67982e783cc36ee974367b04bf328ba765fa8def3fa63f5e162bf3b9c09c7e250c182d6380592bb0ac8a6573ad1c9225d9cf5fa705130a0cc99497724cacd31535c0ac56bd9085670f2a875e90036e96b689f9468e509577d4e9021482cbf4010bb17c0734c7939dd08af633a60d8e307f0b6bf6c60250eda004c3244dcbebfc169ded86681813ed5d3734ff806ac01c440ed489e8048956716678745dfd061385d32d68849476941d5035dba0b3212449188de074fb6767e946eabcb8dd033752ea221c5e4e52bb78547adee7b496d45b68fa20ae3d76c222cfbf2f3f6ded53517eddad9ee71b2b166a85598a95719fd468a82d84947519d48d878668a99451eaa500170672570b3a7fadae4aa8ed4048426290bca51dbbd9b1f7b5d7293deda7484be974737becc44b72857e09bcd77ddd9e84843ed1bb9038ae892561e8d95da8c47bfec2d7b52f77a69225688b2fb1a1eecba364e67582bf8284ebc6be0b23306b125a36ef528aafecdf9e105d7508b50735d3e6a6aa1e483a2a7acf7ff7d5f9e721309828e1ff8d85c13b9e2ec2c0cb76e05ebbd8c5c8e512fe00c6a9d291c7070000" ], "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", - "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:45:07Z", + "anthropic-ratelimit-input-tokens-remaining": "10000000", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:45:07Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:45:07Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-tokens-limit": "12000000", - "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:45:07Z", + "anthropic-ratelimit-tokens-remaining": "12000000", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:07Z", "cache-control": "no-cache", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d54d64e86589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f5c5a57bc7b-PRG", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 06 Jul 2026 08:45:08 GMT", - "request-id": "req_011CckWYyhktjiUUzVGjbaRn", + "date": "Thu, 10 Sep 2026 07:12:08 GMT", + "request-id": "req_011CeuLr6og4ahrpyfnPec45", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-540508076234ed11018ad8b9be047cfd-9a43dd87abb7b745-01", - "transfer-encoding": "chunked", + "traceresponse": "00-db154d560f530266545ecc9baf7015e2-c0470cb196446bc0-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, @@ -96,7 +94,7 @@ "system": [ { "type": "text", - "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies." + "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"\", \"filePath\": \"\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged.\n\n" } ], "messages": [ @@ -114,7 +112,7 @@ "content": [ { "type": "tool_use", - "id": "toolu_01PpE9TfnjhgRCKxrJ5ukSxx", + "id": "toolu_017ccT9FgsRkmMhLq2bsi7nL", "name": "load_skill", "input": { "skillId": "summary-marker" @@ -130,13 +128,8 @@ "content": [ { "type": "tool_result", - "tool_use_id": "toolu_01PpE9TfnjhgRCKxrJ5ukSxx", - "content": [ - { - "type": "text", - "text": "[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged." - } - ] + "tool_use_id": "toolu_017ccT9FgsRkmMhLq2bsi7nL", + "content": "{\"skillId\":\"summary-marker\",\"active\":true}" } ] } @@ -171,34 +164,32 @@ }, "status": 200, "response": [ - "1f8b0800000000000013c555c16e133110bdf72bac39461b940482c0b71c10452d0835a812a2c89aaea71b13afbdd8b36d4a15898fe00bf912e4dd6c9a4d5bcaa5229728f3c69ef7de782674498ea52829462c4845c6c0071a19a5b801beae0824f440c8badf206fa0f49a2c48c82dd69a860b34cb7af862381d4e4693e978341a430646a73b62a146e3d7d5f3cf85bf08a76f4f5787935978f56d8ed34f90ed55820c82b72980319ac8e852d9dc3b26c720bf7ccd20b2af54208cde8174b5b59b50a4ef35b99c7a414d8cc6c62e5677ec8dab6a56ec97e422c8f1643cc920c77c412a0f846cbc53fd9451870742fd10d69d4d15a85a504901ad9a9677f36fd1f1621f5d67e06bde0d4d3288142e4d4e8a0d0590909cd1187432d95d5048c255411e2438cf0a2fd1583cb704ebb5683feb83036a3bbe71539d5b9f2fefeffb3d294d254dab56ee2e9ed46ece31ad5262f32501d6fdc29571c55e250129080f91d36419ff4eae49e9916b237d52dbb40db50ffe4ac4ba2c314067d0233e3d1915f3c3b8425414042f48c4a5b156181739d4797a4b519eb9333718cc1bb6d772303873730e84653a8605398e824ac382bdb7a2e11e7ffffcc50bba1605390ac824d06978ec253c994211c9e9969948d78b401699b460df926ec652182702a11db229291368adbf4a122f7c10a62c499b24c49fa74968a6ec3f8b0a142bef226d5518c714b06d9ad075d8f647d08af23ac59fa55ece8fde1d1fabf7b393a337276a76fcf170d67b830f0faaaf1e9b535fede8e8ddd56df27bfde881b0a3bfb76b819c565c0707ffbc71d74fb872f776e4cbd1deb6b9fdefbaebdb2e065bdbff006bc3e0fa12070000" + "1f8b0800000000000013c555cb8e1a3110bcf315968f082266b3ec616e681569a32551b428a76c6419bb99b1f0d8133f106885948fc817e64bd21e98c0f0489443122e305d6557755334b00213725281f7bc00e60377a12779e03979a1615303cd6907a483f699e62fb4b212345284e651c2b0e46a1987b7c3f1f0667433ce46a30ce94aa63b7cc14659760f71eaee65b40f73fbf1bd288a30dddc21a7ab840567752a70ef15ca9a242bac096896e69f3eef1eb832e0686ea2d603ea83ad9903eeade9943c7c896004748a12f0b0f66d2db6ed2853c7c0825d824130cb5edfa112172530815707650deb52462d8eb0bc86b5679302d42554e0b866e3ea9c7f40b3f214dd0ea88da1e30fbb01b7520258506912348d4a7227d3d4cd025c6a9c156011323630bec2aef91c27bbdd1242b6bd1eecbefefd68d95c5bb1bc1c820b944645c27ad7ea319e3add9f0bb04ec4e62da734e936af8378ad4c71a246682ad26b06317381ffda6043e918dc55bac67ed2f6f6fafdbdc17fae3c8b55c5dd26eff7c92c605e2aec9f602a4df0042a1548b05693c693fffef51b274bd8e0af5694dc285f918575043f4add1cf31b234a678d8d9e28f4e6b848f9f30419a47676a564e2a18c1e0655015900c839174b22a34bc84e6c0d2236b9fd4f3379f56c9ecdecf1ed74cade4d9e1edf3cb1c9f4c3c3e490a1b3289d86d4d6bf8bb1ad8f6c5db9b5dd7f171bed80f4a8b1ce42a260240bd119fa076be964c76dffe2a23ad92cb7d9f5191ffe0dcea77b8cd1c3c91f64c432c465060000" ], "rawHeaders": { "anthropic-ratelimit-input-tokens-limit": "10000000", - "anthropic-ratelimit-input-tokens-remaining": "9999000", - "anthropic-ratelimit-input-tokens-reset": "2026-07-06T08:45:08Z", + "anthropic-ratelimit-input-tokens-remaining": "10000000", + "anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000", - "anthropic-ratelimit-output-tokens-reset": "2026-07-06T08:45:08Z", + "anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-07-06T08:45:08Z", + "anthropic-ratelimit-requests-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-tokens-limit": "12000000", - "anthropic-ratelimit-tokens-remaining": "11999000", - "anthropic-ratelimit-tokens-reset": "2026-07-06T08:45:08Z", + "anthropic-ratelimit-tokens-remaining": "12000000", + "anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:08Z", "cache-control": "no-cache", "cf-cache-status": "DYNAMIC", - "cf-ray": "a16d54dedfda589a-BCN", - "connection": "keep-alive", + "cf-ray": "a38c9f63ce57bc7b-PRG", "content-encoding": "gzip", "content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 06 Jul 2026 08:45:10 GMT", - "request-id": "req_011CckWZ5cCxTdKpxXsHDHfM", + "date": "Thu, 10 Sep 2026 07:12:08 GMT", + "request-id": "req_011CeuLrBuGK4E7uxpMM6NMq", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-c629ba9e76d576cd6e5f26e49c3eccda-a115b633407cc82c-01", - "transfer-encoding": "chunked", + "traceresponse": "00-ae497abd3e1ae1a20e06ccaeba57c754-f8de5c4ed0f0dac4-01", "vary": "Accept-Encoding", "x-robots-tag": "none" }, diff --git a/packages/@n8n/agents/src/index.ts b/packages/@n8n/agents/src/index.ts index 4a412c9c3ea..9dba4bf8796 100644 --- a/packages/@n8n/agents/src/index.ts +++ b/packages/@n8n/agents/src/index.ts @@ -213,6 +213,8 @@ export type { RuntimeSkillRegistry, RuntimeSkillRegistryEntry, RuntimeSkillSource, + RuntimeSkillStateScope, + RuntimeSkillStateStore, RuntimeSkillValidationError, RuntimeSkillValidationResult, } from './skills'; diff --git a/packages/@n8n/agents/src/runtime/__tests__/active-skills.test.ts b/packages/@n8n/agents/src/runtime/__tests__/active-skills.test.ts new file mode 100644 index 00000000000..4a972d77cf6 --- /dev/null +++ b/packages/@n8n/agents/src/runtime/__tests__/active-skills.test.ts @@ -0,0 +1,194 @@ +import { createRuntimeSkillSource, filterRuntimeSkillSource } from '../../skills/registry'; +import type { RuntimeSkillSource } from '../../skills/types'; +import { InMemoryMemory } from '../memory/memory-store'; +import { AgentMessageList } from '../model/message-list'; +import { ActiveSkills } from '../skills/active-skills'; + +const scope = { threadId: 'thread-1', resourceId: 'user-1' }; +const source = createRuntimeSkillSource([ + { + id: 'planning', + name: 'planning', + description: 'Plan tasks.', + instructions: 'Create a task plan.', + }, + { + id: 'builder', + name: 'builder', + description: 'Build workflows.', + instructions: 'Build one workflow.', + }, +]); + +describe('active skills', () => { + it('does not rewrite unchanged stored skills on later turns', async () => { + const memory = new InMemoryMemory(); + await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['builder', 'planning']); + const save = vi.spyOn(memory.skillState, 'save'); + + for (let turn = 0; turn < 2; turn++) { + const active = new ActiveSkills(source, 'assistant', memory.skillState); + await active.restore(new AgentMessageList(), scope); + expect(active.instructions()).toContain('Build one workflow.'); + expect(active.instructions()).toContain('Create a task plan.'); + } + + expect(save).not.toHaveBeenCalled(); + }); + + it('does not rewrite an explicitly empty stored state', async () => { + const memory = new InMemoryMemory(); + await memory.skillState.save({ ...scope, agentName: 'assistant' }, []); + const save = vi.spyOn(memory.skillState, 'save'); + const active = new ActiveSkills(source, 'assistant', memory.skillState); + const list = new AgentMessageList(); + + await active.restore(list, scope); + + expect(list.activeSkillIds).toEqual([]); + expect(active.instructions()).toBeUndefined(); + expect(save).not.toHaveBeenCalled(); + }); + + it('clears stored state when all restored skills are disabled', async () => { + const memory = new InMemoryMemory(); + await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['planning']); + const active = new ActiveSkills( + filterRuntimeSkillSource(source, ['planning']), + 'assistant', + memory.skillState, + ); + + await active.restore(new AgentMessageList(), scope); + + expect(active.instructions()).toBeUndefined(); + await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([]); + }); + + it('initializes stored state from historical skill loads', async () => { + const memory = new InMemoryMemory(); + const list = new AgentMessageList(); + list.addHistory([ + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolName: 'load_skill', + toolCallId: 'old-load', + input: { skillId: 'builder' }, + state: 'resolved', + output: { type: 'content', value: [{ type: 'text', text: 'Build one workflow.' }] }, + }, + ], + }, + ]); + const active = new ActiveSkills(source, 'assistant', memory.skillState); + + await active.restore(list, scope); + + expect(active.instructions()).toContain('Build one workflow.'); + await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([ + 'builder', + ]); + }); + + it('persists checkpoint skills when they differ from stored state', async () => { + const memory = new InMemoryMemory(); + await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['planning']); + const list = new AgentMessageList(); + list.activeSkillIds = ['builder']; + const active = new ActiveSkills(source, 'assistant', memory.skillState); + + await active.restore(list, scope); + + expect(active.instructions()).toContain('Build one workflow.'); + expect(active.instructions()).not.toContain('Create a task plan.'); + await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([ + 'builder', + ]); + }); + + it('removes disabled skills from restored state and from historical tool output', async () => { + const memory = new InMemoryMemory(); + await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['planning', 'builder']); + const list = new AgentMessageList(); + list.addHistory([ + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolName: 'load_skill', + toolCallId: 'old-load', + input: { skillId: 'planning' }, + state: 'resolved', + output: { type: 'content', value: [{ type: 'text', text: 'Create a task plan.' }] }, + }, + ], + }, + ]); + const active = new ActiveSkills( + filterRuntimeSkillSource(source, ['planning']), + 'assistant', + memory.skillState, + ); + await active.restore(list, scope); + + expect(active.instructions()).toContain('Build one workflow.'); + expect(active.instructions()).not.toContain('Create a task plan.'); + expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).not.toContain( + 'Create a task plan.', + ); + await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([ + 'builder', + ]); + }); + + it('keeps threads, resources, and agents isolated', async () => { + const memory = new InMemoryMemory(); + const active = new ActiveSkills(source, 'assistant', memory.skillState); + await active.restore(new AgentMessageList(), scope); + await active.load('builder'); + for (const [name, persistence] of [ + ['other-agent', scope], + ['assistant', { ...scope, resourceId: 'other-user' }], + ['assistant', { ...scope, threadId: 'other-thread' }], + ] as const) { + const other = new ActiveSkills(source, name, memory.skillState); + await other.restore(new AgentMessageList(), persistence); + expect(other.instructions()).toBeUndefined(); + } + await memory.deleteThread(scope.threadId); + await expect( + memory.skillState.load({ ...scope, agentName: 'assistant' }), + ).resolves.toBeUndefined(); + }); + + it('does not activate a missing skill or a failed load', async () => { + const missing: RuntimeSkillSource = { ...source, loadSkill: vi.fn().mockResolvedValue(null) }; + const active = new ActiveSkills(missing, 'assistant'); + const list = new AgentMessageList(); + await active.restore(list); + await expect(active.load('unknown')).resolves.toBeNull(); + await expect(active.load('builder')).resolves.toBeNull(); + expect(list.activeSkillIds).toEqual([]); + expect(active.instructions()).toBeUndefined(); + }); + + it('records concurrent loads without dropping either skill', async () => { + const memory = new InMemoryMemory(); + const active = new ActiveSkills(source, 'assistant', memory.skillState); + const list = new AgentMessageList(); + await active.restore(list, scope); + await Promise.all([active.load('builder'), active.load('planning')]); + await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([ + 'builder', + 'planning', + ]); + expect(AgentMessageList.deserialize(structuredClone(list.serialize())).activeSkillIds).toEqual([ + 'builder', + 'planning', + ]); + }); +}); diff --git a/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts b/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts index 51c88237e13..78ee755c49e 100644 --- a/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts +++ b/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts @@ -8,6 +8,9 @@ import { Agent } from '../../sdk/agent'; import { createCancellation } from '../../sdk/cancellation'; import { isLlmMessage } from '../../sdk/message'; import { Tool, Tool as ToolBuilder } from '../../sdk/tool'; +import { createRuntimeSkillSource } from '../../skills/registry'; +import { createRuntimeSkillTools } from '../../skills/tools'; +import type { RuntimeSkillSource } from '../../skills/types'; import type { CheckpointStore, SerializableAgentState } from '../../types'; import { AgentEvent } from '../../types/runtime/event'; import type { AgentEventData } from '../../types/runtime/event'; @@ -6598,7 +6601,11 @@ describe('AgentRuntime — mid-run observation', () => { function buildMidRunRuntime( memory: InMemoryMemory, - extra?: { tools?: BuiltTool[]; checkpointStorage?: CheckpointStore }, + extra?: { + skillSource?: RuntimeSkillSource; + tools?: BuiltTool[]; + checkpointStorage?: CheckpointStore; + }, ): AgentRuntime { return new AgentRuntime({ name: 'mid-run-agent', @@ -6606,6 +6613,7 @@ describe('AgentRuntime — mid-run observation', () => { instructions: 'You are a test assistant.', memory, tools: extra?.tools ?? [makeStepTool()], + ...(extra?.skillSource ? { skillSource: extra.skillSource } : {}), ...(extra?.checkpointStorage ? { checkpointStorage: extra.checkpointStorage } : {}), observationalMemory: { observerThresholdTokens: 1, @@ -6646,6 +6654,130 @@ describe('AgentRuntime — mid-run observation', () => { expect(await memory.getCursor('thread-1')).not.toBeNull(); }); + it('persists full skill content when the memory adapter has no skill state store', async () => { + const instructions = 'Wait for a real execution before extending the workflow.'; + const source = createRuntimeSkillSource([ + { id: 'builder', name: 'builder', description: 'Build workflows.', instructions }, + ]); + const memory = new InMemoryMemory(); + Object.defineProperty(memory, 'skillState', { value: undefined }); + const runtime = buildMidRunRuntime(memory, { + skillSource: source, + tools: createRuntimeSkillTools(source), + }); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCall('load-builder', 'load_skill', { skillId: 'builder' }), + ) + .mockResolvedValueOnce(makeGenerateSuccess('Please test the first workflow.')); + + await runtime.generate('Build it', { persistence: PERSISTENCE }); + await runtime.dispose(); + + const messages = await memory.getMessages(PERSISTENCE.threadId); + const loads = messages + .flatMap((message) => (isLlmMessage(message) ? message.content : [])) + .filter((part) => part.type === 'tool-call' && part.toolName === 'load_skill'); + expect(loads).toEqual([ + expect.objectContaining({ + output: { + type: 'content', + value: [{ type: 'text', text: expect.stringContaining(instructions) }], + }, + }), + ]); + expect(await memory.getCursor(PERSISTENCE.threadId)).not.toBeNull(); + expect(flattenInstructions(capturedCall(1).instructions)).not.toContain(''); + }); + + it('retains loaded skills through compaction and a new user turn', async () => { + const source = createRuntimeSkillSource([ + { + id: 'builder', + name: 'builder', + description: 'Build workflows.', + instructions: 'Wait for a real execution before extending the workflow.', + }, + ]); + const memory = new InMemoryMemory(); + const options = { skillSource: source, tools: createRuntimeSkillTools(source) }; + const runtime = buildMidRunRuntime(memory, options); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCall('load-builder', 'load_skill', { skillId: 'builder' }), + ) + .mockResolvedValueOnce(makeGenerateSuccess('Please test the first workflow.')); + await runtime.generate('Build it', { persistence: PERSISTENCE }); + await runtime.dispose(); + + expect(capturedCall(1).messages).toEqual([ + { role: 'user', content: OBSERVATION_CONTINUATION_REMINDER }, + ]); + expect(flattenInstructions(capturedCall(1).instructions)).toContain( + 'Wait for a real execution', + ); + const next = buildMidRunRuntime(memory, options); + generateText.mockResolvedValueOnce(makeGenerateSuccess('The live test is still needed.')); + await next.generate('Continue', { persistence: PERSISTENCE }); + await next.dispose(); + expect(flattenInstructions(capturedCall(2).instructions)).toContain( + 'Wait for a real execution', + ); + expect(JSON.stringify(capturedCall(2).messages)).not.toContain('Wait for a real execution'); + }); + + it('restores active skills from a checkpoint with the newly selected content', async () => { + const source = createRuntimeSkillSource([ + { + id: 'builder', + name: 'builder', + description: 'Build workflows.', + instructions: 'Old workflow policy.', + }, + ]); + const checkpointStore = makeClaimingCheckpointStore(); + const memory = new InMemoryMemory(); + const first = buildMidRunRuntime(memory, { + skillSource: source, + tools: [...createRuntimeSkillTools(source), makeInterruptibleTool()], + checkpointStorage: checkpointStore, + }); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCall('load-builder', 'load_skill', { skillId: 'builder' }), + ) + .mockResolvedValueOnce( + makeGenerateWithToolCall('confirm', 'approve', { question: 'Set up?' }), + ); + const result = await first.generate('Build it', { persistence: PERSISTENCE }); + const suspension = result.pendingSuspend?.[0]; + if (!suspension) throw new Error('Expected a setup confirmation'); + + const current = createRuntimeSkillSource([ + { + id: 'builder', + name: 'builder', + description: 'Build workflows.', + instructions: 'Current workflow policy.', + }, + ]); + const resumed = buildMidRunRuntime(memory, { + skillSource: current, + tools: [...createRuntimeSkillTools(current), makeInterruptibleTool()], + checkpointStorage: checkpointStore, + }); + generateText.mockResolvedValueOnce(makeGenerateSuccess('Ready.')); + await resumed.resume( + 'generate', + { approved: true }, + { runId: suspension.runId, toolCallId: suspension.toolCallId }, + ); + await first.dispose(); + await resumed.dispose(); + expect(flattenInstructions(capturedCall(2).instructions)).toContain('Current workflow policy.'); + expect(JSON.stringify(capturedCall(2))).not.toContain('Old workflow policy.'); + }); + it('re-derives the mask from the cursor when resuming a suspended run', async () => { const memory = new InMemoryMemory(); const checkpointStore = makeClaimingCheckpointStore(); diff --git a/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts b/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts index 92693ee289e..8d6a877a51e 100644 --- a/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts +++ b/packages/@n8n/agents/src/runtime/loop/agent-runtime.ts @@ -19,6 +19,7 @@ import { import { StreamSink } from './stream-sink'; import { isCancellation } from '../../sdk/cancellation'; import { computeCost, getModelCost, type ModelCost } from '../../sdk/catalog'; +import type { RuntimeSkillSource } from '../../skills/types'; import type { BuiltFileStore, BuiltMemory, @@ -68,6 +69,7 @@ import { getEffectiveAnthropicCacheTtl, mergeProviderOptions, } from '../model/prompt-cache'; +import { ActiveSkills } from '../skills/active-skills'; import { BackgroundTaskTracker } from '../state/background-task-tracker'; import { AgentEventBus, type AgentAbortScope } from '../state/event-bus'; import { generateRunId, RunStateManager, StaleResumeError } from '../state/run-state'; @@ -92,6 +94,7 @@ export interface AgentRuntimeConfig { */ modelFetch?: FetchFn; instructions: string; + skillSource?: RuntimeSkillSource; instructionProviderOptions?: ProviderOptions; tools?: BuiltTool[]; deferredTools?: BuiltTool[]; @@ -200,9 +203,18 @@ export class AgentRuntime { private context: RuntimeContextBuilder; private toolExecutor: ToolCallExecutor; + private activeSkills?: ActiveSkills; constructor(config: AgentRuntimeConfig) { this.config = config; + // Keep full tool results when the memory backend cannot persist active skill IDs. + if (config.skillSource && (!config.memory || config.memory.skillState)) { + this.activeSkills = new ActiveSkills( + config.skillSource, + config.name, + config.memory?.skillState, + ); + } const tokenCounter = createModelTokenCounter(config.model); this.telemetry = new RuntimeTelemetry(config); this.runId = config.runId ?? generateRunId(); @@ -226,6 +238,7 @@ export class AgentRuntime { onCancelled: () => this.updateState({ status: 'cancelled' }), tokenCounter, ...(config.workspaceFilesystem ? { workspaceFilesystem: config.workspaceFilesystem } : {}), + ...(this.activeSkills ? { loadSkill: this.activeSkills.load.bind(this.activeSkills) } : {}), }); this.modelCost = config.modelCost; this.currentState = { @@ -753,6 +766,7 @@ export class AgentRuntime { */ private async runAgentLoop(ctx: LoopContext, sink: RunOutputSink): Promise { const { list, options, abortScope, pendingResume } = ctx; + await this.activeSkills?.restore(list, options?.persistence); this.context.hydrateDeferredToolsFromList(list); // Inject a model-facing note for any MCP servers that failed to connect // during build(). The agent can mention the outage to the user when @@ -872,7 +886,10 @@ export class AgentRuntime { options?.executionCounter, ); const { system, messages } = list.forLlm( - effectiveInstructions, + // Skill content changes only on activation. Keep it cached when memory compacts. + [effectiveInstructions, this.activeSkills?.instructions()] + .filter(Boolean) + .join('\n\n'), instructionProviderOptions, volatileInstructions, ); @@ -880,7 +897,7 @@ export class AgentRuntime { // only — never persisted back to the message list or tool set. const cached = applyRuntimeCacheBreakpoints({ system, - messages, + messages: this.activeSkills?.modelMessages(messages, list) ?? messages, aiTools, promptCaching: this.config.promptCaching, modelId: this.modelIdString, diff --git a/packages/@n8n/agents/src/runtime/memory/memory-store.ts b/packages/@n8n/agents/src/runtime/memory/memory-store.ts index 6e7fbdb8b54..32a16a62c65 100644 --- a/packages/@n8n/agents/src/runtime/memory/memory-store.ts +++ b/packages/@n8n/agents/src/runtime/memory/memory-store.ts @@ -8,6 +8,7 @@ import { uniqueStrings, } from './memory-lifecycle'; import { normalizeObservationLogReflection } from './observation-log-reflector'; +import type { RuntimeSkillStateStore } from '../../skills/types'; import type { BuiltEpisodicMemoryStore, BuiltMemory, @@ -88,6 +89,23 @@ export class InMemoryMemory private threads = new Map(); private messagesByThread = new Map(); + private skillsByThread = new Map>(); + + readonly skillState: RuntimeSkillStateStore = { + load: async ({ threadId, resourceId, agentName }) => + await Promise.resolve( + this.skillsByThread + .get(threadId) + ?.get(JSON.stringify([resourceId, agentName])) + ?.slice(), + ), + save: async ({ threadId, resourceId, agentName }, ids) => { + const states = this.skillsByThread.get(threadId) ?? new Map(); + states.set(JSON.stringify([resourceId, agentName]), [...ids]); + this.skillsByThread.set(threadId, states); + await Promise.resolve(); + }, + }; private observationLogByScope = new Map(); @@ -143,6 +161,7 @@ export class InMemoryMemory // eslint-disable-next-line @typescript-eslint/require-await async deleteThread(threadId: string): Promise { this.threads.delete(threadId); + this.skillsByThread.delete(threadId); this.messagesByThread.delete(threadId); this.observationLogByScope.delete(threadId); this.cursorsByScope.delete(threadId); diff --git a/packages/@n8n/agents/src/runtime/model/message-list.ts b/packages/@n8n/agents/src/runtime/model/message-list.ts index 303d14082bc..a7f0e7f2806 100644 --- a/packages/@n8n/agents/src/runtime/model/message-list.ts +++ b/packages/@n8n/agents/src/runtime/model/message-list.ts @@ -94,6 +94,8 @@ type MessageSource = 'history' | 'input' | 'response'; * the full three-way source distinction survives a round-trip. */ export class AgentMessageList { + activeSkillIds?: string[]; + private all: AgentDbMessage[] = []; private historySet = new Set(); @@ -414,11 +416,13 @@ export class AgentMessageList { historyIds: toIds(this.historySet), inputIds: toIds(this.inputSet), responseIds: toIds(this.responseSet), + ...(this.activeSkillIds !== undefined ? { activeSkillIds: [...this.activeSkillIds] } : {}), }; } static deserialize(data: SerializedMessageList): AgentMessageList { const list = new AgentMessageList(); + list.activeSkillIds = data.activeSkillIds ? [...data.activeSkillIds] : undefined; const historyIdSet = new Set(data.historyIds); const inputIdSet = new Set(data.inputIds); const responseIdSet = new Set(data.responseIds); diff --git a/packages/@n8n/agents/src/runtime/skills/active-skills.ts b/packages/@n8n/agents/src/runtime/skills/active-skills.ts new file mode 100644 index 00000000000..5f414e290af --- /dev/null +++ b/packages/@n8n/agents/src/runtime/skills/active-skills.ts @@ -0,0 +1,150 @@ +import type { ModelMessage } from 'ai'; + +import { formatActiveSkill } from '../../skills/tools'; +import { + SKILL_LOAD_TOOL_NAME, + type RuntimeSkillContent, + type RuntimeSkillSource, + type RuntimeSkillStateScope, + type RuntimeSkillStateStore, +} from '../../skills/types'; +import type { AgentPersistenceOptions } from '../../types/sdk/agent'; +import type { AgentMessageList } from '../model/message-list'; + +/** Keeps trusted skill instructions outside the conversation's compaction window. */ +export class ActiveSkills { + private readonly loaded = new Map(); + private list?: AgentMessageList; + private scope?: RuntimeSkillStateScope; + private pendingSave = Promise.resolve(); + + constructor( + private readonly source: RuntimeSkillSource, + private readonly agentName: string, + private readonly store?: RuntimeSkillStateStore, + ) {} + + async restore(list: AgentMessageList, persistence?: AgentPersistenceOptions): Promise { + this.loaded.clear(); + this.list = list; + this.scope = persistence + ? { + threadId: persistence.threadId, + resourceId: persistence.resourceId, + agentName: this.agentName, + } + : undefined; + const stored = + list.activeSkillIds === undefined && this.scope + ? await this.store?.load(this.scope) + : undefined; + const ids = list.activeSkillIds ?? stored ?? [...this.recordedLoads(list).values()]; + const registered = new Set(this.source.registry.skills.map(({ id }) => id)); + for (const id of new Set(ids)) { + if (!registered.has(id)) continue; + const skill = await this.source.loadSkill(id); + if (skill) this.loaded.set(id, skill); + } + list.activeSkillIds = [...this.loaded.keys()]; + const storeIsCurrent = + stored !== undefined && + new Set(stored).size === this.loaded.size && + stored.every((id) => this.loaded.has(id)); + if (!storeIsCurrent && (ids.length > 0 || stored !== undefined)) await this.persist(); + } + + async load(skillId: string): Promise { + if (!this.source.registry.skills.some(({ id }) => id === skillId)) return null; + const skill = this.loaded.get(skillId) ?? (await this.source.loadSkill(skillId)); + if (!skill) return null; + if (!this.list) throw new Error('Active skills must be restored before loading'); + if (!this.loaded.has(skillId)) { + this.loaded.set(skillId, skill); + this.list.activeSkillIds = [...this.loaded.keys()]; + await this.persist(); + } + return skill; + } + + instructions(): string | undefined { + if (this.loaded.size === 0) return undefined; + const sections = [...this.loaded].flatMap(([id, skill]) => { + const entry = this.source.registry.skills.find((candidate) => candidate.id === id); + return entry ? [formatActiveSkill(skill, entry)] : []; + }); + return [ + '', + 'Use these current skill versions when relevant to the task. Earlier tool results may describe older versions.', + ...sections, + '', + ].join('\n\n'); + } + + /** Old conversations can contain full skill bodies. Do not replay an obsolete version. */ + modelMessages(messages: ModelMessage[], list: AgentMessageList): ModelMessage[] { + const loads = this.recordedLoads(list); + return messages.map((message) => { + if (message.role !== 'tool') return message; + return { + ...message, + content: message.content.map((part) => { + if (part.type !== 'tool-result') return part; + const skillId = loads.get(part.toolCallId); + if (!skillId) return part; + return { + ...part, + output: { + type: 'json' as const, + value: { skillId, active: this.loaded.has(skillId) }, + }, + }; + }), + }; + }); + } + + private recordedLoads(list: AgentMessageList): Map { + const loads = new Map(); + for (const message of list.messages()) { + if (!('content' in message)) continue; + for (const part of message.content) { + if ( + part.type !== 'tool-call' || + part.toolName !== SKILL_LOAD_TOOL_NAME || + part.state !== 'resolved' || + part.canceled || + !part.input || + typeof part.input !== 'object' || + Array.isArray(part.input) || + !part.output || + typeof part.output !== 'object' || + Array.isArray(part.output) + ) + continue; + if (part.output.success !== true && part.output.type !== 'content') continue; + const { skillId, name, filePath } = part.input; + if (typeof filePath === 'string' && !['', '/', '.', 'SKILL.md'].includes(filePath.trim())) + continue; + const requested = + typeof skillId === 'string' ? skillId : typeof name === 'string' ? name : undefined; + if (!requested) continue; + const entry = this.source.registry.skills.find( + (skill) => skill.id === requested || skill.name === requested, + ); + loads.set(part.toolCallId, entry?.id ?? requested); + } + } + return loads; + } + + private async persist(): Promise { + const scope = this.scope; + const store = this.store; + if (!scope || !store) return; + const ids = [...(this.list?.activeSkillIds ?? [])]; + // Concurrent tool calls must not overwrite a newer activation with an older set. + const save = this.pendingSave.then(async () => await store.save(scope, ids)); + this.pendingSave = save.catch(() => undefined); + await save; + } +} diff --git a/packages/@n8n/agents/src/runtime/tools/tool-adapter.ts b/packages/@n8n/agents/src/runtime/tools/tool-adapter.ts index b86979a5734..6197c7f8538 100644 --- a/packages/@n8n/agents/src/runtime/tools/tool-adapter.ts +++ b/packages/@n8n/agents/src/runtime/tools/tool-adapter.ts @@ -139,6 +139,7 @@ export async function executeTool( toolCallId, toolName: builtTool.name, runId: executionContext.runId, + ...(executionContext.loadSkill ? { loadSkill: executionContext.loadSkill } : {}), persistence: executionContext.persistence, emitEvent: executionContext.emitEvent, abortSignal: executionContext.abortSignal, @@ -155,6 +156,7 @@ export async function executeTool( toolCallId, toolName: builtTool.name, runId: executionContext.runId, + ...(executionContext.loadSkill ? { loadSkill: executionContext.loadSkill } : {}), persistence: executionContext.persistence, emitEvent: executionContext.emitEvent, abortSignal: executionContext.abortSignal, diff --git a/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts b/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts index 89363ee3fcb..b35b3ec3ccc 100644 --- a/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts +++ b/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts @@ -15,6 +15,7 @@ import { import { isAbortError, raceWithAbort } from '../../sdk/abort'; import { isCancellation } from '../../sdk/cancellation'; import { isLlmMessage } from '../../sdk/message'; +import type { RuntimeSkillLoader } from '../../skills/types'; import type { AgentExecutionCounter, BuiltTelemetry, @@ -178,6 +179,7 @@ function getToolResumeJsonSchema( } export interface ToolCallExecutorDeps { + loadSkill?: RuntimeSkillLoader; telemetry: RuntimeTelemetry; eventBus: AgentEventBus; /** Effective tool-call concurrency (default 1 = sequential). */ @@ -980,6 +982,7 @@ export class ToolCallExecutor { await executeTool(input, builtTool, resumeData, resolvedTelemetry, toolCallId, { runId, persistence, + ...(this.deps.loadSkill ? { loadSkill: this.deps.loadSkill } : {}), emitEvent: (event) => this.eventBus.emit(event), abortSignal, executionCounter, diff --git a/packages/@n8n/agents/src/sdk/agent.ts b/packages/@n8n/agents/src/sdk/agent.ts index 45c7a78c0c4..e3cc145151e 100644 --- a/packages/@n8n/agents/src/sdk/agent.ts +++ b/packages/@n8n/agents/src/sdk/agent.ts @@ -1114,6 +1114,7 @@ export class Agent implements BuiltAgent, AgentBuilder { model: modelConfig, ...(this.modelFetchValue !== undefined ? { modelFetch: this.modelFetchValue } : {}), instructions, + ...(this.skillSource ? { skillSource: this.skillSource } : {}), tools: allTools.length > 0 ? allTools : undefined, deferredTools: finalDeferredTools.length > 0 ? finalDeferredTools : undefined, ...(this.workspaceInstance?.filesystem && this.workspaceInstance.filesystem.readOnly !== true diff --git a/packages/@n8n/agents/src/skills/__tests__/runtime-skills.test.ts b/packages/@n8n/agents/src/skills/__tests__/runtime-skills.test.ts index 6a1ad2cd8e6..f2fd14ec223 100644 --- a/packages/@n8n/agents/src/skills/__tests__/runtime-skills.test.ts +++ b/packages/@n8n/agents/src/skills/__tests__/runtime-skills.test.ts @@ -519,6 +519,7 @@ Use the workflow SDK.`, agent as unknown as { build(): Promise } ).build(); const { instructions } = runtimeConfig; + expect(runtimeConfig.skillSource).toBe(source); expect(prepare).toHaveBeenCalledTimes(1); expect(instructions).toContain('name: "Summarize notes"'); diff --git a/packages/@n8n/agents/src/skills/index.ts b/packages/@n8n/agents/src/skills/index.ts index f1fedfce3b5..6bbedc2acb4 100644 --- a/packages/@n8n/agents/src/skills/index.ts +++ b/packages/@n8n/agents/src/skills/index.ts @@ -21,6 +21,8 @@ export type { RuntimeSkillRegistry, RuntimeSkillRegistryEntry, RuntimeSkillSource, + RuntimeSkillStateScope, + RuntimeSkillStateStore, RuntimeSkillValidationError, RuntimeSkillValidationResult, } from './types'; diff --git a/packages/@n8n/agents/src/skills/tools.ts b/packages/@n8n/agents/src/skills/tools.ts index 91e3eb0c549..84432d71926 100644 --- a/packages/@n8n/agents/src/skills/tools.ts +++ b/packages/@n8n/agents/src/skills/tools.ts @@ -10,6 +10,8 @@ import { type RuntimeSkillRegistry, type RuntimeSkillRegistryEntry, type RuntimeSkillSource, + type RuntimeSkillLoader, + type RuntimeSkillContent, } from './types'; const MAX_OUTPUT_BYTES = 64 * 1024; @@ -109,7 +111,10 @@ export function createSkillLoadTool(source: RuntimeSkillSource): BuiltTool { ) .input(skillLoadInputSchema) .output(skillLoadResultSchema) - .handler(async ({ skillId, name }) => await loadSkill(source, { skillId, name })) + .handler( + async ({ skillId, name }, context) => + await loadSkill(source, { skillId, name }, context.loadSkill), + ) .build(); } @@ -121,8 +126,8 @@ export function createSkillLoadTool(source: RuntimeSkillSource): BuiltTool { .input(skillLoadInputWithFilesSchema) .output(skillLoadResultSchema) .handler( - async ({ skillId, name, filePath }) => - await loadSkill(source, { skillId, name, filePath, loadFile }), + async ({ skillId, name, filePath }, context) => + await loadSkill(source, { skillId, name, filePath, loadFile }, context.loadSkill), ) .build(); } @@ -135,6 +140,7 @@ async function loadSkill( filePath?: string; loadFile?: NonNullable; }, + activate?: RuntimeSkillLoader, ): Promise { const { skillId, name, filePath, loadFile } = input; await source.prepare?.(); @@ -205,7 +211,7 @@ async function loadSkill( }; } - const skill = await source.loadSkill(skillEntry.id); + const skill = await (activate ?? source.loadSkill)(skillEntry.id); if (!skill) { return { ok: false, @@ -217,6 +223,26 @@ async function loadSkill( }; } + if (activate) { + return { + success: true, + skillId: skillEntry.id, + name: skillEntry.name, + hash: skillEntry.hash, + content: 'The skill instructions are active. Follow them for this task.', + linkedFiles: skillEntry.linkedFiles, + }; + } + return { + type: 'content', + value: [{ type: 'text', text: formatActiveSkill(skill, skillEntry) }], + }; +} + +export function formatActiveSkill( + skill: RuntimeSkillContent, + skillEntry: RuntimeSkillRegistryEntry, +): string { const content = cap(skill.instructions); const linkedFilePaths = LINKED_FILE_GROUPS.flatMap((group) => skillEntry.linkedFiles[group]).map( (file) => file.path, @@ -229,10 +255,7 @@ async function loadSkill( ] : []), ]; - return { - type: 'content', - value: [{ type: 'text', text: `${header.join('\n')}\n\n${content}` }], - }; + return `${header.join('\n')}\n\n${content}`; } function findSkillEntry( diff --git a/packages/@n8n/agents/src/skills/types.ts b/packages/@n8n/agents/src/skills/types.ts index f817ad9c1c9..2206226ca61 100644 --- a/packages/@n8n/agents/src/skills/types.ts +++ b/packages/@n8n/agents/src/skills/types.ts @@ -155,6 +155,18 @@ export interface RuntimeSkillSource { loadFile?: RuntimeSkillFileLoader; } +export interface RuntimeSkillStateScope { + threadId: string; + resourceId: string; + agentName: string; +} + +/** Stores active IDs separately from conversation text that memory can compact. */ +export interface RuntimeSkillStateStore { + load(scope: RuntimeSkillStateScope): Promise; + save(scope: RuntimeSkillStateScope, skillIds: string[]): Promise; +} + export interface RuntimeSkillValidationError { code: string; message: string; diff --git a/packages/@n8n/agents/src/types/runtime/message-list.ts b/packages/@n8n/agents/src/types/runtime/message-list.ts index 8e454a5522d..1f0e123981d 100644 --- a/packages/@n8n/agents/src/types/runtime/message-list.ts +++ b/packages/@n8n/agents/src/types/runtime/message-list.ts @@ -5,4 +5,6 @@ export interface SerializedMessageList { historyIds: string[]; inputIds: string[]; responseIds: string[]; + /** IDs only. Resumed runs load their content from the current skill catalog. */ + activeSkillIds?: string[]; } diff --git a/packages/@n8n/agents/src/types/sdk/memory.ts b/packages/@n8n/agents/src/types/sdk/memory.ts index 2cb0c4d4d4b..d216d127ce9 100644 --- a/packages/@n8n/agents/src/types/sdk/memory.ts +++ b/packages/@n8n/agents/src/types/sdk/memory.ts @@ -9,6 +9,7 @@ import type { ObservationLogReflectFn, ObservationLogScope, } from './observation-log'; +import type { RuntimeSkillStateStore } from '../../skills/types'; import type { JSONObject } from '../utils/json'; /** @@ -34,6 +35,8 @@ export interface Thread { } export interface BuiltMemory { + /** Retains active skills across separate runs, including compacted conversations. */ + skillState?: RuntimeSkillStateStore; // --- Thread management --- getThread(threadId: string): Promise; saveThread(thread: Omit): Promise; diff --git a/packages/@n8n/agents/src/types/sdk/tool.ts b/packages/@n8n/agents/src/types/sdk/tool.ts index 9d003058185..618b7284060 100644 --- a/packages/@n8n/agents/src/types/sdk/tool.ts +++ b/packages/@n8n/agents/src/types/sdk/tool.ts @@ -3,6 +3,7 @@ import type { ZodType } from 'zod'; import type { AgentExecutionCounter } from './agent'; import type { AgentMessage } from './message'; +import type { RuntimeSkillLoader } from '../../skills/types'; import type { AgentEventData } from '../runtime/event'; import type { BuiltTelemetry } from '../telemetry'; import type { JSONObject, JSONValue } from '../utils/json'; @@ -15,6 +16,8 @@ export interface ToolSuspendOptions { } export interface ToolExecutionContext { + /** Load and retain instructions from this run's selected skill catalog. */ + loadSkill?: RuntimeSkillLoader; /** Agent run ID for the current execution. */ runId?: string; /** @@ -52,6 +55,7 @@ export interface ToolExecutionContext { } export interface ToolContext { + loadSkill?: ToolExecutionContext['loadSkill']; /** AI SDK tool call ID for the current local tool execution. */ toolCallId?: string; /** Exact model-facing name of the tool being executed. */ @@ -71,6 +75,7 @@ export interface ToolContext { } export interface InterruptibleToolContext { + loadSkill?: ToolExecutionContext['loadSkill']; /** * Suspend execution and send a payload to the consumer. * Must be used with `return await` — the branded return type signals diff --git a/packages/@n8n/api-types/src/index.ts b/packages/@n8n/api-types/src/index.ts index bbe3bc05244..d60658a066d 100644 --- a/packages/@n8n/api-types/src/index.ts +++ b/packages/@n8n/api-types/src/index.ts @@ -368,6 +368,8 @@ export { CANVAS_NODE_CONTEXT_FLAG, INSTANCE_AI_CONVERSATION_HISTORY_FLAG, INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, + INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, + INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, INSTANCE_AI_NODE_USAGE_FLAG, INSTANCE_AI_FOLDER_EXPLORATION_FLAG, domainAccessActionSchema, @@ -437,10 +439,14 @@ export { deriveInstanceAiSetupState, INSTANCE_AI_THREAD_SOURCES, INSTANCE_AI_THREAD_SOURCE_FALLBACK, + instanceAiBuildModeSchema, + instanceAiPromptConfigurationSchema, INSTANCE_AI_RUN_LIMIT_REASONS, } from './schemas/instance-ai.schema'; export type { + InstanceAiBuildMode, + InstanceAiPromptConfiguration, InstanceAiThreadSource, InstanceAiThreadSourcePersisted, InstanceAiThreadOrigin, diff --git a/packages/@n8n/api-types/src/schemas/__tests__/instance-ai.schema.test.ts b/packages/@n8n/api-types/src/schemas/__tests__/instance-ai.schema.test.ts index b2141223172..c864ef4eac8 100644 --- a/packages/@n8n/api-types/src/schemas/__tests__/instance-ai.schema.test.ts +++ b/packages/@n8n/api-types/src/schemas/__tests__/instance-ai.schema.test.ts @@ -21,6 +21,7 @@ import { errorPayloadSchema, FETCH_URL_ALLOW_ALL_GRANT_KEY, InstanceAiAdminSettingsUpdateRequest, + InstanceAiSendMessageRequest, instanceAiEventSchema, INSTANCE_AI_EPHEMERAL_EVENT_TYPES, isDisplayableConfirmationRequest, @@ -46,6 +47,22 @@ import { type InstanceAiPermissions, } from '../instance-ai.schema'; +describe('Instance AI prompt version requests', () => { + it('accepts an optional version pin and rejects empty or oversized pins', () => { + const base = { message: 'Build a workflow', timeZone: 'UTC' }; + expect(InstanceAiSendMessageRequest.safeParse(base).success).toBe(true); + expect( + InstanceAiSendMessageRequest.parse({ ...base, promptVersion: ' progressive@1 ' }) + .promptVersion, + ).toBe('progressive@1'); + for (const promptVersion of ['', ' ', 'x'.repeat(129)]) { + expect(InstanceAiSendMessageRequest.safeParse({ ...base, promptVersion }).success).toBe( + false, + ); + } + }); +}); + describe('sandbox provider', () => { it('accepts supported providers', () => { expect(isInstanceAiSandboxProvider('n8n-sandbox')).toBe(true); diff --git a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts index d0bee9d4b12..8b69150c505 100644 --- a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts +++ b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts @@ -1390,12 +1390,33 @@ export const instanceAiHandoffContextSchema = z.discriminatedUnion('source', [ ]); export type InstanceAiHandoffContext = z.infer; +/** + * Build style for a run. `progressive` makes the agent build a minimal working + * slice first, gate increments on real executions, and extend on actual + * execution data. `default` uses the standard building policy. + */ +export const instanceAiBuildModeSchema = z.enum(['default', 'progressive']); +export type InstanceAiBuildMode = z.infer; + +export const instanceAiPromptConfigurationSchema = z.object({ + version: z.string(), + systemPromptVersion: z.string(), + skillVariants: z.array(z.string()), + skillsHash: z.string(), + fallbackFrom: z.string().optional(), +}); +export type InstanceAiPromptConfiguration = z.infer; + export class InstanceAiSendMessageRequest extends Z.class({ message: z.string().default(''), attachments: z.array(instanceAiAttachmentSchema).max(10).optional(), context: instanceAiHandoffContextSchema.optional(), timeZone: TimeZoneSchema, pushRef: z.string().optional(), + /** Explicit override for evals. Omit to use the backend experiment assignment. */ + mode: instanceAiBuildModeSchema.optional(), + /** Pin a published prompt profile. Takes precedence over mode. */ + promptVersion: z.string().trim().min(1).max(128).optional(), }) {} export class InstanceAiCorrectTaskRequest extends Z.class({ @@ -1765,6 +1786,7 @@ export interface InstanceAiMemoryTaskSnapshot { } export interface InstanceAiThreadStatusResponse { + promptConfiguration?: InstanceAiPromptConfiguration; hasActiveRun: boolean; isSuspended: boolean; runId?: string; @@ -2288,6 +2310,10 @@ export const CANVAS_NODE_CONTEXT_FLAG = '104_canvas_aia_node_context'; export const INSTANCE_AI_CONVERSATION_HISTORY_FLAG = '109_instance_ai_conversation_history'; export const INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT = 'variant'; + +export const INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG = '111_instance_ai_progressive_building'; +export const INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT = 'variant'; + /** Enables the node-usage context surface for Instance AI: the `node-usage` * action and the `nodeTypes` filter on `workflows(action="list")`. */ diff --git a/packages/@n8n/instance-ai/docs/prompt-profiles.md b/packages/@n8n/instance-ai/docs/prompt-profiles.md new file mode 100644 index 00000000000..b3d1f9a5f54 --- /dev/null +++ b/packages/@n8n/instance-ai/docs/prompt-profiles.md @@ -0,0 +1,111 @@ +# Prompt profiles + +A prompt profile selects a system prompt version, skill variants, and tool +exclusions together. Definitions live in `src/prompts/prompt-profiles.ts`. +The backend resolves the profile before creating the agent. Prompt rendering +does not read feature flags. + +The initial profiles are `default@1` and `progressive@1`. The general system +prompt is the same in both. The progressive profile changes the workflow +skills and removes the planning skill and `create-tasks` tool. + +## Selection and recovery + +A request's `promptVersion` takes precedence over `mode`. Without a version pin, +an explicit mode selects its current profile. Without either override, the +backend experiment assignment selects the profile. + +Internal follow-ups retain the selected version. Checkpoints store it in +`persistence.hostMetadata.promptVersion`. If that version is no longer registered +after deployment, recovery selects the default profile and records `fallbackFrom`. +Legacy checkpoints use their saved mode, or default when no valid mode exists. +An unknown explicit request pin is rejected instead of silently falling back. + +## Define a variant + +1. Put reusable instructions in a skill fragment. +2. Add a versioned `SkillVariant` with its target skill IDs and fragment IDs. +3. Declare required tools in the fragment's `dependencies.tools`. The composer + rejects a profile that disables a required tool. Recommended tools are optional; + disabled recommendations are removed from the selected catalog. +4. Add the variant to a profile. Put skill and tool exclusions in that same variant. +5. Add selection and behavior tests. Include existing-resource operations when + changing workflow-building behavior. + +Different variants may change different skills. Duplicate variants, overlapping +skill changes, and changes to excluded skills are rejected. Use an explicit +combined variant when two policies must change the same skill. Fragment order +within a skill is its original body followed by the selected fragment. +Fragments do not appear as standalone catalog entries. + +Published version IDs identify immutable definitions. Add a new version when +changing a published policy or system template. Keep the old renderer and fragments +if old runs must remain reproducible. Otherwise remove the old profile and use the +documented default fallback. Content hashes identify the concrete instructions +in addition to their version IDs. + +## Inspect a profile + +From this package: + +```sh +pnpm prompts:print --profile default@1 +pnpm prompts:print --profile progressive@1 +``` + +The output contains the system prompt, selected skill bodies, and a manifest. +This shows the profile before user-specific feature gates and context are applied. + +Include `promptVersion` in a chat request to select an exact profile. Send the +pin with each new user message. Internal follow-ups retain the selected version. + +Traces include `prompt_configuration` and the rendered `system_prompt_hash`. +Thread status includes `promptConfiguration` with the selected version and skill +hash. Both include fallback information when available. + +Trace metadata also includes `prompt_version`, such as `progressive@1`, for +filtering. The same property is sent with `instance_ai_run_finished`, +`Builder sent message`, `Builder satisfied user intent`, and +`Builder generation errored`. It identifies the resolved profile for that run, +including after a resume. Events omit it when no resolved profile is available. + +## Evaluate profiles + +Set `promptVersion` in a local eval case, or set `N8N_EVAL_PROMPT_VERSION` for a +suite. Case version pins take precedence. An explicit case mode also overrides +suite settings. The runner sends the selected pin with every user message and +rejects unknown versions before provisioning resources. + +Eval rows carry the backend's selected version and skill hash in +`run.outputs.buildTrace.promptConfiguration` when the status lookup succeeds. +Keep pinned case files local while the LangTracer case-write contract does not +carry their version field. Existing suites can use the runner's environment +override without changing their stored cases. + +## Active skills and compaction + +The agents SDK retains loaded skill IDs outside the conversation text. It renders +their current bodies in a cached instruction block, so memory compaction can +remove large tool results without removing the active instructions. + +Checkpoints retain the IDs. Memory backends can implement `BuiltMemory.skillState` +to retain them across separate runs. The in-memory backend and Instance AI's +database adapter provide this store. It is scoped by thread, resource, and agent. +Custom memory backends need this optional store for cross-run retention after +compaction. Resumes always load content from the selected catalog; removed skills +are dropped, and a changed version replaces the old body. + +Tools that inline skill guidance should use `ToolContext.loadSkill`. This activates +the selected skill and avoids persisting another copy of its instructions in a +tool result. Ordinary tool output remains untrusted data for memory extraction. + +## Roll back or remove a profile + +Switch the assignment to default for new user messages. Internal follow-ups keep +their selected version until that user turn ends. New turns resolve the assignment +again, and active IDs load their bodies from the newly selected catalog. + +To retire a version, remove its profile entry after deciding whether existing +checkpoints may use the default fallback. Remove unreferenced variants and fragments +and update the mode-to-profile defaults. Run the selection, recovery, and behavior +tests. Do not leave profile-specific conditions in the general system prompt. diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/build-mode.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/build-mode.test.ts new file mode 100644 index 00000000000..ba4d191845f --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/build-mode.test.ts @@ -0,0 +1,89 @@ +import { mock } from 'vitest-mock-extended'; + +import type { CliArgs } from '../cli/args'; +import type { WorkflowTestCaseWithFile } from '../data/workflows'; +import { resolveEvalBuildMode, resolveEvalPromptSettings } from '../harness/build-mode'; +import type { EvalLogger } from '../harness/logger'; +import { selectCases } from '../run/case-selection'; + +const args = mock({ buildViaMcp: false, prebuiltWorkflows: undefined }); +const logger = mock(); +const entry = (buildMode?: 'progressive' | 'default'): WorkflowTestCaseWithFile => ({ + fileSlug: 'test-case', + testCase: { + complexity: 'simple', + tags: [], + datasets: ['full'], + buildMode, + conversation: [{ role: 'user', text: 'Build a workflow' }], + }, +}); + +describe('eval build mode', () => { + beforeEach(() => vi.stubEnv('N8N_EVAL_PROMPT_VERSION', undefined)); + afterEach(() => vi.unstubAllEnvs()); + it('uses control unless the suite explicitly selects progressive', () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', ''); + expect(resolveEvalBuildMode(undefined)).toBe('default'); + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'progressive'); + expect(resolveEvalBuildMode(undefined)).toBe('progressive'); + }); + it('preserves case overrides in both suite modes', () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'progressive'); + expect(resolveEvalBuildMode('default')).toBe('default'); + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'default'); + expect(resolveEvalBuildMode('progressive')).toBe('progressive'); + }); + + it.each([undefined, '', ' ', 'default', ' default '])('uses control for %s', (value) => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', value); + expect(resolveEvalBuildMode(undefined)).toBe('default'); + }); + + it('rejects an invalid environment mode when a selected case needs it', () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'progresssive'); + expect(() => selectCases(args, [entry()], logger)).toThrow('N8N_EVAL_BUILD_MODE'); + }); + + it('preserves explicit case overrides when the environment value is invalid', () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'progresssive'); + expect(resolveEvalBuildMode('default')).toBe('default'); + expect(resolveEvalBuildMode('progressive')).toBe('progressive'); + expect( + selectCases(args, [entry('default'), entry('progressive')], logger).testCasesWithFiles, + ).toHaveLength(2); + }); + + it('ignores the orchestrator mode for MCP builds', () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'progresssive'); + expect( + selectCases({ ...args, buildViaMcp: true }, [entry()], logger).testCasesWithFiles, + ).toHaveLength(1); + }); + + it('pins a case version before suite settings and mode', () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'invalid'); + vi.stubEnv('N8N_EVAL_PROMPT_VERSION', 'default@1'); + expect( + resolveEvalPromptSettings({ promptVersion: 'progressive@1', buildMode: 'default' }), + ).toEqual({ + promptVersion: 'progressive@1', + buildMode: 'progressive', + }); + }); + + it('uses suite versions only for cases without explicit overrides', () => { + vi.stubEnv('N8N_EVAL_PROMPT_VERSION', 'progressive@1'); + expect(resolveEvalPromptSettings({}).promptVersion).toBe('progressive@1'); + expect(resolveEvalPromptSettings({ buildMode: 'default' })).toEqual({ + buildMode: 'default', + promptVersion: undefined, + }); + }); + + it('rejects an unavailable pinned version before running cases', () => { + const pinned = entry(); + pinned.testCase.promptVersion = 'missing@1'; + expect(() => selectCases(args, [pinned], logger)).toThrow('Unknown Instance AI prompt version'); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts index b03a135a7b7..3ff3982bcc3 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/build-workflow-conversation-seed.test.ts @@ -1,8 +1,8 @@ import { vi } from 'vitest'; import type { N8nClient } from '../clients/n8n-client'; -import { buildWorkflow } from '../harness/build-workflow'; -import { recordUserTurn } from '../harness/chat-loop'; +import { buildWorkflow, buildFailedOnInfra } from '../harness/build-workflow'; +import { recordUserTurn, runMultiTurnConversation } from '../harness/chat-loop'; import type { ConversationSeed } from '../harness/conversation-seed'; import type { EvalLogger } from '../harness/logger'; @@ -452,6 +452,7 @@ describe('buildWorkflow with an inline seed', () => { }); describe('buildWorkflow with scenario seed data tables', () => { + afterEach(() => vi.restoreAllMocks()); const jobApplications = { id: 'job-applications-1234', name: 'Job Applications', @@ -459,6 +460,64 @@ describe('buildWorkflow with scenario seed data tables', () => { rows: [{ application_id: 'row_001' }], }; + it.each([false, true])( + 'classifies input reseeding failures (case expired=%s)', + async (expired) => { + let now = 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const client = makeClient( + vi.fn().mockResolvedValue({ + restored: 0, + workflowIds: [], + dataTableIds: ['dt-real-1'], + agentIds: [], + }), + ); + client.seedDataTableRows = vi + .fn() + .mockRejectedValue(new Error('Input rows could not be seeded')); + vi.mocked(runMultiTurnConversation).mockImplementationOnce(async (config) => { + if (!config.beforeUserExecution) throw new Error('Missing input preparation'); + const deadline = config.startTime + config.timeoutMs; + if (expired) now = deadline; + await config.beforeUserExecution(deadline); + }); + const result = await buildWorkflow({ + ...baseConfig, + client, + allowUserExecution: true, + conversation: [ + { role: 'user', text: 'Build a contact log' }, + { role: 'user', text: 'I ran it' }, + ], + executionScenarios: [ + { + name: 'inputs', + description: '', + dataSetup: '', + successCriteria: '', + seedDataTables: [jobApplications], + }, + ], + }); + expect(result.success).toBe(false); + expect(result.seedingFailed).toBe(!expired); + expect(buildFailedOnInfra(result)).toBe(!expired); + expect(result.error).toContain(expired ? 'Case timed out' : 'Input rows could not be seeded'); + }, + ); + + it('rejects an invalid mode before creating a thread', async () => { + vi.stubEnv('N8N_EVAL_BUILD_MODE', 'progresssive'); + try { + const client = makeClient(vi.fn()); + await expect(buildWorkflow({ ...baseConfig, client })).rejects.toThrow('N8N_EVAL_BUILD_MODE'); + expect(client.ensureThread).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + } + }); + it('creates the table under a per-run name and tells the agent THAT name', async () => { // The name is project-unique, so a per-run suffix is what lets two iterations of // one case coexist. The agent binds by discovery, so it has to be told the real diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/event-parser.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/event-parser.test.ts index 6c3a6c82bd4..45697b1de59 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/event-parser.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/event-parser.test.ts @@ -4,9 +4,55 @@ import { extractOutcomeFromEvents, mergeSeededConversationMetrics, seededTurnCounters, + savedWorkflowsFromEvents, + lastSavedWorkflowIdFromEvents, } from '../outcome/event-parser'; import type { CapturedEvent, ConversationMetrics, TranscriptTurn } from '../types'; +describe('saved workflow names', () => { + function saved(id: string, workflowName?: string, success = true): CapturedEvent { + return { + timestamp: 0, + type: 'tool-result', + data: { + payload: { + toolName: 'build-workflow', + result: { success, workflowId: id, workflowName }, + }, + }, + }; + } + + it.each([undefined, '', ' '])('preserves the last name when a later save has name %s', (name) => { + const events = [saved('a', 'Contact log'), saved('b', 'Helper'), saved('a', name)]; + expect(savedWorkflowsFromEvents(events)).toEqual([ + { id: 'a', name: 'Contact log' }, + { id: 'b', name: 'Helper' }, + { id: 'a', name: 'Contact log' }, + ]); + expect(lastSavedWorkflowIdFromEvents(events)).toBe('a'); + }); + + it('updates names only after successful saves and falls back to IDs for unknown names', () => { + expect( + savedWorkflowsFromEvents([ + saved('a'), + saved('a', 'Original'), + saved('a', 'Rejected', false), + saved('a'), + saved('a', 'Renamed'), + saved('a'), + ]), + ).toEqual([ + { id: 'a', name: 'a' }, + { id: 'a', name: 'Original' }, + { id: 'a', name: 'Original' }, + { id: 'a', name: 'Renamed' }, + { id: 'a', name: 'Renamed' }, + ]); + }); +}); + // --------------------------------------------------------------------------- // extractOutcomeFromEvents // --------------------------------------------------------------------------- diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts index 7bde969e57c..100a75c596d 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts @@ -107,6 +107,18 @@ describe('diskCaseToLangTracerCreate', () => { }); describe('unsupportedPushReason', () => { + it('refuses a case whose prompt version would be lost', () => { + expect(unsupportedPushReason(diskCase({ promptVersion: 'progressive@1' }))).toContain( + 'promptVersion', + ); + }); + it.each(['default', 'progressive'] as const)( + 'refuses a case whose %s mode would be lost', + (buildMode) => { + expect(unsupportedPushReason(diskCase({ buildMode }))).toContain('buildMode'); + }, + ); + it('returns null for a plain conversation-driven case', () => { expect(unsupportedPushReason(diskCase())).toBeNull(); }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts index ec45a47f642..2af6ec6ec73 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts @@ -112,6 +112,22 @@ describe('uniqueProjectScopes', () => { }); describe('unsupportedMcpBuildSetupFields', () => { + it.each([undefined, false])('allows user execution setting %s', (allowUserExecution) => { + expect(unsupportedMcpBuildSetupFields(testCase({ allowUserExecution }))).toEqual([]); + }); + + it('rejects cases that enable user execution', () => { + expect(unsupportedMcpBuildSetupFields(testCase({ allowUserExecution: true }))).toEqual([ + 'allowUserExecution', + ]); + }); + + it.each(['default', 'progressive'] as const)( + 'keeps explicit mode %s unsupported', + (buildMode) => { + expect(unsupportedMcpBuildSetupFields(testCase({ buildMode }))).toEqual(['buildMode']); + }, + ); it('classifies every test-case schema key, so adding a field forces a decision', () => { // MCP_BUILD_KEY_SUPPORT must stay in lockstep with the case schema: a new // build-side setup field left unclassified would let --build-via-mcp build diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/scenario-seed-tables.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/scenario-seed-tables.test.ts index 4faa86ca7af..38f61c0f1e6 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/scenario-seed-tables.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/scenario-seed-tables.test.ts @@ -148,6 +148,49 @@ function makeClient(seedDataTableRows: Mock): N8nClient { } describe('reseedScenarioTables', () => { + afterEach(() => vi.restoreAllMocks()); + + it('reduces the next request budget by the time spent seeding earlier tables', async () => { + let now = 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const seedDataTableRows = vi.fn(async () => { + now += 600; + await Promise.resolve(); + }); + + await reseedScenarioTables( + makeClient(seedDataTableRows), + scenario({ seedDataTables: [jobApplications, { ...jobApplications, name: 'Second' }] }), + 'thread-1', + { 'Job Applications': 'dt-real-1', Second: 'dt-real-2' }, + silentLogger, + 2_000, + ); + + expect(seedDataTableRows.mock.calls.map((call) => call[3])).toEqual([1_000, 400]); + }); + + it('does not seed the next table when the previous request consumes the deadline', async () => { + let now = 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const seedDataTableRows = vi.fn(async () => { + now = 2_000; + await Promise.resolve(); + }); + + await expect( + reseedScenarioTables( + makeClient(seedDataTableRows), + scenario({ seedDataTables: [jobApplications, { ...jobApplications, name: 'Second' }] }), + 'thread-1', + { 'Job Applications': 'dt-real-1', Second: 'dt-real-2' }, + silentLogger, + 2_000, + ), + ).rejects.toThrow('Case timed out'); + expect(seedDataTableRows).toHaveBeenCalledTimes(1); + }); + it('clears + seeds each declared table by its bound real id', async () => { const seedDataTableRows = vi.fn().mockResolvedValue(undefined); const client = makeClient(seedDataTableRows); @@ -160,7 +203,12 @@ describe('reseedScenarioTables', () => { silentLogger, ); - expect(seedDataTableRows).toHaveBeenCalledWith('thread-1', 'dt-real-1', jobApplications.rows); + expect(seedDataTableRows).toHaveBeenCalledWith( + 'thread-1', + 'dt-real-1', + jobApplications.rows, + undefined, + ); }); it('seeds an empty row set when a table declares no rows', async () => { @@ -176,7 +224,7 @@ describe('reseedScenarioTables', () => { silentLogger, ); - expect(seedDataTableRows).toHaveBeenCalledWith('thread-1', 'dt-real-1', []); + expect(seedDataTableRows).toHaveBeenCalledWith('thread-1', 'dt-real-1', [], undefined); }); it('does nothing when the scenario declares no seed tables', async () => { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/user-execution.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/user-execution.test.ts new file mode 100644 index 00000000000..ef2f3f577b6 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/user-execution.test.ts @@ -0,0 +1,238 @@ +import { mock } from 'vitest-mock-extended'; + +import { N8nClient, type WorkflowResponse } from '../clients/n8n-client'; +import { runMultiTurnConversation } from '../harness/chat-loop'; +import type { EvalLogger } from '../harness/logger'; +import { reseedScenarioTables } from '../harness/seed-tables'; +import type { CapturedEvent } from '../types'; + +const client = mock(); +const logger = mock(); +const workflow: WorkflowResponse = { + id: 'primary', + name: 'Primary', + active: false, + versionId: 'v1', + connections: {}, + nodes: [{ id: 'trigger', name: 'Start', type: 'n8n-nodes-base.manualTrigger' }], +}; + +function saved(workflowId: string, success = true): CapturedEvent { + return { + timestamp: 0, + type: 'tool-result', + data: { + payload: { + toolCallId: workflowId, + toolName: 'build-workflow', + result: { workflowId, success }, + }, + }, + }; +} + +async function run( + runWorkflowId = 'primary', + events: CapturedEvent[] = [saved('primary'), saved('helper')], + beforeUserExecution?: (deadline: number) => Promise, +) { + const decisions = [ + { kind: 'followUp', message: 'I ran it', runWorkflowId } as const, + { kind: 'done' } as const, + ]; + await runMultiTurnConversation({ + client, + logger, + threadId: 'thread', + buildMode: 'progressive', + promptVersion: 'progressive@1', + allowUserExecution: true, + beforeUserExecution, + events: [ + { timestamp: 0, type: 'run-start', data: { type: 'run-start' } }, + { timestamp: 0, type: 'run-finish', data: { type: 'run-finish' } }, + ...events, + ], + approvedRequests: new Set(), + startTime: Date.now(), + timeoutMs: 10_000, + nextMessageDecider: async () => await Promise.resolve(decisions.shift() ?? { kind: 'done' }), + }); +} + +describe('user execution during a conversation', () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + beforeEach(() => { + vi.clearAllMocks(); + client.getThreadStatus.mockResolvedValue({ backgroundTasks: [] } as never); + client.getWorkflow.mockResolvedValue(workflow); + client.executeWorkflow.mockResolvedValue({ executionId: 'execution' }); + client.getExecution.mockResolvedValue({ + id: 'execution', + workflowId: 'primary', + status: 'success', + data: '', + }); + }); + + it('runs the selected primary workflow before the message, even after a helper was saved', async () => { + await run(); + expect(client.executeWorkflow).toHaveBeenCalledWith('primary', 'Start', expect.any(Number)); + expect(client.executeWithLlmMock).not.toHaveBeenCalled(); + expect(client.getExecution.mock.invocationCallOrder[0]).toBeLessThan( + client.sendMessage.mock.invocationCallOrder[0], + ); + expect(client.sendMessage).toHaveBeenCalledWith( + 'thread', + 'I ran it', + undefined, + 'progressive', + 'progressive@1', + ); + }); + + it.each([{ events: [] }, { events: [saved('primary', false)] }])( + 'rejects a target without a successful save', + async ({ events }) => { + await expect(run('primary', events)).rejects.toThrow('was not saved'); + expect(client.executeWorkflow).not.toHaveBeenCalled(); + expect(client.sendMessage).not.toHaveBeenCalled(); + }, + ); + + it('rejects pinned data instead of presenting it as a live run', async () => { + client.getWorkflow.mockResolvedValue({ ...workflow, pinData: { Start: [{ json: {} }] } }); + await expect(run()).rejects.toThrow('without pinned data'); + expect(client.executeWorkflow).not.toHaveBeenCalled(); + }); + + it('rejects credentialed workflows in the live eval action', async () => { + client.getWorkflow.mockResolvedValue({ + ...workflow, + nodes: [ + ...workflow.nodes, + { + id: 'slack', + name: 'Slack', + type: 'n8n-nodes-base.slack', + credentials: { slackApi: { id: 'credential' } }, + }, + ], + }); + await expect(run()).rejects.toThrow('without credentials'); + }); + + it.each(['new', 'running', 'waiting', 'unknown'] as const)( + 'waits through %s and preserves failures for inspection', + async (status) => { + client.getExecution.mockResolvedValueOnce({ + id: 'execution', + workflowId: 'primary', + status, + data: '', + }); + client.getExecution.mockResolvedValueOnce({ + id: 'execution', + workflowId: 'primary', + status: 'error', + data: '', + }); + await run(); + expect(client.getExecution).toHaveBeenCalledTimes(2); + expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('status=error')); + expect(client.sendMessage).toHaveBeenCalledTimes(1); + }, + ); + + it('stops an indeterminate execution when the case deadline expires', async () => { + let now = 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + client.getExecution.mockImplementationOnce(async () => { + now += 10_000; + await Promise.resolve(); + return { id: 'execution', workflowId: 'primary', status: 'unknown', data: '' }; + }); + client.stopExecution.mockResolvedValueOnce(undefined); + await expect(run()).rejects.toThrow('Case timed out'); + expect(client.stopExecution).toHaveBeenCalledWith('execution'); + expect(client.sendMessage).not.toHaveBeenCalled(); + }); + + it('bounds each execution request by the remaining case time', async () => { + await run(); + const [, , timeoutMs] = client.executeWorkflow.mock.calls[0]; + expect(timeoutMs).toBeGreaterThan(0); + expect(timeoutMs).toBeLessThan(10_000); + }); + + it('prepares the declared data before executing the workflow', async () => { + const prepare = vi.fn().mockResolvedValue(undefined); + await run('primary', [saved('primary')], prepare); + expect(prepare).toHaveBeenCalledTimes(1); + expect(prepare.mock.invocationCallOrder[0]).toBeLessThan( + client.executeWorkflow.mock.invocationCallOrder[0], + ); + }); + + it('stops the execution if waiting fails', async () => { + client.getExecution.mockRejectedValueOnce(new Error('Request timed out')); + client.stopExecution.mockResolvedValueOnce(undefined); + await expect(run()).rejects.toThrow('Request timed out'); + expect(client.stopExecution).toHaveBeenCalledWith('execution'); + expect(client.sendMessage).not.toHaveBeenCalled(); + }); + + it('aborts input seeding at the case deadline before starting another table or workflow', async () => { + let now = 1_000; + vi.spyOn(Date, 'now').mockImplementation(() => now); + const fetchMock = vi.fn(async (_url: string, options: RequestInit) => { + return await new Promise((_resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Request exceeded test allowance')), 1_000); + options.signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + const reason: unknown = options.signal?.reason; + reject(reason instanceof Error ? reason : new Error(String(reason))); + }, + { once: true }, + ); + }); + }); + vi.stubGlobal('fetch', fetchMock); + const seedClient = new N8nClient('http://n8n.test'); + const prepare = async (deadline: number) => { + now += 9_800; + await reseedScenarioTables( + seedClient, + { + name: 'inputs', + description: '', + dataSetup: '', + successCriteria: '', + seedDataTables: [ + { id: 'first-table', name: 'first', columns: [] }, + { id: 'second-table', name: 'second', columns: [] }, + ], + }, + 'thread', + { first: 'first-table', second: 'second-table' }, + logger, + deadline, + ); + }; + + await expect(run('primary', [saved('primary')], prepare)).rejects.toMatchObject({ + name: 'TimeoutError', + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(fetchMock.mock.calls[0][1].signal?.aborted).toBe(true); + expect(client.getWorkflow).not.toHaveBeenCalled(); + expect(client.executeWorkflow).not.toHaveBeenCalled(); + expect(client.sendMessage).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts index f1983aca11b..f687fc7a7c4 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/user-proxy.test.ts @@ -16,7 +16,8 @@ import { UserProxyLlm } from '../utils/user-proxy'; import type { UserProxyAgent } from '../utils/user-proxy/agent'; import { confirmationDecisionSchema, - userTurnDecisionSchema, + createUserTurnDecisionSchema, + userTurnWithoutExecutionSchema, type Decision, type ProxyDecisionMode, } from '../utils/user-proxy/tools'; @@ -75,6 +76,7 @@ function fakeCredentialClient( class FakeAgent implements UserProxyAgent { readonly prompts: string[] = []; readonly modes: ProxyDecisionMode[] = []; + readonly savedWorkflowIds: string[][] = []; private queue: Array = []; enqueue(...decisions: Array): void { @@ -82,9 +84,14 @@ class FakeAgent implements UserProxyAgent { } // eslint-disable-next-line @typescript-eslint/require-await - async decide(userPrompt: string, mode: ProxyDecisionMode): Promise { + async decide( + userPrompt: string, + mode: ProxyDecisionMode, + savedWorkflowIds: string[] = [], + ): Promise { this.prompts.push(userPrompt); this.modes.push(mode); + this.savedWorkflowIds.push(savedWorkflowIds); const next = this.queue.shift(); if (next instanceof Error) throw next; return next; @@ -1835,6 +1842,37 @@ describe('UserProxyLlm.respondToConfirmation', () => { // --------------------------------------------------------------------------- describe('UserProxyLlm.decideFollowUp', () => { + it('shows saved workflow IDs and names for user executions', async () => { + const agent = new FakeAgent(); + agent.enqueue({ action: 'declare_done' }); + const proxy = new UserProxyLlm({ + conversation: [{ role: 'user', text: 'Build a contact log' }], + allowUserExecution: true, + agent, + }); + proxy.ingestEvents([ + { + timestamp: 0, + type: 'tool-result', + data: { + payload: { + toolCallId: 'build', + toolName: 'build-workflow', + result: { + success: true, + workflowId: 'wf-primary', + workflowName: 'Contact log', + }, + }, + }, + }, + ]); + await proxy.decideFollowUp(); + expect(agent.prompts[0]).toContain('wf-primary'); + expect(agent.prompts[0]).toContain('Contact log'); + expect(agent.savedWorkflowIds[0]).toEqual(['wf-primary']); + }); + it('returns done immediately when messageBudget is 0 without invoking the agent', async () => { const agent = new FakeAgent(); const proxy = new UserProxyLlm({ @@ -1992,6 +2030,31 @@ describe('UserProxyLlm.decideFollowUp', () => { // --------------------------------------------------------------------------- describe('mode-scoped decision schemas', () => { + const userTurnDecisionSchema = createUserTurnDecisionSchema(['workflow']); + it('restricts executions to saved IDs and omits the field without candidates', () => { + const schema = createUserTurnDecisionSchema(['primary-id', 'helper-id']); + const decision = { action: 'send_follow_up_message', message: 'I ran it' }; + for (const runWorkflowId of ['primary-id', 'helper-id']) { + expect(schema.safeParse({ ...decision, runWorkflowId }).success).toBe(true); + } + for (const runWorkflowId of ['', 'Contact log', 'unknown-id']) { + expect(schema.safeParse({ ...decision, runWorkflowId }).success).toBe(false); + } + const emptySchema = createUserTurnDecisionSchema([]); + expect(emptySchema.safeParse({ ...decision, runWorkflowId: 'primary-id' }).success).toBe(false); + expect(emptySchema.safeParse(decision).success).toBe(true); + }); + + it('excludes user executions unless the case enables them', () => { + const decision = { + action: 'send_follow_up_message', + message: 'I ran it', + runWorkflowId: 'workflow', + }; + expect(userTurnWithoutExecutionSchema.safeParse(decision).success).toBe(false); + expect(userTurnDecisionSchema.safeParse(decision).success).toBe(true); + }); + it('user-turn schema does not offer confirmation actions', () => { expect( userTurnDecisionSchema.safeParse({ diff --git a/packages/@n8n/instance-ai/evaluations/cli/mcp-builder.ts b/packages/@n8n/instance-ai/evaluations/cli/mcp-builder.ts index 14f2f4e3829..533229eec8b 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/mcp-builder.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/mcp-builder.ts @@ -181,6 +181,12 @@ export const MCP_BUILD_KEY_SUPPORT: Record< // Caps user-proxy follow-ups in the orchestrator chat loop — simply doesn't // apply to a single-shot `claude` build, so it is NOT flagged. messageBudget: 'supported', + // Drives the Instance AI orchestrator's progressive-building loop; a + // single-shot `claude` build has no mode to set, so a case that pins it + // explicitly must run through the orchestrator. + buildMode: 'orchestrator-only', + promptVersion: 'orchestrator-only', + allowUserExecution: 'orchestrator-only', // Judged by the harness after the build (processExpectations are skipped for // transcript-less MCP builds there); declaring them needs no build-side setup. processExpectations: 'supported', @@ -215,6 +221,7 @@ export function unsupportedMcpBuildSetupFields(testCase: WorkflowTestCase): stri const values: Partial> = { ...testCase }; return ORCHESTRATOR_ONLY_KEYS.filter((key) => { const value = values[key]; + if (key === 'allowUserExecution') return value === true; if (value === undefined || value === null || value === '') return false; // An empty array declares nothing to seed. return !Array.isArray(value) || value.length > 0; diff --git a/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts b/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts index bb40ffa7bb2..dcaee8cfb20 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/__tests__/n8n-client.test.ts @@ -40,6 +40,51 @@ function stubFetch(body: unknown) { return fetchMock; } +describe('N8nClient chat build mode', () => { + afterEach(() => vi.unstubAllGlobals()); + + it('sends an explicit prompt version with the chat message', async () => { + const fetchMock = stubFetch({ data: { runId: 'run-1' } }); + await new N8nClient(BASE_URL).sendMessage( + 'thread-1', + 'Build it', + undefined, + 'default', + 'progressive@1', + ); + expect(fetchMock).toHaveBeenCalledWith( + `${BASE_URL}/rest/instance-ai/chat/thread-1`, + expect.objectContaining({ + body: JSON.stringify({ + message: 'Build it', + mode: 'default', + promptVersion: 'progressive@1', + }), + }), + ); + }); + + it.each([undefined, 'default', 'progressive'] as const)( + 'sends an explicit eval mode when the override is %s', + async (mode) => { + const fetchMock = stubFetch({ data: { runId: 'run-1' } }); + const client = new N8nClient(BASE_URL); + + await expect( + client.sendMessage('thread-1', 'Build a workflow', undefined, mode), + ).resolves.toEqual({ runId: 'run-1' }); + + expect(fetchMock).toHaveBeenCalledWith( + `${BASE_URL}/rest/instance-ai/chat/thread-1`, + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ message: 'Build a workflow', mode: mode ?? 'default' }), + }), + ); + }, + ); +}); + describe('N8nClient — TRUST-229 artifact fetch methods', () => { afterEach(() => { vi.unstubAllGlobals(); diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index bc1aa8d9533..74e8db020cb 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -7,6 +7,8 @@ // --------------------------------------------------------------------------- import type { + InstanceAiSendMessageRequest, + InstanceAiBuildMode, InstanceAiConfirmRequest, InstanceAiRichMessagesResponse, InstanceAiEvalAgentExecutionResult, @@ -22,6 +24,7 @@ import type { AgentSkill, EvaluationConfigDto, } from '@n8n/api-types'; +import type { ExecutionStatus } from 'n8n-workflow'; import { Agent, setGlobalDispatcher } from 'undici'; import { z } from 'zod'; @@ -186,7 +189,7 @@ interface ExecutionListItem { export interface ExecutionDetail { id: string; workflowId: string; - status: string; + status: ExecutionStatus; /** Flatted-serialized execution data (contains error details, run data per node) */ data: string; } @@ -304,12 +307,19 @@ export class N8nClient { threadId: string, message: string, attachments?: InstanceAiWorkflowAttachment[], + mode: InstanceAiBuildMode = 'default', + promptVersion?: string, ): Promise<{ runId: string }> { const result = await this.fetch(`/rest/instance-ai/chat/${threadId}`, { method: 'POST', - body: attachments && attachments.length > 0 ? { message, attachments } : { message }, + body: { + message, + ...(attachments?.length ? { attachments } : {}), + mode, + ...(promptVersion ? { promptVersion } : {}), + } satisfies InstanceAiSendMessageRequest, }); - return result as { runId: string }; + return this.unwrapRestData<{ runId: string }>(result); } /** @@ -338,9 +348,12 @@ export class N8nClient { * Get the current status of a thread (active run, suspended, background tasks). * GET /rest/instance-ai/threads/:threadId/status */ - async getThreadStatus(threadId: string): Promise { + async getThreadStatus( + threadId: string, + timeoutMs?: number, + ): Promise { return this.unwrapRestData( - await this.fetch(`/rest/instance-ai/threads/${threadId}/status`), + await this.fetch(`/rest/instance-ai/threads/${threadId}/status`, { timeoutMs }), ); } @@ -519,8 +532,8 @@ export class N8nClient { * Get a single workflow by ID. * GET /rest/workflows/:id */ - async getWorkflow(id: string): Promise { - const result = (await this.fetch(`/rest/workflows/${id}`)) as { + async getWorkflow(id: string, timeoutMs?: number): Promise { + const result = (await this.fetch(`/rest/workflows/${id}`, { timeoutMs })) as { data: WorkflowResponse; }; return result.data; @@ -581,12 +594,14 @@ export class N8nClient { async executeWorkflow( workflowId: string, triggerNodeName?: string, + timeoutMs?: number, ): Promise<{ executionId: string }> { const body: Record = {}; if (triggerNodeName) { body.triggerToStartFrom = { name: triggerNodeName }; } const result = (await this.fetch(`/rest/workflows/${workflowId}/run`, { + timeoutMs, method: 'POST', body, })) as { data: { executionId: string } }; @@ -597,13 +612,17 @@ export class N8nClient { * Get a single execution by ID. * GET /rest/executions/:id */ - async getExecution(executionId: string): Promise { - const result = (await this.fetch(`/rest/executions/${executionId}`)) as { + async getExecution(executionId: string, timeoutMs?: number): Promise { + const result = (await this.fetch(`/rest/executions/${executionId}`, { timeoutMs })) as { data: ExecutionDetail; }; return result.data; } + async stopExecution(executionId: string): Promise { + await this.fetch(`/rest/executions/${executionId}/stop`, { method: 'POST', timeoutMs: 5_000 }); + } + /** * Update a workflow (partial update). * PATCH /rest/workflows/:id -- used to set/restore pin data for execution eval. @@ -928,8 +947,10 @@ export class N8nClient { threadId: string, tableId: string, rows: Array>, + timeoutMs?: number, ): Promise { await this.fetch('/rest/instance-ai/eval/seed-data-table-rows', { + timeoutMs, method: 'POST', body: { threadId, tableId, rows }, }); diff --git a/packages/@n8n/instance-ai/evaluations/harness/build-mode.ts b/packages/@n8n/instance-ai/evaluations/harness/build-mode.ts new file mode 100644 index 00000000000..2f876c95057 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/harness/build-mode.ts @@ -0,0 +1,33 @@ +import type { InstanceAiBuildMode } from '@n8n/api-types'; + +import { + assertInstanceAiPromptVersion, + resolvePromptProfile, +} from '../../src/prompts/prompt-profiles'; +import type { WorkflowTestCase } from '../types'; + +export function resolveEvalPromptSettings( + testCase: Pick, +) { + const promptVersion = + testCase.promptVersion ?? + (testCase.buildMode === undefined ? process.env.N8N_EVAL_PROMPT_VERSION?.trim() : undefined); + if (promptVersion) { + assertInstanceAiPromptVersion(promptVersion); + return { + promptVersion, + buildMode: resolvePromptProfile({ version: promptVersion }).profile.mode, + }; + } + return { promptVersion: undefined, buildMode: resolveEvalBuildMode(testCase.buildMode) }; +} + +/** Case overrides take precedence. Validate the environment only when it is used. */ +export function resolveEvalBuildMode( + buildMode: WorkflowTestCase['buildMode'], +): InstanceAiBuildMode { + const mode = buildMode ?? process.env.N8N_EVAL_BUILD_MODE?.trim(); + if (!mode || mode === 'default') return 'default'; + if (mode === 'progressive') return mode; + throw new Error('N8N_EVAL_BUILD_MODE must be "progressive" or "default", or unset'); +} diff --git a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts index 58473495cc9..dedf0cf1cf7 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/build-workflow.ts @@ -7,11 +7,16 @@ // execution and cleanup. // --------------------------------------------------------------------------- -import type { InstanceAiConfirmRequest, InstanceAiWorkflowAttachment } from '@n8n/api-types'; +import type { + InstanceAiBuildMode, + InstanceAiConfirmRequest, + InstanceAiWorkflowAttachment, +} from '@n8n/api-types'; import { truncate } from '@n8n/utils/string/truncate'; import crypto from 'node:crypto'; import { setTimeout as delay } from 'node:timers/promises'; +import { resolveEvalPromptSettings } from './build-mode'; import { SSE_SETTLE_DELAY_MS, startSseConnection, @@ -51,6 +56,7 @@ import { buildSeededTablesNote, dedupeScenarioSeedTables, evictLeftoverSeedTables, + reseedScenarioTables, uniquifyScenarioTableNames, } from './seed-tables'; import type { CheckOutcome } from '../binaryChecks/types'; @@ -106,6 +112,11 @@ interface MultiTurnDriverConfig { threadId: string; conversation: ConversationTurn[]; messageBudget?: number; + /** Resolved wire value sent with every message (see `resolveEvalBuildMode`). */ + buildMode?: InstanceAiBuildMode; + promptVersion?: string; + allowUserExecution?: boolean; + beforeUserExecution?: (deadline: number) => Promise; events: CapturedEvent[]; approvedRequests: Set; startTime: number; @@ -162,6 +173,7 @@ async function driveMultiTurnConversation( const proxy = new UserProxyLlm({ conversation: proxyConversation, messageBudget: config.messageBudget, + allowUserExecution: config.allowUserExecution, logger: config.logger, ...(config.allowlistedCredentialIds !== undefined ? { @@ -193,6 +205,8 @@ async function driveMultiTurnConversation( config.threadId, openingMessage + (config.openingMessageSuffix ?? ''), config.openingAttachments, + config.buildMode, + config.promptVersion, ); await runMultiTurnConversation({ @@ -206,6 +220,10 @@ async function driveMultiTurnConversation( confirmationStrategy, nextMessageDecider, proxyResponses: config.proxyResponses, + buildMode: config.buildMode, + promptVersion: config.promptVersion, + allowUserExecution: config.allowUserExecution, + beforeUserExecution: config.beforeUserExecution, }); return { ...proxy.getDecisionStats() }; @@ -432,6 +450,10 @@ export interface BuildWorkflowConfig { conversation?: ConversationTurn[]; /** Max follow-up messages the proxy will send. Ignored in auto-approve mode. */ messageBudget?: number; + /** Case-declared build style; resolved via `resolveEvalBuildMode` (absent → default). */ + buildMode?: WorkflowTestCase['buildMode']; + promptVersion?: string; + allowUserExecution?: boolean; /** Credentials this build should see (created for real, view pinned to them). */ credentials?: TestCaseCredential[]; /** Run-level registry the created credential IDs are added to for cleanup. */ @@ -494,6 +516,7 @@ export function workflowExpectedForCase( */ export async function buildWorkflow(config: BuildWorkflowConfig): Promise { const { client, logger } = config; + const { buildMode, promptVersion } = resolveEvalPromptSettings(config); const threadId = crypto.randomUUID(); const startTime = Date.now(); const timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS; @@ -950,6 +973,28 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise { + const scenario = config.executionScenarios?.[0]; + if (scenario) { + try { + await reseedScenarioTables( + client, + scenario, + threadId, + scenarioTableIdsByName, + logger, + deadline, + ); + } catch (error) { + // Keep overall case timeouts separate from input setup failures. + seedingFailed = Date.now() < deadline; + throw error; + } + } + }, events, approvedRequests, startTime, @@ -980,6 +1025,8 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise 0) { + try { + buildTrace.promptConfiguration = ( + await client.getThreadStatus(threadId, metadataBudget) + ).promptConfiguration; + } catch { + logger.verbose('Prompt configuration was not available for this build.'); + } + } const outcome = await buildAgentOutcome( client, { ...eventOutcome, workflowIds: threadWorkflowIds }, diff --git a/packages/@n8n/instance-ai/evaluations/harness/chat-loop.ts b/packages/@n8n/instance-ai/evaluations/harness/chat-loop.ts index 74e0ee185a8..32e439e8a34 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/chat-loop.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/chat-loop.ts @@ -11,14 +11,15 @@ // flows automatically. // --------------------------------------------------------------------------- -import type { InstanceAiConfirmRequest } from '@n8n/api-types'; +import type { InstanceAiBuildMode, InstanceAiConfirmRequest } from '@n8n/api-types'; import { INSTANCE_AI_MEMORY_TASK_WAIT_TIMEOUT_MS } from '@n8n/api-types'; +import { isTerminalExecutionStatus } from 'n8n-workflow'; import { setTimeout as delay } from 'node:timers/promises'; import type { EvalLogger } from './logger'; import type { N8nClient } from '../clients/n8n-client'; import { consumeSseStream } from '../clients/sse-client'; -import { lastSavedWorkflowIdFromEvents } from '../outcome/event-parser'; +import { lastSavedWorkflowIdFromEvents, savedWorkflowsFromEvents } from '../outcome/event-parser'; import type { CapturedEvent } from '../types'; import { USER_TURN_EVENT } from '../types'; import { getEventPayload, tryInfrastructureResponse } from '../utils/confirmation-payload'; @@ -288,11 +289,19 @@ export type NextMessageDecision = * its own setup or credential work happens to advance the checksum. */ renameWorkflowTo?: string; + /** A normal user run, performed before delivering the next message. */ + runWorkflowId?: string; } | { kind: 'done' }; export interface MultiTurnConfig extends WaitConfig { nextMessageDecider: () => Promise; + /** Restore the case's declared input rows before a normal user execution. */ + beforeUserExecution?: (deadline: number) => Promise; + allowUserExecution?: boolean; + /** Repeat the eval override on each message to bypass the backend assignment. */ + buildMode?: InstanceAiBuildMode; + promptVersion?: string; } export async function runMultiTurnConversation(config: MultiTurnConfig): Promise { @@ -319,12 +328,27 @@ export async function runMultiTurnConversation(config: MultiTurnConfig): Promise await applyExternalRename(config, decision.renameWorkflowTo); } + // Before the follow-up is delivered, so a "I just ran it" message is true + // by the time the agent reads it and inspects the executions list. + if (decision.runWorkflowId !== undefined) { + if (!config.allowUserExecution) throw new Error('User executions are disabled for this case'); + await applyUserExecution(config, decision.runWorkflowId); + } + + if (Date.now() - config.startTime >= config.timeoutMs) return; + config.logger.verbose( `[multi-turn] Sending follow-up: ${decision.message.slice(0, 80)}${decision.message.length > 80 ? '...' : ''}`, ); recordUserTurn(config.events, decision.message); try { - await config.client.sendMessage(config.threadId, decision.message); + await config.client.sendMessage( + config.threadId, + decision.message, + undefined, + config.buildMode, + config.promptVersion, + ); } catch (error: unknown) { const msg = error instanceof Error ? error.message : String(error); config.logger.verbose(`[multi-turn] sendMessage failed: ${msg} — exiting loop`); @@ -391,6 +415,54 @@ async function applyExternalRename(config: MultiTurnConfig, rename: string): Pro } } +/** Use the normal execution route so the agent can inspect user-run evidence. */ +async function applyUserExecution(config: MultiTurnConfig, workflowId: string): Promise { + if (!savedWorkflowsFromEvents(config.events).some((workflow) => workflow.id === workflowId)) { + throw new Error(`User-run workflow ${workflowId} was not saved in this conversation`); + } + const remainingMs = () => { + const remaining = config.timeoutMs - (Date.now() - config.startTime); + if (remaining <= 0) throw new Error('Case timed out before the user execution completed'); + return remaining; + }; + remainingMs(); + await config.beforeUserExecution?.(config.startTime + config.timeoutMs); + const workflow = await config.client.getWorkflow(workflowId, remainingMs()); + if (Object.keys(workflow.pinData ?? {}).length > 0) { + throw new Error('User-run evals require a workflow without pinned data'); + } + if (workflow.nodes.some((node) => Object.keys(node.credentials ?? {}).length > 0)) { + throw new Error('User-run evals require a workflow without credentials'); + } + const trigger = workflow.nodes.find((node) => + ['n8n-nodes-base.manualTrigger', 'n8n-nodes-base.scheduleTrigger'].includes(node.type), + ); + if (!trigger) throw new Error('User-run evals require a manual or schedule trigger'); + + const { executionId } = await config.client.executeWorkflow( + workflowId, + trigger.name, + remainingMs(), + ); + try { + while (true) { + const execution = await config.client.getExecution(executionId, remainingMs()); + if (isTerminalExecutionStatus(execution.status)) { + config.logger.info( + `[user-run] Executed ${workflowId}: status=${execution.status} executionId=${executionId}`, + ); + return; + } + await delay(Math.min(POLL_INTERVAL_MS, remainingMs())); + } + } catch (error) { + await config.client.stopExecution(executionId).catch(() => { + config.logger.warn(`[user-run] Could not stop execution ${executionId}`); + }); + throw error; + } +} + // --------------------------------------------------------------------------- // Confirmation auto-approval // --------------------------------------------------------------------------- diff --git a/packages/@n8n/instance-ai/evaluations/harness/schema.ts b/packages/@n8n/instance-ai/evaluations/harness/schema.ts index 2337ee52231..1a314bbf424 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/schema.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/schema.ts @@ -161,6 +161,10 @@ const evalTestCaseObjectSchema = z triggerType: z.enum(['manual', 'webhook', 'schedule', 'form']).optional(), executionScenarios: z.array(ExecutionScenarioSchema).optional(), messageBudget: z.number().int().positive().optional(), + /** Optional case override. Unset cases use the suite mode or control. */ + buildMode: z.enum(['progressive', 'default']).optional(), + promptVersion: z.string().trim().min(1).max(128).optional(), + allowUserExecution: z.boolean().optional(), /** Optional NL assertions about the build CONVERSATION (process: clarifications, push-back, * ordering). LLM-judged from the transcript, so skipped in prebuilt/MCP runs. Counted as units. */ processExpectations: z.array(z.string().min(1)).optional(), diff --git a/packages/@n8n/instance-ai/evaluations/harness/seed-tables.ts b/packages/@n8n/instance-ai/evaluations/harness/seed-tables.ts index f82687da46e..73b5069c445 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/seed-tables.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/seed-tables.ts @@ -167,6 +167,7 @@ export async function reseedScenarioTables( threadId: string, tableIdsByName: Record, logger: EvalLogger, + deadline?: number, ): Promise { for (const table of scenario.seedDataTables ?? []) { const tableId = tableIdsByName[table.name]; @@ -175,7 +176,11 @@ export async function reseedScenarioTables( `Scenario "${scenario.name}" declares seed table "${table.name}" that was not pre-seeded before the build; cannot bind its rows.`, ); } - await client.seedDataTableRows(threadId, tableId, table.rows ?? []); + const timeoutMs = deadline === undefined ? undefined : deadline - Date.now(); + if (timeoutMs !== undefined && timeoutMs <= 0) { + throw new Error('Case timed out while preparing user execution data'); + } + await client.seedDataTableRows(threadId, tableId, table.rows ?? [], timeoutMs); logger.verbose( ` [${scenario.name}] reseeded data table "${table.name}" (${String((table.rows ?? []).length)} row(s))`, ); diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts index 999da9e58de..99d95aae371 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/to-exported.ts @@ -60,6 +60,15 @@ export interface ToLangTracerOptions { * already holds, and such a case is barred from suites anyway. Returns a * human-readable reason, else null. */ export function unsupportedPushReason(testCase: EvalTestCaseInput): string | null { + if (testCase.promptVersion !== undefined) { + return 'pins promptVersion, which the current case-write contract does not carry. Keep the case on disk.'; + } + if (testCase.buildMode !== undefined) { + return 'pins buildMode, which the current LangTracer write/export contract does not carry. Keep the case on disk until that contract supports the mode.'; + } + if (testCase.allowUserExecution) { + return 'enables user execution, which the current LangTracer write/export contract does not carry.'; + } const seed = testCase.seed; switch (seed?.mode) { case undefined: diff --git a/packages/@n8n/instance-ai/evaluations/outcome/event-parser.ts b/packages/@n8n/instance-ai/evaluations/outcome/event-parser.ts index 3867e18a081..9a74cd026cd 100644 --- a/packages/@n8n/instance-ai/evaluations/outcome/event-parser.ts +++ b/packages/@n8n/instance-ai/evaluations/outcome/event-parser.ts @@ -628,36 +628,27 @@ function extractIdFromRecord(record: Record, keys: string[]): s return undefined; } -/** - * The workflow id of the MOST RECENT build that actually SAVED, or undefined if - * this run has saved none. - * - * Deliberately narrower than `extractOutcomeFromEvents().workflowIds`, which - * includes ids from FAILED builds on purpose: a save that parsed but didn't - * persist still returns `{ success: false, workflowId }`, and discovery wants - * those so cleanup can claim and delete whatever the run touched. - * - * That's the wrong set for any caller that MUTATES a workflow. A failed build - * against an attached or pre-existing workflow would otherwise look like a - * build this run performed, and the caller would act on state it never created. - * Requires `success === true` rather than `!== false`, so a result shape - * without the flag is treated as not-saved — the safe direction when the - * consequence is writing to someone else's workflow. - * - * Returns the last id directly rather than a deduped list for the caller to - * index. Dedupe keeps FIRST-seen order, so a run that saved A, then B, then A - * again yields [A, B] and the last element is B — while the workflow most - * recently written is A. Anything mutating "the workflow under discussion" has - * to follow save order, not first-appearance order. - */ -export function lastSavedWorkflowIdFromEvents(events: CapturedEvent[]): string | undefined { - const saved = extractOutcomeFromEvents(events) - .toolCalls.filter((call) => WORKFLOW_TOOLS.has(call.toolName)) - .filter((call) => toResultRecord(call.result)?.success === true) - .map((call) => extractIdFromResult(call.result, 'workflowId', 'id')) - .filter((id): id is string => id !== undefined); +/** Successful saves in event order. Failed saves can also contain IDs, so + * they must not authorize later harness mutations or executions. */ +export function savedWorkflowsFromEvents( + events: CapturedEvent[], +): Array<{ id: string; name: string }> { + const names = new Map(); + return extractOutcomeFromEvents(events).toolCalls.flatMap((call) => { + if (!WORKFLOW_TOOLS.has(call.toolName)) return []; + const result = toResultRecord(call.result); + if (result?.success !== true) return []; + const id = extractIdFromResult(result, 'workflowId', 'id'); + if (!id) return []; + if (typeof result.workflowName === 'string' && result.workflowName.trim()) { + names.set(id, result.workflowName); + } + return [{ id, name: names.get(id) ?? id }]; + }); +} - return saved.at(-1); +export function lastSavedWorkflowIdFromEvents(events: CapturedEvent[]): string | undefined { + return savedWorkflowsFromEvents(events).at(-1)?.id; } function dedupe(arr: string[]): string[] { diff --git a/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts b/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts index c4bc14b1e51..99af67b78f6 100644 --- a/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts +++ b/packages/@n8n/instance-ai/evaluations/run/build-orchestrator.ts @@ -106,6 +106,9 @@ export type BuildArgs = Pick< WorkflowTestCase, | 'conversation' | 'messageBudget' + | 'buildMode' + | 'promptVersion' + | 'allowUserExecution' | 'credentials' | 'seed' | 'executionScenarios' @@ -644,6 +647,9 @@ export function createBuildOrchestrator(deps: BuildOrchestratorDeps): BuildOrche build = await lane.tracedBuild({ conversation: entry.conversation, messageBudget: entry.messageBudget, + buildMode: entry.buildMode, + promptVersion: entry.promptVersion, + allowUserExecution: entry.allowUserExecution, credentials: entry.credentials, seed: entry.seed, executionScenarios: entry.executionScenarios, diff --git a/packages/@n8n/instance-ai/evaluations/run/case-selection.ts b/packages/@n8n/instance-ai/evaluations/run/case-selection.ts index 41c76c3b36e..01e5c236f4d 100644 --- a/packages/@n8n/instance-ai/evaluations/run/case-selection.ts +++ b/packages/@n8n/instance-ai/evaluations/run/case-selection.ts @@ -7,6 +7,7 @@ import type { CliArgs } from '../cli/args'; import { unsupportedMcpBuildSetupFields } from '../cli/mcp-builder'; import type { WorkflowTestCaseWithFile } from '../data/workflows'; +import { resolveEvalPromptSettings } from '../harness/build-mode'; import type { EvalLogger } from '../harness/logger'; import { loadPrebuiltManifest, @@ -87,5 +88,9 @@ export function selectCases( ); } } + if (!args.buildViaMcp && !prebuiltManifest) { + // Reject invalid configuration before provisioning lanes or recording results. + for (const { testCase } of testCasesWithFiles) resolveEvalPromptSettings(testCase); + } return { testCasesWithFiles, prebuiltManifest }; } diff --git a/packages/@n8n/instance-ai/evaluations/run/eval-session.ts b/packages/@n8n/instance-ai/evaluations/run/eval-session.ts index 652356d7a3e..370ee2df711 100644 --- a/packages/@n8n/instance-ai/evaluations/run/eval-session.ts +++ b/packages/@n8n/instance-ai/evaluations/run/eval-session.ts @@ -145,6 +145,9 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { client: lane.client, conversation: buildArgs.conversation, messageBudget: buildArgs.messageBudget, + buildMode: buildArgs.buildMode, + promptVersion: buildArgs.promptVersion, + allowUserExecution: buildArgs.allowUserExecution, credentials: buildArgs.credentials, seed: buildArgs.seed, executionScenarios: buildArgs.executionScenarios, diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 30a5be38405..e4eb9d246df 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -7,6 +7,7 @@ import type { InstanceAiEvalExecutionResult, InstanceAiEvalSeedDataTable, InstanceAiRunDebugResponse, + InstanceAiPromptConfiguration, } from '@n8n/api-types'; import type { CheckOutcome } from './binaryChecks/types'; @@ -161,6 +162,7 @@ export interface EventOutcome { } export interface BuildTrace { + promptConfiguration?: InstanceAiPromptConfiguration; finalText: string; toolCalls: CapturedToolCall[]; agentActivities: AgentActivity[]; @@ -249,6 +251,11 @@ export interface WorkflowTestCase { executionScenarios?: ExecutionScenario[]; /** Max follow-up messages the proxy will send. Ignored in auto-approve mode. */ messageBudget?: number; + /** Optional case override. Unset cases use the suite mode or control. */ + buildMode?: 'progressive' | 'default'; + promptVersion?: string; + /** Enable the user-run action for credential-free execution cases. */ + allowUserExecution?: boolean; /** Optional NL assertions about the build CONVERSATION (process: clarifications, push-back, * ordering). LLM-judged from the transcript; requires a transcript, so skipped in * prebuilt/MCP runs. Counted toward the per-case + headline pass rate alongside scenarios. */ diff --git a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts index 541f95269c7..533a4b8cea3 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts @@ -3,7 +3,8 @@ import { CONFIRMATION_TOOL_DESCRIPTIONS, confirmationDecisionSchema, USER_TURN_TOOL_DESCRIPTIONS, - userTurnDecisionSchema, + createUserTurnDecisionSchema, + userTurnWithoutExecutionSchema, type Decision, type ProxyDecisionMode, } from './tools'; @@ -13,18 +14,32 @@ import type { EvalLogger } from '../../harness/logger'; export interface UserProxyAgentConfig { modelId?: string; logger?: EvalLogger; + allowUserExecution?: boolean; } export interface UserProxyAgent { - decide(userPrompt: string, mode: ProxyDecisionMode): Promise; + decide( + userPrompt: string, + mode: ProxyDecisionMode, + savedWorkflowIds?: string[], + ): Promise; } export function createUserProxyAgent(config: UserProxyAgentConfig = {}): UserProxyAgent { return { - async decide(userPrompt: string, mode: ProxyDecisionMode): Promise { + async decide( + userPrompt: string, + mode: ProxyDecisionMode, + savedWorkflowIds: string[] = [], + ): Promise { // The schema handed to the model is the action menu for this moment in // the conversation — actions that cannot function now are not offered. - const schema = mode === 'user-turn' ? userTurnDecisionSchema : confirmationDecisionSchema; + const schema = + mode === 'user-turn' + ? config.allowUserExecution + ? createUserTurnDecisionSchema(savedWorkflowIds) + : userTurnWithoutExecutionSchema + : confirmationDecisionSchema; const toolDescriptions = mode === 'user-turn' ? USER_TURN_TOOL_DESCRIPTIONS : CONFIRMATION_TOOL_DESCRIPTIONS; const agent = createEvalAgent('eval-user-proxy', { diff --git a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts index 7a237bbb3a3..61e0d05a1ed 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/index.ts @@ -19,6 +19,7 @@ import { createOneCredential } from '../../credentials/seeder'; import { buildAutoApprovePayload } from '../../harness/chat-loop'; import type { NextMessageDecision } from '../../harness/chat-loop'; import type { EvalLogger } from '../../harness/logger'; +import { savedWorkflowsFromEvents } from '../../outcome/event-parser'; import type { CapturedEvent, ConversationTurn } from '../../types'; import { getEventPayload } from '../confirmation-payload'; import { getNestedRecord, getString } from '../safe-extract'; @@ -92,6 +93,7 @@ const DEFAULT_MESSAGE_BUDGET = 5; export interface UserProxyConfig { conversation: ConversationTurn[]; messageBudget?: number; + allowUserExecution?: boolean; modelId?: string; logger?: EvalLogger; /** Test seam — inject a fake agent. */ @@ -123,6 +125,8 @@ export class UserProxyLlm { private readonly responseByRequestId = new Map(); private readonly sentScriptUserTurnIndexes = new Set(); private readonly decisionStats: ProxyDecisionStats = {}; + private readonly allowUserExecution: boolean; + private savedWorkflows: ReturnType = []; private readonly credentialCreation?: CredentialCreationConfig; /** Mutable running copy of `credentialCreation.allowlistedCredentialIds` — @@ -139,10 +143,16 @@ export class UserProxyLlm { constructor(config: UserProxyConfig) { this.script = config.conversation; + this.allowUserExecution = config.allowUserExecution ?? false; this.messageBudget = config.messageBudget ?? DEFAULT_MESSAGE_BUDGET; this.logger = config.logger; this.agent = - config.agent ?? createUserProxyAgent({ modelId: config.modelId, logger: config.logger }); + config.agent ?? + createUserProxyAgent({ + modelId: config.modelId, + logger: config.logger, + allowUserExecution: config.allowUserExecution, + }); this.credentialCreation = config.credentialCreation; this.allowlistedCredentialIds = config.credentialCreation?.allowlistedCredentialIds ?? []; this.bypassCredentialTestIds = config.credentialCreation?.bypassCredentialTestIds ?? []; @@ -161,6 +171,14 @@ export class UserProxyLlm { } ingestEvents(events: CapturedEvent[]): void { + if (this.allowUserExecution) { + this.savedWorkflows = [ + ...new Map( + savedWorkflowsFromEvents(events).map((workflow) => [workflow.id, workflow]), + ).values(), + ]; + } + const newEvents = events.slice(this.ingestedEventCount); this.ingestedEventCount = events.length; @@ -310,7 +328,11 @@ export class UserProxyLlm { } const prompt = buildFollowUpPrompt(this.promptContext()); - const decision = await this.agent.decide(prompt, 'user-turn'); + const decision = await this.agent.decide( + prompt, + 'user-turn', + this.savedWorkflows.map(({ id }) => id), + ); if (!decision) { const [next] = this.remainingUserScriptTurns(); if (!next || hasStageDirection(next.text)) { @@ -337,6 +359,9 @@ export class UserProxyLlm { kind: 'followUp', message, ...(decision.renameWorkflowTo ? { renameWorkflowTo: decision.renameWorkflowTo } : {}), + // Like the rename, the mid-run execution is a harness side effect at + // this turn boundary, not something the user says. + ...(decision.runWorkflowId ? { runWorkflowId: decision.runWorkflowId } : {}), }; } if (decision.action !== 'declare_done') { @@ -357,6 +382,7 @@ export class UserProxyLlm { return { script: this.script, actualTranscript: this.actualTranscript, + savedWorkflows: this.allowUserExecution ? this.savedWorkflows : undefined, }; } diff --git a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/prompts.ts b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/prompts.ts index 03cb3437039..62fe608cd0d 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/prompts.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/prompts.ts @@ -9,6 +9,7 @@ export interface PromptContext { script: ConversationTurn[]; /** What's actually been said this run, both sides. */ actualTranscript: ConversationTurn[]; + savedWorkflows?: Array<{ id: string; name: string }>; } export const SYSTEM_PROMPT = `You are simulating a real user in a workflow-building conversation with an AI assistant. @@ -103,6 +104,11 @@ export function buildFollowUpPrompt(ctx: PromptContext): string { return [ formatScriptSection(ctx), formatTranscriptSection(ctx), + ...(ctx.savedWorkflows?.length + ? [ + `Saved workflows available to run. Select the intended workflow, not a prerequisite or cleanup workflow.\n${JSON.stringify(ctx.savedWorkflows)}`, + ] + : []), "It is now the user's turn: the agent finished its run and is waiting, and no widget is on screen. Decide what the user does — send a chat message or end the conversation.", 'Pick `send_follow_up_message` when the agent\'s last response leaves anything open — it asked a question, requested approval, presented a plan to react to, or stalled and needs unblocking. Approving or rejecting a plan the agent presented in plain text IS a follow-up message (e.g. "No — two changes first: …" / "Yes, go ahead."). If the script answers the open point, deliver it with concrete values verbatim; if the script doesn\'t cover it and credentials aren\'t involved, give a brief plausible reply.', 'If a stage direction tells the user to keep requesting changes or stay in the conversation, pick `send_follow_up_message` with the NEXT change — even after a successful build — until the change list is exhausted.', diff --git a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/tools.ts b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/tools.ts index 0259c7606a0..2e1d4913fee 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/user-proxy/tools.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/user-proxy/tools.ts @@ -126,6 +126,9 @@ const chooseCredentialSetupOptionDecisionSchema = z.object({ existingCredentialId: z.string().optional(), }); +const RUN_WORKFLOW_DESCRIPTION = + 'Workflow to run before sending this message. Set only when the director says the user runs it. Select the intended workflow, not a prerequisite or cleanup workflow. This runs normally, without mocks, and supports credential-free manual or schedule workflows only.'; + const sendFollowUpMessageDecisionSchema = z.object({ action: z.literal('send_follow_up_message'), message: z.string(), @@ -147,6 +150,7 @@ const sendFollowUpMessageDecisionSchema = z.object({ * that workflow already carries this name. */ renameWorkflowTo: z.string().min(1).optional(), + runWorkflowId: z.string().min(1).optional().describe(RUN_WORKFLOW_DESCRIPTION), }); const declareDoneDecisionSchema = z.object({ @@ -162,6 +166,11 @@ const declareDoneDecisionSchema = z.object({ * The schema handed to the model per mode IS the action menu — actions that * cannot function at that moment are not offered at all. */ +export const userTurnWithoutExecutionSchema = z.discriminatedUnion('action', [ + sendFollowUpMessageDecisionSchema.omit({ runWorkflowId: true }).strict(), + declareDoneDecisionSchema, +]); + export type ProxyDecisionMode = 'confirmation' | 'user-turn'; export const confirmationDecisionSchema = z.discriminatedUnion('action', [ @@ -173,10 +182,20 @@ export const confirmationDecisionSchema = z.discriminatedUnion('action', [ chooseCredentialSetupOptionDecisionSchema, ]); -export const userTurnDecisionSchema = z.discriminatedUnion('action', [ - sendFollowUpMessageDecisionSchema, - declareDoneDecisionSchema, -]); +export function createUserTurnDecisionSchema(savedWorkflowIds: string[]) { + const [firstId, ...remainingIds] = savedWorkflowIds; + if (firstId === undefined) return userTurnWithoutExecutionSchema; + + return z.discriminatedUnion('action', [ + sendFollowUpMessageDecisionSchema.extend({ + runWorkflowId: z + .enum([firstId, ...remainingIds]) + .optional() + .describe(RUN_WORKFLOW_DESCRIPTION), + }), + declareDoneDecisionSchema, + ]); +} /** Full union — the type every decision consumer handles. Agents are only ever * offered the mode-scoped subsets above. */ diff --git a/packages/@n8n/instance-ai/scripts/print-prompts.ts b/packages/@n8n/instance-ai/scripts/print-prompts.ts index 0d3cfa64a32..d78cc8298c2 100644 --- a/packages/@n8n/instance-ai/scripts/print-prompts.ts +++ b/packages/@n8n/instance-ai/scripts/print-prompts.ts @@ -14,6 +14,13 @@ import { join, resolve } from 'path'; import { buildSubAgentPrompt } from '../src/agent/sub-agent-factory'; import { getSystemPrompt } from '../src/agent/system-prompt'; +import { + assertInstanceAiPromptVersion, + describePromptProfile, + getVersionedSystemPrompt, + resolvePromptProfile, +} from '../src/prompts/prompt-profiles'; +import { loadInstanceAiPromptSkills } from '../src/skills/runtime-skills'; interface Variant { /** File name (without extension) inside the agent's folder. */ @@ -31,9 +38,10 @@ interface AgentEntry { variants: Variant[]; } -function parseArgs(argv: string[]): { outDir: string } { +function parseArgs(argv: string[]): { outDir: string; promptVersion?: string } { const args = argv.slice(2); let outDir = resolve(__dirname, '..', '.output', 'prompts'); + let promptVersion: string | undefined; for (let i = 0; i < args.length; i++) { if (args[i] === '--out' || args[i] === '-o') { const next = args[i + 1]; @@ -43,13 +51,18 @@ function parseArgs(argv: string[]): { outDir: string } { } outDir = resolve(next); i++; + } else if (args[i] === '--profile') { + promptVersion = args[++i]; + if (!promptVersion) throw new Error('--profile requires a version'); + assertInstanceAiPromptVersion(promptVersion); } else if (args[i] === '--help' || args[i] === '-h') { - console.log('Usage: pnpm prompts:print [--out ]'); + console.log('Usage: pnpm prompts:print [--out ] [--profile ]'); console.log(' --out, -o Output directory (default: /.output/prompts)'); + console.log(' --profile Export a prompt profile and its selected skills'); process.exit(0); } } - return { outDir }; + return { outDir, promptVersion }; } function collectAgents(): AgentEntry[] { @@ -133,8 +146,36 @@ function renderFile(agent: AgentEntry, variant: Variant): string { return header.join('\n') + variant.body; } -function main(): void { - const { outDir } = parseArgs(process.argv); +async function main(): Promise { + const { outDir, promptVersion } = parseArgs(process.argv); + if (promptVersion) { + const selected = resolvePromptProfile({ version: promptVersion }); + const { source, disabledTools } = await loadInstanceAiPromptSkills(selected.profile); + const directory = join(outDir, promptVersion); + mkdirSync(join(directory, 'skills'), { recursive: true }); + writeFileSync( + join(directory, 'system.md'), + getVersionedSystemPrompt(selected.profile.systemPromptVersion, {}), + ); + writeFileSync( + join(directory, 'manifest.json'), + JSON.stringify( + { + ...describePromptProfile(selected, source), + disabledTools, + skills: source.registry.skills.map(({ id, version, hash }) => ({ id, version, hash })), + }, + null, + 2, + ), + ); + for (const entry of source.registry.skills) { + const skill = await source.loadSkill(entry.id); + if (skill) writeFileSync(join(directory, 'skills', `${entry.id}.md`), skill.instructions); + } + console.log(`Wrote profile ${promptVersion} to ${directory}`); + return; + } const agents = collectAgents(); const written: Array<{ relPath: string; chars: number }> = []; @@ -159,4 +200,4 @@ function main(): void { } } -main(); +void main(); diff --git a/packages/@n8n/instance-ai/skills/planning/SKILL.md b/packages/@n8n/instance-ai/skills/planning/SKILL.md index faf0cc2f8d4..5b7f7b95a15 100644 --- a/packages/@n8n/instance-ai/skills/planning/SKILL.md +++ b/packages/@n8n/instance-ai/skills/planning/SKILL.md @@ -1,5 +1,8 @@ --- name: planning +dependencies: + tools: + - create-tasks description: >- ONLY for coordinated multi-artifact work: multiple workflows with dependencies, shared data-table schema/migration across tasks, or the user explicitly asked diff --git a/packages/@n8n/instance-ai/skills/progressive-building/SKILL.md b/packages/@n8n/instance-ai/skills/progressive-building/SKILL.md new file mode 100644 index 00000000000..5676c76fcd4 --- /dev/null +++ b/packages/@n8n/instance-ai/skills/progressive-building/SKILL.md @@ -0,0 +1,112 @@ +--- +name: progressive-building +dependencies: + tools: + - build-workflow + - verify-built-workflow + - workflows + - executions +description: >- + Load before build-workflow and before scoping or planning new workflows and + feature additions, including requests spanning multiple workflows. Implement + one increment per user message. Finish setup and inspect a successful real + execution before offering another increment. Then wait for the next user + reply. Partial setup is incomplete. A first setup or test refusal pauses + building. Follow the skill's full-build exceptions. Also supports workflow edits + and repairs. For workflows that create or write Data Tables, load + data-table-manager first. Requests only to + run, inspect, or manage existing resources use their normal tools and skills. +--- + +# Progressive building + +Apply this policy when creating or extending workflows and repairing those +builds. It overrides conflicting scoping and setup narration in the building +and post-build guidance above. Keep their validation, approval, credential, +publish, and cleanup rules. Write to the user in their conversation language. + +Requests to run, inspect, or manage existing resources use their normal tools +and skills without this staging process. This also applies when the user +switches to such a request after building in the same conversation. + +## Choose the first version + +Acknowledge the full request before selecting the smallest useful outcome. +Build a working part of the requested workflow. Do not create a throwaway demo. +Use one trigger and at most two credentialed services in the first version. +Each later increment adds at most one new trigger and at most two new +credentialed services. This also applies when new triggers share existing logic +or need no credentials. Count the trigger's credential. Count services even +when their credentials are connected. +Parameters and placeholders do not count. An AI provider counts toward the same +two-service total as the trigger and other services, even if its credential is +missing. Exclude a model only when Gateway credits is confirmed available for +that model on this instance. If it needs a provider API key, count it. + +Keep this limit internal. Explain what the first version does and what comes +next. Preserve the user's services. Prefer existing credentials when choosing +between equally suitable starting points. Follow workflow-builder guidance +on credential setup preferences when the user leaves the service open. + +Ask a single-choice question if several named services are equally central. +Otherwise state a reasonable starting assumption. Do not offer a multi-select +list that adds services or triggers to the first version. + +The `planning` skill and `create-tasks` tool are unavailable in this mode. +Keep additional workflows as later roadmap items. + +## Build, set up, and run + +1. Build the selected version. Use the existing verification and setup flow. +2. Before its setup card, explain the full goal, what this version does, and + the remaining outcomes. Do this on automatic `` + turns too. Then open setup. Do not add more work while setup is incomplete. +3. Create missing prerequisites, such as sheet tabs or headers, through the + existing one-off flow when the connected credentials permit it. +4. After setup, offer only a live run for manual or schedule triggers. The execution + approval card supplies consent. For event triggers, explain how to start + listening for a test event and perform the event, then ask the user to report + back. Inspect the resulting execution with `executions`. +5. Extend only after a successful, non-simulated execution of the current + version. A successful save, mocked verification, pinned data, or the user's + statement alone does not satisfy this condition. Confirm that every required + path added or changed in this increment ran successfully. Unchanged paths do + not need another run. + Report simulated verification as simulated, not as end-to-end + success. Repair failures before extending. +6. After one increment's verification and setup, end the turn. Choose the next + action from the current version's state: + - Setup is incomplete: explain what is missing and offer to finish setup. + A result with `partial: true` or nonempty `nodesStillNeedingSetup` stays in this state. + Respect skipped credentials. If setup was deferred, pause without reopening it. + - Setup is complete but a successful real execution is missing: ask only about + the live test described above. Do not offer to build the next increment instead. + - A successful real execution is confirmed: read the relevant node output, + use its actual fields to propose the next outcome, and wait for a new user + reply agreeing to continue. Edit the same workflow and source file when + extending it. + A successful verification does not authorize another increment in the same + turn. The original list of requested outcomes does not replace this pause. + +Keep a short Done/Next roadmap in substantive replies about this build. Mark an +outcome as done only after execution evidence confirms it. Do not claim the +whole request is complete while outcomes remain. +Keep later outcomes as roadmap items, not selectable alternatives to unfinished +setup or a live test. A reply such as "continue" or "what's next?" keeps the +current setup or live-test step. It is not a full-build request or another decline. + +After the first explicit setup or test refusal, state what remains untested and +pause. Do not offer another increment or ask the user to repeat the refusal. +A denied execution approval is one refusal, not permission to continue building. + +## Finish without staging when asked + +Build a precise, complete implementation specification in one pass. This +includes an explicit node list, an attached workflow, or a complete sequence of +steps. A list of desired capabilities or entry points alone does not qualify. +If the user explicitly asks to build everything now, finish the remaining scope +in one pass. Do not infer this instruction from a list of requested additions. + +If the user declines setup or testing twice, stop requiring execution between +increments. Finish the requested scope and offer setup at the end. Respect +previously skipped credentials. State which parts remain untested. diff --git a/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts b/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts index 737e24c524a..5a03828a164 100644 --- a/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts +++ b/packages/@n8n/instance-ai/src/agent/__tests__/instance-agent.test.ts @@ -89,6 +89,7 @@ vi.mock('../../tools/filesystem/create-tools-from-mcp-server', () => ({ vi.mock('../../tracing/langsmith-tracing', () => ({ buildAgentTraceInputs: vi.fn().mockReturnValue({}), mergeTraceRunInputs: vi.fn(), + setTracePromptVersion: vi.fn(), })); vi.mock('../system-prompt', () => ({ @@ -99,6 +100,7 @@ import { Agent as AgentImport, Memory as MemoryImport } from '@n8n/agents'; import { createOrchestratorDomainTools as createOrchestratorDomainToolsImport } from '../../tools'; import { createToolsFromLocalMcpServer as createToolsFromLocalMcpServerImport } from '../../tools/filesystem/create-tools-from-mcp-server'; +import { setTracePromptVersion } from '../../tracing/langsmith-tracing'; import { createInstanceAgent } from '../instance-agent'; import { getSystemPrompt as getSystemPromptImport } from '../system-prompt'; @@ -202,6 +204,22 @@ describe('createInstanceAgent', () => { expect(secondRunAttachedTools['nodes-run-2']).toMatchObject({ name: 'nodes-run-2' }); }); + it('applies the selected profile exclusions to domain and orchestration tools', async () => { + await createInstanceAgent({ + modelId: 'test-model', + context: { runLabel: 'profile' }, + orchestrationContext: { + runId: 'profile', + disabledToolNames: new Set(['create-tasks', 'nodes']), + }, + memoryConfig: {}, + mcpManager: createMcpManagerStub(), + } as never); + expect(getDeferredTools()).not.toHaveProperty('create-tasks-profile'); + expect(getAttachedTools()).not.toHaveProperty('nodes-profile'); + expect(getAttachedTools()).toHaveProperty('build-workflow-profile'); + }); + it('requires MCP tool approval unless the executeMcpTool permission is always_allow', async () => { const baseOptions = (executeMcpTool?: string) => ({ @@ -369,6 +387,10 @@ describe('createInstanceAgent', () => { it('attaches native telemetry from the trace context when present', async () => { const telemetry = { provider: 'langsmith' }; + const tracing = { + getTelemetry: vi.fn().mockReturnValue(telemetry), + wrapTools: vi.fn((tools: unknown) => tools), + }; await createInstanceAgent({ modelId: 'test-model', @@ -380,16 +402,15 @@ describe('createInstanceAgent', () => { }, orchestrationContext: { runId: 'trace-test', - tracing: { - getTelemetry: vi.fn().mockReturnValue(telemetry), - wrapTools: vi.fn((tools: unknown) => tools), - }, + promptConfiguration: { version: 'default@1' }, + tracing, }, memoryConfig: {}, mcpManager: createMcpManagerStub(), } as never); expect(mockAgentInstances[0]?.telemetry).toHaveBeenCalledWith(telemetry); + expect(setTracePromptVersion).toHaveBeenCalledWith(tracing, 'default@1'); }); it('attaches runtime skills to the orchestrator when provided by the context', async () => { @@ -434,6 +455,32 @@ describe('createInstanceAgent', () => { } as never); expect(mockAgentInstances[0]?.skills).toHaveBeenCalledWith(runtimeSkills); + expect(createOrchestratorDomainTools).toHaveBeenLastCalledWith( + expect.objectContaining({ runtimeSkillCatalog: runtimeSkills }), + ); + }); + + it('passes the selected catalog to domain tools before workspace materialization', async () => { + const runtimeSkillCatalog = { + registry: { schemaVersion: 1, skillsHash: 'selected-skills', skills: [] }, + loadSkill: vi.fn(), + }; + const runtimeSkills = { + registry: { schemaVersion: 1, skillsHash: 'workspace-skills', skills: [] }, + loadSkill: vi.fn(), + }; + + await createInstanceAgent({ + modelId: 'test-model', + context: {}, + orchestrationContext: { runId: 'skills-test', runtimeSkillCatalog, runtimeSkills }, + memoryConfig: {}, + mcpManager: createMcpManagerStub(), + } as never); + + expect(createOrchestratorDomainTools).toHaveBeenLastCalledWith( + expect.objectContaining({ runtimeSkillCatalog }), + ); }); it('exposes browser_connect and browser_navigate from localMcpServer in the agent toolset', async () => { diff --git a/packages/@n8n/instance-ai/src/agent/instance-agent.ts b/packages/@n8n/instance-ai/src/agent/instance-agent.ts index 58eeb085449..7a8e4a111da 100644 --- a/packages/@n8n/instance-ai/src/agent/instance-agent.ts +++ b/packages/@n8n/instance-ai/src/agent/instance-agent.ts @@ -12,8 +12,8 @@ import { type McpToolNameValidationError, } from './mcp-tool-name-validation'; import { attachRuntimeWorkspaceCapabilities } from './runtime-workspace'; -import { getSystemPrompt } from './system-prompt'; import { listConnectedMcpServices } from '../mcp/connected-mcp-services'; +import { getVersionedSystemPrompt, resolvePromptProfile } from '../prompts/prompt-profiles'; import { hasRuntimeSkills } from '../skills/runtime-skills'; import { createToolRegistry, mergeToolRegistries, toolRegistryValues } from '../tool-registry'; import { createOrchestratorDomainTools, createOrchestrationTools } from '../tools'; @@ -24,7 +24,11 @@ import { DOMAIN_TOOL_IDS, ORCHESTRATION_TOOL_IDS, } from '../tools/tool-ids'; -import { buildAgentTraceInputs, mergeTraceRunInputs } from '../tracing/langsmith-tracing'; +import { + buildAgentTraceInputs, + mergeTraceRunInputs, + setTracePromptVersion, +} from '../tracing/langsmith-tracing'; import type { CreateInstanceAgentOptions, InstanceAiContext, @@ -93,8 +97,14 @@ export async function createInstanceAgent( // Thread the trace handle in so domain tools (e.g. build-workflow) can emit // explicit child runs that land on the active trace — orchestration tools // (e.g. verify) already get it via OrchestrationContext. - const domainContext: InstanceAiContext = { ...context, tracing: orchestrationContext?.tracing }; - + const domainContext: InstanceAiContext = { + ...context, + tracing: orchestrationContext?.tracing, + runtimeSkillCatalog: + orchestrationContext?.runtimeSkillCatalog ?? + context.runtimeSkillCatalog ?? + orchestrationContext?.runtimeSkills, + }; // Load MCP tools (cached by config hash inside the manager — only spawns // processes / opens connections on first call or config change). The manager // returns per-server connection failures alongside the tools so they travel @@ -192,6 +202,8 @@ export async function createInstanceAgent( safeLocalMcpTools, safeMcpTools, ); + for (const name of orchestrationContext?.disabledToolNames ?? []) + allOrchestratorTools.delete(name); const tracedOrchestratorTools = orchestrationContext?.tracing?.wrapTools(allOrchestratorTools, { agentRole: 'orchestrator', @@ -204,25 +216,33 @@ export async function createInstanceAgent( const hasDeferredExternalMcpTools = hasDeferrableTools && Array.from(safeMcpTools.keys()).some((name) => deferredTools.has(name)); const runtimeTools = hasDeferrableTools ? coreTools : tracedOrchestratorTools; - const systemPrompt = getSystemPrompt({ - webhookBaseUrl: orchestrationContext?.webhookBaseUrl, - formBaseUrl: orchestrationContext?.formBaseUrl, - localGateway: context.localGatewayStatus, - toolSearchEnabled: hasDeferrableTools, - mcpToolSearchEnabled: hasDeferredExternalMcpTools, - licenseHints: context.licenseHints, - browserAvailable: browserToolNames.size > 0, - branchReadOnly: context.branchReadOnly, - projectId: context.projectId, - // Presence of the service IS the experiment gate — the host only wires it - // for flagged-in users on project-bound runs. - conversationHistoryEnabled: Boolean(context.conversationHistoryService), - workspaceRoot: - orchestrationContext?.workspace && orchestrationContext.workspaceRoot - ? orchestrationContext.workspaceRoot - : undefined, - }); + const systemPrompt = getVersionedSystemPrompt( + orchestrationContext?.promptConfiguration?.systemPromptVersion ?? + resolvePromptProfile({}).profile.systemPromptVersion, + { + webhookBaseUrl: orchestrationContext?.webhookBaseUrl, + formBaseUrl: orchestrationContext?.formBaseUrl, + localGateway: context.localGatewayStatus, + toolSearchEnabled: hasDeferrableTools, + mcpToolSearchEnabled: hasDeferredExternalMcpTools, + licenseHints: context.licenseHints, + browserAvailable: browserToolNames.size > 0, + branchReadOnly: context.branchReadOnly, + projectId: context.projectId, + // Presence of the service IS the experiment gate — the host only wires it + // for flagged-in users on project-bound runs. + conversationHistoryEnabled: Boolean(context.conversationHistoryService), + workspaceRoot: + orchestrationContext?.workspace && orchestrationContext.workspaceRoot + ? orchestrationContext.workspaceRoot + : undefined, + }, + ); + setTracePromptVersion( + orchestrationContext?.tracing, + orchestrationContext?.promptConfiguration?.version, + ); const telemetry = orchestrationContext?.tracing?.getTelemetry?.({ agentRole: 'orchestrator', functionId: 'instance-ai.orchestrator', @@ -285,6 +305,7 @@ export async function createInstanceAgent( orchestrationContext?.tracing?.actorRun, buildAgentTraceInputs({ systemPrompt, + promptConfiguration: orchestrationContext?.promptConfiguration, tools: runtimeTools, deferredTools: hasDeferrableTools ? deferredTools : undefined, modelId, diff --git a/packages/@n8n/instance-ai/src/index.ts b/packages/@n8n/instance-ai/src/index.ts index ff674d9446a..a92c54ca8c7 100644 --- a/packages/@n8n/instance-ai/src/index.ts +++ b/packages/@n8n/instance-ai/src/index.ts @@ -14,6 +14,7 @@ import type * as StructuredFileParserMod from './parsers/structured-file-parser' import type * as ValidateAttachmentsMod from './parsers/validate-attachments'; import type * as PlannedTaskPermissionsMod from './planned-tasks/planned-task-permissions'; import type * as PlannedTaskServiceMod from './planned-tasks/planned-task-service'; +import type * as PromptProfilesMod from './prompts/prompt-profiles'; import type * as BackgroundTaskManagerMod from './runtime/background-task-manager'; import type * as LivenessPolicyMod from './runtime/liveness-policy'; import type * as ResumableStreamExecutorMod from './runtime/resumable-stream-executor'; @@ -127,6 +128,9 @@ const loadUsageAccumulator = lazyModule( const loadRuntimeSkills = lazyModule( () => require('./skills/runtime-skills') as typeof RuntimeSkillsMod, ); +const loadPromptProfiles = lazyModule( + () => require('./prompts/prompt-profiles') as typeof PromptProfilesMod, +); const loadMaterializeRuntimeSkills = lazyModule( () => require('./skills/materialize-runtime-skills') as typeof MaterializeRuntimeSkillsMod, ); @@ -269,6 +273,10 @@ export const createInternalOperationTraceContext: typeof LangsmithTracingMod.cre export const createTraceReplayOnlyContext: typeof LangsmithTracingMod.createTraceReplayOnlyContext = lazyFunction(() => loadLangsmithTracing().createTraceReplayOnlyContext); +export const setTracePromptVersion: typeof LangsmithTracingMod.setTracePromptVersion = lazyFunction( + () => loadLangsmithTracing().setTracePromptVersion, +); + export const continueInstanceAiTraceContext: typeof LangsmithTracingMod.continueInstanceAiTraceContext = lazyFunction(() => loadLangsmithTracing().continueInstanceAiTraceContext); @@ -306,6 +314,18 @@ export type { SubAgentOptions } from './agent/sub-agent-factory'; export declare const INSTANCE_AI_SKILLS_DIR: typeof RuntimeSkillsMod.INSTANCE_AI_SKILLS_DIR; export const loadInstanceAiRuntimeSkillSource: typeof RuntimeSkillsMod.loadInstanceAiRuntimeSkillSource = lazyFunction(() => loadRuntimeSkills().loadInstanceAiRuntimeSkillSource); +export const loadInstanceAiRuntimeSkillSourceForBuildMode: typeof RuntimeSkillsMod.loadInstanceAiRuntimeSkillSourceForBuildMode = + lazyFunction(() => loadRuntimeSkills().loadInstanceAiRuntimeSkillSourceForBuildMode); +export const loadInstanceAiPromptSkills: typeof RuntimeSkillsMod.loadInstanceAiPromptSkills = + lazyFunction(() => loadRuntimeSkills().loadInstanceAiPromptSkills); +export const resolvePromptProfile: typeof PromptProfilesMod.resolvePromptProfile = lazyFunction( + () => loadPromptProfiles().resolvePromptProfile, +); +export const assertInstanceAiPromptVersion: typeof PromptProfilesMod.assertInstanceAiPromptVersion = + lazyFunction(() => loadPromptProfiles().assertInstanceAiPromptVersion); +export const describePromptProfile: typeof PromptProfilesMod.describePromptProfile = lazyFunction( + () => loadPromptProfiles().describePromptProfile, +); export const createLazyWorkspaceRuntimeSkillSource: typeof MaterializeRuntimeSkillsMod.createLazyWorkspaceRuntimeSkillSource = lazyFunction(() => loadMaterializeRuntimeSkills().createLazyWorkspaceRuntimeSkillSource); export { diff --git a/packages/@n8n/instance-ai/src/prompts/__tests__/prompt-profiles.test.ts b/packages/@n8n/instance-ai/src/prompts/__tests__/prompt-profiles.test.ts new file mode 100644 index 00000000000..86e302114f6 --- /dev/null +++ b/packages/@n8n/instance-ai/src/prompts/__tests__/prompt-profiles.test.ts @@ -0,0 +1,175 @@ +import { createRuntimeSkillSource, filterRuntimeSkillSource } from '@n8n/agents'; + +import { getSystemPrompt } from '../../agent/system-prompt'; +import { + loadInstanceAiPromptSkills, + loadInstanceAiRuntimeSkillSource, +} from '../../skills/runtime-skills'; +import { + assertInstanceAiPromptVersion, + getVersionedSystemPrompt, + describePromptProfile, + resolvePromptProfile, + INSTANCE_AI_PROMPT_PROFILES, + PROMPT_FRAGMENT_SKILLS, +} from '../prompt-profiles'; +import { composeSkillVariants, type SkillVariant } from '../skill-variants'; + +describe('prompt profiles', () => { + it('preserves the default prompt, skills, and tools', async () => { + const selected = resolvePromptProfile({}); + const original = filterRuntimeSkillSource( + loadInstanceAiRuntimeSkillSource(), + PROMPT_FRAGMENT_SKILLS, + ); + const result = await loadInstanceAiPromptSkills(selected.profile); + expect(result.source.registry).toEqual(original.registry); + expect(result.disabledTools).toEqual([]); + expect(getVersionedSystemPrompt(selected.profile.systemPromptVersion, {})).toBe( + getSystemPrompt({}), + ); + }); + + it('selects skills and tool exclusions together', async () => { + const selected = resolvePromptProfile({ mode: 'progressive' }); + const result = await loadInstanceAiPromptSkills(selected.profile); + expect(result.disabledTools).toEqual(['create-tasks']); + await expect(result.source.loadSkill('planning')).resolves.toBeNull(); + await expect(result.source.loadSkill('progressive-building')).resolves.toBeNull(); + expect((await result.source.loadSkill('workflow-builder'))?.version).toBe( + 'progressive-building@1', + ); + expect(describePromptProfile(selected, result.source)).toEqual({ + version: 'progressive@1', + systemPromptVersion: 'instance-agent@1', + skillVariants: ['progressive-building@1'], + skillsHash: result.source.registry.skillsHash, + }); + }); + + it('honors explicit versions before mode and reports retired-version fallback', () => { + expect(resolvePromptProfile({ version: 'default@1', mode: 'progressive' }).profile.mode).toBe( + 'default', + ); + expect(resolvePromptProfile({ version: 'progressive@1', mode: 'default' }).profile.mode).toBe( + 'progressive', + ); + const retired = resolvePromptProfile({ version: 'retired@1', mode: 'progressive' }); + expect(retired.profile.version).toBe('default@1'); + expect(retired.fallbackFrom).toBe('retired@1'); + expect(() => assertInstanceAiPromptVersion('retired@1')).toThrow( + 'Unknown Instance AI prompt version', + ); + }); + + it('rejects duplicate profile identifiers', () => { + expect(() => + resolvePromptProfile({}, [...INSTANCE_AI_PROMPT_PROFILES, resolvePromptProfile({}).profile]), + ).toThrow('must be unique'); + }); + + it('registers another profile without changing the system prompt', async () => { + const fixture: SkillVariant = { + id: 'concise@1', + changes: [{ skillId: 'answers', appendFrom: 'concise' }], + }; + const profiles = [ + ...INSTANCE_AI_PROMPT_PROFILES, + { + ...resolvePromptProfile({}).profile, + version: 'concise@1', + variants: [fixture], + }, + ]; + const selected = resolvePromptProfile({ version: 'concise@1' }, profiles); + const source = createRuntimeSkillSource([ + { + id: 'answers', + name: 'answers', + description: 'Answer questions.', + instructions: 'Answer from the evidence.', + }, + { + id: 'concise', + name: 'concise', + description: 'Keep answers short.', + instructions: 'Use one paragraph.', + }, + ]); + const result = await composeSkillVariants(source, selected.profile.variants); + expect((await result.source.loadSkill('answers'))?.instructions).toBe( + 'Answer from the evidence.\n\nUse one paragraph.', + ); + await expect(result.source.loadSkill('concise')).resolves.toBeNull(); + expect(getVersionedSystemPrompt(selected.profile.systemPromptVersion, {})).toBe( + getSystemPrompt({}), + ); + }); +}); + +describe('skill variant composition', () => { + const source = createRuntimeSkillSource([ + { + id: 'build', + name: 'build', + description: 'Build.', + instructions: 'Build carefully.', + dependencies: { tools: ['save'] }, + }, + { id: 'review', name: 'review', description: 'Review.', instructions: 'Inspect first.' }, + { id: 'policy', name: 'policy', description: 'Policy.', instructions: 'Check the result.' }, + ]); + const build: SkillVariant = { + id: 'build@1', + changes: [{ skillId: 'build', appendFrom: 'policy' }], + }; + const review: SkillVariant = { + id: 'review@1', + changes: [{ skillId: 'review', appendFrom: 'policy' }], + }; + + it('composes independent changes without changing the original source', async () => { + const result = await composeSkillVariants(source, [build, review]); + expect((await result.source.loadSkill('build'))?.instructions).toContain('Check the result.'); + expect((await result.source.loadSkill('review'))?.instructions).toContain('Check the result.'); + expect((await source.loadSkill('build'))?.instructions).toBe('Build carefully.'); + }); + + it('rejects duplicate, overlapping, disabled, and missing targets', async () => { + await expect(composeSkillVariants(source, [build, build])).rejects.toThrow('Duplicate'); + await expect( + composeSkillVariants(source, [build, { ...build, id: 'other@1' }]), + ).rejects.toThrow('Conflicting'); + await expect( + composeSkillVariants(source, [{ ...build, disabledSkills: ['build'] }]), + ).rejects.toThrow('disabled skill'); + await expect( + composeSkillVariants(source, [ + { id: 'missing@1', changes: [{ skillId: 'missing', appendFrom: 'policy' }] }, + ]), + ).rejects.toThrow('Unknown variant target'); + }); + + it('rejects removing a tool that a remaining skill requires', async () => { + await expect( + composeSkillVariants(source, [{ id: 'no-save@1', changes: [], disabledTools: ['save'] }]), + ).rejects.toThrow('requires disabled tool'); + }); + + it('removes disabled recommendations without treating them as required dependencies', async () => { + const optional = createRuntimeSkillSource([ + { + id: 'review', + name: 'review', + description: 'Review work.', + instructions: 'Review the result.', + recommendedTools: ['inspect', 'save'], + }, + ]); + const result = await composeSkillVariants(optional, [ + { id: 'read-only@1', changes: [], disabledTools: ['save'] }, + ]); + expect((await result.source.loadSkill('review'))?.recommendedTools).toEqual(['inspect']); + expect((await result.source.loadSkill('review'))?.instructions).toBe('Review the result.'); + }); +}); diff --git a/packages/@n8n/instance-ai/src/prompts/prompt-profiles.ts b/packages/@n8n/instance-ai/src/prompts/prompt-profiles.ts new file mode 100644 index 00000000000..b853ef223fb --- /dev/null +++ b/packages/@n8n/instance-ai/src/prompts/prompt-profiles.ts @@ -0,0 +1,102 @@ +import type { RuntimeSkillSource } from '@n8n/agents'; +import type { InstanceAiBuildMode, InstanceAiPromptConfiguration } from '@n8n/api-types'; +import { UnexpectedError, UserError } from 'n8n-workflow'; + +import type { SkillVariant } from './skill-variants'; +import { getSystemPrompt } from '../agent/system-prompt'; +import { ORCHESTRATION_TOOL_IDS } from '../tools/tool-ids'; + +export interface PromptProfile { + version: string; + mode: InstanceAiBuildMode; + systemPromptVersion: string; + variants: readonly SkillVariant[]; +} + +export const DEFAULT_PROMPT_VERSION = 'default@1'; +export const PROGRESSIVE_PROMPT_VERSION = 'progressive@1'; +const SYSTEM_PROMPT_VERSION = 'instance-agent@1'; + +const progressiveBuilding: SkillVariant = { + id: 'progressive-building@1', + changes: [ + { skillId: 'workflow-builder', appendFrom: 'progressive-building', useDescription: true }, + { skillId: 'post-build-flow', appendFrom: 'progressive-building' }, + ], + disabledSkills: ['planning'], + disabledTools: [ORCHESTRATION_TOOL_IDS.CREATE_TASKS], +}; + +/** Published versions are immutable. Add a new entry to change a profile. */ +export const INSTANCE_AI_PROMPT_PROFILES: readonly PromptProfile[] = [ + { + version: DEFAULT_PROMPT_VERSION, + mode: 'default', + systemPromptVersion: SYSTEM_PROMPT_VERSION, + variants: [], + }, + { + version: PROGRESSIVE_PROMPT_VERSION, + mode: 'progressive', + systemPromptVersion: SYSTEM_PROMPT_VERSION, + variants: [progressiveBuilding], + }, +]; + +export const PROMPT_FRAGMENT_SKILLS = [ + ...new Set( + INSTANCE_AI_PROMPT_PROFILES.flatMap((profile) => + profile.variants.flatMap((variant) => variant.changes.map((change) => change.appendFrom)), + ), + ), +]; + +export function assertInstanceAiPromptVersion(version: string): void { + if (!INSTANCE_AI_PROMPT_PROFILES.some((profile) => profile.version === version)) { + throw new UserError(`Unknown Instance AI prompt version "${version}"`); + } +} + +/** Explicit versions take precedence. Retired persisted versions recover with the default profile. */ +export function resolvePromptProfile( + selection: { version?: string; mode?: InstanceAiBuildMode }, + profiles: readonly PromptProfile[] = INSTANCE_AI_PROMPT_PROFILES, +): { profile: PromptProfile; fallbackFrom?: string } { + if (new Set(profiles.map((profile) => profile.version)).size !== profiles.length) { + throw new UnexpectedError('Prompt profile versions must be unique'); + } + const version = + selection.version ?? + (selection.mode === 'progressive' ? PROGRESSIVE_PROMPT_VERSION : DEFAULT_PROMPT_VERSION); + const selected = profiles.find((profile) => profile.version === version); + if (selected) return { profile: selected }; + const fallback = profiles.find((profile) => profile.version === DEFAULT_PROMPT_VERSION); + if (!fallback) throw new UnexpectedError('The default prompt profile is missing'); + return { profile: fallback, fallbackFrom: version }; +} + +export function describePromptProfile( + selection: ReturnType, + source: RuntimeSkillSource, +): InstanceAiPromptConfiguration { + return { + version: selection.profile.version, + systemPromptVersion: selection.profile.systemPromptVersion, + skillVariants: selection.profile.variants.map((variant) => variant.id), + skillsHash: source.registry.skillsHash, + ...(selection.fallbackFrom ? { fallbackFrom: selection.fallbackFrom } : {}), + }; +} + +const systemPrompts = new Map([ + [SYSTEM_PROMPT_VERSION, getSystemPrompt], +]); + +export function getVersionedSystemPrompt( + version: string, + options: Parameters[0], +): string { + const render = systemPrompts.get(version); + if (!render) throw new UnexpectedError(`Unknown system prompt version "${version}"`); + return render(options); +} diff --git a/packages/@n8n/instance-ai/src/prompts/skill-variants.ts b/packages/@n8n/instance-ai/src/prompts/skill-variants.ts new file mode 100644 index 00000000000..6da938c21d9 --- /dev/null +++ b/packages/@n8n/instance-ai/src/prompts/skill-variants.ts @@ -0,0 +1,99 @@ +import { + createRuntimeSkillSource, + filterRuntimeSkillSource, + type RuntimeSkillSource, +} from '@n8n/agents'; +import { UnexpectedError } from 'n8n-workflow'; + +export interface SkillVariant { + id: string; + changes: ReadonlyArray<{ + skillId: string; + appendFrom: string; + useDescription?: boolean; + }>; + disabledSkills?: readonly string[]; + disabledTools?: readonly string[]; +} + +/** Variants may change different skills. Overlapping changes require an explicit combined variant. */ +export async function composeSkillVariants( + source: RuntimeSkillSource, + variants: readonly SkillVariant[], + hiddenSkills: readonly string[] = [], +): Promise<{ source: RuntimeSkillSource; disabledTools: string[] }> { + const changes = new Map< + string, + { variant: SkillVariant; change: SkillVariant['changes'][number] } + >(); + const excluded = new Set(hiddenSkills); + const disabledTools = new Set(); + const variantIds = new Set(); + for (const variant of variants) { + if (variantIds.has(variant.id)) + throw new UnexpectedError(`Duplicate skill variant "${variant.id}"`); + variantIds.add(variant.id); + for (const id of variant.disabledSkills ?? []) excluded.add(id); + for (const name of variant.disabledTools ?? []) disabledTools.add(name); + for (const change of variant.changes) { + if (changes.has(change.skillId)) + throw new UnexpectedError(`Conflicting variants for skill "${change.skillId}"`); + changes.set(change.skillId, { variant, change }); + excluded.add(change.appendFrom); + } + } + for (const id of changes.keys()) { + if (excluded.has(id)) throw new UnexpectedError(`Variant changes disabled skill "${id}"`); + if (!source.registry.skills.some((skill) => skill.id === id)) { + throw new UnexpectedError(`Unknown variant target skill "${id}"`); + } + } + + const skills = await Promise.all( + source.registry.skills + .filter(({ id }) => !excluded.has(id)) + .map(async ({ id }) => { + const original = await source.loadSkill(id); + if (!original) throw new UnexpectedError(`Runtime skill "${id}" is missing`); + const selected = changes.get(id); + if (!selected) return original; + const fragment = await source.loadSkill(selected.change.appendFrom); + if (!fragment) + throw new UnexpectedError(`Skill fragment "${selected.change.appendFrom}" is missing`); + return { + ...original, + version: selected.variant.id, + description: selected.change.useDescription ? fragment.description : original.description, + instructions: `${original.instructions}\n\n${fragment.instructions}`, + ...(fragment.dependencies?.tools?.length + ? { + dependencies: { + ...original.dependencies, + tools: [ + ...new Set([ + ...(original.dependencies?.tools ?? []), + ...fragment.dependencies.tools, + ]), + ], + }, + } + : {}), + }; + }), + ); + const availableSkills = skills.map((skill) => { + const disabled = skill.dependencies?.tools?.find((tool) => disabledTools.has(tool)); + if (disabled) + throw new UnexpectedError(`Skill "${skill.id}" requires disabled tool "${disabled}"`); + const recommended = skill.recommendedTools?.filter((tool) => !disabledTools.has(tool)); + return recommended?.length !== skill.recommendedTools?.length + ? { ...skill, recommendedTools: recommended } + : skill; + }); + return { + source: filterRuntimeSkillSource({ ...source, ...createRuntimeSkillSource(availableSkills) }, [ + ...excluded, + ]), + disabledTools: [...disabledTools].sort(), + }; +} diff --git a/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts b/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts index efd518f0291..7ef101be4f3 100644 --- a/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts +++ b/packages/@n8n/instance-ai/src/runtime/__tests__/run-state-registry.test.ts @@ -72,6 +72,38 @@ describe('RunStateRegistry', () => { mockedNanoid.mockImplementation(() => `id-${++nanoidCounter}`); }); + it('retains mode across internal runs and clears it on a control request', () => { + registry.setBuildMode('thread-1', 'progressive'); + registry.startRun({ threadId: 'thread-1', user: { id: 'user-1', name: 'Alice' } }); + expect(registry.getBuildMode('thread-1')).toBe('progressive'); + registry.setBuildMode('thread-1', undefined); + expect(registry.getBuildMode('thread-1')).toBeUndefined(); + }); + + it('removes retained mode when a thread is cleared', () => { + registry.setBuildMode('thread-1', 'progressive'); + registry.clearThread('thread-1'); + expect(registry.getBuildMode('thread-1')).toBeUndefined(); + }); + + it('retains the selected prompt version and metadata until the next explicit selection', () => { + const metadata = { + version: 'progressive@1', + systemPromptVersion: 'instance-agent@1', + skillVariants: ['progressive-building@1'], + skillsHash: 'selected-skills', + }; + registry.setPromptConfiguration('thread-1', metadata); + registry.startRun({ threadId: 'thread-1', user: { id: 'user-1', name: 'Alice' } }); + expect(registry.getPromptVersion('thread-1')).toBe('progressive@1'); + expect(registry.getPromptConfiguration('thread-1')).toEqual(metadata); + registry.setPromptVersion('thread-1', 'default@1'); + expect(registry.getPromptConfiguration('thread-1')).toBeUndefined(); + expect(registry.getPromptVersion('thread-1')).toBe('default@1'); + registry.clearThread('thread-1'); + expect(registry.getPromptVersion('thread-1')).toBeUndefined(); + }); + // ── startRun ────────────────────────────────────────────────────────────── describe('startRun', () => { @@ -1034,16 +1066,31 @@ describe('RunStateRegistry', () => { describe('shutdown', () => { it('clears everything and returns all active and suspended runs', () => { registry.startRun({ threadId: 'thread-1', user: { id: 'u1', name: 'A' } }); + registry.setPromptConfiguration('thread-1', { + version: 'progressive@1', + systemPromptVersion: 'instance-agent@1', + skillVariants: ['progressive-building@1'], + skillsHash: 'selected-skills', + }); registry.startRun({ threadId: 'thread-2', user: { id: 'u2', name: 'B' } }); registry.suspendRun( 'thread-2', createSuspendedRunState({ threadId: 'thread-2', runId: 'run_suspended' }), ); + registry.startRun({ threadId: 'thread-3', user: { id: 'u3', name: 'C' } }); + registry.setPromptVersion('thread-3', 'default@1'); + expect(registry.getPromptConfiguration('thread-3')).toBeUndefined(); const result = registry.shutdown(); - expect(result.activeRuns).toHaveLength(1); + expect(result.activeRuns).toHaveLength(2); expect(result.activeRuns[0].runId).toBe('run_id-1'); + expect(result.activeRuns[0].promptVersion).toBe('progressive@1'); + expect(result.activeRuns.find((run) => run.threadId === 'thread-3')?.promptVersion).toBe( + 'default@1', + ); + expect(registry.getPromptConfiguration('thread-1')).toBeUndefined(); + expect(registry.getPromptVersion('thread-3')).toBeUndefined(); expect(result.suspendedRuns).toHaveLength(1); expect(result.suspendedRuns[0].runId).toBe('run_suspended'); }); diff --git a/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts b/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts index 5e678609ef0..8601764b858 100644 --- a/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts +++ b/packages/@n8n/instance-ai/src/runtime/run-state-registry.ts @@ -1,4 +1,6 @@ import type { + InstanceAiBuildMode, + InstanceAiPromptConfiguration, InstanceAiCredentialDestinationDecision, InstanceAiThreadStatusResponse, } from '@n8n/api-types'; @@ -147,6 +149,12 @@ export class RunStateRegistry { /** IANA time zone captured at initial-run entry and reused by follow-up runs. */ private readonly threadTimeZones = new Map(); + /** Build mode captured at user-run entry and reused by follow-up runs. */ + private readonly threadBuildModes = new Map(); + private readonly threadPromptSelections = new Map< + string, + { version: string; metadata?: InstanceAiPromptConfiguration } + >(); /** * Resolves a user id from the opaque `TUser` the registry is parameterised over. * Required rather than optional: per-user concurrency counting depends on it, and a @@ -487,6 +495,33 @@ export class RunStateRegistry { return this.threadTimeZones.get(threadId); } + /** Retain the request mode for internal follow-ups. An omitted mode clears it. */ + setBuildMode(threadId: string, buildMode: InstanceAiBuildMode | undefined): void { + if (buildMode === undefined) this.threadBuildModes.delete(threadId); + else this.threadBuildModes.set(threadId, buildMode); + } + + getBuildMode(threadId: string): InstanceAiBuildMode | undefined { + return this.threadBuildModes.get(threadId); + } + + setPromptVersion(threadId: string, version: string | undefined): void { + if (version === undefined) this.threadPromptSelections.delete(threadId); + else this.threadPromptSelections.set(threadId, { version }); + } + + getPromptVersion(threadId: string): string | undefined { + return this.threadPromptSelections.get(threadId)?.version; + } + + setPromptConfiguration(threadId: string, metadata: InstanceAiPromptConfiguration): void { + this.threadPromptSelections.set(threadId, { version: metadata.version, metadata }); + } + + getPromptConfiguration(threadId: string): InstanceAiPromptConfiguration | undefined { + return this.threadPromptSelections.get(threadId)?.metadata; + } + /** * Find active/suspended runs and pending confirmations that timed out under policy. * Returns thread IDs and request IDs that should be cancelled/rejected. @@ -637,6 +672,8 @@ export class RunStateRegistry { this.threadUsers.delete(threadId); this.threadTimeZones.delete(threadId); + this.threadBuildModes.delete(threadId); + this.threadPromptSelections.delete(threadId); const groupId = this.threadMessageGroupId.get(threadId); if (groupId) this.runIdsByMessageGroup.delete(groupId); @@ -666,11 +703,15 @@ export class RunStateRegistry { * confirmation Promise. */ shutdown(): { - activeRuns: ActiveRunState[]; + activeRuns: Array; suspendedRuns: Array>; pendingThreadIds: string[]; } { - const activeRuns = [...this.activeRuns.values()]; + // Capture the selected version before clearing per-thread state. + const activeRuns = [...this.activeRuns.values()].map((run) => ({ + ...run, + promptVersion: this.getPromptVersion(run.threadId), + })); const suspendedRuns = [...this.suspendedRuns.values()]; const pendingThreadIds = [ ...new Set([...this.pendingConfirmations.values()].map((p) => p.threadId)), @@ -681,6 +722,8 @@ export class RunStateRegistry { this.pendingConfirmations.clear(); this.threadUsers.clear(); this.threadTimeZones.clear(); + this.threadBuildModes.clear(); + this.threadPromptSelections.clear(); this.threadMessageGroupId.clear(); this.runIdsByMessageGroup.clear(); diff --git a/packages/@n8n/instance-ai/src/skills/__tests__/progressive-building.test.ts b/packages/@n8n/instance-ai/src/skills/__tests__/progressive-building.test.ts new file mode 100644 index 00000000000..27e72f585a9 --- /dev/null +++ b/packages/@n8n/instance-ai/src/skills/__tests__/progressive-building.test.ts @@ -0,0 +1,101 @@ +import { createSkillLoadTool, filterRuntimeSkillSource } from '@n8n/agents'; +import { mock } from 'vitest-mock-extended'; + +import { getSystemPrompt } from '../../agent/system-prompt'; +import type { Logger } from '../../logger'; +import { buildRuntimeSkillWorkspaceBundle } from '../materialize-runtime-skills'; +import { + loadInstanceAiRuntimeSkillSource, + loadInstanceAiRuntimeSkillSourceForBuildMode, +} from '../runtime-skills'; +import { disabledInstanceAiSkillIds } from '../skill-gates'; + +const logger = mock(); + +describe('progressive workflow skill variants', () => { + it('excludes planning and changes only the workflow-building and post-build instructions', async () => { + const source = loadInstanceAiRuntimeSkillSource(); + const originalRegistry = structuredClone(source.registry); + const progressive = await loadInstanceAiRuntimeSkillSourceForBuildMode('progressive'); + const policy = await source.loadSkill('progressive-building'); + if (!policy) throw new Error('Expected the progressive policy'); + + for (const entry of source.registry.skills) { + const original = await source.loadSkill(entry.id); + const selected = await progressive.loadSkill(entry.id); + const selectedEntry = progressive.registry.skills.find(({ id }) => id === entry.id); + if (entry.id === 'planning' || entry.id === 'progressive-building') { + expect(original).not.toBeNull(); + expect(selected).toBeNull(); + expect(selectedEntry).toBeUndefined(); + } else if (entry.id === 'workflow-builder' || entry.id === 'post-build-flow') { + expect(selected?.instructions).toContain(original?.instructions); + expect(selected?.instructions).toContain(policy.instructions); + expect(selectedEntry?.hash).not.toBe(entry.hash); + } else if (entry.id === 'planned-task-runtime') { + expect(selected?.instructions).toBe(original?.instructions); + expect(selected?.recommendedTools).not.toContain('create-tasks'); + } else { + expect(selected).toEqual(original); + expect(selectedEntry).toEqual(entry); + } + } + + expect(progressive.registry.skillsHash).not.toBe(source.registry.skillsHash); + expect(source.registry).toEqual(originalRegistry); + const control = await loadInstanceAiRuntimeSkillSourceForBuildMode('default'); + expect(control.registry).toEqual( + filterRuntimeSkillSource(source, ['progressive-building']).registry, + ); + await expect(loadInstanceAiRuntimeSkillSourceForBuildMode(undefined)).resolves.toBe(control); + }); + + it('serves the selected skills through both workspace files and load_skill', async () => { + const policy = await loadInstanceAiRuntimeSkillSource().loadSkill('progressive-building'); + if (!policy) throw new Error('Expected the progressive policy'); + const excluded = disabledInstanceAiSkillIds({ + configEvalsEnabled: true, + }); + const control = filterRuntimeSkillSource( + await loadInstanceAiRuntimeSkillSourceForBuildMode('default'), + excluded, + ); + const progressive = filterRuntimeSkillSource( + await loadInstanceAiRuntimeSkillSourceForBuildMode('progressive'), + excluded, + ); + const root = '/home/daytona/workspace'; + const [controlBundle, progressiveBundle] = await Promise.all( + [control, progressive].map( + async (source) => await buildRuntimeSkillWorkspaceBundle({ source, root, logger }), + ), + ); + if (!controlBundle || !progressiveBundle) throw new Error('Expected skill bundles'); + + expect(progressiveBundle.manifest.skillsHash).not.toBe(controlBundle.manifest.skillsHash); + for (const skillId of ['workflow-builder', 'post-build-flow']) { + const skillPath = `${root}/skills/${skillId}/SKILL.md`; + expect(progressiveBundle.files.get(skillPath)).toContain('# Progressive building'); + expect(controlBundle.files.get(skillPath)).not.toContain('# Progressive building'); + const loadTool = createSkillLoadTool(progressiveBundle.source); + const loaded = await loadTool.handler?.({ skillId }, {}); + expect(loaded).toHaveProperty('value.0.text', expect.stringContaining(policy.instructions)); + } + + const reference = `${root}/skills/post-build-flow/references/trigger-input-data-shapes.md`; + expect(progressiveBundle.files.get(reference)).toBeTruthy(); + expect(progressiveBundle.files.get(reference)).toBe(controlBundle.files.get(reference)); + expect(controlBundle.files.has(`${root}/skills/planning/SKILL.md`)).toBe(true); + expect(progressiveBundle.files.has(`${root}/skills/planning/SKILL.md`)).toBe(false); + await expect(progressive.loadSkill('planning')).resolves.toBeNull(); + await expect( + createSkillLoadTool(progressiveBundle.source).handler?.({ skillId: 'planning' }, {}), + ).resolves.toMatchObject({ success: false }); + await expect(progressive.loadSkill('progressive-building')).resolves.toBeNull(); + expect(progressiveBundle.files.has(`${root}/skills/progressive-building/SKILL.md`)).toBe(false); + }); + + it('keeps the progressive policy out of the general system prompt', () => { + expect(getSystemPrompt()).not.toContain('# Progressive building'); + }); +}); diff --git a/packages/@n8n/instance-ai/src/skills/__tests__/runtime-skills.test.ts b/packages/@n8n/instance-ai/src/skills/__tests__/runtime-skills.test.ts index 219fae04827..e43b624a0eb 100644 --- a/packages/@n8n/instance-ai/src/skills/__tests__/runtime-skills.test.ts +++ b/packages/@n8n/instance-ai/src/skills/__tests__/runtime-skills.test.ts @@ -2,7 +2,11 @@ import { createSkillLoadTool } from '@n8n/agents'; import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { INSTANCE_AI_SKILLS_DIR, loadInstanceAiRuntimeSkillSource } from '../runtime-skills'; +import { + INSTANCE_AI_SKILLS_DIR, + loadInstanceAiRuntimeSkillSource, + loadInstanceAiRuntimeSkillSourceForBuildMode, +} from '../runtime-skills'; import { CONFIG_EVALS_SKILL_ID, disabledInstanceAiSkillIds } from '../skill-gates'; const ORIGINAL_ENABLED_MODULES = process.env.N8N_ENABLED_MODULES; @@ -171,6 +175,19 @@ describe('Instance AI runtime skills', () => { expect(configEvals?.id).toBe(CONFIG_EVALS_SKILL_ID); }); + it('keeps the progressive-building fragment out of both profile catalogs', async () => { + const source = loadInstanceAiRuntimeSkillSource(); + const progressive = source.registry.skills.find( + (skill) => skill.name === 'progressive-building', + ); + expect(progressive?.id).toBe('progressive-building'); + for (const mode of ['default', 'progressive'] as const) { + const selected = await loadInstanceAiRuntimeSkillSourceForBuildMode(mode); + expect(selected.registry.skills.map(({ id }) => id)).not.toContain('progressive-building'); + await expect(selected.loadSkill('progressive-building')).resolves.toBeNull(); + } + }); + it('excludes bundled Agents module skills unless the module is enabled', async () => { const source = await loadRuntimeSkillSourceWithEnabledModules('instance-ai'); diff --git a/packages/@n8n/instance-ai/src/skills/runtime-skills.ts b/packages/@n8n/instance-ai/src/skills/runtime-skills.ts index 4c0ff050f40..68798347d27 100644 --- a/packages/@n8n/instance-ai/src/skills/runtime-skills.ts +++ b/packages/@n8n/instance-ai/src/skills/runtime-skills.ts @@ -1,12 +1,21 @@ import { loadRuntimeSkillSourceFromDirectory, type RuntimeSkillSource } from '@n8n/agents'; +import type { InstanceAiBuildMode } from '@n8n/api-types'; import { resolve } from 'node:path'; import { isAgentFeatureEnabled } from '@/utils/agent-feature-enabled'; +import { + PROMPT_FRAGMENT_SKILLS, + resolvePromptProfile, + type PromptProfile, +} from '../prompts/prompt-profiles'; +import { composeSkillVariants } from '../prompts/skill-variants'; + export const INSTANCE_AI_SKILLS_DIR = resolve(__dirname, '..', '..', 'skills'); const AGENTS_MODULE_RUNTIME_SKILLS = new Set(['agent-builder', 'intent-recognition']); let cachedRuntimeSkillSource: RuntimeSkillSource | undefined; +const cachedProfiles = new Map>(); export function loadInstanceAiRuntimeSkillSource(): RuntimeSkillSource { cachedRuntimeSkillSource ??= loadRuntimeSkillSourceFromDirectory(INSTANCE_AI_SKILLS_DIR, { @@ -15,6 +24,25 @@ export function loadInstanceAiRuntimeSkillSource(): RuntimeSkillSource { return cachedRuntimeSkillSource; } +export async function loadInstanceAiRuntimeSkillSourceForBuildMode( + mode: InstanceAiBuildMode | undefined, +): Promise { + return (await loadInstanceAiPromptSkills(resolvePromptProfile({ mode }).profile)).source; +} + +export async function loadInstanceAiPromptSkills(profile: PromptProfile) { + let pending = cachedProfiles.get(profile.version); + if (!pending) { + pending = composeSkillVariants( + loadInstanceAiRuntimeSkillSource(), + profile.variants, + PROMPT_FRAGMENT_SKILLS, + ); + cachedProfiles.set(profile.version, pending); + } + return await pending; +} + export function hasRuntimeSkills( source: RuntimeSkillSource | undefined, ): source is RuntimeSkillSource { diff --git a/packages/@n8n/instance-ai/src/tools/__tests__/index.test.ts b/packages/@n8n/instance-ai/src/tools/__tests__/index.test.ts index 862d7eac895..a46433e08d3 100644 --- a/packages/@n8n/instance-ai/src/tools/__tests__/index.test.ts +++ b/packages/@n8n/instance-ai/src/tools/__tests__/index.test.ts @@ -1,6 +1,8 @@ +import { mock } from 'vitest-mock-extended'; + import { createAllTools, createOrchestrationTools, createOrchestratorDomainTools } from '..'; import { isParseableAttachment } from '../../parsers/structured-file-parser'; -import type { InstanceAiContext } from '../../types'; +import type { InstanceAiContext, OrchestrationContext } from '../../types'; import { ALWAYS_LOADED_TOOL_NAMES } from '../tool-ids'; vi.mock('../../parsers/structured-file-parser', () => ({ @@ -241,13 +243,10 @@ describe('domain tool construction', () => { ); }); - it('registers create-tasks but not the removed plan orchestration tool', () => { - const context = makeContext({ - workflowTaskService: {}, - domainContext: {}, - } as Partial); + it('constructs create-tasks for the agent to apply profile exclusions', () => { + const context = mock(); - const orchestrationTools = createOrchestrationTools(context as never); + const orchestrationTools = createOrchestrationTools(context); expect(orchestrationTools.has('create-tasks')).toBe(true); expect(orchestrationTools.has('plan')).toBe(false); diff --git a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts index 362603fbefe..4e45160d1bd 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/__tests__/build-workflow.tool.test.ts @@ -2,6 +2,10 @@ import { executeTool } from '../../../__tests__/tool-test-utils'; import { FolderResolutionError } from '../../../errors/folder-resolution.error'; import { WorkflowNotFoundError } from '../../../errors/workflow-not-found.error'; import { WorkflowSaveConflictError } from '../../../errors/workflow-save-conflict.error'; +import { + loadInstanceAiRuntimeSkillSource, + loadInstanceAiRuntimeSkillSourceForBuildMode, +} from '../../../skills/runtime-skills'; import { emitTraceOnlyChildRun } from '../../../tracing/langsmith-tracing'; import type { InstanceAiContext } from '../../../types'; import type { WorkflowBuildOutcome } from '../../../workflow-loop/workflow-loop-state'; @@ -422,36 +426,92 @@ describe('createBuildWorkflowTool', () => { }); }); - it('hands one-off builds to the one-off-operations skill with optional verification', async () => { - const source = 'workflow source from workspace'; - const { context, filePath } = makeContext({ source }); + it('uses the selected post-build instructions without sharing them across modes', async () => { + const source = loadInstanceAiRuntimeSkillSource(); + const policy = await source.loadSkill('progressive-building'); + if (!policy) throw new Error('Expected the progressive policy'); + for (const mode of ['default', 'progressive', 'default'] as const) { + const { context, filePath } = makeContext({ + source: 'workflow source', + overrides: { + runtimeSkillCatalog: await loadInstanceAiRuntimeSkillSourceForBuildMode(mode), + }, + }); - const result = await executeTool(createBuildWorkflowTool(context), { - filePath, - name: 'One-off attendee export', - executionIntent: 'one-off', - }); + const result = await executeTool(createBuildWorkflowTool(context), { + filePath, + name: 'Request tracker', + }); - expect(result).toMatchObject({ - success: true, - // One-off intent rides on executionIntent, NOT on a new readiness - // status — the readiness union is persisted and old readers hard-fail - // on unknown variants (rollback safety). - verificationReadiness: { status: 'ready' }, - executionIntent: 'one-off', - postBuildFlow: { - required: true, - skillId: 'one-off-operations', - reason: 'direct-one-off-build-succeeded', - }, - }); - expect(result.postBuildFlow?.guidance).toContain('Simulated verification is NOT required'); - expect(result.postBuildFlow?.instructions).toContain('# One-Off Operations'); - expect(result.postBuildFlow?.instructions).not.toContain('recommended_tools'); - // The verify-biased post-build-flow body must NOT ride along on a one-off build. - expect(result.postBuildFlow?.instructions).not.toContain('# Post-Build Flow'); + expect(result.success).toBe(true); + expect(result.postBuildFlow?.skillId).toBe('post-build-flow'); + expect(result.postBuildFlow?.instructions).toContain('# Post-Build Flow'); + expect(result.postBuildFlow?.instructions?.includes(policy.instructions)).toBe( + mode === 'progressive', + ); + expect(result.postBuildFlow?.instructions).not.toContain('## Verification follow-up'); + expect(result.postBuildFlow?.instructions).not.toContain('## Setup follow-up'); + } }); + it.each([ + ['reusable', 'post-build-flow'], + ['one-off', 'one-off-operations'], + ] as const)('activates the selected skill for a %s build', async (executionIntent, skillId) => { + const source = await loadInstanceAiRuntimeSkillSourceForBuildMode('progressive'); + const loadSkill = vi.fn(source.loadSkill); + const { context, filePath } = makeContext({ source: 'workflow source' }); + const result = await executeTool( + createBuildWorkflowTool(context), + { filePath, name: 'Request tracker', executionIntent }, + { loadSkill }, + ); + expect(result.success).toBe(true); + expect(loadSkill).toHaveBeenCalledWith(skillId); + expect(result.postBuildFlow?.instructions).toBe( + `Follow the active ${skillId} skill instructions.`, + ); + }); + + it.each(['default', 'progressive'] as const)( + 'hands one-off builds to the one-off-operations skill in %s mode', + async (mode) => { + const source = 'workflow source from workspace'; + const { context, filePath } = makeContext({ + source, + overrides: { + runtimeSkillCatalog: await loadInstanceAiRuntimeSkillSourceForBuildMode(mode), + }, + }); + + const result = await executeTool(createBuildWorkflowTool(context), { + filePath, + name: 'One-off attendee export', + executionIntent: 'one-off', + }); + + expect(result).toMatchObject({ + success: true, + // One-off intent rides on executionIntent, NOT on a new readiness + // status — the readiness union is persisted and old readers hard-fail + // on unknown variants (rollback safety). + verificationReadiness: { status: 'ready' }, + executionIntent: 'one-off', + postBuildFlow: { + required: true, + skillId: 'one-off-operations', + reason: 'direct-one-off-build-succeeded', + }, + }); + expect(result.postBuildFlow?.guidance).toContain('Simulated verification is NOT required'); + expect(result.postBuildFlow?.instructions).toContain('# One-Off Operations'); + expect(result.postBuildFlow?.instructions).not.toContain('recommended_tools'); + // The verify-biased post-build-flow body must NOT ride along on a one-off build. + expect(result.postBuildFlow?.instructions).not.toContain('# Post-Build Flow'); + expect(result.postBuildFlow?.instructions).not.toContain('# Progressive building'); + }, + ); + it('drops invalid node groups before saving and reports the drop', async () => { const source = 'workflow source from workspace'; const { context, filePath, trackTelemetry } = makeContext({ source }); diff --git a/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts b/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts index 9d6125f4cf5..cff4fd4d77f 100644 --- a/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts +++ b/packages/@n8n/instance-ai/src/tools/workflows/build-workflow.tool.ts @@ -1,4 +1,4 @@ -import { Tool } from '@n8n/agents'; +import { Tool, type RuntimeSkillSource, type RuntimeSkillLoader } from '@n8n/agents'; import { instanceAiApprovalResumeSchema, instanceAiConfirmationSeveritySchema, @@ -9,10 +9,8 @@ import { SDK_IMPORTABLE_FUNCTIONS, type WorkflowJSON, } from '@n8n/workflow-sdk'; -import { makeGetNodeTypeForGrouping } from 'n8n-workflow'; +import { makeGetNodeTypeForGrouping, UnexpectedError } from 'n8n-workflow'; import { nanoid } from 'nanoid'; -import { readFileSync } from 'node:fs'; -import { join } from 'node:path'; import { z } from 'zod'; import { computeChatModelValidationIssues } from './chat-model-validation'; @@ -85,7 +83,7 @@ import { } from './workflow-validation-warnings'; import { FolderResolutionError } from '../../errors/folder-resolution.error'; import { WorkflowSaveConflictError } from '../../errors/workflow-save-conflict.error'; -import { INSTANCE_AI_SKILLS_DIR } from '../../skills/runtime-skills'; +import { loadInstanceAiRuntimeSkillSource } from '../../skills/runtime-skills'; import { emitTraceOnlyChildRun } from '../../tracing/langsmith-tracing'; import type { FolderResolutionFailure, InstanceAiContext, WorkflowFolderRef } from '../../types'; import { BuildFailureTracker } from '../../workflow-builder/build-failure-tracker'; @@ -114,6 +112,7 @@ const confirmationSuspendSchema = z.object({ const confirmationResumeSchema = instanceAiApprovalResumeSchema; interface BuildCtx { + loadSkill?: RuntimeSkillLoader; toolCallId?: string; resumeData?: z.infer; suspend?: (payload: z.infer) => Promise; @@ -320,9 +319,6 @@ const ONE_OFF_OPERATIONS_GUIDANCE = const POST_BUILD_FLOW_GUIDANCE = 'This direct build is not complete yet. Follow the post-build instructions in `instructions` now (do NOT load the post-build-flow skill — they are the same instructions) before verification, setup, error-workflow follow-up, publishing, testing, or any final user-visible summary. Follow-up order is verification/setup first, then mocked/no-mock live-test when latest verification used mocks or simulations, then generic testing prompts. Until a non-simulated execution succeeds, never offer publishing as an alternative to the live test. A user-run execution counts only after `executions(action="list")` and `executions(action="get")` confirm that it succeeded and ran the required path; the user\'s statement alone is not execution evidence. Honor an explicit publish request before live execution only after warning that the live path remains untested. Offer the explicit error-workflow opt-in for direct new primary workflows only after the primary workflow is successfully published. Do not replace the error-workflow opt-in with a generic add-anything, publish, or test question.'; -// Inlined into successful build results; the skills stay registered for tag-driven follow-up turns. -const inlineSkillInstructionsCache = new Map(); - /** Tag-turn-only sections, stripped from the inline copy; follow-up turns load the full skill. */ const INLINE_SKIPPED_SECTIONS = [ '## Verification follow-up', @@ -330,20 +326,19 @@ const INLINE_SKIPPED_SECTIONS = [ '## Credentials before build', ]; -function getInlineSkillInstructions(skillId: string): string { - let instructions = inlineSkillInstructionsCache.get(skillId); - if (instructions === undefined) { - const raw = readFileSync(join(INSTANCE_AI_SKILLS_DIR, skillId, 'SKILL.md'), 'utf-8'); - // Strip the YAML front-matter; catalog metadata is noise in a tool result. - const body = raw.replace(/^---\n[\s\S]*?\n---\n/, '').trim(); - instructions = body - .split(/\n(?=## )/) - .filter((section) => !INLINE_SKIPPED_SECTIONS.some((title) => section.startsWith(title))) - .join('\n') - .trim(); - inlineSkillInstructionsCache.set(skillId, instructions); - } - return instructions; +async function getInlineSkillInstructions( + skillId: string, + source: RuntimeSkillSource = loadInstanceAiRuntimeSkillSource(), + activate?: RuntimeSkillLoader, +): Promise { + const skill = await (activate ?? source.loadSkill)(skillId); + if (!skill) throw new UnexpectedError(`Runtime skill "${skillId}" is missing`); + if (activate) return `Follow the active ${skillId} skill instructions.`; + return skill.instructions + .split(/\n(?=#{1,2} )/) + .filter((section) => !INLINE_SKIPPED_SECTIONS.some((title) => section.startsWith(title))) + .join('\n') + .trim(); } // Discriminated on skillId so a mismatched skillId/reason pair cannot validate. @@ -365,11 +360,13 @@ const postBuildFlowOutputSchema = z.discriminatedUnion('skillId', [ }), ]); -function directPostBuildFlowHandoff( +async function directPostBuildFlowHandoff( owner: ReturnType['owner'], isAuxiliarySupportingWorkflow: boolean, outcome: WorkflowBuildOutcome, -): z.infer | undefined { + skills?: RuntimeSkillSource, + activate?: RuntimeSkillLoader, +): Promise | undefined> { if (owner?.type !== 'direct' || isAuxiliarySupportingWorkflow) return undefined; // One-off instructions only apply when the workflow can actually run — their @@ -382,7 +379,7 @@ function directPostBuildFlowHandoff( skillId: ONE_OFF_OPERATIONS_SKILL_ID, reason: 'direct-one-off-build-succeeded', guidance: ONE_OFF_OPERATIONS_GUIDANCE, - instructions: getInlineSkillInstructions(ONE_OFF_OPERATIONS_SKILL_ID), + instructions: await getInlineSkillInstructions(ONE_OFF_OPERATIONS_SKILL_ID, skills, activate), }; } @@ -391,7 +388,7 @@ function directPostBuildFlowHandoff( skillId: POST_BUILD_FLOW_SKILL_ID, reason: 'direct-build-succeeded', guidance: POST_BUILD_FLOW_GUIDANCE, - instructions: getInlineSkillInstructions(POST_BUILD_FLOW_SKILL_ID), + instructions: await getInlineSkillInstructions(POST_BUILD_FLOW_SKILL_ID, skills, activate), }; } @@ -1308,10 +1305,12 @@ export function createBuildWorkflowTool(context: InstanceAiContext) { executionIntent, summary, }); - const postBuildFlow = directPostBuildFlowHandoff( + const postBuildFlow = await directPostBuildFlowHandoff( owner, isAuxiliarySupportingWorkflow, outcome, + context.runtimeSkillCatalog, + ctx.loadSkill, ); await promoteMainWorkflow(context, saved.id); diff --git a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts index 6cf88761ae4..ad113ca133c 100644 --- a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts +++ b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts @@ -20,6 +20,7 @@ import { redactLangSmithTelemetrySpan, releaseTraceClient, shutdownProductTelemetryProviders, + setTracePromptVersion, submitLangsmithUserFeedback, withCurrentTraceSpan, } from '../langsmith-tracing'; @@ -443,6 +444,8 @@ describe('createInstanceAiTraceContext', () => { }); expect(tracing?.getTelemetry).toBeDefined(); + setTracePromptVersion(tracing, undefined); + expect(tracing?.rootRun.metadata).not.toHaveProperty('prompt_version'); const telemetryOrBuilder = tracing!.getTelemetry!({ agentRole: 'orchestrator', @@ -457,6 +460,7 @@ describe('createInstanceAiTraceContext', () => { expect(telemetry.recordInputs).toBe(true); expect(telemetry.recordOutputs).toBe(true); expect(telemetry.runtimeRootSpanEnabled).toBe(false); + expect(telemetry.metadata).not.toHaveProperty('prompt_version'); expect(telemetry.metadata).toEqual( expect.objectContaining({ thread_id: 'thread-1', @@ -566,6 +570,7 @@ describe('createInstanceAiTraceContext', () => { input: { message: 'What workflows do I have?' }, }); const actorRun = await startForegroundActor(tracing!); + setTracePromptVersion(tracing, 'default@1'); const telemetryOrBuilder = tracing!.getTelemetry!({ agentRole: 'orchestrator', @@ -578,8 +583,15 @@ describe('createInstanceAiTraceContext', () => { expect.objectContaining({ langsmith_root_run_id: tracing?.rootRun.id, langsmith_actor_run_id: actorRun.id, + prompt_version: 'default@1', }), ); + expect(tracing?.rootRun.metadata).toHaveProperty('prompt_version', 'default@1'); + expect(actorRun.metadata).toHaveProperty('prompt_version', 'default@1'); + for (const run of [tracing!.rootRun, actorRun]) { + const span = agentsMock.getSpans().find((entry) => entry.id === run.otelSpanId); + expect(span?.attributes).toHaveProperty('langsmith.metadata.prompt_version', 'default@1'); + } await telemetry.provider?.shutdown(); }); @@ -1338,6 +1350,7 @@ describe('createInstanceAiTraceContext', () => { }); expect(tracing).toBeDefined(); + setTracePromptVersion(tracing, 'progressive@1'); const continuedTracing = await continueInstanceAiTraceContext(tracing!, { threadId: 'thread-1', @@ -1360,6 +1373,7 @@ describe('createInstanceAiTraceContext', () => { expect(continuedTracing.rootRun.metadata).toEqual( expect.objectContaining({ 'instance_ai.canonical_name': 'instance-ai.orchestrator_resume', + prompt_version: 'progressive@1', }), ); expect(continuedTracing.rootRun.metadata).toEqual( @@ -1379,6 +1393,7 @@ describe('createInstanceAiTraceContext', () => { expect(continuedTracing.orchestratorRun.metadata).toEqual( expect.objectContaining({ 'instance_ai.canonical_name': 'instance-ai.agent.orchestrator', + prompt_version: 'progressive@1', }), ); }); @@ -1693,6 +1708,22 @@ describe('createInstanceAiTraceContext', () => { expect(JSON.stringify(inputs)).not.toContain('Full skill instructions'); }); + it('records the selected profile and system prompt hash in trace inputs', () => { + const inputs = buildAgentTraceInputs({ + systemPrompt: 'Test instructions.', + promptConfiguration: { + version: 'default@1', + systemPromptVersion: 'instance-agent@1', + skillVariants: [], + skillsHash: 'selected-skills', + fallbackFrom: 'retired@1', + }, + }); + expect(inputs).toHaveProperty('prompt_configuration.version', 'default@1'); + expect(inputs).toHaveProperty('prompt_configuration.fallbackFrom', 'retired@1'); + expect(inputs.system_prompt_hash).toMatch(/^[a-f0-9]{64}$/); + }); + it('redacts model secrets from trace metadata', async () => { const tracing = await createInstanceAiTraceContext({ threadId: 'thread-1', diff --git a/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts b/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts index 93ddbb7291e..b7fae33152c 100644 --- a/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts +++ b/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts @@ -915,6 +915,18 @@ export function appendRootRunMetadata( } } +export function setTracePromptVersion( + tracing: InstanceAiTraceContext | undefined, + version: string | undefined, +): void { + if (!tracing || version === undefined) return; + const metadata = { prompt_version: version }; + appendRootRunMetadata(tracing.rootRun, metadata); + if (tracing.actorRun.id !== tracing.rootRun.id) { + appendRootRunMetadata(tracing.actorRun, metadata); + } +} + export function appendGeneratedWorkflowIdToRootMetadata( root: InstanceAiTraceRun, workflowId: string, @@ -1715,14 +1727,21 @@ function createTelemetryFactory(options: { const executionMode = telemetryOptions.executionMode ?? (options.traceKind === 'background_subagent' ? 'background_subagent' : 'foreground'); - const metadata = toTelemetryMetadata(options.baseMetadata, telemetryOptions.metadata, { - agent_role: agentRole, - execution_mode: executionMode, - trace_kind: options.traceKind, - langsmith_trace_id: options.rootRun.traceId, - langsmith_root_run_id: options.rootRun.id, - langsmith_actor_run_id: actorRun.id, - }); + const metadata = toTelemetryMetadata( + options.baseMetadata, + { + prompt_version: options.rootRun.metadata?.prompt_version, + }, + telemetryOptions.metadata, + { + agent_role: agentRole, + execution_mode: executionMode, + trace_kind: options.traceKind, + langsmith_trace_id: options.rootRun.traceId, + langsmith_root_run_id: options.rootRun.id, + langsmith_actor_run_id: actorRun.id, + }, + ); const functionId = telemetryOptions.functionId ?? formatTelemetryFunctionId(agentRole); if (options.baseTelemetry) { @@ -1880,7 +1899,14 @@ export async function continueInstanceAiTraceContext( return existingContext; } - const baseMetadata = await buildBaseMetadata(options); + const promptVersion = existingContext?.rootRun.metadata?.prompt_version; + const baseMetadata = await buildBaseMetadata({ + ...options, + metadata: { + ...(typeof promptVersion === 'string' ? { prompt_version: promptVersion } : {}), + ...options.metadata, + }, + }); const projectName = existingContext?.projectName ?? options.projectName ?? resolveDefaultProjectName(); const continuedMetadata = diff --git a/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts b/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts index c9e38f003d9..a28d61651d2 100644 --- a/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts +++ b/packages/@n8n/instance-ai/src/tracing/trace-payloads.ts @@ -5,6 +5,7 @@ import { type RedactionOptions, type RuntimeSkillRegistry, } from '@n8n/agents'; +import type { InstanceAiPromptConfiguration } from '@n8n/api-types'; import { isRecord } from '@n8n/utils/is-record'; import { createHash } from 'node:crypto'; @@ -98,6 +99,7 @@ const LLM_AI_SDK_OPERATION_IDS = new Set([ ]); export interface AgentTraceInputOptions { + promptConfiguration?: InstanceAiPromptConfiguration; systemPrompt?: string; tools?: InstanceAiToolRegistry; deferredTools?: InstanceAiToolRegistry; @@ -1369,6 +1371,14 @@ export function mergeTraceInputs( export function buildAgentTraceInputs(options: AgentTraceInputOptions): Record { return sanitizeTracePayload({ + ...(options.promptConfiguration + ? { + prompt_configuration: options.promptConfiguration, + system_prompt_hash: createHash('sha256') + .update(options.systemPrompt ?? '') + .digest('hex'), + } + : {}), ...(options.systemPrompt ? { system_prompt: serializeTraceText(options.systemPrompt) } : {}), ...(options.modelId !== undefined ? { model: serializeModelIdForTrace(options.modelId) } : {}), ...(options.toolSearchEnabled !== undefined diff --git a/packages/@n8n/instance-ai/src/types.ts b/packages/@n8n/instance-ai/src/types.ts index b4022f3a20f..b7785bd9460 100644 --- a/packages/@n8n/instance-ai/src/types.ts +++ b/packages/@n8n/instance-ai/src/types.ts @@ -18,6 +18,7 @@ import type { ChatIntegrationDescriptor, EvaluationMetric, TaskList, + InstanceAiPromptConfiguration, InstanceAiFileAttachment, InstanceAiPermissions, InstanceAiSetupItem, @@ -1275,6 +1276,8 @@ export interface InstanceAiContext { * that land on the active trace. Absent outside a traced run. */ tracing?: InstanceAiTraceContext; + /** Selected skill source for inline tool guidance. */ + runtimeSkillCatalog?: RuntimeSkillSource; projectId?: string; /** * Per-run folder-exploration gate, resolved by the host before the context @@ -1912,6 +1915,9 @@ export interface OrchestrationContext { messageGroupId?: string; userId: string; projectId?: string; + /** The selected prompt profile owns its skill and tool exclusions. */ + promptConfiguration?: InstanceAiPromptConfiguration; + disabledToolNames?: ReadonlySet; orchestratorAgentId: string; modelId: ModelConfig; checkpointStore?: CheckpointStore; diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-temporary-workflow.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-temporary-workflow.service.test.ts index 45436fce1bb..4cd6c72572a 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-temporary-workflow.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-temporary-workflow.service.test.ts @@ -45,6 +45,7 @@ function createService() { configEvalsEnabled: false, mcpConnectionsEnabled: false, conversationHistoryEnabled: false, + progressiveBuildingEnabled: false, nodeUsageEnabled: false, folderExplorationEnabled: false, }); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts index 6631d8dfef1..26b1cc5ed8b 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai-terminal-outcome.service.test.ts @@ -502,11 +502,19 @@ describe('InstanceAiTerminalOutcomeService — terminal response guard wiring', const finalization = await service.finishInvalidConfirmationRun({ threadId: 'thread-a', runId: 'run-1', + promptVersion: 'progressive@1', abortController, }); expect(finalization.status).toBe('error'); expect(finalization.reason).toBe('invalid_confirmation_payload'); + expect(deps.publishRunFinish).toHaveBeenCalledWith( + 'thread-a', + 'run-1', + 'errored', + expect.any(String), + 'progressive@1', + ); expect(deps.runState.cancelThread).toHaveBeenCalledWith('thread-a'); expect(deps.suspendedThreads.dropPendingConfirmationsForThread).toHaveBeenCalledWith( 'thread-a', diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts index 105bd5d2d7e..6cce13026e1 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.adapter.service.test.ts @@ -68,6 +68,8 @@ import { INSTANCE_AI_CONVERSATION_HISTORY_FLAG, INSTANCE_AI_NODE_USAGE_FLAG, INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, + INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, + INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, CONFIG_EVALUATIONS_ENABLED_VARIANT, INSTANCE_AI_MCP_CONNECTIONS_FLAG, INSTANCE_AI_MCP_CONNECTIONS_ENABLED_VARIANT, @@ -5103,6 +5105,7 @@ describe('resolveExperimentGates', () => { [CONFIG_EVALUATIONS_FLAG]: CONFIG_EVALUATIONS_ENABLED_VARIANT, [INSTANCE_AI_MCP_CONNECTIONS_FLAG]: INSTANCE_AI_MCP_CONNECTIONS_ENABLED_VARIANT, [INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, + [INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, [INSTANCE_AI_NODE_USAGE_FLAG]: true, [INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: true, }; @@ -5114,6 +5117,7 @@ describe('resolveExperimentGates', () => { configEvalsEnabled: true, mcpConnectionsEnabled: true, conversationHistoryEnabled: true, + progressiveBuildingEnabled: true, nodeUsageEnabled: true, folderExplorationEnabled: true, }); @@ -5126,6 +5130,7 @@ describe('resolveExperimentGates', () => { [CONFIG_EVALUATIONS_FLAG]: 'control', [INSTANCE_AI_MCP_CONNECTIONS_FLAG]: 'control', [INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: 'control', + [INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: 'control', [INSTANCE_AI_NODE_USAGE_FLAG]: false, [INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: false, }); @@ -5134,6 +5139,7 @@ describe('resolveExperimentGates', () => { configEvalsEnabled: false, mcpConnectionsEnabled: false, conversationHistoryEnabled: false, + progressiveBuildingEnabled: false, nodeUsageEnabled: false, folderExplorationEnabled: false, }); @@ -5146,6 +5152,7 @@ describe('resolveExperimentGates', () => { configEvalsEnabled: false, mcpConnectionsEnabled: false, conversationHistoryEnabled: false, + progressiveBuildingEnabled: false, nodeUsageEnabled: false, folderExplorationEnabled: false, }); @@ -5159,6 +5166,7 @@ describe('resolveExperimentGates', () => { configEvalsEnabled: false, mcpConnectionsEnabled: false, conversationHistoryEnabled: false, + progressiveBuildingEnabled: false, nodeUsageEnabled: false, folderExplorationEnabled: false, }); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts index 3240f95baa1..bec319c5a6d 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts @@ -207,6 +207,8 @@ describe('InstanceAiController', () => { payload.context, payload.timeZone, payload.pushRef, + payload.mode, + payload.promptVersion, ); }); @@ -241,6 +243,35 @@ describe('InstanceAiController', () => { payloadWithPushRef.context, payloadWithPushRef.timeZone, 'iframe-push-ref-123', + payloadWithPushRef.mode, + payloadWithPushRef.promptVersion, + ); + }); + + it('should forward the build mode and prompt version to startRun', async () => { + const payloadWithMode = mock({ + message: 'build me a workflow', + timeZone: 'UTC', + attachments: undefined, + mode: 'progressive', + promptVersion: 'progressive@1', + }); + memoryService.checkThreadOwnership.mockResolvedValue('owned'); + instanceAiService.hasActiveRun.mockReturnValue(false); + instanceAiService.startRun.mockReturnValue('run-5'); + + await controller.chat(req, res, THREAD_ID, payloadWithMode); + + expect(instanceAiService.startRun).toHaveBeenCalledWith( + req.user, + THREAD_ID, + payloadWithMode.message, + payloadWithMode.attachments, + payloadWithMode.context, + payloadWithMode.timeZone, + payloadWithMode.pushRef, + 'progressive', + 'progressive@1', ); }); @@ -273,6 +304,8 @@ describe('InstanceAiController', () => { payloadWithContext.context, payloadWithContext.timeZone, payloadWithContext.pushRef, + payloadWithContext.mode, + payloadWithContext.promptVersion, ); }); @@ -371,6 +404,8 @@ describe('InstanceAiController', () => { nodesPayload.context, nodesPayload.timeZone, nodesPayload.pushRef, + nodesPayload.mode, + nodesPayload.promptVersion, ); }); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.build-mode.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.build-mode.test.ts new file mode 100644 index 00000000000..12d2b4c993f --- /dev/null +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.build-mode.test.ts @@ -0,0 +1,133 @@ +import type { SerializableAgentState } from '@n8n/agents'; +import type { InstanceAiBuildMode } from '@n8n/api-types'; +import type { User } from '@n8n/db'; +import { RunStateRegistry } from '@n8n/instance-ai'; +import { mock } from 'vitest-mock-extended'; + +import { InstanceAiService } from '../instance-ai.service'; +import type { + RebuildSuspendedRunOutcome, + ResumableOrphan, +} from '../suspended-run-restorer.service'; + +type Persistence = NonNullable; +type ServiceInternals = { + instanceAiConfig: { runDebugEnabled: boolean }; + aiConfig: { modelStreamIdleTimeoutMs: number; modelStreamFirstOutputTimeoutMs: number }; + runState: RunStateRegistry; + modeWhenEnvironmentBuilt?: InstanceAiBuildMode; + threadPushRef: Map; + checkpointStore: { load: (key: string) => Promise }; + revalidateActiveUser: (userId: string) => Promise; + createExecutionEnvironment: () => Promise<{ + orchestrationContext: object; + }>; + createAgentFromEnvironment: () => Promise; + buildOrchestratorAgentStreamOptions: ( + user: User, + threadId: string, + runId: string, + signal: AbortSignal, + ) => { persistence: Persistence }; + buildOrchestratorResumeAgentOptions: ( + user: User, + threadId: string, + runId: string, + agentRunId: string, + toolCallId: string, + signal: AbortSignal, + ) => { persistence: Persistence }; + rebuildSuspendedRunFromCheckpoint: ( + orphan: ResumableOrphan, + ) => Promise; +}; + +const user = mock({ id: 'user-1' }); +const orphan = mock({ + userId: user.id, + threadId: 'thread-1', + runId: 'run-1', + checkpointKey: 'checkpoint-1', + toolCallId: 'tool-1', + requestId: 'request-1', + checkpointTaskId: null, +}); + +function createService(checkpoint?: SerializableAgentState): ServiceInternals { + const service = Object.create(InstanceAiService.prototype) as ServiceInternals; + service.instanceAiConfig = { runDebugEnabled: false }; + service.aiConfig = { modelStreamIdleTimeoutMs: 90_000, modelStreamFirstOutputTimeoutMs: 180_000 }; + service.runState = new RunStateRegistry((owner: User) => owner.id); + service.threadPushRef = new Map(); + service.checkpointStore = { load: vi.fn(async () => checkpoint) }; + service.revalidateActiveUser = vi.fn(async () => user); + service.createExecutionEnvironment = vi.fn(async () => { + service.modeWhenEnvironmentBuilt = service.runState.getBuildMode(orphan.threadId); + return { orchestrationContext: {} }; + }); + service.createAgentFromEnvironment = vi.fn(async () => ({})); + return service; +} + +describe('InstanceAiService build mode recovery', () => { + it.each(['default', 'progressive'] as const)( + 'preserves mode %s when a saved run is rebuilt in a fresh service', + async (mode) => { + const original = createService(); + original.runState.setBuildMode(orphan.threadId, mode); + original.runState.setPromptVersion(orphan.threadId, `${mode}@1`); + const signal = new AbortController().signal; + const { persistence } = original.buildOrchestratorAgentStreamOptions( + user, + orphan.threadId, + orphan.runId, + signal, + ); + expect(persistence.hostMetadata).toEqual({ buildMode: mode, promptVersion: `${mode}@1` }); + + const checkpoint = mock({ + persistence: structuredClone(persistence), + }); + const restored = createService(checkpoint); + const result = await restored.rebuildSuspendedRunFromCheckpoint(orphan); + expect(result.kind).toBe('ready'); + if (result.kind !== 'ready') throw new Error('Expected a restored run'); + expect(restored.runState.getBuildMode(orphan.threadId)).toBe(mode); + expect(restored.modeWhenEnvironmentBuilt).toBe(mode); + expect(restored.runState.getPromptVersion(orphan.threadId)).toBe(`${mode}@1`); + + const resumed = restored.buildOrchestratorResumeAgentOptions( + user, + orphan.threadId, + orphan.runId, + orphan.checkpointKey, + orphan.toolCallId, + signal, + ); + expect(resumed.persistence.hostMetadata).toEqual({ + buildMode: mode, + promptVersion: `${mode}@1`, + }); + }, + ); + + it.each([undefined, null, 'unknown'])( + 'uses control for checkpoint mode %s', + async (buildMode) => { + const checkpoint = mock({ + persistence: { + resourceId: user.id, + threadId: orphan.threadId, + ...(buildMode !== undefined ? { hostMetadata: { buildMode } } : {}), + }, + }); + const restored = createService(checkpoint); + restored.runState.setBuildMode(orphan.threadId, 'progressive'); + const result = await restored.rebuildSuspendedRunFromCheckpoint(orphan); + expect(result.kind).toBe('ready'); + if (result.kind !== 'ready') throw new Error('Expected a restored run'); + expect(restored.runState.getBuildMode(orphan.threadId)).toBe('default'); + expect(restored.modeWhenEnvironmentBuilt).toBe('default'); + }, + ); +}); diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts index 9cf03e28f60..940dc50810c 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.service.test.ts @@ -6,7 +6,12 @@ vi.mock('@n8n/agents', async (importOriginal) => ({ vi.mock('@n8n/instance-ai', async () => { const { z } = await vi.importActual('zod'); + const profiles = await vi.importActual('@n8n/instance-ai'); return { + resolvePromptProfile: profiles.resolvePromptProfile, + assertInstanceAiPromptVersion: profiles.assertInstanceAiPromptVersion, + describePromptProfile: profiles.describePromptProfile, + setTracePromptVersion: vi.fn(), // Wiring-only stub: the real mapping has its own unit tests // (instance-ai/src/tracing/__tests__/thread-provenance.test.ts). What the // service tests pin is that its OUTPUT reaches the trace — spreading an @@ -37,12 +42,15 @@ vi.mock('@n8n/instance-ai', async () => { getPromptWorkspaceRoot: vi.fn(() => '/home/daytona/workspace'), getWorkspaceRoot: vi.fn(async () => '/home/daytona/workspace'), setupSandboxWorkspace: vi.fn(), - loadInstanceAiRuntimeSkillSource: vi.fn(() => ({ - registry: { - skillsHash: 'runtime-skills-hash', - skills: [{ id: 'data-table-manager' }], + loadInstanceAiPromptSkills: vi.fn(() => ({ + disabledTools: [], + source: { + registry: { + skillsHash: 'runtime-skills-hash', + skills: [{ id: 'data-table-manager' }], + }, + loadSkill: vi.fn(), }, - loadSkill: vi.fn(), })), disabledInstanceAiSkillIds: vi.fn(() => []), workflowBuildOutcomeSchema: z.object({}), @@ -214,7 +222,7 @@ import { createSandbox, createWorkspace, createInstanceAiTraceContext, - loadInstanceAiRuntimeSkillSource, + loadInstanceAiPromptSkills, resumeAgentRun, setupSandboxWorkspace, shutdownProductTelemetryProviders, @@ -436,6 +444,8 @@ type StartRunServiceInternals = { } >; setTimeZone: MockedFunction<(threadId: string, timeZone: string) => void>; + setBuildMode: MockedFunction<(threadId: string, mode: string | undefined) => void>; + setPromptVersion: Mock; activeRunCount: MockedFunction<() => number>; activeRunCountForUser: MockedFunction<(userId: string) => number>; }; @@ -459,6 +469,8 @@ function createStartRunService(): StartRunServiceInternals { messageGroupId: 'group-1', })), setTimeZone: vi.fn(), + setBuildMode: vi.fn(), + setPromptVersion: vi.fn(), activeRunCount: vi.fn(() => 0), activeRunCountForUser: vi.fn(() => 0), }; @@ -648,6 +660,10 @@ type ShutdownServiceInternals = { type TerminalGuardOrderServiceInternals = { terminalOutcome: InstanceAiTerminalOutcomeService; runState: { + getBuildMode: Mock; + getPromptVersion: Mock; + getPromptConfiguration: Mock; + setPromptConfiguration: Mock; getRunIdsForMessageGroup: Mock; cancelThread: Mock; clearActiveRun: Mock; @@ -752,6 +768,10 @@ function createTerminalGuardOrderService(): TerminalGuardOrderServiceInternals { InstanceAiService.prototype, ) as unknown as TerminalGuardOrderServiceInternals; service.runState = { + getBuildMode: vi.fn(() => undefined), + getPromptVersion: vi.fn(), + getPromptConfiguration: vi.fn(), + setPromptConfiguration: vi.fn(), getRunIdsForMessageGroup: vi.fn(() => ['run-1']), cancelThread: vi.fn(), clearActiveRun: vi.fn(), @@ -838,12 +858,15 @@ describe('InstanceAiService — runtime workspace setup', () => { }), ); (createLazyWorkspaceRuntimeSkillSource as Mock).mockImplementation(({ source }) => source); - (loadInstanceAiRuntimeSkillSource as Mock).mockImplementation(() => ({ - registry: { - skillsHash: 'runtime-skills-hash', - skills: [{ id: 'data-table-manager' }], + (loadInstanceAiPromptSkills as Mock).mockImplementation(() => ({ + disabledTools: [], + source: { + registry: { + skillsHash: 'runtime-skills-hash', + skills: [{ id: 'data-table-manager' }], + }, + loadSkill: vi.fn(), }, - loadSkill: vi.fn(), })); }); @@ -896,7 +919,16 @@ describe('InstanceAiService — runtime workspace setup', () => { oauth2CallbackUrl: string; webhookBaseUrl: string; formBaseUrl: string; - runState: { touchActiveRun: Mock; registerPendingConfirmation: Mock }; + runState: { + touchActiveRun: Mock; + registerPendingConfirmation: Mock; + getBuildMode: Mock; + getPromptVersion: Mock; + getPromptConfiguration: Mock; + setPromptConfiguration: Mock; + setBuildMode: Mock; + setPromptVersion: Mock; + }; spawnBackgroundTask: Mock; cancelBackgroundTask: Mock; backgroundTasks: { touchTask: Mock }; @@ -955,6 +987,12 @@ describe('InstanceAiService — runtime workspace setup', () => { service.runState = { touchActiveRun: vi.fn(), registerPendingConfirmation: vi.fn(), + getBuildMode: vi.fn(() => undefined), + getPromptVersion: vi.fn(), + getPromptConfiguration: vi.fn(), + setPromptConfiguration: vi.fn(), + setBuildMode: vi.fn(), + setPromptVersion: vi.fn(), }; service.spawnBackgroundTask = vi.fn(); service.cancelBackgroundTask = vi.fn(); @@ -1008,7 +1046,7 @@ describe('InstanceAiService — runtime workspace setup', () => { expect.objectContaining({ id: 'instance-ai-runtime-skill-workspace' }), ); expect(createLazyWorkspaceRuntimeSkillSource).toHaveBeenCalledTimes(1); - expect(loadInstanceAiRuntimeSkillSource).toHaveBeenCalledTimes(1); + expect(loadInstanceAiPromptSkills).toHaveBeenCalledTimes(1); expect(environment.orchestrationContext.runtimeSkills?.registry.skills).toEqual([ { id: 'data-table-manager' }, ]); @@ -1097,7 +1135,7 @@ describe('InstanceAiService — runtime workspace setup', () => { (createLazyWorkspaceRuntimeSkillSource as Mock).mockClear(); (createSandbox as Mock).mockClear(); (setupSandboxWorkspace as Mock).mockClear(); - (loadInstanceAiRuntimeSkillSource as Mock).mockClear(); + (loadInstanceAiPromptSkills as Mock).mockClear(); service.settingsService.getSandboxStatus.mockReturnValue({ enabled: true, provider: 'n8n-sandbox', @@ -1124,7 +1162,15 @@ describe('InstanceAiService — runtime workspace setup', () => { expect(setupSandboxWorkspace).not.toHaveBeenCalled(); }); - it('resolves the folder-exploration gate once and passes it into the context', async () => { + it.each([ + [false, undefined, 'default', undefined], + [true, undefined, 'progressive', undefined], + [true, 'default', 'default', undefined], + [false, 'progressive', 'progressive', undefined], + [true, 'progressive', 'default', 'default@1'], + [false, 'default', 'progressive', 'progressive@1'], + [true, 'progressive', 'default', 'retired@1'], + ] as const)('selects mode (%s, %s, %s, %s)', async (enabled, override, expected, version) => { const service = Object.create(InstanceAiService.prototype) as unknown as { createExecutionEnvironment: ( user: User, @@ -1175,7 +1221,16 @@ describe('InstanceAiService — runtime workspace setup', () => { oauth2CallbackUrl: string; webhookBaseUrl: string; formBaseUrl: string; - runState: { touchActiveRun: Mock; registerPendingConfirmation: Mock }; + runState: { + touchActiveRun: Mock; + registerPendingConfirmation: Mock; + getBuildMode: Mock; + getPromptVersion: Mock; + getPromptConfiguration: Mock; + setPromptConfiguration: Mock; + setBuildMode: Mock; + setPromptVersion: Mock; + }; spawnBackgroundTask: Mock; cancelBackgroundTask: Mock; backgroundTasks: { touchTask: Mock }; @@ -1210,6 +1265,7 @@ describe('InstanceAiService — runtime workspace setup', () => { configEvalsEnabled: true, mcpConnectionsEnabled: false, conversationHistoryEnabled: false, + progressiveBuildingEnabled: enabled, nodeUsageEnabled: false, folderExplorationEnabled: true, }), @@ -1235,6 +1291,12 @@ describe('InstanceAiService — runtime workspace setup', () => { service.runState = { touchActiveRun: vi.fn(), registerPendingConfirmation: vi.fn(), + getBuildMode: vi.fn(() => override), + getPromptVersion: vi.fn(() => version), + getPromptConfiguration: vi.fn(), + setPromptConfiguration: vi.fn(), + setBuildMode: vi.fn(), + setPromptVersion: vi.fn(), }; service.spawnBackgroundTask = vi.fn(); service.cancelBackgroundTask = vi.fn(); @@ -1273,7 +1335,7 @@ describe('InstanceAiService — runtime workspace setup', () => { }); (setupSandboxWorkspace as Mock).mockResolvedValue(undefined); - await service.createExecutionEnvironment( + const environment = await service.createExecutionEnvironment( fakeUser, 'thread-1', 'run-1', @@ -1281,6 +1343,13 @@ describe('InstanceAiService — runtime workspace setup', () => { ); expect(service.adapterService.resolveExperimentGates).toHaveBeenCalledTimes(1); + expect(service.runState.setBuildMode).toHaveBeenCalledWith('thread-1', expected); + expect(loadInstanceAiPromptSkills).toHaveBeenCalledWith( + expect.objectContaining({ mode: expected, version: `${expected}@1` }), + ); + expect(environment.orchestrationContext).toMatchObject({ + promptConfiguration: { version: `${expected}@1` }, + }); expect(service.adapterService.createContext).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ folderExplorationEnabled: true }), @@ -1600,6 +1669,26 @@ describe('InstanceAiService — background task auto-follow-up', () => { expect(service.executeRun).toHaveBeenCalled(); }); + it('records each request mode for later internal runs', () => { + const service = createStartRunService(); + service.startRun( + fakeUser, + 'thread-a', + 'build', + undefined, + undefined, + 'UTC', + undefined, + 'progressive', + 'progressive@1', + ); + expect(service.runState.setPromptVersion).toHaveBeenLastCalledWith('thread-a', 'progressive@1'); + expect(service.runState.setBuildMode).toHaveBeenLastCalledWith('thread-a', 'progressive'); + service.startRun(fakeUser, 'thread-a', 'continue'); + expect(service.runState.setBuildMode).toHaveBeenLastCalledWith('thread-a', undefined); + expect(service.runState.setPromptVersion).toHaveBeenLastCalledWith('thread-a', undefined); + }); + it('passes handoff context into executeRun', () => { const service = createStartRunService(); const context = { @@ -1627,6 +1716,24 @@ describe('InstanceAiService — background task auto-follow-up', () => { ); }); + it('rejects an unknown explicit prompt version before starting a run', () => { + const service = createStartRunService(); + expect(() => + service.startRun( + fakeUser, + 'thread-a', + 'build', + undefined, + undefined, + 'UTC', + undefined, + undefined, + 'missing@1', + ), + ).toThrow('Unknown Instance AI prompt version'); + expect(service.runState.startRun).not.toHaveBeenCalled(); + }); + it('passes agent-preview handoff context into executeRun', () => { const service = createStartRunService(); const context = { @@ -3158,6 +3265,7 @@ describe('InstanceAiService — terminal response guard wiring', () => { it('persists the resumed-run fallback error before cleanup', async () => { const service = createTerminalGuardOrderService(); + service.runState.getPromptConfiguration.mockReturnValue({ version: 'progressive@1' }); const abortController = new AbortController(); vi.mocked(resumeAgentRun).mockImplementationOnce(async (_agent, _data, resumeOptions) => { const onResumeClaimed = resumeOptions.onResumeClaimed; @@ -3186,6 +3294,7 @@ describe('InstanceAiService — terminal response guard wiring', () => { run_id: 'run-1', status: 'error', user_id: 'user-1', + prompt_version: 'progressive@1', }); expect(service.telemetry.track).toHaveBeenCalledWith('Builder generation errored', { thread_id: 'thread-a', @@ -3193,6 +3302,7 @@ describe('InstanceAiService — terminal response guard wiring', () => { error_message: 'provider failed', error_source: 'exception', user_id: 'user-1', + prompt_version: 'progressive@1', }); }); @@ -3259,6 +3369,7 @@ describe('InstanceAiService — terminal response guard wiring', () => { it('tracks "Builder generation errored" when a resumed stream reports an error', async () => { const service = createTerminalGuardOrderService(); + service.runState.getPromptConfiguration.mockReturnValue({ version: 'default@1' }); const abortController = new AbortController(); mockClaimedResumeResult({ status: 'errored', @@ -3288,6 +3399,7 @@ describe('InstanceAiService — terminal response guard wiring', () => { error_message: 'model overloaded', error_source: 'stream', user_id: 'user-1', + prompt_version: 'default@1', }); }); @@ -3328,7 +3440,11 @@ describe('InstanceAiService — terminal response guard wiring', () => { it('claims credits when a resumed run completes', async () => { const service = createTerminalGuardOrderService(); + service.runState.getPromptConfiguration.mockReturnValue({ version: 'progressive@1' }); const abortController = new AbortController(); + service.creditService.claimRunUsage.mockImplementationOnce(async () => { + service.runState.getPromptConfiguration.mockReturnValue({ version: 'default@1' }); + }); mockClaimedResumeResult({ status: 'completed', agentRunId: 'agent-run-1', @@ -3357,12 +3473,19 @@ describe('InstanceAiService — terminal response guard wiring', () => { [], 'completed', ); + expect(service.telemetry.track).toHaveBeenCalledWith('Builder sent message', { + thread_id: 'thread-a', + message: 'done', + prompt_version: 'progressive@1', + }); expect(service.telemetry.track).toHaveBeenCalledWith('Builder satisfied user intent', { thread_id: 'thread-a', + prompt_version: 'progressive@1', }); // user_id must be present so the heartbeat event reaches PostHog expect(service.telemetry.track).toHaveBeenCalledWith('instance_ai_run_finished', { thread_id: 'thread-a', + prompt_version: 'progressive@1', run_id: 'run-1', status: 'completed', user_id: 'user-1', @@ -4213,6 +4336,7 @@ describe('InstanceAiService — run error reporter lifecycle', () => { it('does not acquire local ownership when checkpoint claiming returns an error stream', async () => { const service = createTerminalGuardOrderService(); + service.runState.getPromptConfiguration.mockReturnValue({ version: 'progressive@1' }); const abortController = new AbortController(); const suspendedTracing = { id: 'suspended-trace' }; const resumeTracing = { @@ -4268,6 +4392,21 @@ describe('InstanceAiService — run error reporter lifecycle', () => { errorCode: undefined, }); expect(service.eventBus.events.map((event) => event.type)).toEqual(['error', 'run-finish']); + expect(service.telemetry.track).toHaveBeenCalledWith('instance_ai_run_finished', { + thread_id: 'thread-a', + run_id: 'run-1', + status: 'error', + user_id: 'user-1', + prompt_version: 'progressive@1', + }); + expect(service.telemetry.track).toHaveBeenCalledWith('Builder generation errored', { + thread_id: 'thread-a', + run_id: 'run-1', + error_message: 'checkpoint unavailable', + error_source: 'stream', + user_id: 'user-1', + prompt_version: 'progressive@1', + }); expect(service.tracing.finalizeRunTracing).not.toHaveBeenCalled(); expect(service.tracing.finalizeMessageTraceRoot).not.toHaveBeenCalled(); expect(service.schedulePlannedTasks).not.toHaveBeenCalled(); diff --git a/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts b/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts index 8a45ccc9f0b..428ef5cef86 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/run-debug-gating.test.ts @@ -1,11 +1,12 @@ import type { User } from '@n8n/db'; -import { RunDebugBuffer } from '@n8n/instance-ai'; +import { RunDebugBuffer, RunStateRegistry } from '@n8n/instance-ai'; import { InstanceAiService } from '../instance-ai.service'; type RunDebugGatingInternals = { instanceAiConfig: { runDebugEnabled: boolean }; runDebugBuffer: RunDebugBuffer; + runState: RunStateRegistry; buildOrchestratorAgentStreamOptions: ( user: User, threadId: string, @@ -27,6 +28,7 @@ function createRunDebugGatingService(runDebugEnabled: boolean): RunDebugGatingIn const service = Object.create(InstanceAiService.prototype) as RunDebugGatingInternals; service.instanceAiConfig = { runDebugEnabled }; service.runDebugBuffer = new RunDebugBuffer(); + service.runState = new RunStateRegistry((user: User) => user.id); return service; } diff --git a/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts b/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts index 3f68dec6448..71c9ea056b7 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai-terminal-outcome.service.ts @@ -84,6 +84,7 @@ export interface InstanceAiTerminalOutcomeServiceOptions { runId: string, status: 'completed' | 'cancelled' | 'errored', reason?: string, + promptVersion?: string, ) => void; } @@ -267,6 +268,7 @@ export class InstanceAiTerminalOutcomeService { async finishInvalidConfirmationRun(args: { threadId: string; runId: string; + promptVersion?: string; abortController: AbortController; tracing?: InstanceAiTraceContext; }): Promise { @@ -282,6 +284,7 @@ export class InstanceAiTerminalOutcomeService { args.runId, 'errored', 'I need your input to continue, but I could not display the prompt. Please try again.', + args.promptVersion, ); return { status: 'error', diff --git a/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts index 215ed92f03f..e9477a69823 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.adapter.service.ts @@ -11,6 +11,8 @@ import { INSTANCE_AI_MCP_CONNECTIONS_ENABLED_VARIANT, INSTANCE_AI_CONVERSATION_HISTORY_FLAG, INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, + INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, + INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, } from '@n8n/api-types'; import type { AiGatewayConfigDto } from '@n8n/api-types'; import { Logger, ModuleRegistry } from '@n8n/backend-common'; @@ -556,6 +558,8 @@ export class InstanceAiAdapterService { mcpConnectionsEnabled: boolean; /** Past-conversation recall: tool, prompt section and first-turn hint. */ conversationHistoryEnabled: boolean; + /** Progressive workflow policy and planning-tool selection. */ + progressiveBuildingEnabled: boolean; /** Node-usage context surface: the `node-usage` action and the `nodeTypes` filter on `list`. */ nodeUsageEnabled: boolean; /** Per-user folder-exploration gate, passed into `createContext`. Fails @@ -578,6 +582,9 @@ export class InstanceAiAdapterService { conversationHistoryEnabled: flags[INSTANCE_AI_CONVERSATION_HISTORY_FLAG] === INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, + progressiveBuildingEnabled: + flags[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG] === + INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, nodeUsageEnabled: flags[INSTANCE_AI_NODE_USAGE_FLAG] === true, folderExplorationEnabled: flags[INSTANCE_AI_FOLDER_EXPLORATION_FLAG] === true, }; diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index ff9b8eedc09..cba5f420f94 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -226,6 +226,8 @@ export class InstanceAiController { payload.context, payload.timeZone, payload.pushRef, + payload.mode, + payload.promptVersion, ); return { runId }; } diff --git a/packages/cli/src/modules/instance-ai/instance-ai.service.ts b/packages/cli/src/modules/instance-ai/instance-ai.service.ts index c60f7015a2d..3f3ae5765da 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.service.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.service.ts @@ -12,8 +12,10 @@ import { mcpConnectRequestSchema, credentialSetupHintSchema, formatAttachmentSizeLimit, + instanceAiBuildModeSchema, TEMPLATED_CUSTOM_AUTH_CREDENTIAL_TYPE, type InstanceAiAttachment, + type InstanceAiBuildMode, type InstanceAiHandoffContext, type InstanceAiAgentAttachment, type InstanceAiFileAttachment, @@ -40,7 +42,10 @@ import { createLazyWorkspaceRuntimeSkillSource, getPromptWorkspaceRoot, getWorkspaceRoot, - loadInstanceAiRuntimeSkillSource, + loadInstanceAiPromptSkills, + resolvePromptProfile, + describePromptProfile, + setTracePromptVersion, disabledInstanceAiSkillIds, createInstanceAiTraceContext, threadProvenanceMetadata, @@ -123,6 +128,7 @@ import { OperationalError, UnexpectedError, UserError } from 'n8n-workflow'; import { nanoid } from 'nanoid'; import { N8N_VERSION, WORKFLOW_SDK_VERSION } from '@/constants'; +import { BadRequestError } from '@/errors/response-errors/bad-request.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; import { EventService } from '@/events/event.service'; import { InstanceAiBuilderDelegateAdapterService } from '@/modules/agents/instance-ai-builder-delegate.adapter'; @@ -626,6 +632,8 @@ type RunFinishErrorInfo = { errorSource?: 'stream' | 'exception'; }; +type RunFinishMetadata = RunFinishErrorInfo & { promptVersion?: string }; + type UnclaimedResumeContext = { threadId: string; runId: string; @@ -927,8 +935,10 @@ export class InstanceAiService { runState: this.runState, suspendedThreads: this.suspendedThreads, tracing: this.tracing, - publishRunFinish: (threadId, runId, status, reason) => { - this.publishRunFinish(threadId, runId, status, reason); + publishRunFinish: (threadId, runId, status, reason, promptVersion) => { + this.publishRunFinish(threadId, runId, status, reason, undefined, undefined, { + promptVersion, + }); }, }); this.defaultTimeZone = globalConfig.generic.timezone; @@ -1202,7 +1212,12 @@ export class InstanceAiService { this.backgroundTasks.getTaskSnapshots(threadId), ); const memoryTasks = this.memoryTaskRegistry.getTasks(threadId); - return { ...status, memoryTasks }; + const selectedPrompt = this.runState.getPromptConfiguration(threadId); + return { + ...status, + memoryTasks, + ...(selectedPrompt ? { promptConfiguration: selectedPrompt } : {}), + }; } private memoryTaskObserverFor( @@ -1283,6 +1298,10 @@ export class InstanceAiService { // Host run id, persisted with checkpoints so the interrupted-run // sweep can match a crashed run's checkpoint exactly. hostRunId: runId, + hostMetadata: { + buildMode: this.runState.getBuildMode(threadId) ?? null, + promptVersion: this.runState.getPromptVersion(threadId) ?? null, + }, }, providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } }, @@ -1310,7 +1329,15 @@ export class InstanceAiService { abortSignal: signal, // Keep billing stopped/errored resumed runs (see stream-options builder). recoverUsageOnAbort: true, - persistence: { resourceId: user.id, threadId, hostRunId: runId }, + persistence: { + resourceId: user.id, + threadId, + hostRunId: runId, + hostMetadata: { + buildMode: this.runState.getBuildMode(threadId) ?? null, + promptVersion: this.runState.getPromptVersion(threadId) ?? null, + }, + }, // Must mirror buildOrchestratorAgentStreamOptions: without this request-level // cache directive, resumed (HITL) turns send no cache_control, so Anthropic // reprocesses the whole conversation uncached on every resume (~100K tokens). @@ -1419,7 +1446,15 @@ export class InstanceAiService { context?: InstanceAiHandoffContext, timeZone?: string, pushRef?: string, + mode?: InstanceAiBuildMode, + promptVersion?: string, ): string { + if ( + promptVersion !== undefined && + resolvePromptProfile({ version: promptVersion }).fallbackFrom + ) { + throw new BadRequestError(`Unknown Instance AI prompt version "${promptVersion}"`); + } this.assertRunAdmissible(user); this.liveness.clearThreadState(threadId); const { runId, abortController, messageGroupId } = this.runState.startRun({ @@ -1434,6 +1469,11 @@ export class InstanceAiService { this.runState.setTimeZone(threadId, timeZone); } + // A new user message resets selection. Explicit eval modes take precedence; + // otherwise environment creation selects and stores the backend assignment. + this.runState.setBuildMode(threadId, mode); + this.runState.setPromptVersion(threadId, promptVersion); + if (pushRef !== undefined) { this.threadPushRef.set(threadId, pushRef); } @@ -1937,7 +1977,15 @@ export class InstanceAiService { // Truly mid-stream run: the durable run-finish is all history needs — // the fold derives the terminal tree from the log after restart. - this.publishRunFinish(run.threadId, run.runId, 'cancelled', 'service_shutdown'); + this.publishRunFinish( + run.threadId, + run.runId, + 'cancelled', + 'service_shutdown', + undefined, + undefined, + { promptVersion: run.promptVersion }, + ); await this.tracing.finalizeRunTracing(run.runId, run.tracing, { status: 'cancelled', reason: 'service_shutdown', @@ -2435,6 +2483,7 @@ export class InstanceAiService { configEvalsEnabled, mcpConnectionsEnabled, conversationHistoryEnabled, + progressiveBuildingEnabled, nodeUsageEnabled, folderExplorationEnabled, } = await this.adapterService.resolveExperimentGates(user); @@ -2442,6 +2491,15 @@ export class InstanceAiService { const conversationHistory = conversationHistoryEnabled ? this.conversationHistoryService.forContext(user.id, boundProjectId, threadId) : undefined; + // Follow-ups and resumed runs retain the selected mode if flags change. + const selectedPrompt = resolvePromptProfile({ + version: this.runState.getPromptVersion(threadId), + mode: + this.runState.getBuildMode(threadId) ?? + (progressiveBuildingEnabled ? 'progressive' : 'default'), + }); + const buildMode = selectedPrompt.profile.mode; + this.runState.setBuildMode(threadId, buildMode); const context = this.adapterService.createContext(user, { searchProxyConfig, pushRef, @@ -2587,10 +2645,14 @@ export class InstanceAiService { // Per-user skill gate: hide flag-gated skills (filtered copy, cache // preserved) so every derived skill source inherits the exclusion. const flagDisabledSkillIds = disabledInstanceAiSkillIds({ configEvalsEnabled }); + const selectedSkills = await loadInstanceAiPromptSkills(selectedPrompt.profile); + const selectedRuntimeSkills = selectedSkills.source; const allRuntimeSkills = flagDisabledSkillIds.length > 0 - ? filterRuntimeSkillSource(loadInstanceAiRuntimeSkillSource(), flagDisabledSkillIds) - : loadInstanceAiRuntimeSkillSource(); + ? filterRuntimeSkillSource(selectedRuntimeSkills, flagDisabledSkillIds) + : selectedRuntimeSkills; + const promptMetadata = describePromptProfile(selectedPrompt, allRuntimeSkills); + this.runState.setPromptConfiguration(threadId, promptMetadata); let runtimeSkills = allRuntimeSkills; let runtimeWorkspace: Workspace | undefined; let workspaceRoot: string | undefined; @@ -2672,6 +2734,8 @@ export class InstanceAiService { messageGroupId, userId: user.id, projectId: boundProjectId, + promptConfiguration: promptMetadata, + disabledToolNames: new Set(selectedSkills.disabledTools), orchestratorAgentId: orchestratorAgentId(runId), modelId, checkpointStore: this.checkpointStore, @@ -3677,6 +3741,7 @@ export class InstanceAiService { const signal = abortController.signal; let tracing: InstanceAiTraceContext | undefined; + let promptVersion: string | undefined; let messageTraceFinalization: MessageTraceFinalization | undefined; let aiCreatedWorkflowIds: Set | undefined; let messageId = ''; @@ -3835,6 +3900,8 @@ export class InstanceAiService { orchestrationContext, conversationHistory, } = environment; + promptVersion = orchestrationContext.promptConfiguration?.version; + setTracePromptVersion(tracing, promptVersion); aiCreatedWorkflowIds = context.aiCreatedWorkflowIds ??= new Set(); const isPostPlanFollowUp = isReplanFollowUp || checkpoint?.isCheckpointFollowUp === true; // Make the current user message available since memory history only @@ -4220,6 +4287,7 @@ export class InstanceAiService { if (intermediateText) { this.telemetry.track('Builder sent message', { thread_id: threadId, + ...(promptVersion ? { prompt_version: promptVersion } : {}), message: redactTelemetryText(intermediateText), is_intermediate: true, }); @@ -4239,6 +4307,7 @@ export class InstanceAiService { messageTraceFinalization = await this.terminalOutcome.finishInvalidConfirmationRun({ threadId, runId, + promptVersion, abortController, tracing, }); @@ -4361,6 +4430,7 @@ export class InstanceAiService { this.backgroundTasks.getRunningTasks(threadId).length, ); await this.finalizeRun(threadId, runId, result.status, { + promptVersion, userId: user.id, modelId, archivedWorkflowIds, @@ -4392,10 +4462,12 @@ export class InstanceAiService { if (result.status === 'completed') { this.telemetry.track('Builder sent message', { thread_id: threadId, + ...(promptVersion ? { prompt_version: promptVersion } : {}), message: redactTelemetryText(outputText), }); this.telemetry.track('Builder satisfied user intent', { thread_id: threadId, + ...(promptVersion ? { prompt_version: promptVersion } : {}), }); } } catch (error) { @@ -4449,6 +4521,7 @@ export class InstanceAiService { cancellationReason, archivedWorkflowIds, user.id, + { promptVersion }, ); return; } @@ -4521,7 +4594,7 @@ export class InstanceAiService { userFacingErrorMessage, archivedWorkflowIds, user.id, - { errorMessage, errorSource: 'exception' }, + { errorMessage, errorSource: 'exception', promptVersion }, ); } finally { this.runState.clearActiveRun(threadId); @@ -5041,6 +5114,14 @@ export class InstanceAiService { try { const state = await this.checkpointStore.load(orphan.checkpointKey); if (!state) return { kind: 'no-checkpoint' }; + // Restore before rebuilding the prompt and tools. Older checkpoints use control. + const mode = instanceAiBuildModeSchema.safeParse(state.persistence?.hostMetadata?.buildMode); + this.runState.setBuildMode(orphan.threadId, mode.success ? mode.data : 'default'); + const version = state.persistence?.hostMetadata?.promptVersion; + this.runState.setPromptVersion( + orphan.threadId, + typeof version === 'string' ? version : undefined, + ); } catch (error: unknown) { return { kind: 'no-checkpoint', error }; } @@ -5425,6 +5506,10 @@ export class InstanceAiService { }, ): Promise { let messageTraceFinalization: MessageTraceFinalization | undefined; + const promptVersion = + opts.orchestrationContext?.promptConfiguration?.version ?? + this.runState.getPromptConfiguration(opts.threadId)?.version; + setTracePromptVersion(opts.tracing, promptVersion); let completedSetupWorkflowId: string | undefined; let skipPostRunCleanup = false; let resumeClaimed = false; @@ -5525,7 +5610,7 @@ export class InstanceAiService { if (!resumeClaimed) { skipPostRunCleanup = true; const claimError = result.error ?? new Error('Resume checkpoint claim did not complete'); - await this.settleUnclaimedResume(opts, claimError, 'stream'); + await this.settleUnclaimedResume(opts, claimError, 'stream', promptVersion); return; } if (result.status === 'suspended') { @@ -5591,6 +5676,7 @@ export class InstanceAiService { if (intermediateText) { this.telemetry.track('Builder sent message', { thread_id: opts.threadId, + ...(promptVersion ? { prompt_version: promptVersion } : {}), message: redactTelemetryText(intermediateText), is_intermediate: true, }); @@ -5608,6 +5694,7 @@ export class InstanceAiService { messageTraceFinalization = await this.terminalOutcome.finishInvalidConfirmationRun({ threadId: opts.threadId, runId: opts.runId, + promptVersion, abortController: opts.abortController, tracing: opts.tracing, }); @@ -5733,6 +5820,7 @@ export class InstanceAiService { this.backgroundTasks.getRunningTasks(opts.threadId).length, ); await this.finalizeRun(opts.threadId, opts.runId, result.status, { + promptVersion, userId: opts.user.id, // Forward modelId so title refinement fires on the resume path too — a run // that suspends for HITL and completes here would otherwise never be titled. @@ -5769,16 +5857,18 @@ export class InstanceAiService { ); this.telemetry.track('Builder sent message', { thread_id: opts.threadId, + ...(promptVersion ? { prompt_version: promptVersion } : {}), message: redactTelemetryText(outputText), }); this.telemetry.track('Builder satisfied user intent', { thread_id: opts.threadId, + ...(promptVersion ? { prompt_version: promptVersion } : {}), }); } } catch (error) { if (!resumeClaimed) { skipPostRunCleanup = true; - await this.settleUnclaimedResume(opts, error, 'exception'); + await this.settleUnclaimedResume(opts, error, 'exception', promptVersion); return; } @@ -5828,6 +5918,7 @@ export class InstanceAiService { cancellationReason, archivedWorkflowIds, opts.user.id, + { promptVersion }, ); return; } @@ -5903,7 +5994,7 @@ export class InstanceAiService { userFacingErrorMessage, archivedWorkflowIds, opts.user.id, - { errorMessage, errorSource: 'exception' }, + { errorMessage, errorSource: 'exception', promptVersion }, ); } finally { this.runState.clearActiveRun(opts.threadId, opts.resumeExecutionToken); @@ -5978,6 +6069,7 @@ export class InstanceAiService { opts: UnclaimedResumeContext, error: unknown, errorSource: NonNullable, + promptVersion?: string, ): Promise { const outcome = classifyUnclaimedResume(error, { aborted: opts.signal.aborted, @@ -6016,6 +6108,7 @@ export class InstanceAiService { await this.emitTerminalRun({ threadId: opts.threadId, runId: opts.runId, + promptVersion, status: 'cancelled', reason, messageGroupId: opts.messageGroupId, @@ -6046,6 +6139,7 @@ export class InstanceAiService { await this.emitTerminalRun({ threadId: opts.threadId, runId: opts.runId, + promptVersion, status: 'errored', reason: outcome.reason, errorCode: outcome.errorCode, @@ -6070,6 +6164,7 @@ export class InstanceAiService { private async emitTerminalRun(args: { threadId: string; runId: string; + promptVersion?: string; status: 'cancelled' | 'errored'; reason: string; errorCode?: TerminalErrorCode; @@ -6103,15 +6198,10 @@ export class InstanceAiService { ), )) ?? []; - this.publishRunFinish( - threadId, - runId, - status, - args.reason, - archivedWorkflowIds, - args.user.id, - args.errorInfo, - ); + this.publishRunFinish(threadId, runId, status, args.reason, archivedWorkflowIds, args.user.id, { + ...args.errorInfo, + promptVersion: args.promptVersion, + }); } private async bestEffort( @@ -6528,6 +6618,7 @@ export class InstanceAiService { reason, archivedWorkflowIds, suspended.user.id, + { promptVersion: suspended.orchestrationContext?.promptConfiguration?.version }, ); await this.tracing.maybeFinalizeRunTraceRoot(suspended.runId, { @@ -6550,7 +6641,7 @@ export class InstanceAiService { reason?: string, archivedWorkflowIds?: string[], userId?: string, - errorInfo?: RunFinishErrorInfo, + metadata?: RunFinishMetadata, ): void { const effectiveStatus = status === 'errored' ? 'error' : status; const hasArchived = archivedWorkflowIds && archivedWorkflowIds.length > 0; @@ -6572,6 +6663,7 @@ export class InstanceAiService { this.telemetry.track('instance_ai_run_finished', { thread_id: threadId, run_id: runId, + ...(metadata?.promptVersion ? { prompt_version: metadata.promptVersion } : {}), status: effectiveStatus, ...(userId ? { user_id: userId } : {}), }); @@ -6580,8 +6672,9 @@ export class InstanceAiService { this.telemetry.track('Builder generation errored', { thread_id: threadId, run_id: runId, - error_message: redactTelemetryText(errorInfo?.errorMessage ?? reason ?? 'unknown'), - ...(errorInfo?.errorSource ? { error_source: errorInfo.errorSource } : {}), + ...(metadata?.promptVersion ? { prompt_version: metadata.promptVersion } : {}), + error_message: redactTelemetryText(metadata?.errorMessage ?? reason ?? 'unknown'), + ...(metadata?.errorSource ? { error_source: metadata.errorSource } : {}), ...(userId ? { user_id: userId } : {}), }); } @@ -6700,6 +6793,7 @@ export class InstanceAiService { status: 'completed' | 'cancelled' | 'errored', options?: { userId?: string; + promptVersion?: string; modelId?: ModelConfig; archivedWorkflowIds?: string[]; workSummary?: WorkSummary; @@ -6715,7 +6809,7 @@ export class InstanceAiService { options?.errorReason, options?.archivedWorkflowIds, options?.userId, - options?.errorInfo, + { ...options?.errorInfo, promptVersion: options?.promptVersion }, ); this.emitRunMetrics(threadId, status, options); if (status === 'completed' && options?.userId && options?.modelId) { diff --git a/packages/cli/src/modules/instance-ai/storage/__tests__/typeorm-agent-memory.test.ts b/packages/cli/src/modules/instance-ai/storage/__tests__/typeorm-agent-memory.test.ts index 456bb497974..3422741c802 100644 --- a/packages/cli/src/modules/instance-ai/storage/__tests__/typeorm-agent-memory.test.ts +++ b/packages/cli/src/modules/instance-ai/storage/__tests__/typeorm-agent-memory.test.ts @@ -62,6 +62,46 @@ function getToolInputs(message: AgentDbMessage | undefined): unknown[] { } describe('TypeORMAgentMemory', () => { + it('persists active skills without replacing other thread metadata or another agent state', async () => { + const thread = mock({ + id: 'thread-1', + resourceId: 'user-1', + metadata: { titleSource: 'user' }, + }); + const threadRepo = mock(); + threadRepo.findOneBy.mockResolvedValue(thread); + threadRepo.save.mockResolvedValue(thread); + const { memory } = createMemory({ threadRepo }); + const scope = { threadId: 'thread-1', resourceId: 'user-1', agentName: 'builder' }; + await Promise.all([ + memory.skillState.save(scope, ['workflow-builder']), + memory.skillState.save({ ...scope, agentName: 'reviewer' }, ['review']), + ]); + await expect(memory.skillState.load(scope)).resolves.toEqual(['workflow-builder']); + await expect(memory.skillState.load({ ...scope, agentName: 'reviewer' })).resolves.toEqual([ + 'review', + ]); + await expect( + memory.skillState.load({ ...scope, resourceId: 'other-user' }), + ).resolves.toBeUndefined(); + expect(thread.metadata).toHaveProperty('titleSource', 'user'); + + await memory.skillState.save(scope, []); + await expect(memory.skillState.load(scope)).resolves.toEqual([]); + await expect(memory.skillState.load({ ...scope, agentName: 'reviewer' })).resolves.toEqual([ + 'review', + ]); + }); + + it('treats missing skill metadata as a legacy thread', async () => { + const threadRepo = mock(); + threadRepo.findOneBy.mockResolvedValue(mock({ metadata: null })); + const { memory } = createMemory({ threadRepo }); + await expect( + memory.skillState.load({ threadId: 'thread-1', resourceId: 'user-1', agentName: 'builder' }), + ).resolves.toBeUndefined(); + }); + it('logs and skips invalid native message rows', async () => { const messageRepo = mock(); messageRepo.find.mockResolvedValueOnce([makeMessageRow()]); diff --git a/packages/cli/src/modules/instance-ai/storage/typeorm-agent-memory.ts b/packages/cli/src/modules/instance-ai/storage/typeorm-agent-memory.ts index 6fee0ca19c1..d9dc608bbee 100644 --- a/packages/cli/src/modules/instance-ai/storage/typeorm-agent-memory.ts +++ b/packages/cli/src/modules/instance-ai/storage/typeorm-agent-memory.ts @@ -13,6 +13,7 @@ import { type ObservationLogTaskLockHandle, type JSONObject, type JSONValue, + type RuntimeSkillStateStore, } from '@n8n/agents'; import { Logger } from '@n8n/backend-common'; import { Service } from '@n8n/di'; @@ -28,6 +29,7 @@ import { } from '@n8n/instance-ai'; import { In, LessThan, Like } from '@n8n/typeorm'; import { UnexpectedError } from 'n8n-workflow'; +import { z } from 'zod'; import { ConflictError } from '@/errors/response-errors/conflict.error'; import { ForbiddenError } from '@/errors/response-errors/forbidden.error'; @@ -50,6 +52,19 @@ function parseJsonSafe(text: string): unknown { } } +const activeSkillStatesSchema = z.array( + z.object({ + agentName: z.string(), + resourceId: z.string(), + skillIds: z.array(z.string()), + }), +); + +function activeSkillStates(metadata: Thread['metadata']) { + const parsed = activeSkillStatesSchema.safeParse(metadata?.activeSkillStates); + return parsed.success ? parsed.data : []; +} + function isAgentMessage(value: unknown): value is AgentMessage { if (!isRecord(value)) return false; @@ -147,6 +162,7 @@ function workingMemoryKey(params: { } const PATCH_ONLY_METADATA_KEYS = new Set([ + 'activeSkillStates', 'instanceAiIterationLog', 'instanceAiPlannedTasks', 'instanceAiTasks', @@ -179,6 +195,32 @@ function mergeSaveThreadMetadata( export class TypeORMAgentMemory implements BuiltMemory, BuiltObservationLogStore, BuiltObservationLogTaskLockStore { + readonly skillState: RuntimeSkillStateStore = { + load: async ({ threadId, resourceId, agentName }) => { + const thread = await this.getThread(threadId); + return activeSkillStates(thread?.metadata).find( + (state) => state.resourceId === resourceId && state.agentName === agentName, + )?.skillIds; + }, + save: async ({ threadId, resourceId, agentName }, skillIds) => { + const updated = await this.patchThread({ + threadId, + update: (thread) => ({ + metadata: { + ...thread.metadata, + activeSkillStates: [ + ...activeSkillStates(thread.metadata).filter( + (state) => state.resourceId !== resourceId || state.agentName !== agentName, + ), + { resourceId, agentName, skillIds }, + ], + }, + }), + }); + if (!updated) throw new UnexpectedError('Cannot save active skills for a missing thread'); + }, + }; + private readonly threadMutationQueues = new Map>(); private readonly observationLog: TypeORMObservationLogStore; diff --git a/packages/frontend/editor-ui/src/app/constants/experiments.ts b/packages/frontend/editor-ui/src/app/constants/experiments.ts index 1c4fad1bcd0..35cbcb519f4 100644 --- a/packages/frontend/editor-ui/src/app/constants/experiments.ts +++ b/packages/frontend/editor-ui/src/app/constants/experiments.ts @@ -1,4 +1,7 @@ -import { INSTANCE_AI_MCP_CONNECTIONS_FLAG } from '@n8n/api-types'; +import { + INSTANCE_AI_MCP_CONNECTIONS_FLAG, + INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, +} from '@n8n/api-types'; function createExperiment< const TName extends string, @@ -143,7 +146,12 @@ export const OPEN_WORKFLOW_IN_ASSISTANT_EXPERIMENT = createExperiment( '108_open_workflow_in_assistant', ); +export const INSTANCE_AI_PROGRESSIVE_BUILDING_EXPERIMENT = createExperiment( + INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, +); + export const EXPERIMENTS_TO_TRACK = [ + INSTANCE_AI_PROGRESSIVE_BUILDING_EXPERIMENT.name, EXTRA_TEMPLATE_LINKS_EXPERIMENT.name, TEMPLATE_ONBOARDING_EXPERIMENT.name, BATCH_11AUG_EXPERIMENT.name, diff --git a/packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/agentTimeline.utils.test.ts b/packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/agentTimeline.utils.test.ts index 64855bbf8d7..60aa3fbdf89 100644 --- a/packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/agentTimeline.utils.test.ts +++ b/packages/frontend/editor-ui/src/features/ai/instanceAi/__tests__/agentTimeline.utils.test.ts @@ -3,6 +3,7 @@ import type { InstanceAiAgentNode, InstanceAiTimelineEntry, InstanceAiToolCallState, + InstanceAiWorkflowSetupNode, } from '@n8n/api-types'; import { buildTimelineBlocks, @@ -382,6 +383,52 @@ describe('buildTimelineBlocks', () => { expect(blocks[0].type === 'thinking' && blocks[0].entries).toHaveLength(3); }); + test('text before a tool call that suspended on a setup card stays user-facing', () => { + const setupCard = makeToolCall({ + toolCallId: 'tc-setup', + toolName: 'workflows', + args: { action: 'setup', workflowId: 'wf-1' }, + isLoading: true, + confirmation: { + requestId: 'req-1', + severity: 'info', + message: 'Set up credentials', + setupRequests: [{ nodeName: 'Slack' } as unknown as InstanceAiWorkflowSetupNode], + }, + }); + const blocks = blocksOf( + [ + reasoning('r1'), + text('This first version posts to Slack; Gmail comes next.', 'r1'), + toolEntry('tc-setup', 'r1'), + ], + [setupCard], + 'active', + ); + + expect(blocks.map((b) => b.type)).toEqual(['thinking', 'text', 'thinking']); + }); + + test('text before an approval-gated tool call stays user-facing', () => { + const approval = makeToolCall({ + toolCallId: 'tc-run', + toolName: 'executions', + args: { action: 'run', workflowId: 'wf-1' }, + confirmation: { + requestId: 'req-1', + severity: 'info', + message: 'Run?', + inputType: 'approval', + }, + }); + const blocks = blocksOf( + [reasoning('r1'), text('Running it live now.', 'r1'), toolEntry('tc-run', 'r1')], + [approval], + ); + + expect(blocks.map((b) => b.type)).toEqual(['thinking', 'text', 'thinking']); + }); + test('trailing text of a response is user-facing and splits blocks', () => { const blocks = blocksOf( [reasoning('r1'), toolEntry('tc-1', 'r1'), text('Here is your answer.', 'r1')], diff --git a/packages/frontend/editor-ui/src/features/ai/instanceAi/agentTimeline.utils.ts b/packages/frontend/editor-ui/src/features/ai/instanceAi/agentTimeline.utils.ts index 108fdab829e..fe40348035e 100644 --- a/packages/frontend/editor-ui/src/features/ai/instanceAi/agentTimeline.utils.ts +++ b/packages/frontend/editor-ui/src/features/ai/instanceAi/agentTimeline.utils.ts @@ -130,6 +130,8 @@ export function buildTimelineBlocks( if ( tc && classifyToolCall(tc) === 'trace' && + // Keep the explanation before a confirmation outside the trace. + tc.confirmation === undefined && !( tc.toolName === 'build-agent' && hasBuilderChildInResponse(entry.responseId, builderChildResponseIds)