Compare commits

...

13 Commits

Author SHA1 Message Date
n8n-assistant[bot]
7f7a8ac25b
🚀 Release 2.41.3 (#39574)
Co-authored-by: n8n-release-helper[bot] <265227495+n8n-release-helper[bot]@users.noreply.github.com>
2026-09-25 07:15:53 +00:00
n8n-assistant[bot]
29afe517b3
fix(core): Propagate project span attributes to node spans (backport to release-candidate/2.41.x) (#39456)
Co-authored-by: Theodore Nastos <15264814+theonastos@users.noreply.github.com>
2026-09-24 08:05:59 +00:00
n8n-assistant[bot]
2db9ab8dce
🚀 Release 2.41.2 (#39447)
Co-authored-by: n8n-release-helper[bot] <265227495+n8n-release-helper[bot]@users.noreply.github.com>
2026-09-24 07:16:20 +00:00
n8n-assistant[bot]
56f9e97b8e
fix(core): Retry instance reports that cross the UTC midnight boundary (backport to release-candidate/2.41.x) (#39433)
Co-authored-by: Dmitrii <dmitrii.kulikov@n8n.io>
2026-09-23 21:28:35 +00:00
n8n-assistant[bot]
2de7d47c2e
fix(core): Keep OpenTelemetry export working after a restart when Sentry is enabled (backport to release-candidate/2.41.x) (#39367)
Co-authored-by: ocdejong <52361003+ocdejong@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-23 16:02:14 +02:00
n8n-assistant[bot]
b4e38b3cc7
feat(core): Track n8n Assistant workflow setup (no-changelog) (backport to release-candidate/2.41.x) (#39376)
Co-authored-by: oleg <me@olegivaniv.com>
2026-09-23 13:12:43 +00:00
n8n-assistant[bot]
665bcd9af7
fix(editor): Improve n8n Assistant async credential setup (no-changelog) (backport to release-candidate/2.41.x) (#39391)
Co-authored-by: oleg <me@olegivaniv.com>
2026-09-23 14:51:46 +02:00
n8n-assistant[bot]
07f2d3675a
fix(core): Record live n8n Assistant test runs as verification evidence (backport to release-candidate/2.41.x) (#39383)
Co-authored-by: Jaakko Husso <jaakko@n8n.io>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-23 11:30:55 +00:00
n8n-assistant[bot]
4c5ecc22fc
feat(core): Authenticate instance reports with the license certificate (backport to release-candidate/2.41.x) (#39380)
Co-authored-by: Konstantin Tieber <46342664+konstantintieber@users.noreply.github.com>
2026-09-23 11:24:49 +00:00
n8n-assistant[bot]
397e4f2eb7
🚀 Release 2.41.1 (#39351)
Co-authored-by: n8n-release-helper[bot] <265227495+n8n-release-helper[bot]@users.noreply.github.com>
2026-09-23 07:14:32 +00:00
n8n-assistant[bot]
d8c49e2a3e
feat(core): Enable Agents by default (backport to release-candidate/2.41.x) (#39328)
Co-authored-by: Michael Drury <me@michaeldrury.co.uk>
2026-09-22 19:15:52 +00:00
n8n-assistant[bot]
69022dc863
fix(core): Keep skill activation from invalidating the prompt cache (no-changelog) (backport to release-candidate/2.41.x) (#39295)
Co-authored-by: Riqwan Thamir <rmthamir@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-22 13:41:43 +00:00
n8n-assistant[bot]
b3196fb520
fix(core): Restore pubsub subscriptions after Redis reconnect and detect half-open subscriber connections (backport to release-candidate/2.41.x) (#39241)
Co-authored-by: Mike Repeć <mike.repec@n8n.io>
2026-09-22 09:14:54 +00:00
151 changed files with 5149 additions and 804 deletions

View File

@ -1,3 +1,39 @@
## [2.41.3](https://github.com/n8n-io/n8n/compare/n8n@2.41.2...n8n@2.41.3) (2026-09-25)
### Bug Fixes
* **core:** Propagate project span attributes to node spans ([#39456](https://github.com/n8n-io/n8n/issues/39456)) ([29afe51](https://github.com/n8n-io/n8n/commit/29afe517b3a6954a6070c13e26dde9e9d648bfe3))
## [2.41.2](https://github.com/n8n-io/n8n/compare/n8n@2.41.1...n8n@2.41.2) (2026-09-24)
### Bug Fixes
* **core:** Keep OpenTelemetry export working after a restart when Sentry is enabled ([#39367](https://github.com/n8n-io/n8n/issues/39367)) ([2de7d47](https://github.com/n8n-io/n8n/commit/2de7d47c2e079bd4474984b11e038877ac5fd7cf))
* **core:** Record live n8n Assistant test runs as verification evidence ([#39383](https://github.com/n8n-io/n8n/issues/39383)) ([07f2d36](https://github.com/n8n-io/n8n/commit/07f2d3675a31fbdc2b82d3e5c205669102d9bb2c))
* **core:** Retry instance reports that cross the UTC midnight boundary ([#39433](https://github.com/n8n-io/n8n/issues/39433)) ([56f9e97](https://github.com/n8n-io/n8n/commit/56f9e97b8e026d072e98c0eea6ce17f3c6076b4c))
### Features
* **core:** Authenticate instance reports with the license certificate ([#39380](https://github.com/n8n-io/n8n/issues/39380)) ([4c5ecc2](https://github.com/n8n-io/n8n/commit/4c5ecc22fce7990ae4ff9ba44bfb99f3c202b61d))
## [2.41.1](https://github.com/n8n-io/n8n/compare/n8n@2.41.0...n8n@2.41.1) (2026-09-23)
### Bug Fixes
* **core:** Restore pubsub subscriptions after Redis reconnect and detect half-open subscriber connections ([#39241](https://github.com/n8n-io/n8n/issues/39241)) ([b3196fb](https://github.com/n8n-io/n8n/commit/b3196fb520d4011ed66fdfcceb8da5da2b2e819d))
### Features
* **core:** Enable Agents by default ([#39328](https://github.com/n8n-io/n8n/issues/39328)) ([d8c49e2](https://github.com/n8n-io/n8n/commit/d8c49e2a3e85344e3d651bcf26dea90fa3707d2c))
# [2.41.0](https://github.com/n8n-io/n8n/compare/n8n@2.40.0...n8n@2.41.0) (2026-09-22) # [2.41.0](https://github.com/n8n-io/n8n/compare/n8n@2.40.0...n8n@2.41.0) (2026-09-22)

View File

@ -1,6 +1,6 @@
{ {
"name": "n8n-monorepo", "name": "n8n-monorepo",
"version": "2.41.0", "version": "2.41.3",
"private": true, "private": true,
"engines": { "engines": {
"node": ">=24.0.0", "node": ">=24.0.0",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/agents", "name": "@n8n/agents",
"version": "0.26.0", "version": "0.26.2",
"description": "AI agent SDK for n8n's code-first execution engine", "description": "AI agent SDK for n8n's code-first execution engine",
"engines": { "engines": {
"node": ">=24.0.0" "node": ">=24.0.0"

View File

@ -67,29 +67,31 @@
} }
}, },
"status": 200, "status": 200,
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqshPA7QoqLUtLRhcA\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Krds9KGve8gJNTNa9jn6N7\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"engineering-status\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1045,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCd2zmwW8c3i9MDr6fG\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"I'll load the engineering-status skill first, then use it to format the status token.\"},{\"type\":\"tool_use\",\"id\":\"toolu_019R6LssCYfGrz8ddVcPvYQC\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"engineering-status\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1045,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":76,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "9999000", "anthropic-ratelimit-input-tokens-remaining": "9999000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:55Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:55Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:55Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "11999000", "anthropic-ratelimit-tokens-remaining": "11999000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:04Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:55Z",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f485dd8bc7b-PRG", "cf-ray": "a3f01ad7ea6cbbdd-FRA",
"connection": "keep-alive",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "application/json", "content-type": "application/json",
"date": "Thu, 10 Sep 2026 07:12:04 GMT", "date": "Tue, 22 Sep 2026 08:57:55 GMT",
"request-id": "req_011CeuLqs6fxp1AkSuMSRWYb", "request-id": "req_011CfJCd2nsYW1GzkhCoPFhw",
"server": "cloudflare", "server": "cloudflare",
"server-timing": "x-originResponse;dur=667",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-77c37e49f4c62f7623d8cc66c4f3a26a-6bf7325e3f3f69d3-01", "traceresponse": "00-234e70eda24ff35d6c847bca1695098f-c9010c0bb2f634e5-01",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },
@ -105,7 +107,7 @@
"system": [ "system": [
{ {
"type": "text", "type": "text",
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually.\n\n</active_skills>" "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
} }
], ],
"messages": [ "messages": [
@ -121,9 +123,13 @@
{ {
"role": "assistant", "role": "assistant",
"content": [ "content": [
{
"type": "text",
"text": "I'll load the engineering-status skill first, then use it to format the status token."
},
{ {
"type": "tool_use", "type": "tool_use",
"id": "toolu_01Krds9KGve8gJNTNa9jn6N7", "id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
"name": "load_skill", "name": "load_skill",
"input": { "input": {
"skillId": "engineering-status" "skillId": "engineering-status"
@ -139,8 +145,17 @@
"content": [ "content": [
{ {
"type": "tool_result", "type": "tool_result",
"tool_use_id": "toolu_01Krds9KGve8gJNTNa9jn6N7", "tool_use_id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
"content": "{\"skillId\":\"engineering-status\",\"active\":true}" "content": [
{
"type": "text",
"text": "{\"skillId\":\"engineering-status\",\"active\":true}"
},
{
"type": "text",
"text": "[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually."
}
]
} }
] ]
} }
@ -189,29 +204,30 @@
} }
}, },
"status": 200, "status": 200,
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqxFjGLPD3QE1zh7Ef\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Now I'll format the status token:\"},{\"type\":\"tool_use\",\"id\":\"toolu_01CKRsNPiekmX9TWf3ooQdHU\",\"name\":\"format_status\",\"input\":{\"status\":\"waiting_on_vendor_info\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1244,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":68,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCd6hmu4KWXr78SxvJK\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Now I'll format the status token using the format_status tool:\"},{\"type\":\"tool_use\",\"id\":\"toolu_011UF3b8wMRddXy2WYPGRYZe\",\"name\":\"format_status\",\"input\":{\"status\":\"waiting_on_vendor_info\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1230,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":74,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "9999000", "anthropic-ratelimit-input-tokens-remaining": "9999000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:57Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:57Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:56Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "11999000", "anthropic-ratelimit-tokens-remaining": "11999000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:05Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:57Z",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f4f38babc7b-PRG", "cf-ray": "a3f01add2e2bbbdd-FRA",
"connection": "keep-alive",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "application/json", "content-type": "application/json",
"date": "Thu, 10 Sep 2026 07:12:06 GMT", "date": "Tue, 22 Sep 2026 08:57:57 GMT",
"request-id": "req_011CeuLqwpw3fPBk6xt6nrrm", "request-id": "req_011CfJCd6Vcovu2SLcUWAPC4",
"server": "cloudflare", "server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-d17795404368021a8e502b8755cc4b35-c750063766d5eb87-01", "traceresponse": "00-e98096358e475d79d338f7d1cf690dbb-aa6a8d0521eacc76-01",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },
@ -227,7 +243,7 @@
"system": [ "system": [
{ {
"type": "text", "type": "text",
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually.\n\n</active_skills>" "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
} }
], ],
"messages": [ "messages": [
@ -243,9 +259,13 @@
{ {
"role": "assistant", "role": "assistant",
"content": [ "content": [
{
"type": "text",
"text": "I'll load the engineering-status skill first, then use it to format the status token."
},
{ {
"type": "tool_use", "type": "tool_use",
"id": "toolu_01Krds9KGve8gJNTNa9jn6N7", "id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
"name": "load_skill", "name": "load_skill",
"input": { "input": {
"skillId": "engineering-status" "skillId": "engineering-status"
@ -261,8 +281,17 @@
"content": [ "content": [
{ {
"type": "tool_result", "type": "tool_result",
"tool_use_id": "toolu_01Krds9KGve8gJNTNa9jn6N7", "tool_use_id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
"content": "{\"skillId\":\"engineering-status\",\"active\":true}" "content": [
{
"type": "text",
"text": "{\"skillId\":\"engineering-status\",\"active\":true}"
},
{
"type": "text",
"text": "[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually."
}
]
} }
] ]
}, },
@ -271,11 +300,11 @@
"content": [ "content": [
{ {
"type": "text", "type": "text",
"text": "Now I'll format the status token:" "text": "Now I'll format the status token using the format_status tool:"
}, },
{ {
"type": "tool_use", "type": "tool_use",
"id": "toolu_01CKRsNPiekmX9TWf3ooQdHU", "id": "toolu_011UF3b8wMRddXy2WYPGRYZe",
"name": "format_status", "name": "format_status",
"input": { "input": {
"status": "waiting_on_vendor_info" "status": "waiting_on_vendor_info"
@ -291,7 +320,7 @@
"content": [ "content": [
{ {
"type": "tool_result", "type": "tool_result",
"tool_use_id": "toolu_01CKRsNPiekmX9TWf3ooQdHU", "tool_use_id": "toolu_011UF3b8wMRddXy2WYPGRYZe",
"content": "{\"formatted\":\"STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO\"}" "content": "{\"formatted\":\"STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO\"}"
} }
] ]
@ -341,29 +370,30 @@
} }
}, },
"status": 200, "status": 200,
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLr3AQ92UTrYgeFRerd\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The status token has been formatted successfully:\\n\\n**STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO**\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1346,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":32,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCdCtqry5T2NJGAMCjV\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Perfect! The engineering-status skill has been loaded and the status token has been formatted.\\n\\n**Formatted Status:**\\nSTATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1338,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":46,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "9999000", "anthropic-ratelimit-input-tokens-remaining": "9999000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:57Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:58Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:57Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "11999000", "anthropic-ratelimit-tokens-remaining": "11999000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:06Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:57Z",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f561a95bc7b-PRG", "cf-ray": "a3f01ae61be4bbdd-FRA",
"connection": "keep-alive",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "application/json", "content-type": "application/json",
"date": "Thu, 10 Sep 2026 07:12:07 GMT", "date": "Tue, 22 Sep 2026 08:57:58 GMT",
"request-id": "req_011CeuLr2Whz42pkAm7AzGhs", "request-id": "req_011CfJCdCcii1dW1VhpqXjuj",
"server": "cloudflare", "server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-80319e2d05f81fe007810b9d0f60ed01-c7956b0a3a60554e-01", "traceresponse": "00-4d1157be05f9b0ab551736730e6ba0e2-b17f94e742ef7628-01",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },

View File

@ -51,29 +51,30 @@
} }
}, },
"status": 200, "status": 200,
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqj5DrEFutdjSD8ETs\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Eq5aFLJhGriLQF4UBg7tyT\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"summary-marker\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":957,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCcuKuoVBeB7Ake3FNv\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Rhg9DwH7n3MzSh9iz6TFzh\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"summary-marker\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":957,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "9999000", "anthropic-ratelimit-input-tokens-remaining": "9999000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:53Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:54Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:53Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "11999000", "anthropic-ratelimit-tokens-remaining": "11999000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:53Z",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f3bd9d2bc7b-PRG", "cf-ray": "a3f01acbda36bbdd-FRA",
"connection": "keep-alive",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "application/json", "content-type": "application/json",
"date": "Thu, 10 Sep 2026 07:12:02 GMT", "date": "Tue, 22 Sep 2026 08:57:54 GMT",
"request-id": "req_011CeuLqiWV7tDxhVGbXmda8", "request-id": "req_011CfJCcteyXnhh5BmvQdDTp",
"server": "cloudflare", "server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-7cf7d7c3cfa9378a22214da3c62a1ea0-1ad2b8e7d42a1569-01", "traceresponse": "00-66ff12c75cd9a23f50952244bb3cba2c-133f511c645be506-01",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },
@ -89,7 +90,7 @@
"system": [ "system": [
{ {
"type": "text", "type": "text",
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged.\n\n</active_skills>" "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
} }
], ],
"messages": [ "messages": [
@ -107,7 +108,7 @@
"content": [ "content": [
{ {
"type": "tool_use", "type": "tool_use",
"id": "toolu_01Eq5aFLJhGriLQF4UBg7tyT", "id": "toolu_01Rhg9DwH7n3MzSh9iz6TFzh",
"name": "load_skill", "name": "load_skill",
"input": { "input": {
"skillId": "summary-marker" "skillId": "summary-marker"
@ -123,8 +124,17 @@
"content": [ "content": [
{ {
"type": "tool_result", "type": "tool_result",
"tool_use_id": "toolu_01Eq5aFLJhGriLQF4UBg7tyT", "tool_use_id": "toolu_01Rhg9DwH7n3MzSh9iz6TFzh",
"content": "{\"skillId\":\"summary-marker\",\"active\":true}" "content": [
{
"type": "text",
"text": "{\"skillId\":\"summary-marker\",\"active\":true}"
},
{
"type": "text",
"text": "[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged."
}
]
} }
] ]
} }
@ -157,29 +167,30 @@
} }
}, },
"status": 200, "status": 200,
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqo1c45tLcGPFM5dEA\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"**Summary:** n8n agents maintain efficient workflow operations by keeping context compact, reducing overhead and improving performance.\\n\\nSKILL_MARKER_ALPHA\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1139,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":34,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}", "response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCcxfLB1kLZMwT9hmcx\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Based on the Summary marker skill instructions, here's the summary:\\n\\n**Summary:** n8n agents maintain workflow context efficiency by keeping data structures and variables compact, reducing memory overhead and improving performance during workflow execution.\\n\\nSKILL_MARKER_ALPHA\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1106,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":55,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "9999000", "anthropic-ratelimit-input-tokens-remaining": "9999000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:03Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:54Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:03Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:55Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:02Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:54Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "11999000", "anthropic-ratelimit-tokens-remaining": "11999000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:03Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:54Z",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f417db9bc7b-PRG", "cf-ray": "a3f01ad17e0abbdd-FRA",
"connection": "keep-alive",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "application/json", "content-type": "application/json",
"date": "Thu, 10 Sep 2026 07:12:03 GMT", "date": "Tue, 22 Sep 2026 08:57:55 GMT",
"request-id": "req_011CeuLqnRP3vkdg9opi7gNC", "request-id": "req_011CfJCcxUAmJjy1fhrHrwCP",
"server": "cloudflare", "server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-86beb3374d817d1d427aabf5c903ca34-f5a5fa52f1cb2789-01", "traceresponse": "00-7063877d53ab0eae4686a22ce533a4ec-69260e5b962e00c7-01",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },

View File

@ -53,32 +53,34 @@
}, },
"status": 200, "status": 200,
"response": [ "response": [
"1f8b0800000000000013c5554d8bdb3010bde757189d1348c2ba21be9585d2427ae852e8a12942b1a6896259f2ea23ec12f4df3b72e2f823d96d2f253e18346fa479ef69d0c00194cb9212ac655ba0d631e3469c39962547e25e2b2019e98164dcac497624a5e620312597cc7398ec9828fce461924ee6d3793a9b4e67982e783cc36ee974367b04bf328ba765fa8def3fa63f5e162bf3b9c09c7e250c182d6380592bb0ac8a6573ad1c9225d9cf5fa705130a0cc99497724cacd31535c0ac56bd9085670f2a875e90036e96b689f9468e509577d4e9021482cbf4010bb17c0734c7939dd08af633a60d8e307f0b6bf6c60250eda004c3244dcbebfc169ded86681813ed5d3734ff806ac01c440ed489e8048956716678745dfd061385d32d68849476941d5035dba0b3212449188de074fb6767e946eabcb8dd033752ea221c5e4e52bb78547adee7b496d45b68fa20ae3d76c222cfbf2f3f6ded53517eddad9ee71b2b166a85598a95719fd468a82d84947519d48d878668a99451eaa500170672570b3a7fadae4aa8ed4048426290bca51dbbd9b1f7b5d7293deda7484be974737becc44b72857e09bcd77ddd9e84843ed1bb9038ae892561e8d95da8c47bfec2d7b52f77a69225688b2fb1a1eecba364e67582bf8284ebc6be0b23306b125a36ef528aafecdf9e105d7508b50735d3e6a6aa1e483a2a7acf7ff7d5f9e721309828e1ff8d85c13b9e2ec2c0cb76e05ebbd8c5c8e512fe00c6a9d291c7070000" "1f8b0800000000000013cd544d6fd34010bdf757587376243b6a5a756f10a440c501845490085a4dbdd364f17ad7ec47441bf9bfa3b5ebc63629f482826f3b6fc6f3dedbd9a11d69cf928a9cc30d71e7d1fa33811e59b2077f5f1330188190f667607ba88c20050c0a8541d06c8bb20cb3f3d96236cfe68b3ccb7248418af80fb7e1599e2fefae9762e5de5cac1e56e2cb03bef5b95c423ae9042958a362009d93cea38e6d0ba33d690fecebb7ee80529305a6835229386f6a6e099dd1a390a31f817441a3a0208f52b93e167a3952d7c1736f4ad20ed8d5e23c85028b2df1c2127a69341f67643d6e09c573585f1b1b50bda58a2c2abea87ecf3fa0f9768a362998e087a1f9450a8eec4e16c4bd8c4e40b44aa015d1757d47360ae71b32c0401bcf718752e1ad22689a2469cecea8bbfd4767f9ad3245797c068ea4b44d04fdeca40ef1a8f4b1ce1ba37870d4cf413c079ee5d757bbf2e6f2755d7d78f5193fdd6cdf7f9ccf439c048d55ac53060577a554aa6d53070f6cdf444b958a529f1a0869a9f0ada0f63ba8aaa5de4c64241083f09c7241cae39f95b72923e55de440a8bbb7efcee8a7e41aad97a8da2030809eeb98ef49b8ecd7d0b9dcfc0764de8935b0640dae65736232a1aad0decf0e9775623e15da92ec1a9a178d4f5c737f7bc3a61e509afcaddff947c58d401888192de1e1db7ff12a9eecf5e6df2de7c9365d5c0e6d3de2c431478718b445bf00ccc6474350070000"
], ],
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "10000000", "anthropic-ratelimit-input-tokens-remaining": "10000000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:58Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:58Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:58Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "12000000", "anthropic-ratelimit-tokens-remaining": "12000000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:07Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:58Z",
"cache-control": "no-cache", "cache-control": "no-cache",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f5c5a57bc7b-PRG", "cf-ray": "a3f01aebf8c4bbdd-FRA",
"connection": "keep-alive",
"content-encoding": "gzip", "content-encoding": "gzip",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "text/event-stream; charset=utf-8", "content-type": "text/event-stream; charset=utf-8",
"date": "Thu, 10 Sep 2026 07:12:08 GMT", "date": "Tue, 22 Sep 2026 08:57:59 GMT",
"request-id": "req_011CeuLr6og4ahrpyfnPec45", "request-id": "req_011CfJCdGeJvd8FiNh3bikfH",
"server": "cloudflare", "server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-db154d560f530266545ecc9baf7015e2-c0470cb196446bc0-01", "traceresponse": "00-5b8f08c4391de2116ab063ddaa4cdd99-24fa357e98008a81-01",
"transfer-encoding": "chunked",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },
@ -94,7 +96,7 @@
"system": [ "system": [
{ {
"type": "text", "type": "text",
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged.\n\n</active_skills>" "text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
} }
], ],
"messages": [ "messages": [
@ -112,7 +114,7 @@
"content": [ "content": [
{ {
"type": "tool_use", "type": "tool_use",
"id": "toolu_017ccT9FgsRkmMhLq2bsi7nL", "id": "toolu_01J9vkV7BpmPAWaSVhLQ22uC",
"name": "load_skill", "name": "load_skill",
"input": { "input": {
"skillId": "summary-marker" "skillId": "summary-marker"
@ -128,8 +130,17 @@
"content": [ "content": [
{ {
"type": "tool_result", "type": "tool_result",
"tool_use_id": "toolu_017ccT9FgsRkmMhLq2bsi7nL", "tool_use_id": "toolu_01J9vkV7BpmPAWaSVhLQ22uC",
"content": "{\"skillId\":\"summary-marker\",\"active\":true}" "content": [
{
"type": "text",
"text": "{\"skillId\":\"summary-marker\",\"active\":true}"
},
{
"type": "text",
"text": "[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged."
}
]
} }
] ]
} }
@ -164,32 +175,34 @@
}, },
"status": 200, "status": 200,
"response": [ "response": [
"1f8b0800000000000013c555cb8e1a3110bcf315968f082266b3ec616e681569a32551b428a76c6419bb99b1f0d8133f106885948fc817e64bd21e98c0f0489443122e305d6557755334b00213725281f7bc00e60377a12779e03979a1615303cd6907a483f699e62fb4b212345284e651c2b0e46a1987b7c3f1f0667433ce46a30ce94aa63b7cc14659760f71eaee65b40f73fbf1bd288a30dddc21a7ab840567752a70ef15ca9a242bac096896e69f3eef1eb832e0686ea2d603ea83ad9903eeade9943c7c896004748a12f0b0f66d2db6ed2853c7c0825d824130cb5edfa112172530815707650deb52462d8eb0bc86b5679302d42554e0b866e3ea9c7f40b3f214dd0ea88da1e30fbb01b7520258506912348d4a7227d3d4cd025c6a9c156011323630bec2aef91c27bbdd1242b6bd1eecbefefd68d95c5bb1bc1c820b944645c27ad7ea319e3add9f0bb04ec4e62da734e936af8378ad4c71a246682ad26b06317381ffda6043e918dc55bac67ed2f6f6fafdbdc17fae3c8b55c5dd26eff7c92c605e2aec9f602a4df0042a1548b05693c693fffef51b274bd8e0af5694dc285f918575043f4add1cf31b234a678d8d9e28f4e6b848f9f30419a47676a564e2a18c1e0655015900c839174b22a34bc84e6c0d2236b9fd4f3379f56c9ecdecf1ed74cade4d9e1edf3cb1c9f4c3c3e490a1b3289d86d4d6bf8bb1ad8f6c5db9b5dd7f171bed80f4a8b1ce42a260240bd119fa076be964c76dffe2a23ad92cb7d9f5191ffe0dcea77b8cd1c3c91f64c432c465060000" "1f8b0800000000000013c598514fdb3010c7dff914d6bd4c4201356c4c53de109a36a09b26d8db98a2233e1aab8e9dda97b608e5bb4f4e5b9ab4c0b60792bc44b9bbe47ef78fedf84273329c8882bcc709a59ed1f18144c6443c023f940409749c106dae217984c24ad29040a6b1927494a39a56471f8e4e8f4e4627a7f1681443044a8667f8493a8ae3f3fbcb73399ebae5a74b9fcf5c7c15cfe4178876324104ceea6040ef956734216d660d9361487efd5e5da032e4203195d61178b665ea08bd351d93a7594526a38e5112a3d27e63ab36e52853569cb29d92f190c4f1e87d04196639a5992364654dda0d196dfc8e50bee4dbdc1b32509953410e757a5aecc76fbd71beebad23b01577f822f0e4e62aa3945550028254129d0caa9b7b72a1f0744216123096539ca3d278a709ea5aac8ffae080566360ad6f7aa76d367d7e243c13d2a492b45cd5dbf68772d7f7312d4360734a00ea76da5299c94e1e01c1082fa149d28cafa335211db495a58bf414b606fb6e1750bf26ca9b651617efb416a5b37325a961e81f81f355e6f6d13f85af8a02dd835828cebb388328221ccd2ae548b658fa0729d04dc925b7e6d61c1e6e487ac7b859bd9a648b30941e37ec080b6526022764d80b2a140bb6568b86c41f0f358f7fe6e4879ac0e868b0b1b1d6bd3d30fa6798902187bc9dacfd23dc3dac87e47008b60c22341fd0810894197a81f04f0b44d8a146629193a36681189a4c9939d4fde7b559b305f502cd80b3c391af34fbb05289cc1645655416e6ab400f75ff3852f9cc11ef6d7b865a3d8543cec909ced18805aad6241e84eadebaf09a4abdafd100122d29ab9a36aa1e8ae1b8dbb9f49435ecfa6eae2ec6e3f4dbd9f5d5e7ebf46cfce3ebd9bf7c6943affbb716ce962da01777ff9b5f00cf16d97142aba84e4f0e6464ca9533f01f9df94e9b5fbf61afbed35c7f3cdd5b25b7ff41f6656dfb605fc63f07b52da767110000"
], ],
"rawHeaders": { "rawHeaders": {
"anthropic-ratelimit-input-tokens-limit": "10000000", "anthropic-ratelimit-input-tokens-limit": "10000000",
"anthropic-ratelimit-input-tokens-remaining": "10000000", "anthropic-ratelimit-input-tokens-remaining": "10000000",
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:59Z",
"anthropic-ratelimit-output-tokens-limit": "2000000", "anthropic-ratelimit-output-tokens-limit": "2000000",
"anthropic-ratelimit-output-tokens-remaining": "2000000", "anthropic-ratelimit-output-tokens-remaining": "2000000",
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:59Z",
"anthropic-ratelimit-requests-limit": "20000", "anthropic-ratelimit-requests-limit": "20000",
"anthropic-ratelimit-requests-remaining": "19999", "anthropic-ratelimit-requests-remaining": "19999",
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-requests-reset": "2026-09-22T08:57:59Z",
"anthropic-ratelimit-tokens-limit": "12000000", "anthropic-ratelimit-tokens-limit": "12000000",
"anthropic-ratelimit-tokens-remaining": "12000000", "anthropic-ratelimit-tokens-remaining": "12000000",
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:08Z", "anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:59Z",
"cache-control": "no-cache", "cache-control": "no-cache",
"cf-cache-status": "DYNAMIC", "cf-cache-status": "DYNAMIC",
"cf-ray": "a38c9f63ce57bc7b-PRG", "cf-ray": "a3f01af1acffbbdd-FRA",
"connection": "keep-alive",
"content-encoding": "gzip", "content-encoding": "gzip",
"content-security-policy": "default-src 'none'; frame-ancestors 'none'", "content-security-policy": "default-src 'none'; frame-ancestors 'none'",
"content-type": "text/event-stream; charset=utf-8", "content-type": "text/event-stream; charset=utf-8",
"date": "Thu, 10 Sep 2026 07:12:08 GMT", "date": "Tue, 22 Sep 2026 08:57:59 GMT",
"request-id": "req_011CeuLrBuGK4E7uxpMM6NMq", "request-id": "req_011CfJCdLVz3EfLeiVHjxihN",
"server": "cloudflare", "server": "cloudflare",
"strict-transport-security": "max-age=31536000; includeSubDomains; preload", "strict-transport-security": "max-age=31536000; includeSubDomains; preload",
"traceresponse": "00-ae497abd3e1ae1a20e06ccaeba57c754-f8de5c4ed0f0dac4-01", "traceresponse": "00-0dcb0fbb57a4e0ed3374476fe9b454e3-795c47a4b1e4f26c-01",
"transfer-encoding": "chunked",
"vary": "Accept-Encoding", "vary": "Accept-Encoding",
"x-robots-tag": "none" "x-robots-tag": "none"
}, },

View File

@ -5,6 +5,19 @@ import { AgentMessageList } from '../model/message-list';
import { ActiveSkills } from '../skills/active-skills'; import { ActiveSkills } from '../skills/active-skills';
const scope = { threadId: 'thread-1', resourceId: 'user-1' }; const scope = { threadId: 'thread-1', resourceId: 'user-1' };
/**
* Mark the list as observation-masked, so recovered skills render in the
* `<active_skills>` system block. Without a mask the block stays empty and
* skills ride on their still-visible activating tool result.
*/
function mask(list: AgentMessageList): AgentMessageList {
list.maskObservedMessages({
lastObservedAt: new Date(Date.now() + 60_000),
lastObservedMessageId: 'observation-cursor',
});
return list;
}
const source = createRuntimeSkillSource([ const source = createRuntimeSkillSource([
{ {
id: 'planning', id: 'planning',
@ -87,12 +100,46 @@ describe('active skills', () => {
await active.restore(list, scope); await active.restore(list, scope);
expect(active.instructions()).toContain('Build one workflow.'); // The load_skill result is still visible, so the body rides on it and the
// system block stays empty — no cached-prefix rewrite.
expect(active.instructions()).toBeUndefined();
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).toContain(
'Build one workflow.',
);
await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([ await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([
'builder', 'builder',
]); ]);
}); });
it('recovers a skill into the system block once observation masks its load', async () => {
const memory = new InMemoryMemory();
const list = new AgentMessageList();
list.addHistory([
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolName: 'load_skill',
toolCallId: 'old-load',
input: { skillId: 'builder' },
state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'Build one workflow.' }] },
},
],
},
]);
const active = new ActiveSkills(source, 'assistant', memory.skillState);
await active.restore(list, scope);
// Visible load → block empty.
expect(active.instructions()).toBeUndefined();
// Observation hides the load result → the skill moves into the block.
mask(list);
expect(active.instructions()).toContain('Build one workflow.');
});
it('persists checkpoint skills when they differ from stored state', async () => { it('persists checkpoint skills when they differ from stored state', async () => {
const memory = new InMemoryMemory(); const memory = new InMemoryMemory();
await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['planning']); await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['planning']);
@ -120,7 +167,15 @@ describe('active skills', () => {
{ {
type: 'tool-call', type: 'tool-call',
toolName: 'load_skill', toolName: 'load_skill',
toolCallId: 'old-load', toolCallId: 'builder-load',
input: { skillId: 'builder' },
state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'Build one workflow.' }] },
},
{
type: 'tool-call',
toolName: 'load_skill',
toolCallId: 'planning-load',
input: { skillId: 'planning' }, input: { skillId: 'planning' },
state: 'resolved', state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'Create a task plan.' }] }, output: { type: 'content', value: [{ type: 'text', text: 'Create a task plan.' }] },
@ -135,16 +190,132 @@ describe('active skills', () => {
); );
await active.restore(list, scope); await active.restore(list, scope);
expect(active.instructions()).toContain('Build one workflow.'); // Both loads are still visible, so the block stays empty; the enabled skill
expect(active.instructions()).not.toContain('Create a task plan.'); // rides on its result while the disabled one is collapsed out of it.
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).not.toContain( expect(active.instructions()).toBeUndefined();
'Create a task plan.', const rendered = JSON.stringify(active.modelMessages(list.forLlm('').messages, list));
); expect(rendered).toContain('Build one workflow.');
expect(rendered).not.toContain('Create a task plan.');
await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([ await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([
'builder', 'builder',
]); ]);
}); });
it('re-anchors a stamped tool activation on a later turn instead of the block', async () => {
const memory = new InMemoryMemory();
await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['builder']);
const list = new AgentMessageList();
// Turn 1 activated the skill from a tool and stamped its result. On the
// next turn the body must ride on that result again, not the system block.
list.addHistory([
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolName: 'build_workflow',
toolCallId: 'build-1',
input: {},
state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
activatedSkillIds: ['builder'],
},
],
},
]);
const active = new ActiveSkills(source, 'assistant', memory.skillState);
await active.restore(list, scope);
expect(active.instructions()).toBeUndefined();
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).toContain(
'Build one workflow.',
);
});
it('persists a tool activation stamp across serialization', async () => {
const memory = new InMemoryMemory();
const active = new ActiveSkills(source, 'assistant', memory.skillState);
const list = new AgentMessageList();
list.addResponse([
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolName: 'build_workflow',
toolCallId: 'build-1',
input: {},
state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
},
],
},
]);
await active.restore(list, scope);
await active.load('builder', { toolCallId: 'build-1' });
const roundTripped = AgentMessageList.deserialize(structuredClone(list.serialize()));
const stamped = roundTripped
.messages()
.flatMap((message) => ('content' in message ? message.content : []))
.find((part) => part.type === 'tool-call' && part.toolCallId === 'build-1');
expect(stamped).toMatchObject({ activatedSkillIds: ['builder'] });
});
it('recovers a restored programmatic skill through the block when no stamp survives', async () => {
const memory = new InMemoryMemory();
await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['builder']);
const list = new AgentMessageList();
// Legacy data: a prior programmatic activation left no stamp and no
// `load_skill` record. The skill must still reach the model via the block.
list.addHistory([
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolName: 'build_workflow',
toolCallId: 'build-1',
input: {},
state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
},
],
},
]);
const active = new ActiveSkills(source, 'assistant', memory.skillState);
await active.restore(list, scope);
expect(active.instructions()).toContain('Build one workflow.');
});
it('recovers a skill through the block when its activating tool call failed', async () => {
const memory = new InMemoryMemory();
const active = new ActiveSkills(source, 'assistant', memory.skillState);
const list = new AgentMessageList();
list.addResponse([
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolName: 'build_workflow',
toolCallId: 'build-fail',
input: {},
state: 'rejected',
error: 'boom',
},
],
},
]);
await active.restore(list, scope);
// The tool activated the skill, then failed — its error result cannot
// carry the body, so the skill belongs in the block, not silently dropped.
await active.load('builder', { toolCallId: 'build-fail' });
expect(active.instructions()).toContain('Build one workflow.');
});
it('keeps threads, resources, and agents isolated', async () => { it('keeps threads, resources, and agents isolated', async () => {
const memory = new InMemoryMemory(); const memory = new InMemoryMemory();
const active = new ActiveSkills(source, 'assistant', memory.skillState); const active = new ActiveSkills(source, 'assistant', memory.skillState);
@ -165,6 +336,40 @@ describe('active skills', () => {
).resolves.toBeUndefined(); ).resolves.toBeUndefined();
}); });
it('keeps the system block empty for an anchored activation until observation masks it', async () => {
const memory = new InMemoryMemory();
const active = new ActiveSkills(source, 'assistant', memory.skillState);
const list = new AgentMessageList();
list.addResponse([
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolName: 'build_workflow',
toolCallId: 'build-1',
input: {},
state: 'resolved',
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
},
],
},
]);
await active.restore(list, scope);
// A programmatic activation anchored to the calling tool result stays out
// of the system prompt — the body rides on that visible result instead.
await active.load('builder', { toolCallId: 'build-1' });
expect(active.instructions()).toBeUndefined();
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).toContain(
'Build one workflow.',
);
// Only once observation hides the anchor does the skill move into the block.
mask(list);
expect(active.instructions()).toContain('Build one workflow.');
});
it('does not activate a missing skill or a failed load', async () => { it('does not activate a missing skill or a failed load', async () => {
const missing: RuntimeSkillSource = { ...source, loadSkill: vi.fn().mockResolvedValue(null) }; const missing: RuntimeSkillSource = { ...source, loadSkill: vi.fn().mockResolvedValue(null) };
const active = new ActiveSkills(missing, 'assistant'); const active = new ActiveSkills(missing, 'assistant');

View File

@ -295,6 +295,24 @@ export class AgentMessageList {
block.suspension = suspension; block.suspension = suspension;
} }
/**
* Record that a tool programmatically activated a skill on its own result, so
* `ActiveSkills` can re-anchor the skill body there on a later turn instead
* of falling back to the `<active_skills>` system prompt. The stamp rides on
* the persisted message; it is metadata only and never reaches the model
* (`toAiMessages` ignores it). No-op when the tool call is unknown.
*/
stampActivatedSkill(toolCallId: string, skillId: string): void {
const host = this.findToolCallHost(toolCallId);
if (!host) return;
const block = this.findToolCallBlock(host, toolCallId);
if (!block) return;
const current = block.activatedSkillIds ?? [];
if (current.includes(skillId)) return;
block.activatedSkillIds = [...current, skillId];
this.responseSet.add(host);
}
private findToolCallHost(toolCallId: string): AgentDbMessage | undefined { private findToolCallHost(toolCallId: string): AgentDbMessage | undefined {
// Start from the last message and go backwards to find the host message // Start from the last message and go backwards to find the host message
for (let i = this.all.length - 1; i >= 0; i--) { for (let i = this.all.length - 1; i >= 0; i--) {

View File

@ -1,4 +1,4 @@
import type { ModelMessage } from 'ai'; import type { ModelMessage, ToolResultPart } from 'ai';
import { formatActiveSkill } from '../../skills/tools'; import { formatActiveSkill } from '../../skills/tools';
import { import {
@ -11,7 +11,30 @@ import {
import type { AgentPersistenceOptions } from '../../types/sdk/agent'; import type { AgentPersistenceOptions } from '../../types/sdk/agent';
import type { AgentMessageList } from '../model/message-list'; import type { AgentMessageList } from '../model/message-list';
/** Keeps trusted skill instructions outside the conversation's compaction window. */ /**
* Keeps trusted skill instructions available without invalidating the cached
* prompt prefix.
*
* A skill body rides on the tool result that activated it (the recorded result
* is collapsed first, so an obsolete persisted body is never replayed). The
* top-level system prompt does not mention the skill, so activating and
* carrying a skill never rewrites the cached prefix — within a run or across
* runs.
*
* The `<active_skills>` block of the system prompt is a recovery path only.
* Anthropic renders the prompt as `tools → system → messages`, so any edit to
* `system` invalidates the tool block and every message after it. The block
* therefore holds only the skills the anchored path cannot deliver on this
* call — their result masked by observation memory, their activating call
* failed, or the activation predates this window with no `load_skill` record:
*
* - Anchor still deliverable: the skill rides on its resolved tool result, so
* `system` is unchanged and the cached prefix stays warm.
* - Anchor gone or unusable: the skill moves into the block so its guidance is
* never lost. When observation memory replaced the history it already rewrote
* the prefix, so the move is free; otherwise it costs one rewrite, the price
* of not dropping an active skill.
*/
export class ActiveSkills { export class ActiveSkills {
private readonly loaded = new Map<string, RuntimeSkillContent>(); private readonly loaded = new Map<string, RuntimeSkillContent>();
private list?: AgentMessageList; private list?: AgentMessageList;
@ -38,7 +61,11 @@ export class ActiveSkills {
list.activeSkillIds === undefined && this.scope list.activeSkillIds === undefined && this.scope
? await this.store?.load(this.scope) ? await this.store?.load(this.scope)
: undefined; : undefined;
const ids = list.activeSkillIds ?? stored ?? [...this.recordedLoads(list).values()]; const ids = list.activeSkillIds ??
stored ?? [
...this.recordedLoads(list).values(),
...this.stampedActivations(list).map(([, skillId]) => skillId),
];
const registered = new Set(this.source.registry.skills.map(({ id }) => id)); const registered = new Set(this.source.registry.skills.map(({ id }) => id));
for (const id of new Set(ids)) { for (const id of new Set(ids)) {
if (!registered.has(id)) continue; if (!registered.has(id)) continue;
@ -53,7 +80,10 @@ export class ActiveSkills {
if (!storeIsCurrent && (ids.length > 0 || stored !== undefined)) await this.persist(); if (!storeIsCurrent && (ids.length > 0 || stored !== undefined)) await this.persist();
} }
async load(skillId: string): Promise<RuntimeSkillContent | null> { async load(
skillId: string,
anchor?: { toolCallId: string },
): Promise<RuntimeSkillContent | null> {
if (!this.source.registry.skills.some(({ id }) => id === skillId)) return null; if (!this.source.registry.skills.some(({ id }) => id === skillId)) return null;
const skill = this.loaded.get(skillId) ?? (await this.source.loadSkill(skillId)); const skill = this.loaded.get(skillId) ?? (await this.source.loadSkill(skillId));
if (!skill) return null; if (!skill) return null;
@ -61,6 +91,9 @@ export class ActiveSkills {
if (!this.loaded.has(skillId)) { if (!this.loaded.has(skillId)) {
this.loaded.set(skillId, skill); this.loaded.set(skillId, skill);
this.list.activeSkillIds = [...this.loaded.keys()]; this.list.activeSkillIds = [...this.loaded.keys()];
// Stamp the activating tool result so the body re-anchors there on a
// later turn instead of falling back into the system prompt.
if (anchor) this.list.stampActivatedSkill(anchor.toolCallId, skillId);
await this.persist(); await this.persist();
} }
return skill; return skill;
@ -72,12 +105,19 @@ export class ActiveSkills {
]; ];
} }
/**
* The active skills the anchored path cannot deliver on this call, as one
* block for the top-level system prompt. A skill enters the block only when
* it has no successfully resolved, visible tool result to ride on, so a
* still-anchored skill never edits `system` or breaks a warm cache.
*/
instructions(): string | undefined { instructions(): string | undefined {
if (this.loaded.size === 0) return undefined; const { inBlock } = this.placement();
const sections = [...this.loaded].flatMap(([id, skill]) => { const sections = inBlock.flatMap((id) => {
const entry = this.source.registry.skills.find((candidate) => candidate.id === id); const text = this.formatSkill(id);
return entry ? [formatActiveSkill(skill, entry)] : []; return text ? [text] : [];
}); });
if (sections.length === 0) return undefined;
return [ return [
'<active_skills>', '<active_skills>',
'Use these current skill versions when relevant to the task. Earlier tool results may describe older versions.', 'Use these current skill versions when relevant to the task. Earlier tool results may describe older versions.',
@ -86,9 +126,23 @@ export class ActiveSkills {
].join('\n\n'); ].join('\n\n');
} }
/** Old conversations can contain full skill bodies. Do not replay an obsolete version. */ /**
* Per-call view of the conversation for the model.
*
* Collapses recorded `load_skill` results to `{ skillId, active }`: old
* conversations can contain full skill bodies, and an obsolete version must
* not be replayed. Skills anchored to a visible tool call then get their
* current body appended to that result, keeping them out of the system
* block for as long as observational memory keeps the result visible.
*/
modelMessages(messages: ModelMessage[], list: AgentMessageList): ModelMessage[] { modelMessages(messages: ModelMessage[], list: AgentMessageList): ModelMessage[] {
const loads = this.recordedLoads(list); const loads = this.recordedLoads(list);
const { anchors } = this.placement(loads);
const appendices = new Map<string, string[]>();
for (const [skillId, toolCallId] of anchors) {
const text = this.formatSkill(skillId);
if (text) appendices.set(toolCallId, [...(appendices.get(toolCallId) ?? []), text]);
}
return messages.map((message) => { return messages.map((message) => {
if (message.role !== 'tool') return message; if (message.role !== 'tool') return message;
return { return {
@ -96,19 +150,105 @@ export class ActiveSkills {
content: message.content.map((part) => { content: message.content.map((part) => {
if (part.type !== 'tool-result') return part; if (part.type !== 'tool-result') return part;
const skillId = loads.get(part.toolCallId); const skillId = loads.get(part.toolCallId);
if (!skillId) return part; const collapsed: ToolResultPart = skillId
return { ? {
...part, ...part,
output: { output: {
type: 'json' as const, type: 'json' as const,
value: { skillId, active: this.loaded.has(skillId) }, value: { skillId, active: this.loaded.has(skillId) },
}, },
}; }
: part;
const texts = appendices.get(part.toolCallId);
return texts ? appendToolResultText(collapsed, texts) : collapsed;
}), }),
}; };
}); });
} }
/**
* Where each active skill is delivered on this call: anchored to a visible
* tool result, or in the system block. The anchor is the first visible
* recorded `load_skill` call for the skill, else this run's activating
* call. `instructions()` and `modelMessages()` share this so a skill is
* always delivered exactly once.
*
* An anchor must be a **successfully resolved** tool result: the skill body
* is appended to it (see `appendToolResultText`, which skips error outputs),
* so a pending, canceled, or failed call cannot deliver it. A skill with no
* such anchor — its result masked by observation memory, or its activating
* call failed, or the activation predates this window with no `load_skill`
* record — falls back to the `<active_skills>` block, which never drops it.
* The block only edits `system` when the anchored path cannot deliver, so an
* unchanged, still-anchored skill never rewrites the cached prefix.
*/
private placement(loads?: Map<string, string>): {
anchors: Map<string, string>;
inBlock: string[];
} {
const anchors = new Map<string, string>();
if (this.list) {
const deliverable = this.deliverableToolResults();
// `load_skill` results and tool results that stamped a programmatic
// activation are both valid anchors; the stamp is what survives a turn.
const sources: Array<[string, string]> = [
...(loads ?? this.recordedLoads(this.list)),
...this.stampedActivations(this.list),
];
for (const [toolCallId, skillId] of sources) {
if (!anchors.has(skillId) && this.loaded.has(skillId) && deliverable.has(toolCallId)) {
anchors.set(skillId, toolCallId);
}
}
}
const inBlock = [...this.loaded.keys()].filter((id) => !anchors.has(id));
return { anchors, inBlock };
}
/**
* `[toolCallId, skillId]` pairs for tool results that stamped a programmatic
* skill activation (see `stampActivatedSkill`). Unlike `load_skill` records,
* these carry activations a tool started itself, so a post-build skill
* re-anchors to its tool result on the next turn instead of the system block.
*/
private stampedActivations(list: AgentMessageList): Array<[string, string]> {
const pairs: Array<[string, string]> = [];
for (const message of list.messages()) {
if (!('content' in message)) continue;
for (const part of message.content) {
if (part.type === 'tool-call' && Array.isArray(part.activatedSkillIds)) {
for (const skillId of part.activatedSkillIds) pairs.push([part.toolCallId, skillId]);
}
}
}
return pairs;
}
/**
* Tool-call ids whose result is visible and successfully resolved, so a skill
* body can ride on it. Pending, canceled, and failed calls are excluded — a
* skill anchored there would silently vanish, so it belongs in the block.
*/
private deliverableToolResults(): Set<string> {
const ids = new Set<string>();
if (!this.list) return ids;
for (const message of this.list.llmVisibleMessages()) {
if (!('content' in message)) continue;
for (const part of message.content) {
if (part.type === 'tool-call' && part.state === 'resolved' && !part.canceled) {
ids.add(part.toolCallId);
}
}
}
return ids;
}
private formatSkill(skillId: string): string | undefined {
const skill = this.loaded.get(skillId);
const entry = this.source.registry.skills.find((candidate) => candidate.id === skillId);
return skill && entry ? formatActiveSkill(skill, entry) : undefined;
}
private recordedLoads(list: AgentMessageList): Map<string, string> { private recordedLoads(list: AgentMessageList): Map<string, string> {
const loads = new Map<string, string>(); const loads = new Map<string, string>();
for (const message of list.messages()) { for (const message of list.messages()) {
@ -154,3 +294,32 @@ export class ActiveSkills {
await save; await save;
} }
} }
/**
* Append trusted text parts after a tool result's own output. Text and JSON
* outputs become content parts so the skill text is a separate block from the
* (possibly untrusted-wrapped) tool output. Error outputs are left alone.
*/
function appendToolResultText(part: ToolResultPart, texts: string[]): ToolResultPart {
const extra = texts.map((text) => ({ type: 'text' as const, text }));
const { output } = part;
switch (output.type) {
case 'content':
return { ...part, output: { type: 'content', value: [...output.value, ...extra] } };
case 'text':
return {
...part,
output: { type: 'content', value: [{ type: 'text', text: output.value }, ...extra] },
};
case 'json':
return {
...part,
output: {
type: 'content',
value: [{ type: 'text', text: JSON.stringify(output.value) }, ...extra],
},
};
default:
return part;
}
}

View File

@ -117,6 +117,15 @@ export async function executeTool(
throw new Error(`No handler found for tool "${builtTool.name}"`); throw new Error(`No handler found for tool "${builtTool.name}"`);
} }
// Anchor programmatic skill activations to the calling tool's result by
// default, so the skill body rides on that result (kept out of `system`)
// instead of forcing a full prompt-prefix rewrite. An explicit anchor wins.
const baseLoadSkill = executionContext.loadSkill;
const loadSkill: ToolExecutionContext['loadSkill'] = baseLoadSkill
? async (skillId, anchor) =>
await baseLoadSkill(skillId, anchor ?? (toolCallId ? { toolCallId } : undefined))
: undefined;
if (builtTool.suspendSchema) { if (builtTool.suspendSchema) {
const isCancelled = isCancellation(resumeData); const isCancelled = isCancellation(resumeData);
const ctx: InterruptibleToolContext = { const ctx: InterruptibleToolContext = {
@ -139,7 +148,7 @@ export async function executeTool(
toolCallId, toolCallId,
toolName: builtTool.name, toolName: builtTool.name,
runId: executionContext.runId, runId: executionContext.runId,
...(executionContext.loadSkill ? { loadSkill: executionContext.loadSkill } : {}), ...(loadSkill ? { loadSkill } : {}),
persistence: executionContext.persistence, persistence: executionContext.persistence,
emitEvent: executionContext.emitEvent, emitEvent: executionContext.emitEvent,
abortSignal: executionContext.abortSignal, abortSignal: executionContext.abortSignal,
@ -156,7 +165,7 @@ export async function executeTool(
toolCallId, toolCallId,
toolName: builtTool.name, toolName: builtTool.name,
runId: executionContext.runId, runId: executionContext.runId,
...(executionContext.loadSkill ? { loadSkill: executionContext.loadSkill } : {}), ...(loadSkill ? { loadSkill } : {}),
persistence: executionContext.persistence, persistence: executionContext.persistence,
emitEvent: executionContext.emitEvent, emitEvent: executionContext.emitEvent,
abortSignal: executionContext.abortSignal, abortSignal: executionContext.abortSignal,

View File

@ -137,7 +137,15 @@ export interface RuntimeSkillContent extends RuntimeSkillIndexEntry {
linkedFiles?: RuntimeSkillLinkedFiles; linkedFiles?: RuntimeSkillLinkedFiles;
} }
export type RuntimeSkillLoader = (skillId: string) => Promise<RuntimeSkillContent | null>; export type RuntimeSkillLoader = (
skillId: string,
/**
* Tool result the activation rides on. The skill body is appended to this
* result so the top-level system prompt stays byte-identical (no cache
* invalidation). Defaults to the calling tool's own result when omitted.
*/
anchor?: { toolCallId: string },
) => Promise<RuntimeSkillContent | null>;
export interface RuntimeSkillFileContent { export interface RuntimeSkillFileContent {
skillId: string; skillId: string;

View File

@ -133,6 +133,13 @@ export type ContentToolCall = ContentMetadata & {
input: JSONValue; input: JSONValue;
providerExecuted?: boolean; providerExecuted?: boolean;
/**
* Skills this tool activated programmatically (via `ctx.loadSkill`), so the
* skill body can re-anchor to this result on a later turn instead of falling
* back to the system prompt. Metadata only — never sent to the model.
*/
activatedSkillIds?: string[];
} & ( } & (
| { state: 'pending'; suspension?: ToolCallSuspensionInfo } | { state: 'pending'; suspension?: ToolCallSuspensionInfo }
| { state: 'resolved'; output: JSONValue; canceled?: boolean } | { state: 'resolved'; output: JSONValue; canceled?: boolean }

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/ai-node-sdk", "name": "@n8n/ai-node-sdk",
"version": "0.31.0", "version": "0.31.2",
"description": "SDK for building AI nodes in n8n", "description": "SDK for building AI nodes in n8n",
"types": "dist/esm/index.d.ts", "types": "dist/esm/index.d.ts",
"module": "dist/esm/index.js", "module": "dist/esm/index.js",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/ai-utilities", "name": "@n8n/ai-utilities",
"version": "0.34.0", "version": "0.34.2",
"description": "Utilities for building AI nodes in n8n", "description": "Utilities for building AI nodes in n8n",
"types": "dist/esm/index.d.ts", "types": "dist/esm/index.d.ts",
"module": "dist/esm/index.js", "module": "dist/esm/index.js",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/ai-workflow-builder", "name": "@n8n/ai-workflow-builder",
"version": "1.41.0", "version": "1.41.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/api-types", "name": "@n8n/api-types",
"version": "1.41.0", "version": "1.41.1",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -391,8 +391,6 @@ export type FrontendModuleSettings = {
runDebugEnabled: boolean; runDebugEnabled: boolean;
/** Whether this instance is in the activation-capped trial cohort (`N8N_INSTANCE_AI_ACTIVATION_CAPPED`). Optional. */ /** Whether this instance is in the activation-capped trial cohort (`N8N_INSTANCE_AI_ACTIVATION_CAPPED`). Optional. */
activationCapped?: boolean; activationCapped?: boolean;
/** Whether the non-blocking setup panel replaces the suspending setup wizard (`N8N_INSTANCE_AI_SETUP_PANEL_ENABLED`). */
instanceAiSetupPanelEnabled?: boolean;
}; };
/** /**

View File

@ -425,6 +425,8 @@ export {
INSTANCE_AI_CONVERSATION_HISTORY_FLAG, INSTANCE_AI_CONVERSATION_HISTORY_FLAG,
INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
INSTANCE_AI_SETUP_PANEL_FLAG,
INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
INSTANCE_AI_NODE_USAGE_FLAG, INSTANCE_AI_NODE_USAGE_FLAG,
INSTANCE_AI_FOLDER_EXPLORATION_FLAG, INSTANCE_AI_FOLDER_EXPLORATION_FLAG,

View File

@ -2556,6 +2556,9 @@ export const INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT = 'variant';
export const INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG = '111_instance_ai_progressive_building'; export const INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG = '111_instance_ai_progressive_building';
export const INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT = 'variant'; export const INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT = 'variant';
export const INSTANCE_AI_SETUP_PANEL_FLAG = '118_instance_ai_setup_overhaul';
export const INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT = 'variant';
/** Enables the node-usage context surface for Instance AI: the `node-usage` /** Enables the node-usage context surface for Instance AI: the `node-usage`
* action and the `nodeTypes` filter on `workflows(action="list")`. */ * action and the `nodeTypes` filter on `workflows(action="list")`. */

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/backend-common", "name": "@n8n/backend-common",
"version": "1.41.0", "version": "1.41.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -30,7 +30,6 @@ describe('getModuleEntryUrl', () => {
describe('eligibleModules', () => { describe('eligibleModules', () => {
it('should not include opt-in modules by default', () => { it('should not include opt-in modules by default', () => {
const eligible = Container.get(ModuleRegistry).eligibleModules; const eligible = Container.get(ModuleRegistry).eligibleModules;
expect(eligible).not.toContain('agents');
expect(eligible).not.toContain('type-availability-policies'); expect(eligible).not.toContain('type-availability-policies');
}); });
@ -43,14 +42,17 @@ describe('eligibleModules', () => {
expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('policy-infrastructure'); expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('policy-infrastructure');
}); });
it('should include instance-ai by default', () => { it.each(['instance-ai', 'agents'])('should include %s by default', (moduleName) => {
expect(Container.get(ModuleRegistry).eligibleModules).toContain('instance-ai'); expect(Container.get(ModuleRegistry).eligibleModules).toContain(moduleName);
}); });
it('should allow opting out of a default module via env var', () => { it.each(['instance-ai', 'agents'])(
process.env.N8N_DISABLED_MODULES = 'instance-ai'; 'should allow opting out of the default %s module via env var',
expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('instance-ai'); (moduleName) => {
}); process.env.N8N_DISABLED_MODULES = moduleName;
expect(Container.get(ModuleRegistry).eligibleModules).not.toContain(moduleName);
},
);
it('should consider a module ineligible if it was disabled via env var', () => { it('should consider a module ineligible if it was disabled via env var', () => {
process.env.N8N_DISABLED_MODULES = 'insights'; process.env.N8N_DISABLED_MODULES = 'insights';
@ -85,11 +87,12 @@ describe('eligibleModules', () => {
'mcp-registry', 'mcp-registry',
'workflow-reviews', 'workflow-reviews',
'instance-ai', 'instance-ai',
'agents',
]); ]);
}); });
it('should consider a module eligible if it was enabled via env var', () => { it('should consider a module eligible if it was enabled via env var', () => {
process.env.N8N_ENABLED_MODULES = 'agents'; process.env.N8N_ENABLED_MODULES = 'type-availability-policies';
expect(Container.get(ModuleRegistry).eligibleModules).toEqual([ expect(Container.get(ModuleRegistry).eligibleModules).toEqual([
'policy-infrastructure', 'policy-infrastructure',
'insights', 'insights',
@ -123,6 +126,7 @@ describe('eligibleModules', () => {
'workflow-reviews', 'workflow-reviews',
'instance-ai', 'instance-ai',
'agents', 'agents',
'type-availability-policies',
]); ]);
}); });

View File

@ -77,6 +77,7 @@ export class ModuleRegistry {
'mcp-registry', 'mcp-registry',
'workflow-reviews', 'workflow-reviews',
'instance-ai', 'instance-ai',
'agents',
]; ];
private readonly activeModules: string[] = []; private readonly activeModules: string[] = [];

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/backend-network", "name": "@n8n/backend-network",
"version": "1.15.0", "version": "1.15.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/backend-test-utils", "name": "@n8n/backend-test-utils",
"version": "1.41.0", "version": "1.41.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/blob-storage", "name": "@n8n/blob-storage",
"version": "1.9.0", "version": "1.9.2",
"description": "Blob storage layer for n8n, with filesystem, S3, and Azure Blob backends", "description": "Blob storage layer for n8n, with filesystem, S3, and Azure Blob backends",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/chat-hub", "name": "@n8n/chat-hub",
"version": "1.34.0", "version": "1.34.1",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/client-oauth2", "name": "@n8n/client-oauth2",
"version": "1.23.0", "version": "1.23.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/config", "name": "@n8n/config",
"version": "2.39.0", "version": "2.39.1",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -206,15 +206,6 @@ export class InstanceAiConfig {
@Env('N8N_INSTANCE_AI_NODE_CONTEXT_ENABLED') @Env('N8N_INSTANCE_AI_NODE_CONTEXT_ENABLED')
canvasNodeContextEnabled: boolean = false; canvasNodeContextEnabled: boolean = false;
/**
* Non-blocking setup panel (setup panel v2): the persistent checklist above
* the chat input replaces the suspending setup wizard. Env-settable so eval
* lanes can exercise both paths; a managed rollout flag may layer on top
* later behind the same accessors.
*/
@Env('N8N_INSTANCE_AI_SETUP_PANEL_ENABLED')
instanceAiSetupPanelEnabled: boolean = false;
/** /**
* Force-enable the node-usage context surface for Instance AI — the `node-usage` action and * Force-enable the node-usage context surface for Instance AI — the `node-usage` action and
* the `nodeTypes` filter on `workflows(action="list")`. * the `nodeTypes` filter on `workflows(action="list")`.

View File

@ -379,7 +379,6 @@ describe('GlobalConfig', () => {
thinkingEnabled: true, thinkingEnabled: true,
mcpConnectionsEnabled: false, mcpConnectionsEnabled: false,
canvasNodeContextEnabled: false, canvasNodeContextEnabled: false,
instanceAiSetupPanelEnabled: false,
nodeUsageEnabled: false, nodeUsageEnabled: false,
folderExplorationEnabled: false, folderExplorationEnabled: false,
activationCapped: false, activationCapped: false,

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/create-node", "name": "@n8n/create-node",
"version": "0.49.0", "version": "0.49.2",
"description": "Official CLI to create new community nodes for n8n", "description": "Official CLI to create new community nodes for n8n",
"bin": { "bin": {
"create-node": "bin/create-node.cjs" "create-node": "bin/create-node.cjs"

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/db", "name": "@n8n/db",
"version": "1.41.0", "version": "1.41.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo .tmp-schema-docs", "clean": "rimraf dist .turbo .tmp-schema-docs",
"gen:migration-index": "node scripts/generate-migration-index.mjs", "gen:migration-index": "node scripts/generate-migration-index.mjs",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/decorators", "name": "@n8n/decorators",
"version": "1.41.0", "version": "1.41.1",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/engine", "name": "@n8n/engine",
"version": "0.21.0", "version": "0.21.1",
"description": "n8n workflow execution engine (v2)", "description": "n8n workflow execution engine (v2)",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo compiled", "clean": "rimraf dist .turbo compiled",

View File

@ -47,10 +47,16 @@ For built-in providers, the setup service recognizes `ANTHROPIC_API_KEY`,
| `N8N_INSTANCE_AI_NODE_CONTEXT_ENABLED` | boolean | `false` | Force-enable canvas node context. `false` falls back to the PostHog flag. | | `N8N_INSTANCE_AI_NODE_CONTEXT_ENABLED` | boolean | `false` | Force-enable canvas node context. `false` falls back to the PostHog flag. |
| `N8N_INSTANCE_AI_FOLDER_EXPLORATION_ENABLED` | boolean | `false` | Force-enable folder exploration (folder attribution and folder scoping on `workflows(action="list")`). `false` falls back to the PostHog flag. | | `N8N_INSTANCE_AI_FOLDER_EXPLORATION_ENABLED` | boolean | `false` | Force-enable folder exploration (folder attribution and folder scoping on `workflows(action="list")`). `false` falls back to the PostHog flag. |
| `N8N_INSTANCE_AI_BROWSER_USE_ENABLED` | boolean | `true` | Computer Use browser tooling, used for credential setup. | | `N8N_INSTANCE_AI_BROWSER_USE_ENABLED` | boolean | `true` | Computer Use browser tooling, used for credential setup. |
| `N8N_INSTANCE_AI_SETUP_PANEL_ENABLED` | boolean | `false` | Non-blocking setup panel (setup panel v2) instead of the suspending setup wizard. |
| `N8N_INSTANCE_AI_ACTIVATION_CAPPED` | boolean | `false` | Activation capping. | | `N8N_INSTANCE_AI_ACTIVATION_CAPPED` | boolean | `false` | Activation capping. |
| `N8N_INSTANCE_AI_ACTIVATION_LOCK_MESSAGE_THRESHOLD` | number | `1` | Assistant messages that must be sent, in addition to instance activation, before an activation lock applies. | | `N8N_INSTANCE_AI_ACTIVATION_LOCK_MESSAGE_THRESHOLD` | number | `1` | Assistant messages that must be sent, in addition to instance activation, before an activation lock applies. |
The PostHog flag `118_instance_ai_setup_overhaul` selects the setup flow.
`variant` enables the async panel. `control` or an unavailable assignment uses
the wizard. For local testing, set
`N8N_FEATURE_FLAG_OVERRIDES='{"118_instance_ai_setup_overhaul":"variant"}'`
on the n8n server, then restart it and reload the editor. Use `control` to test
the wizard.
### Debugging ### Debugging
| Variable | Type | Default | Description | | Variable | Type | Default | Description |

View File

@ -474,7 +474,7 @@ confirmation card.
`nodesStillNeedingSetup` is what nobody has configured yet, `skippedByUser` what the user `nodesStillNeedingSetup` is what nobody has configured yet, `skippedByUser` what the user
actively dismissed and the agent must not re-open (see `reopenSkipped`). actively dismissed and the agent must not re-open (see `reopenSkipped`).
**Setup panel** (`N8N_INSTANCE_AI_SETUP_PANEL_ENABLED`): the normal setup call **Setup panel** (`118_instance_ai_setup_overhaul: variant`): the normal setup call
analyzes the whole workflow, including bound slots. It publishes the `setup-items` analyzes the whole workflow, including bound slots. It publishes the `setup-items`
snapshot and confirms that it reached storage. It then saves the build's setup snapshot and confirms that it reached storage. It then saves the build's setup
routing marker. Only after both steps succeed does it return routing marker. Only after both steps succeed does it return
@ -609,7 +609,15 @@ Default timeout: 5 minutes; max: 10 minutes. On timeout, execution is cancelled.
| `timeout` | number | no | 300000 | Max wait time in ms (max 600000) | | `timeout` | number | no | 300000 | Max wait time in ms (max 600000) |
| `triggerNodeName` | string | no | — | Trigger node to use when a workflow has more than one trigger | | `triggerNodeName` | string | no | — | Trigger node to use when a workflow has more than one trigger |
**Returns**: `{ executionId, status, data?, error?, startedAt?, finishedAt? }` **Returns**: `{ executionId, status, data?, error?, startedAt?, finishedAt?, verificationClaim? }`
**Live test evidence**: `verify-built-workflow` always simulates destructive
nodes, so a live test runs through this action. When a successful run reaches
every planned node of the latest build, with no saved pins and no injected
trigger input, the run is recorded as a `verified` claim on the build outcome.
The publish gate then reads that claim. The result carries it as
`verificationClaim`. Other runs leave the stored claim unchanged: a live run
can raise the verdict but never lower it.
**Type-aware pin data**: Constructs proper pin data per trigger type: **Type-aware pin data**: Constructs proper pin data per trigger type:
- **Chat trigger**: `{ chatInput, sessionId, action }` - **Chat trigger**: `{ chatInput, sessionId, action }`
@ -831,7 +839,7 @@ a service. When `needsBrowserSetup=true`, the orchestrator should load the
directly, then call `credentials(action="setup")` again to select the created directly, then call `credentials(action="setup")` again to select the created
credential. credential.
**Setup panel** (`N8N_INSTANCE_AI_SETUP_PANEL_ENABLED`): when the call belongs **Setup panel** (`118_instance_ai_setup_overhaul: variant`): when the call belongs
to a workflow (`workflowId`, or the workflow this run last saved) and the stage to a workflow (`workflowId`, or the workflow this run last saved) and the stage
is not `finalize`, the tool does not suspend. It merges the credential types is not `finalize`, the tool does not suspend. It merges the credential types
into the workflow's durable `setup-items` snapshot and returns into the workflow's durable `setup-items` snapshot and returns

View File

@ -310,12 +310,13 @@ The harness remaps that ID and sends the normal chat request with the workflow
attachment and structured handoff context. The transcript records the Execute attachment and structured handoff context. The transcript records the Execute
action so process expectations can check the response. action so process expectations can check the response.
Start each panel eval instance with `N8N_INSTANCE_AI_SETUP_PANEL_ENABLED=true`. Start each panel eval instance with
`N8N_FEATURE_FLAG_OVERRIDES='{"118_instance_ai_setup_overhaul":"variant"}'`.
Set this variable on the n8n server process or in the lane's environment file. Set this variable on the n8n server process or in the lane's environment file.
Setting it only on the eval client does not enable the server feature. Setting it only on the eval client does not enable the server feature.
Run the normal PR tier with the flag on and off. For panel cases, load the Run the normal PR tier with `control` and `variant`. For panel cases, load the
external suite with `--source langtracer --suite <suite-id>`, or stage a local external suite with `--source langtracer --suite <suite-id>`, or stage a local
case and select it with `--filter <case-slug>`. Run those cases with the flag on. case and select it with `--filter <case-slug>`. Run those cases with `variant`.
The repository does not include a `setup-panel-v2` tier. The repository does not include a `setup-panel-v2` tier.
Remote Execute cases require LangTracer to preserve `attach.source` when it Remote Execute cases require LangTracer to preserve `attach.source` when it

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/instance-ai", "name": "@n8n/instance-ai",
"version": "1.26.0", "version": "1.26.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"typecheck": "tsc --noEmit && tsc -p evaluations/tsconfig.json", "typecheck": "tsc --noEmit && tsc -p evaluations/tsconfig.json",

View File

@ -1024,7 +1024,11 @@ describe('RunStateRegistry', () => {
createdAt: Date.now(), createdAt: Date.now(),
}); });
registry.setSetupPanelEnabled('thread-1', true);
expect(registry.isSetupPanelEnabled('thread-1')).toBe(true);
expect(registry.isSetupPanelEnabled('thread-2')).toBe(false);
const result = registry.clearThread('thread-1'); const result = registry.clearThread('thread-1');
expect(registry.isSetupPanelEnabled('thread-1')).toBe(false);
// Confirmations resolved // Confirmations resolved
expect(resolve).toHaveBeenCalledWith({ approved: false }); expect(resolve).toHaveBeenCalledWith({ approved: false });

View File

@ -157,6 +157,7 @@ export class RunStateRegistry<TUser = unknown> {
/** Build mode captured at user-run entry and reused by follow-up runs. */ /** Build mode captured at user-run entry and reused by follow-up runs. */
private readonly threadBuildModes = new Map<string, InstanceAiBuildMode>(); private readonly threadBuildModes = new Map<string, InstanceAiBuildMode>();
private readonly threadSetupPanelEnabled = new Map<string, boolean>();
private readonly threadPromptSelections = new Map< private readonly threadPromptSelections = new Map<
string, string,
{ version: string; metadata?: InstanceAiPromptConfiguration } { version: string; metadata?: InstanceAiPromptConfiguration }
@ -521,6 +522,14 @@ export class RunStateRegistry<TUser = unknown> {
return this.threadBuildModes.get(threadId); return this.threadBuildModes.get(threadId);
} }
setSetupPanelEnabled(threadId: string, enabled: boolean): void {
this.threadSetupPanelEnabled.set(threadId, enabled);
}
isSetupPanelEnabled(threadId: string): boolean {
return this.threadSetupPanelEnabled.get(threadId) === true;
}
setPromptVersion(threadId: string, version: string | undefined): void { setPromptVersion(threadId: string, version: string | undefined): void {
if (version === undefined) this.threadPromptSelections.delete(threadId); if (version === undefined) this.threadPromptSelections.delete(threadId);
else this.threadPromptSelections.set(threadId, { version }); else this.threadPromptSelections.set(threadId, { version });
@ -690,6 +699,7 @@ export class RunStateRegistry<TUser = unknown> {
this.threadTimeZones.delete(threadId); this.threadTimeZones.delete(threadId);
this.threadComputerUseChannels.delete(threadId); this.threadComputerUseChannels.delete(threadId);
this.threadBuildModes.delete(threadId); this.threadBuildModes.delete(threadId);
this.threadSetupPanelEnabled.delete(threadId);
this.threadPromptSelections.delete(threadId); this.threadPromptSelections.delete(threadId);
const groupId = this.threadMessageGroupId.get(threadId); const groupId = this.threadMessageGroupId.get(threadId);
@ -741,6 +751,7 @@ export class RunStateRegistry<TUser = unknown> {
this.threadTimeZones.clear(); this.threadTimeZones.clear();
this.threadComputerUseChannels.clear(); this.threadComputerUseChannels.clear();
this.threadBuildModes.clear(); this.threadBuildModes.clear();
this.threadSetupPanelEnabled.clear();
this.threadPromptSelections.clear(); this.threadPromptSelections.clear();
this.threadMessageGroupId.clear(); this.threadMessageGroupId.clear();
this.runIdsByMessageGroup.clear(); this.runIdsByMessageGroup.clear();

View File

@ -4,7 +4,13 @@ import type { z } from 'zod';
import { executeTool } from '../../__tests__/tool-test-utils'; import { executeTool } from '../../__tests__/tool-test-utils';
import type { InstanceAiContext, ExecutionResult } from '../../types'; import type { InstanceAiContext, ExecutionResult } from '../../types';
import type { VerificationClaim } from '../../workflow-loop/workflow-loop-state';
import { createExecutionsTool } from '../executions.tool'; import { createExecutionsTool } from '../executions.tool';
import { recordLiveRunVerification } from '../orchestration/verification/record-live-run';
vi.mock('../orchestration/verification/record-live-run', () => ({
recordLiveRunVerification: vi.fn().mockResolvedValue(undefined),
}));
// ── Mock helpers ─────────────────────────────────────────────────────────────── // ── Mock helpers ───────────────────────────────────────────────────────────────
@ -461,6 +467,63 @@ describe('executions tool', () => {
); );
}); });
describe('live run verification', () => {
const runResult = { executionId: 'exec-1', status: 'success' as const };
function createAllowedContext() {
const context = createMockContext({
permissions: { runWorkflow: 'always_allow' },
aiCreatedWorkflowIds: new Set(['wf-1']),
});
(context.executionService.run as Mock).mockResolvedValue(runResult);
return context;
}
it('returns the recorded claim with the run result', async () => {
const context = createAllowedContext();
const claim: VerificationClaim = {
level: 'verified',
plannedNodeCount: 1,
reachedNodeCount: 1,
nodesNotReached: [],
simulatedNodes: [],
pinnedNodes: [],
unprovenTargets: [],
publishReady: true,
liveTestRecommended: false,
};
vi.mocked(recordLiveRunVerification).mockResolvedValueOnce(claim);
const tool = createExecutionsTool(context);
const result = await executeTool(
tool,
{ action: 'run' as const, workflowId: 'wf-1', triggerNodeName: 'Every Morning' },
createAgentCtx() as never,
);
expect(result).toEqual({ ...runResult, verificationClaim: claim });
expect(recordLiveRunVerification).toHaveBeenCalledWith({
context,
workflowId: 'wf-1',
triggerNodeName: 'Every Morning',
result: runResult,
});
});
it('returns the plain run result when no claim was recorded', async () => {
const context = createAllowedContext();
const tool = createExecutionsTool(context);
const result = await executeTool(
tool,
{ action: 'run' as const, workflowId: 'wf-1' },
createAgentCtx() as never,
);
expect(result).toEqual(runResult);
});
});
describe('session grant (always allow)', () => { describe('session grant (always allow)', () => {
it('runs without HITL when the workflow has a session grant', async () => { it('runs without HITL when the workflow has a session grant', async () => {
const context = createMockContext({ const context = createMockContext({

View File

@ -17,6 +17,7 @@ import { z } from 'zod';
import { sanitizeInputSchema } from '../agent/sanitize-mcp-schemas'; import { sanitizeInputSchema } from '../agent/sanitize-mcp-schemas';
import type { InstanceAiContext } from '../types'; import type { InstanceAiContext } from '../types';
import { approvalSummarySchema, formatApprovalMessage } from './approval-copy'; import { approvalSummarySchema, formatApprovalMessage } from './approval-copy';
import { recordLiveRunVerification } from './orchestration/verification/record-live-run';
// ── Constants ────────────────────────────────────────────────────────────── // ── Constants ──────────────────────────────────────────────────────────────
@ -416,11 +417,20 @@ async function handleRun(
} }
// Approved or always_allow — execute // Approved or always_allow — execute
return await context.executionService.run(workflowId, input.inputData, { const result = await context.executionService.run(workflowId, input.inputData, {
timeout: input.timeout, timeout: input.timeout,
triggerNodeName: input.triggerNodeName, triggerNodeName: input.triggerNodeName,
abortSignal, abortSignal,
}); });
// A live test is the evidence verification cannot produce itself. Record it
// so the publish gate stops disclosing simulations that this run replaced.
const verificationClaim = await recordLiveRunVerification({
context,
workflowId,
triggerNodeName: input.triggerNodeName,
result,
});
return verificationClaim ? { ...result, verificationClaim } : result;
} }
/** /**

View File

@ -0,0 +1,223 @@
import type { InstanceAiContext } from '../../../../types';
import type { WorkflowBuildOutcome } from '../../../../workflow-loop/workflow-loop-state';
import { recordLiveRunVerification } from '../record-live-run';
import type { ExecutionRunResult } from '../types';
function makeOutcome(overrides: Partial<WorkflowBuildOutcome> = {}): WorkflowBuildOutcome {
return {
workItemId: 'wi_1',
taskId: 'task_1',
workflowId: 'wf_1',
submitted: true,
triggerType: 'trigger_only',
needsUserInput: false,
summary: 'Built',
nodeSimulationPlan: [
{
nodeName: 'Send Telegram Message',
verdict: 'simulate',
reason: 'Sends a message to a Telegram chat.',
confidence: 'high',
source: 'deterministic',
},
],
verification: {
attempted: true,
success: true,
status: 'success',
claim: {
level: 'partial',
plannedNodeCount: 1,
reachedNodeCount: 1,
nodesNotReached: [],
simulatedNodes: [
{ nodeName: 'Send Telegram Message', reason: 'Sends a message to a Telegram chat.' },
],
pinnedNodes: [],
unprovenTargets: [],
publishReady: false,
liveTestRecommended: true,
},
},
...overrides,
};
}
function makeRunResult(overrides: Partial<ExecutionRunResult> = {}): ExecutionRunResult {
return {
executionId: 'exec-2',
status: 'success',
data: { 'Every Morning': [{}], 'Send Telegram Message': [{ ok: true }] },
executedNodeNames: ['Every Morning', 'Send Telegram Message'],
workflowVersionId: 'version-2',
...overrides,
};
}
function createContext({
outcome = makeOutcome(),
}: { outcome?: WorkflowBuildOutcome | null } = {}) {
const workflowTaskService = {
getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue(outcome ?? undefined),
recordVerification: vi.fn(
async (_workItemId: string, verification: { claim: unknown }) =>
await Promise.resolve(verification.claim),
),
};
const context = {
userId: 'user-1',
runId: 'run-2',
workflowService: {
getWorkflowHead: vi
.fn()
.mockResolvedValue({ versionId: 'version-2', activeVersionId: null, updatedAt: 0 }),
},
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
workflowBuildContext: {
threadId: 'thread-1',
runId: 'run-2',
taskId: 'task_1',
workItemId: 'wi_1',
workflowTaskService,
},
} as unknown as InstanceAiContext;
return { context, workflowTaskService };
}
describe('recordLiveRunVerification', () => {
it('records a verified claim for the executed version when the run covers every planned node', async () => {
const { context, workflowTaskService } = createContext();
const claim = await recordLiveRunVerification({
context,
workflowId: 'wf_1',
result: makeRunResult(),
});
expect(claim).toMatchObject({
level: 'verified',
publishReady: true,
simulatedNodes: [],
verifiedVersionId: 'version-2',
liveState: 'unpublished',
});
expect(workflowTaskService.recordVerification).toHaveBeenCalledWith(
'wi_1',
expect.objectContaining({
attempted: true,
success: true,
executionId: 'exec-2',
status: 'success',
claim: expect.objectContaining({ level: 'verified' }),
evidence: expect.objectContaining({
nodesExecuted: ['Every Morning', 'Send Telegram Message'],
}),
}),
);
});
it.each([
['used injected trigger input', { injectedTriggerNodeName: 'Every Morning' }],
['read saved pinned data', { workflowPinnedNodeNames: ['Send Telegram Message'] }],
[
'did not reach a planned node',
{ executedNodeNames: ['Every Morning'], data: { 'Every Morning': [{}] } },
],
['failed', { status: 'error' as const, error: 'Bad Request: chat not found' }],
[
'reported a node error',
{ nodeErrors: [{ nodeName: 'Send Telegram Message', message: 'Bad Request' }] },
],
['ran an unknown version', { workflowVersionId: undefined }],
])('keeps the stored claim when the run %s', async (_label, overrides) => {
const { context, workflowTaskService } = createContext();
const claim = await recordLiveRunVerification({
context,
workflowId: 'wf_1',
result: makeRunResult(overrides),
});
expect(claim).toBeUndefined();
expect(workflowTaskService.recordVerification).not.toHaveBeenCalled();
});
it.each([
['there is no build outcome', null],
['the build outcome has no simulation plan', makeOutcome({ nodeSimulationPlan: undefined })],
[
'a verification is still running',
makeOutcome({ verification: { attempted: false, success: false, status: 'running' } }),
],
])('does not record when %s', async (_label, outcome) => {
const { context, workflowTaskService } = createContext({ outcome });
const claim = await recordLiveRunVerification({
context,
workflowId: 'wf_1',
result: makeRunResult(),
});
expect(claim).toBeUndefined();
expect(workflowTaskService.recordVerification).not.toHaveBeenCalled();
});
it('does not record when the draft was saved during the run', async () => {
const { context, workflowTaskService } = createContext();
vi.mocked(context.workflowService.getWorkflowHead).mockResolvedValueOnce({
versionId: 'version-3',
activeVersionId: null,
updatedAt: 0,
});
const claim = await recordLiveRunVerification({
context,
workflowId: 'wf_1',
result: makeRunResult(),
});
expect(claim).toBeUndefined();
expect(workflowTaskService.recordVerification).not.toHaveBeenCalled();
});
it('does not record outside a workflow build context', async () => {
const { context } = createContext();
context.workflowBuildContext = undefined;
const claim = await recordLiveRunVerification({
context,
workflowId: 'wf_1',
result: makeRunResult(),
});
expect(claim).toBeUndefined();
});
it.each([
['reading the build outcome', 'outcome'],
['reading the workflow head', 'head'],
['recording the claim', 'record'],
] as const)('logs and returns undefined when %s fails', async (_label, failingCall) => {
const { context, workflowTaskService } = createContext();
const error = new Error('storage down');
if (failingCall === 'outcome') {
workflowTaskService.getLatestBuildOutcomeForWorkflow.mockRejectedValueOnce(error);
} else if (failingCall === 'head') {
vi.mocked(context.workflowService.getWorkflowHead).mockRejectedValueOnce(error);
} else {
workflowTaskService.recordVerification.mockRejectedValueOnce(error);
}
const claim = await recordLiveRunVerification({
context,
workflowId: 'wf_1',
result: makeRunResult(),
});
expect(claim).toBeUndefined();
expect(context.logger.warn).toHaveBeenCalledWith(
'Failed to record a live run as verification evidence',
{ workflowId: 'wf_1', error: 'storage down' },
);
});
});

View File

@ -0,0 +1,32 @@
import type { VerificationPublishState } from './claim';
import type { Logger } from '../../../logger';
import type { InstanceAiWorkflowService } from '../../../types';
/**
* Version pair behind `claim.liveState`. The executed version has to come from
* the execution record: the workflow head moves when anybody saves, so
* substituting it would let the claim describe a version this run never ran.
* Without that record there is no publish state — an unknown run version must
* not become `live-current`, which reads as "production is proven".
*/
export async function resolvePublishState(args: {
workflowService: InstanceAiWorkflowService;
workflowId: string;
executedVersionId: string | null | undefined;
logger: Logger;
}): Promise<VerificationPublishState | undefined> {
const { workflowService, workflowId, executedVersionId, logger } = args;
if (!executedVersionId) return undefined;
try {
const head = await workflowService.getWorkflowHead(workflowId);
return { activeVersionId: head.activeVersionId, draftVersionId: executedVersionId };
} catch (error) {
logger.warn('Failed to read publish state for the verification claim', {
workflowId,
error: error instanceof Error ? error.message : String(error),
});
return undefined;
}
}

View File

@ -0,0 +1,84 @@
/**
* Live runs as verification evidence.
*
* `verify-built-workflow` always simulates nodes with side effects, so the live
* test it recommends runs through `executions(action="run")`. Without this, the
* stored claim keeps the simulated verdict and the publish gate warns about a
* path the user just saw succeed.
*/
import { analyzeVerificationResult, countProducedOutputRows } from './analyze-result';
import { deriveVerificationClaim } from './claim';
import type { ExecutionRunResult } from './types';
import type { InstanceAiContext } from '../../../types';
import type { VerificationClaim } from '../../../workflow-loop/workflow-loop-state';
/**
* Record a live run on the workflow's latest build outcome when it proves the
* whole workflow. Returns the stored claim, or undefined when nothing changed.
*
* Only a `verified` run is recorded: a live run can raise the verdict but never
* lower it. A narrow or failed ad hoc run (one branch, injected trigger input,
* saved pins) says less than the verification it would replace.
*/
export async function recordLiveRunVerification(args: {
context: InstanceAiContext;
workflowId: string;
triggerNodeName?: string;
result: ExecutionRunResult;
}): Promise<VerificationClaim | undefined> {
const { context, workflowId, triggerNodeName, result } = args;
const buildContext = context.workflowBuildContext;
const workflowTaskService = buildContext?.workflowTaskService;
if (!buildContext || !workflowTaskService || result.status !== 'success') return undefined;
try {
const outcome = await workflowTaskService.getLatestBuildOutcomeForWorkflow(workflowId);
const plan = outcome?.nodeSimulationPlan;
// A verification in flight owns the record; do not race its write.
if (!outcome || !plan || outcome.verification?.status === 'running') return undefined;
// A save during the run moves the draft past the version that ran, and
// this run proves nothing about the newer draft.
const executedVersionId = result.workflowVersionId;
const head = await context.workflowService.getWorkflowHead(workflowId);
if (!executedVersionId || head.versionId !== executedVersionId) return undefined;
// No planned simulations: this run used no verification pin data. Saved
// pins and injected trigger input still count as simulated in the analysis.
const analysis = analyzeVerificationResult({
result,
buildOutcome: outcome,
simulatedNodes: [],
stateBefore: undefined,
runId: context.runId ?? buildContext.runId,
triggerNodeName,
});
const claim = deriveVerificationClaim({
analysis,
plannedNodeCount: plan.length,
publishState: { activeVersionId: head.activeVersionId, draftVersionId: executedVersionId },
});
if (claim.level !== 'verified') return undefined;
return await workflowTaskService.recordVerification(outcome.workItemId, {
attempted: true,
success: true,
executionId: result.executionId || undefined,
status: result.status,
claim,
evidence: {
nodesExecuted: analysis.nodesExecuted,
producedOutputRows: countProducedOutputRows(result.data),
},
verifiedAt: new Date().toISOString(),
});
} catch (error) {
// Advisory: the run itself succeeded, and the stored verdict stays as it was.
context.logger.warn('Failed to record a live run as verification evidence', {
workflowId,
error: error instanceof Error ? error.message : String(error),
});
return undefined;
}
}

View File

@ -10,16 +10,16 @@ import { Tool } from '@n8n/agents';
import { isTriggerNodeType } from 'n8n-workflow'; import { isTriggerNodeType } from 'n8n-workflow';
import { z } from 'zod'; import { z } from 'zod';
import type { InstanceAiWorkflowService, OrchestrationContext } from '../../types'; import type { OrchestrationContext } from '../../types';
import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result'; import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result';
import { deriveVerificationClaim } from './verification/claim'; import { deriveVerificationClaim } from './verification/claim';
import type { VerificationPublishState } from './verification/claim';
import { import {
handleMissingSimulationPlan, handleMissingSimulationPlan,
handleBlockedVerification, handleBlockedVerification,
persistVerificationOutcome, persistVerificationOutcome,
} from './verification/finalize-result'; } from './verification/finalize-result';
import { prepareVerificationRun } from './verification/prepare-run'; import { prepareVerificationRun } from './verification/prepare-run';
import { resolvePublishState } from './verification/publish-state';
import { reconcileStaleCredentialPlan } from './verification/reconcile-plan'; import { reconcileStaleCredentialPlan } from './verification/reconcile-plan';
import { resolveVerificationTarget } from './verification/resolve-target'; import { resolveVerificationTarget } from './verification/resolve-target';
import { import {
@ -64,35 +64,6 @@ function formatLiveStateNote(claim: VerificationClaim | undefined): string | und
: fact; : fact;
} }
/**
* Version pair behind `claim.liveState`. The executed version has to come from
* the execution record: the workflow head moves when anybody saves, so
* substituting it would let the claim describe a version this run never ran.
* Without that record there is no publish state — an unknown run version must
* not become `live-current`, which reads as "production is proven".
*/
async function resolvePublishState(args: {
workflowService: InstanceAiWorkflowService;
workflowId: string;
executedVersionId: string | null | undefined;
logger: OrchestrationContext['logger'];
}): Promise<VerificationPublishState | undefined> {
const { workflowService, workflowId, executedVersionId, logger } = args;
if (!executedVersionId) return undefined;
try {
const head = await workflowService.getWorkflowHead(workflowId);
return { activeVersionId: head.activeVersionId, draftVersionId: executedVersionId };
} catch (error) {
logger.warn('Failed to read publish state for the verification claim', {
workflowId,
error: error instanceof Error ? error.message : String(error),
});
return undefined;
}
}
export const verifyBuiltWorkflowInputSchema = z.object({ export const verifyBuiltWorkflowInputSchema = z.object({
workItemId: z workItemId: z
.string() .string()

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/mcp-apps", "name": "@n8n/mcp-apps",
"version": "0.18.0", "version": "0.18.1",
"description": "MCP Apps UI resources and server helpers for n8n", "description": "MCP Apps UI resources and server helpers for n8n",
"main": "dist/server/index.js", "main": "dist/server/index.js",
"types": "dist/server/index.d.ts", "types": "dist/server/index.d.ts",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/node-cli", "name": "@n8n/node-cli",
"version": "0.50.0", "version": "0.50.2",
"description": "Official CLI for developing community nodes for n8n", "description": "Official CLI for developing community nodes for n8n",
"bin": { "bin": {
"n8n-node": "bin/n8n-node.mjs" "n8n-node": "bin/n8n-node.mjs"

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/node-engine-compatibility", "name": "@n8n/node-engine-compatibility",
"version": "0.9.0", "version": "0.9.2",
"description": "Adapts v1 nodes and workflows to the Engine 2.0 execution model", "description": "Adapts v1 nodes and workflows to the Engine 2.0 execution model",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/n8n-nodes-langchain", "name": "@n8n/n8n-nodes-langchain",
"version": "2.41.0", "version": "2.41.2",
"description": "", "description": "",
"main": "index.js", "main": "index.js",
"exports": { "exports": {

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/task-runner", "name": "@n8n/task-runner",
"version": "2.41.0", "version": "2.41.2",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"start": "node dist/start.js", "start": "node dist/start.js",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/telemetry", "name": "@n8n/telemetry",
"version": "1.9.0", "version": "1.9.1",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm watch", "dev": "pnpm watch",

View File

@ -1,6 +1,7 @@
import { z } from 'zod/v4'; import { z } from 'zod/v4';
import { defineTelemetryEvents } from '../define'; import { defineTelemetryEvents } from '../define';
import { setupTelemetryProperties } from '../setup-properties';
const descriptionProperties = { const descriptionProperties = {
has_description: z.boolean().describe('Whether the saved credential has a nonempty description.'), has_description: z.boolean().describe('Whether the saved credential has a nonempty description.'),
@ -62,6 +63,7 @@ export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({
description: description:
'The user started a connection attempt or selected an existing credential in the setup panel. Includes retries.', 'The user started a connection attempt or selected an existing credential in the setup panel. Includes retries.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
source: z.literal('instance_ai_setup_panel'), source: z.literal('instance_ai_setup_panel'),
workflow_id: z.string(), workflow_id: z.string(),
thread_id: z.string(), thread_id: z.string(),
@ -74,6 +76,7 @@ export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({
description: description:
'A setup-panel credential was selected or connected and its workflow binding was applied or queued. This does not report workflow validation.', 'A setup-panel credential was selected or connected and its workflow binding was applied or queued. This does not report workflow validation.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
source: z.literal('instance_ai_setup_panel'), source: z.literal('instance_ai_setup_panel'),
workflow_id: z.string(), workflow_id: z.string(),
thread_id: z.string(), thread_id: z.string(),

View File

@ -7,6 +7,7 @@ import {
import { z } from 'zod/v4'; import { z } from 'zod/v4';
import { defineTelemetryEvents } from '../define'; import { defineTelemetryEvents } from '../define';
import { setupItemProperties, setupTelemetryProperties } from '../setup-properties';
/** /**
* How each n8n Assistant setup component is configured. Source (who set it) and * How each n8n Assistant setup component is configured. Source (who set it) and
@ -57,6 +58,7 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
description: description:
'The setup panel observed a new requirement snapshot. Also fires when no setup is needed, including workflows whose credentials were already connected.', 'The setup panel observed a new requirement snapshot. Also fires when no setup is needed, including workflows whose credentials were already connected.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
workflow_id: z.string(), workflow_id: z.string(),
thread_id: z.string(), thread_id: z.string(),
credential_count: z.number(), credential_count: z.number(),
@ -70,11 +72,13 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
description: description:
'A setup checklist row became visible. Reopening the same row within the mounted panel does not emit another event.', 'A setup checklist row became visible. Reopening the same row within the mounted panel does not emit another event.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
workflow_id: z.string(), workflow_id: z.string(),
thread_id: z.string(), thread_id: z.string(),
kind: z.enum(['credential', 'parameters', 'details']), kind: z.enum(['credential', 'parameters', 'details']),
credential_type: z.string().optional(), credential_type: z.string().optional(),
parameter_count: z.number(), parameter_count: z.number(),
items: z.array(z.object(setupItemProperties)).optional(),
}), }),
}, },
SETUP_PANEL_DISMISSED: { SETUP_PANEL_DISMISSED: {
@ -82,6 +86,7 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
description: description:
'A visible setup panel closed. The reason separates navigation and removed requirements from a finished execution or an explicit dismissal.', 'A visible setup panel closed. The reason separates navigation and removed requirements from a finished execution or an explicit dismissal.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
workflow_id: z.string(), workflow_id: z.string(),
thread_id: z.string(), thread_id: z.string(),
reason: z.enum([ reason: z.enum([
@ -421,6 +426,10 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
description: description:
'The user sent a message to the n8n Assistant. Fires once per message on the optimistic send, before the request is admitted, so a refused send still counts as an attempt. Carries who wrote the text: a pre-fill is an opener n8n composed (a failed execution, a credential modal, a template card, a suggestion chip) that the user accepted or edited, so pre-fill share must be read from prefill_type rather than matched against the message body.', 'The user sent a message to the n8n Assistant. Fires once per message on the optimistic send, before the request is admitted, so a refused send still counts as an attempt. Carries who wrote the text: a pre-fill is an opener n8n composed (a failed execution, a credential modal, a template card, a suggestion chip) that the user accepted or edited, so pre-fill share must be read from prefill_type rather than matched against the message body.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
workflow_id: z.string().optional(),
pending_credential_count: z.number().optional(),
pending_parameter_count: z.number().optional(),
thread_id: z.string(), thread_id: z.string(),
instance_id: z.string(), instance_id: z.string(),
is_first_message: z is_first_message: z

View File

@ -1,6 +1,7 @@
import { z } from 'zod/v4'; import { z } from 'zod/v4';
import { defineTelemetryEvents } from '../define'; import { defineTelemetryEvents } from '../define';
import { setupItemProperties, setupTelemetryProperties } from '../setup-properties';
export const WORKFLOW_TELEMETRY = defineTelemetryEvents({ export const WORKFLOW_TELEMETRY = defineTelemetryEvents({
USER_REQUESTED_WORKFLOW_TEST: { USER_REQUESTED_WORKFLOW_TEST: {
@ -8,9 +9,40 @@ export const WORKFLOW_TELEMETRY = defineTelemetryEvents({
description: description:
'The user started a manual workflow test from the AI Assistant setup panel. This event does not report execution success.', 'The user started a manual workflow test from the AI Assistant setup panel. This event does not report execution success.',
properties: z.object({ properties: z.object({
...setupTelemetryProperties,
source: z.literal('instance_ai_setup_panel'), source: z.literal('instance_ai_setup_panel'),
workflow_id: z.string(), workflow_id: z.string(),
thread_id: z.string().optional(), thread_id: z.string().optional(),
test_request_id: z.string().optional(),
}),
},
SETUP_TEST_FINISHED: {
name: 'AI Assistant setup test finished',
description:
'The setup panel observed a terminal test result or a failed run request. A failed request does not prove that execution did not start. Leaving setup before the result can omit this event.',
properties: z.object({
...setupTelemetryProperties,
source: z.literal('instance_ai_setup_panel'),
workflow_id: z.string(),
thread_id: z.string(),
test_request_id: z.string(),
execution_id: z.string().optional(),
status: z.enum(['success', 'error', 'crashed', 'canceled', 'request_failed']),
}),
},
SETUP_SAVED: {
name: 'AI Assistant workflow setup saved',
description:
'A setup surface observed saved requirements after applying changes. Completion covers the observed setup requirements, not credential validity or edits made while setup is closed.',
properties: z.object({
...setupTelemetryProperties,
instance_id: z.string(),
workflow_id: z.string(),
thread_id: z.string(),
source: z.enum(['instance_ai_setup_wizard', 'instance_ai_setup_panel']),
request_id: z.string().optional(),
items: z.array(z.object({ ...setupItemProperties, completed: z.boolean() })),
setup_complete: z.boolean(),
}), }),
}, },
MULTIPLE_NODES_SELECTED: { MULTIPLE_NODES_SELECTED: {

View File

@ -0,0 +1,18 @@
import { z } from 'zod/v4';
const setupVariant = z.enum(['control', 'variant']).optional();
export const setupTelemetryProperties = {
session_id: z.string().optional(),
variant: setupVariant,
'$feature/118_instance_ai_setup_overhaul': setupVariant,
};
// A requirement is identified by its node, kind, and credential type or parameter name.
export const setupItemProperties = {
node_id: z.string(),
node_type: z.string().optional(),
kind: z.enum(['credential', 'parameter']),
credential_type: z.string().optional(),
parameter_name: z.string().optional(),
};

View File

@ -1,6 +1,6 @@
{ {
"name": "n8n", "name": "n8n",
"version": "2.41.0", "version": "2.41.3",
"description": "n8n Workflow Automation Tool", "description": "n8n Workflow Automation Tool",
"main": "dist/index", "main": "dist/index",
"types": "dist/index.d.ts", "types": "dist/index.d.ts",
@ -189,7 +189,6 @@
"@opentelemetry/exporter-trace-otlp-grpc": "^0.221.0", "@opentelemetry/exporter-trace-otlp-grpc": "^0.221.0",
"@opentelemetry/exporter-trace-otlp-proto": "^0.221.0", "@opentelemetry/exporter-trace-otlp-proto": "^0.221.0",
"@opentelemetry/resources": "^2.10.0", "@opentelemetry/resources": "^2.10.0",
"@opentelemetry/sdk-node": "^0.221.0",
"@opentelemetry/sdk-trace-base": "^2.10.0", "@opentelemetry/sdk-trace-base": "^2.10.0",
"@opentelemetry/sdk-trace-node": "^2.10.0", "@opentelemetry/sdk-trace-node": "^2.10.0",
"@opentelemetry/semantic-conventions": "^1.43.0", "@opentelemetry/semantic-conventions": "^1.43.0",

View File

@ -28,6 +28,7 @@ import { License } from '@/license';
import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config'; import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config';
import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service'; import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service';
import { OtelService } from '@/modules/otel/otel.service';
import { NodeTypes } from '@/node-types'; import { NodeTypes } from '@/node-types';
import { PostHogClient } from '@/posthog'; import { PostHogClient } from '@/posthog';
import { PollJobProvider } from '@/scheduling/poll-trigger-node/poll-job-provider'; import { PollJobProvider } from '@/scheduling/poll-trigger-node/poll-job-provider';
@ -72,6 +73,7 @@ mockInstance(NodeTypes);
const shutdownService = mockInstance(ShutdownService); const shutdownService = mockInstance(ShutdownService);
shutdownService.validate.mockReturnValue(undefined); shutdownService.validate.mockReturnValue(undefined);
mockInstance(PostHogClient); mockInstance(PostHogClient);
mockInstance(OtelService);
mockInstance(TelemetryEventRelay); mockInstance(TelemetryEventRelay);
mockInstance(ActivityEventRelay); mockInstance(ActivityEventRelay);
mockInstance(WorkflowFailureNotificationEventRelay); mockInstance(WorkflowFailureNotificationEventRelay);
@ -126,6 +128,7 @@ describe('Start - AuthRolesService initialization', () => {
Container.set(MultiMainSetup, multiMainSetup); Container.set(MultiMainSetup, multiMainSetup);
Container.set(AuthHandlerRegistry, authHandlerRegistry); Container.set(AuthHandlerRegistry, authHandlerRegistry);
Container.set(PostHogClient, mockInstance(PostHogClient)); Container.set(PostHogClient, mockInstance(PostHogClient));
Container.set(OtelService, mockInstance(OtelService));
Container.set(TelemetryEventRelay, mockInstance(TelemetryEventRelay)); Container.set(TelemetryEventRelay, mockInstance(TelemetryEventRelay));
Container.set(ActivityEventRelay, mockInstance(ActivityEventRelay)); Container.set(ActivityEventRelay, mockInstance(ActivityEventRelay));
Container.set( Container.set(

View File

@ -24,6 +24,7 @@ import { WorkflowFailureNotificationEventRelay } from '@/events/relays/workflow-
import type { ExecutionPersistence } from '@/executions/execution-persistence'; import type { ExecutionPersistence } from '@/executions/execution-persistence';
import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials'; import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config'; import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config';
import { OtelService } from '@/modules/otel/otel.service';
import { NodeTypes } from '@/node-types'; import { NodeTypes } from '@/node-types';
import { PostHogClient } from '@/posthog'; import { PostHogClient } from '@/posthog';
import { PubSubRegistry } from '@/scaling/pubsub/pubsub.registry'; import { PubSubRegistry } from '@/scaling/pubsub/pubsub.registry';
@ -61,6 +62,7 @@ mockInstance(NodeTypes);
mockInstance(ShutdownService); mockInstance(ShutdownService);
mockInstance(MessageEventBus); mockInstance(MessageEventBus);
mockInstance(PostHogClient); mockInstance(PostHogClient);
mockInstance(OtelService);
mockInstance(TelemetryEventRelay); mockInstance(TelemetryEventRelay);
mockInstance(ActivityEventRelay); mockInstance(ActivityEventRelay);
mockInstance(WorkflowFailureNotificationEventRelay); mockInstance(WorkflowFailureNotificationEventRelay);

View File

@ -4,13 +4,22 @@
import type { Logger } from '@n8n/backend-common'; import type { Logger } from '@n8n/backend-common';
import { ExpressionEngineConfig, type GlobalConfig } from '@n8n/config'; import { ExpressionEngineConfig, type GlobalConfig } from '@n8n/config';
import { EXPRESSION_METRICS } from '@n8n/expression-runtime'; import { EXPRESSION_METRICS } from '@n8n/expression-runtime';
import { trace } from '@opentelemetry/api'; import type { Tracer } from '@opentelemetry/api';
import promClient from 'prom-client'; import promClient from 'prom-client';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import { ExpressionObservabilityProvider } from '../expression-observability.provider'; import { ExpressionObservabilityProvider } from '../expression-observability.provider';
import type { OtelService } from '@/modules/otel/otel.service';
const scopedLogger = mock<Logger>(); const scopedLogger = mock<Logger>();
const otelService = mock<OtelService>();
const startSpanMock = vi.fn().mockReturnValue({
setStatus: vi.fn(),
setAttribute: vi.fn(),
recordException: vi.fn(),
end: vi.fn(),
});
function buildConfig(overrides: Partial<ExpressionEngineConfig> = {}): ExpressionEngineConfig { function buildConfig(overrides: Partial<ExpressionEngineConfig> = {}): ExpressionEngineConfig {
const config = new ExpressionEngineConfig(); const config = new ExpressionEngineConfig();
@ -32,6 +41,7 @@ function buildGlobalConfig(prefix = 'n8n_'): GlobalConfig {
describe('ExpressionObservabilityProvider', () => { describe('ExpressionObservabilityProvider', () => {
beforeEach(() => { beforeEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
otelService.getTracer.mockReturnValue({ startSpan: startSpanMock } as unknown as Tracer);
promClient.register.clear(); promClient.register.clear();
}); });
@ -41,6 +51,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig({ observabilityEnabled: false }), buildConfig({ observabilityEnabled: false }),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
expect(() => { expect(() => {
@ -57,6 +68,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig({ engine: 'legacy' }), buildConfig({ engine: 'legacy' }),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.counter(EXPRESSION_METRICS.poolAcquired.name, 1); provider.metrics.counter(EXPRESSION_METRICS.poolAcquired.name, 1);
@ -72,6 +84,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.counter(EXPRESSION_METRICS.poolAcquired.name, 2); provider.metrics.counter(EXPRESSION_METRICS.poolAcquired.name, 2);
@ -86,6 +99,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.gauge(EXPRESSION_METRICS.codeCacheSize.name, 42); provider.metrics.gauge(EXPRESSION_METRICS.codeCacheSize.name, 42);
@ -98,6 +112,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 3, { provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 3, {
status: 'success', status: 'success',
@ -116,6 +131,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
promClient.register.clear(); promClient.register.clear();
@ -140,6 +156,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
promClient.register.clear(); promClient.register.clear();
@ -153,25 +170,12 @@ describe('ExpressionObservabilityProvider', () => {
}); });
describe('tail sampling', () => { describe('tail sampling', () => {
const startSpanMock = vi.fn().mockReturnValue({
setStatus: vi.fn(),
setAttribute: vi.fn(),
recordException: vi.fn(),
end: vi.fn(),
});
beforeEach(() => {
startSpanMock.mockClear();
vi.spyOn(trace, 'getTracer').mockReturnValue({
startSpan: startSpanMock,
} as unknown as ReturnType<typeof trace.getTracer>);
});
it('drops healthy spans under the slow threshold', () => { it('drops healthy spans under the slow threshold', () => {
const provider = new ExpressionObservabilityProvider( const provider = new ExpressionObservabilityProvider(
buildConfig({ slowEvaluationThresholdMs: 50, tracesSampleRate: 0 }), buildConfig({ slowEvaluationThresholdMs: 50, tracesSampleRate: 0 }),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.01, { provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.01, {
status: 'success', status: 'success',
@ -185,6 +189,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig({ slowEvaluationThresholdMs: 50 }), buildConfig({ slowEvaluationThresholdMs: 50 }),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.1, { provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.1, {
status: 'success', status: 'success',
@ -206,6 +211,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig({ slowEvaluationThresholdMs: 50 }), buildConfig({ slowEvaluationThresholdMs: 50 }),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.005, { provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.005, {
status: 'error', status: 'error',
@ -227,6 +233,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig({ tracesEnabled: false, slowEvaluationThresholdMs: 10 }), buildConfig({ tracesEnabled: false, slowEvaluationThresholdMs: 10 }),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.5, { provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.5, {
status: 'error', status: 'error',
@ -242,6 +249,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
expect(() => { expect(() => {
provider.metrics.counter('test.unknown', 1); provider.metrics.counter('test.unknown', 1);
@ -256,6 +264,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.01, { provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.01, {
@ -285,6 +294,7 @@ describe('ExpressionObservabilityProvider', () => {
buildConfig(), buildConfig(),
buildLogger(), buildLogger(),
buildGlobalConfig(), buildGlobalConfig(),
otelService,
); );
provider.logs.info('hello', { k: 'v' }); provider.logs.info('hello', { k: 'v' });
provider.logs.warn('warn', { k: 'v' }); provider.logs.warn('warn', { k: 'v' });

View File

@ -10,8 +10,7 @@ import type {
TracesAPI, TracesAPI,
} from '@n8n/expression-runtime'; } from '@n8n/expression-runtime';
import { EXPRESSION_METRICS, NoOpProvider } from '@n8n/expression-runtime'; import { EXPRESSION_METRICS, NoOpProvider } from '@n8n/expression-runtime';
import { SpanStatusCode, trace } from '@opentelemetry/api'; import { SpanStatusCode } from '@opentelemetry/api';
import type { Tracer } from '@opentelemetry/api';
import { UnexpectedError } from 'n8n-workflow'; import { UnexpectedError } from 'n8n-workflow';
import promClient, { type Counter, type Gauge, type Histogram } from 'prom-client'; import promClient, { type Counter, type Gauge, type Histogram } from 'prom-client';
@ -26,6 +25,8 @@ import {
toPromName, toPromName,
} from './expression-observability.formatters'; } from './expression-observability.formatters';
import { OtelService } from '@/modules/otel/otel.service';
type TailSampleDecision = 'drop' | 'keep'; type TailSampleDecision = 'drop' | 'keep';
@Service() @Service()
@ -40,8 +41,6 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
private readonly prefix!: string; private readonly prefix!: string;
private tracer?: Tracer;
private readonly metricDefs = new Map<string, MetricDef>( private readonly metricDefs = new Map<string, MetricDef>(
(Object.values(EXPRESSION_METRICS) as MetricDef[]).map((def) => [def.name, def]), (Object.values(EXPRESSION_METRICS) as MetricDef[]).map((def) => [def.name, def]),
); );
@ -50,6 +49,7 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
private readonly config: ExpressionEngineConfig, private readonly config: ExpressionEngineConfig,
private readonly logger: Logger, private readonly logger: Logger,
globalConfig: GlobalConfig, globalConfig: GlobalConfig,
private readonly otelService: OtelService,
) { ) {
this.scopedLogger = this.logger.scoped('expression-engine'); this.scopedLogger = this.logger.scoped('expression-engine');
@ -167,7 +167,7 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
? 'slow' ? 'slow'
: 'healthy'; : 'healthy';
const tracer = this.getTracer(); const tracer = this.otelService.getTracer(TRACER_NAME);
const errorType = tags?.type && tags.type !== 'none' ? tags.type : undefined; const errorType = tags?.type && tags.type !== 'none' ? tags.type : undefined;
const span = tracer.startSpan('expression.evaluate', { const span = tracer.startSpan('expression.evaluate', {
attributes: { attributes: {
@ -194,7 +194,7 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
private startSpan(name: string, attributes?: Record<string, unknown>): Span { private startSpan(name: string, attributes?: Record<string, unknown>): Span {
if (!this.config.tracesEnabled) return NoOpProvider.traces.startSpan(name, attributes); if (!this.config.tracesEnabled) return NoOpProvider.traces.startSpan(name, attributes);
const tracer = this.getTracer(); const tracer = this.otelService.getTracer(TRACER_NAME);
const otelSpan = tracer.startSpan(name, { const otelSpan = tracer.startSpan(name, {
attributes: normalizeAttributes(attributes), attributes: normalizeAttributes(attributes),
}); });
@ -212,9 +212,4 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
end: () => otelSpan.end(), end: () => otelSpan.end(),
}; };
} }
private getTracer(): Tracer {
this.tracer ??= trace.getTracer(TRACER_NAME);
return this.tracer;
}
} }

View File

@ -1,19 +1,22 @@
import type { Logger } from '@n8n/backend-common'; import type { Logger } from '@n8n/backend-common';
import type { OutboundHttp } from '@n8n/backend-network';
import type { AgentsConfig } from '@n8n/config'; import type { AgentsConfig } from '@n8n/config';
import { context, type Tracer } from '@opentelemetry/api'; import { context, type Tracer } from '@opentelemetry/api';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import type { InstanceSettings } from 'n8n-core';
import { OtelTestProvider } from '@/modules/otel/__tests__/support/otel-test-provider'; import { OtelTestProvider } from '@/modules/otel/__tests__/support/otel-test-provider';
import { ExecutionLevelTracer } from '@/modules/otel/execution-level-tracer'; import { ExecutionLevelTracer } from '@/modules/otel/execution-level-tracer';
import type { OtelSettingsService } from '@/modules/otel/otel-settings.service'; import type { OtelSettingsService } from '@/modules/otel/otel-settings.service';
import { OtelService } from '@/modules/otel/otel.service';
import { AgentRunTracingService } from '../agent-run-tracing.service'; import { AgentRunTracingService } from '../agent-run-tracing.service';
// Deliberately does not mock '@opentelemetry/api': agent-run-tracing.service.test.ts // Deliberately uses a real tracer provider: agent-run-tracing.service.test.ts
// already covers the built-metadata/branching logic against a stubbed tracer. This // already covers the built-metadata/branching logic against a stubbed tracer. This
// file proves the real OTel path end-to-end — that a registered provider actually // file proves the real OTel path end-to-end — that the module's provider actually
// records the span the service's tracer produces, and that a disabled OTel module // records the span the service's tracer produces, and that a disabled OTel module
// (no provider registered) or disabled agent tracing both emit nothing. // (no provider started) or disabled agent tracing both emit nothing.
describe('AgentRunTracingService (real OTel provider)', () => { describe('AgentRunTracingService (real OTel provider)', () => {
const baseMetadata = { const baseMetadata = {
agentId: 'agent-1', agentId: 'agent-1',
@ -26,7 +29,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
const otel = OtelTestProvider.create(); const otel = OtelTestProvider.create();
try { try {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otel.asOtelService());
const built = await service.build(baseMetadata); const built = await service.build(baseMetadata);
expect(built).toBeDefined(); expect(built).toBeDefined();
@ -57,6 +60,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
const otel = OtelTestProvider.create({ withContextManager: true }); const otel = OtelTestProvider.create({ withContextManager: true });
try { try {
const executionLevelTracer = new ExecutionLevelTracer( const executionLevelTracer = new ExecutionLevelTracer(
otel.asOtelService(),
mock<OtelSettingsService>({ getSettings: () => ({ injectOutbound: false }) as never }), mock<OtelSettingsService>({ getSettings: () => ({ injectOutbound: false }) as never }),
mock<Logger>(), mock<Logger>(),
); );
@ -70,7 +74,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
executionLevelTracer.startNode({ executionId, node }); executionLevelTracer.startNode({ executionId, node });
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const agentRunTracingService = new AgentRunTracingService(agentsConfig); const agentRunTracingService = new AgentRunTracingService(agentsConfig, otel.asOtelService());
const parentCtx = executionLevelTracer.getActiveContext(executionId, node.name); const parentCtx = executionLevelTracer.getActiveContext(executionId, node.name);
expect(parentCtx).toBeDefined(); expect(parentCtx).toBeDefined();
@ -118,15 +122,21 @@ describe('AgentRunTracingService (real OTel provider)', () => {
} }
}); });
it('emits no spans when the OTel module has no provider registered', async () => { it('emits no spans when the OTel module has not started a provider', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const otelService = new OtelService(
mock<OtelSettingsService>(),
mock<InstanceSettings>(),
mock<Logger>(),
mock<OutboundHttp>(),
);
const service = new AgentRunTracingService(agentsConfig, otelService);
const built = await service.build(baseMetadata); const built = await service.build(baseMetadata);
expect(built).toBeDefined(); expect(built).toBeDefined();
// No provider registered — the global tracer falls back to OTel's // No provider started — the module hands out OTel's no-op tracer, so
// no-op implementation, so starting a span records nothing anywhere. // starting a span records nothing anywhere.
(built?.tracer as Tracer).startActiveSpan('test-span', (span) => { (built?.tracer as Tracer).startActiveSpan('test-span', (span) => {
expect(span.isRecording()).toBe(false); expect(span.isRecording()).toBe(false);
span.end(); span.end();
@ -137,7 +147,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
const otel = OtelTestProvider.create(); const otel = OtelTestProvider.create();
try { try {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: false }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: false });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otel.asOtelService());
const built = await service.build(baseMetadata); const built = await service.build(baseMetadata);
expect(built).toBeUndefined(); expect(built).toBeUndefined();

View File

@ -1,16 +1,17 @@
import type { AgentsConfig } from '@n8n/config'; import type { AgentsConfig } from '@n8n/config';
import { trace } from '@opentelemetry/api'; import type { Tracer } from '@opentelemetry/api';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import type { OtelService } from '@/modules/otel/otel.service';
import { AgentRunTracingService, modelIdFromSnapshot } from '../agent-run-tracing.service'; import { AgentRunTracingService, modelIdFromSnapshot } from '../agent-run-tracing.service';
vi.mock('@opentelemetry/api', () => ({ const otelService = mock<OtelService>();
trace: { getTracer: vi.fn(() => ({ startSpan: vi.fn(), startActiveSpan: vi.fn() })) },
}));
describe('AgentRunTracingService', () => { describe('AgentRunTracingService', () => {
beforeEach(() => { beforeEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
otelService.getTracer.mockReturnValue(mock<Tracer>());
}); });
const baseMetadata = { const baseMetadata = {
@ -21,38 +22,39 @@ describe('AgentRunTracingService', () => {
}; };
it('exposes whether tracing is enabled via the `enabled` getter', () => { it('exposes whether tracing is enabled via the `enabled` getter', () => {
expect(new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: true })).enabled).toBe( expect(
true, new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: true }), otelService).enabled,
); ).toBe(true);
expect(new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: false })).enabled).toBe( expect(
false, new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: false }), otelService)
); .enabled,
).toBe(false);
}); });
it('returns undefined when agent tracing is disabled, without touching the tracer', async () => { it('returns undefined when agent tracing is disabled, without touching the tracer', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: false }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: false });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
const built = await service.build(baseMetadata); const built = await service.build(baseMetadata);
expect(built).toBeUndefined(); expect(built).toBeUndefined();
expect(trace.getTracer).not.toHaveBeenCalled(); expect(otelService.getTracer).not.toHaveBeenCalled();
}); });
it('builds telemetry from a fresh tracer fetch when tracing is enabled', async () => { it('builds telemetry from a fresh tracer fetch when tracing is enabled', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
await service.build(baseMetadata); await service.build(baseMetadata);
await service.build(baseMetadata); await service.build(baseMetadata);
expect(trace.getTracer).toHaveBeenCalledTimes(2); expect(otelService.getTracer).toHaveBeenCalledTimes(2);
expect(trace.getTracer).toHaveBeenCalledWith('@n8n/agents'); expect(otelService.getTracer).toHaveBeenCalledWith('@n8n/agents');
}); });
it('never marks the built telemetry as LangSmith, so root spans stay generic', async () => { it('never marks the built telemetry as LangSmith, so root spans stay generic', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
const built = await service.build(baseMetadata); const built = await service.build(baseMetadata);
@ -62,7 +64,7 @@ describe('AgentRunTracingService', () => {
it('includes execution_id/workflow_id/node_id only when source is "workflow"', async () => { it('includes execution_id/workflow_id/node_id only when source is "workflow"', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
const workflowBuilt = await service.build({ const workflowBuilt = await service.build({
...baseMetadata, ...baseMetadata,
@ -95,7 +97,7 @@ describe('AgentRunTracingService', () => {
it('omits execution_id/workflow_id/node_id individually when not provided, even for a workflow source', async () => { it('omits execution_id/workflow_id/node_id individually when not provided, even for a workflow source', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
const noneProvided = await service.build({ const noneProvided = await service.build({
...baseMetadata, ...baseMetadata,
@ -150,7 +152,7 @@ describe('AgentRunTracingService', () => {
it('rejects workflow-only fields at compile time for a non-workflow source', async () => { it('rejects workflow-only fields at compile time for a non-workflow source', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
// @ts-expect-error executionId only exists on the 'workflow' branch of // @ts-expect-error executionId only exists on the 'workflow' branch of
// the AgentRunTracingMetadata union — a non-workflow source can't carry // the AgentRunTracingMetadata union — a non-workflow source can't carry
@ -160,7 +162,7 @@ describe('AgentRunTracingService', () => {
it('marks workflow-sourced runs as rootAnchored: false only when a parent context was found, and leaves other sources anchored', async () => { it('marks workflow-sourced runs as rootAnchored: false only when a parent context was found, and leaves other sources anchored', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
const workflowBuilt = await service.build({ const workflowBuilt = await service.build({
...baseMetadata, ...baseMetadata,
@ -184,7 +186,7 @@ describe('AgentRunTracingService', () => {
it('includes user_id and model_id only when provided', async () => { it('includes user_id and model_id only when provided', async () => {
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true }); const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
const service = new AgentRunTracingService(agentsConfig); const service = new AgentRunTracingService(agentsConfig, otelService);
const built = await service.build({ const built = await service.build({
...baseMetadata, ...baseMetadata,

View File

@ -2,7 +2,8 @@ import type { AttributeValue, BuiltTelemetry } from '@n8n/agents';
import { Telemetry } from '@n8n/agents'; import { Telemetry } from '@n8n/agents';
import { AgentsConfig } from '@n8n/config'; import { AgentsConfig } from '@n8n/config';
import { Service } from '@n8n/di'; import { Service } from '@n8n/di';
import { trace } from '@opentelemetry/api';
import { OtelService } from '@/modules/otel/otel.service';
const AGENTS_TRACER_NAME = '@n8n/agents'; const AGENTS_TRACER_NAME = '@n8n/agents';
@ -49,13 +50,16 @@ function isWorkflowTracingMetadata(
} }
/** /**
* Builds the per-run OTel telemetry handed to an agent run, riding along with * Builds the per-run OTel telemetry handed to an agent run on the workflow OTel
* the workflow OTel module's already-registered global tracer provider — this * module's tracer provider — this service never creates, registers, or shuts
* service never creates, registers, or shuts down a provider itself. * down a provider itself.
*/ */
@Service() @Service()
export class AgentRunTracingService { export class AgentRunTracingService {
constructor(private readonly agentsConfig: AgentsConfig) {} constructor(
private readonly agentsConfig: AgentsConfig,
private readonly otelService: OtelService,
) {}
/** Whether agent tracing is enabled — lets callers skip tracing-only work (e.g. a DB lookup) upfront. */ /** Whether agent tracing is enabled — lets callers skip tracing-only work (e.g. a DB lookup) upfront. */
get enabled(): boolean { get enabled(): boolean {
@ -67,9 +71,8 @@ export class AgentRunTracingService {
* `ExecutionOptions.telemetry` entirely in that case, indistinguishable * `ExecutionOptions.telemetry` entirely in that case, indistinguishable
* from "OTel module off" to the SDK (both are no-ops). * from "OTel module off" to the SDK (both are no-ops).
* *
* Fetches the tracer fresh on every call rather than caching it: a tracer * Fetches the tracer fresh on every call rather than caching it, so a
* obtained before any provider is registered would otherwise go stale * tracer never goes stale across an OTel module restart.
* across an OTel module restart. Fetching fresh sidesteps that entirely.
*/ */
async build(metadata: AgentRunTracingMetadata): Promise<BuiltTelemetry | undefined> { async build(metadata: AgentRunTracingMetadata): Promise<BuiltTelemetry | undefined> {
if (!this.agentsConfig.tracingEnabled) return undefined; if (!this.agentsConfig.tracingEnabled) return undefined;
@ -91,7 +94,7 @@ export class AgentRunTracingService {
}; };
const built = await new Telemetry() const built = await new Telemetry()
.tracer(trace.getTracer(AGENTS_TRACER_NAME)) .tracer(this.otelService.getTracer(AGENTS_TRACER_NAME))
.metadata(attributes) .metadata(attributes)
.recordInputs(this.agentsConfig.tracingRecordInputs) .recordInputs(this.agentsConfig.tracingRecordInputs)
.recordOutputs(this.agentsConfig.tracingRecordOutputs) .recordOutputs(this.agentsConfig.tracingRecordOutputs)

View File

@ -46,6 +46,7 @@ function createService() {
mcpConnectionsEnabled: false, mcpConnectionsEnabled: false,
conversationHistoryEnabled: false, conversationHistoryEnabled: false,
progressiveBuildingEnabled: false, progressiveBuildingEnabled: false,
setupPanelEnabled: false,
nodeUsageEnabled: false, nodeUsageEnabled: false,
folderExplorationEnabled: false, folderExplorationEnabled: false,
aiPreferencesEnabled: false, aiPreferencesEnabled: false,

View File

@ -72,6 +72,8 @@ import {
INSTANCE_ACTIVITY_CONTEXT_FLAG, INSTANCE_ACTIVITY_CONTEXT_FLAG,
INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
INSTANCE_AI_SETUP_PANEL_FLAG,
INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
CONFIG_EVALUATIONS_ENABLED_VARIANT, CONFIG_EVALUATIONS_ENABLED_VARIANT,
INSTANCE_AI_MCP_CONNECTIONS_FLAG, INSTANCE_AI_MCP_CONNECTIONS_FLAG,
@ -2287,6 +2289,7 @@ describe('createDataTableAdapter', () => {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function createWorkflowAdapterForTests(overrides?: { function createWorkflowAdapterForTests(overrides?: {
setupPanelVariant?: 'control' | 'variant';
namedVersionsLicensed?: boolean; namedVersionsLicensed?: boolean;
foldersLicensed?: boolean; foldersLicensed?: boolean;
branchReadOnly?: boolean; branchReadOnly?: boolean;
@ -2465,6 +2468,7 @@ function createWorkflowAdapterForTests(overrides?: {
threadId: 'thread-1', threadId: 'thread-1',
projectId: boundProjectId, projectId: boundProjectId,
folderExplorationEnabled: overrides?.folderExploration ?? false, folderExplorationEnabled: overrides?.folderExploration ?? false,
setupPanelVariant: overrides?.setupPanelVariant,
}); });
const adapter = context.workflowService; const adapter = context.workflowService;
@ -3584,6 +3588,38 @@ describe('createWorkflowAdapter', () => {
}); });
}); });
it.each(['control', 'variant', undefined] as const)(
'carries setup assignment %s on workflow creation, updates, and publishing',
async (setupPanelVariant) => {
const { adapter, mockTelemetry } = createWorkflowAdapterForTests({ setupPanelVariant });
await adapter.createFromWorkflowJSON(minimalWorkflowJSON);
await adapter.updateFromWorkflowJSON('wf-new', minimalWorkflowJSON);
await adapter.publish('wf-new');
for (const event of [
'Builder created workflow',
'Builder modified workflow',
'Builder published workflow',
]) {
const properties = mockTelemetry.track.mock.calls.find(([name]) => name === event)?.[1];
expect(properties).toMatchObject({
user_id: 'user-1',
thread_id: 'thread-1',
workflow_id: 'wf-new',
});
if (setupPanelVariant) {
expect(properties).toMatchObject({
variant: setupPanelVariant,
[`$feature/${INSTANCE_AI_SETUP_PANEL_FLAG}`]: setupPanelVariant,
});
} else {
expect(properties).not.toHaveProperty('variant');
expect(properties).not.toHaveProperty(`$feature/${INSTANCE_AI_SETUP_PANEL_FLAG}`);
}
}
},
);
it('marks the workflow as AI-builder temporary when markAsAiTemporary is true', async () => { it('marks the workflow as AI-builder temporary when markAsAiTemporary is true', async () => {
const { const {
adapter, adapter,
@ -6040,6 +6076,7 @@ describe('resolveExperimentGates', () => {
[INSTANCE_AI_MCP_CONNECTIONS_FLAG]: INSTANCE_AI_MCP_CONNECTIONS_ENABLED_VARIANT, [INSTANCE_AI_MCP_CONNECTIONS_FLAG]: INSTANCE_AI_MCP_CONNECTIONS_ENABLED_VARIANT,
[INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, [INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, [INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
[INSTANCE_AI_SETUP_PANEL_FLAG]: INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
[INSTANCE_AI_NODE_USAGE_FLAG]: true, [INSTANCE_AI_NODE_USAGE_FLAG]: true,
[INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: INSTANCE_AI_FOLDER_EXPLORATION_ENABLED_VARIANT, [INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: INSTANCE_AI_FOLDER_EXPLORATION_ENABLED_VARIANT,
[CONTEXT_PREFERENCES_FLAG]: CONTEXT_PREFERENCES_ENABLED_VARIANT, [CONTEXT_PREFERENCES_FLAG]: CONTEXT_PREFERENCES_ENABLED_VARIANT,
@ -6053,6 +6090,8 @@ describe('resolveExperimentGates', () => {
mcpConnectionsEnabled: true, mcpConnectionsEnabled: true,
conversationHistoryEnabled: true, conversationHistoryEnabled: true,
progressiveBuildingEnabled: true, progressiveBuildingEnabled: true,
setupPanelEnabled: true,
setupPanelVariant: 'variant',
nodeUsageEnabled: true, nodeUsageEnabled: true,
folderExplorationEnabled: true, folderExplorationEnabled: true,
aiPreferencesEnabled: true, aiPreferencesEnabled: true,
@ -6119,6 +6158,7 @@ describe('resolveExperimentGates', () => {
[INSTANCE_AI_MCP_CONNECTIONS_FLAG]: 'control', [INSTANCE_AI_MCP_CONNECTIONS_FLAG]: 'control',
[INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: 'control', [INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: 'control',
[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: 'control', [INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: 'control',
[INSTANCE_AI_SETUP_PANEL_FLAG]: 'control',
[INSTANCE_AI_NODE_USAGE_FLAG]: false, [INSTANCE_AI_NODE_USAGE_FLAG]: false,
[INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: 'control', [INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: 'control',
[CONTEXT_PREFERENCES_FLAG]: CONTEXT_PREFERENCES_CONTROL_VARIANT, [CONTEXT_PREFERENCES_FLAG]: CONTEXT_PREFERENCES_CONTROL_VARIANT,
@ -6129,6 +6169,8 @@ describe('resolveExperimentGates', () => {
mcpConnectionsEnabled: false, mcpConnectionsEnabled: false,
conversationHistoryEnabled: false, conversationHistoryEnabled: false,
progressiveBuildingEnabled: false, progressiveBuildingEnabled: false,
setupPanelEnabled: false,
setupPanelVariant: 'control',
nodeUsageEnabled: false, nodeUsageEnabled: false,
folderExplorationEnabled: false, folderExplorationEnabled: false,
aiPreferencesEnabled: false, aiPreferencesEnabled: false,
@ -6136,6 +6178,18 @@ describe('resolveExperimentGates', () => {
}); });
}); });
it.each([true, 'unexpected'])(
'keeps setup disabled for invalid assignment %s',
async (assignment) => {
stubContainer({ [INSTANCE_AI_SETUP_PANEL_FLAG]: assignment });
const gates = await createAdapter().resolveExperimentGates(user);
expect(gates.setupPanelEnabled).toBe(false);
expect(gates).not.toHaveProperty('setupPanelVariant');
},
);
// Regression guard for the shipped bug: the flag is multivariate, so a // Regression guard for the shipped bug: the flag is multivariate, so a
// boolean `true` is not a value PostHog can return for it. Reading it as one // boolean `true` is not a value PostHog can return for it. Reading it as one
// left the gate shut at every rollout percentage. // left the gate shut at every rollout percentage.
@ -6165,6 +6219,7 @@ describe('resolveExperimentGates', () => {
mcpConnectionsEnabled: false, mcpConnectionsEnabled: false,
conversationHistoryEnabled: false, conversationHistoryEnabled: false,
progressiveBuildingEnabled: false, progressiveBuildingEnabled: false,
setupPanelEnabled: false,
nodeUsageEnabled: false, nodeUsageEnabled: false,
folderExplorationEnabled: false, folderExplorationEnabled: false,
aiPreferencesEnabled: false, aiPreferencesEnabled: false,
@ -6181,6 +6236,7 @@ describe('resolveExperimentGates', () => {
mcpConnectionsEnabled: false, mcpConnectionsEnabled: false,
conversationHistoryEnabled: false, conversationHistoryEnabled: false,
progressiveBuildingEnabled: false, progressiveBuildingEnabled: false,
setupPanelEnabled: false,
nodeUsageEnabled: false, nodeUsageEnabled: false,
folderExplorationEnabled: false, folderExplorationEnabled: false,
aiPreferencesEnabled: false, aiPreferencesEnabled: false,

View File

@ -729,7 +729,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
getSandboxStatus: Mock; getSandboxStatus: Mock;
isLocalGatewayDisabledForUser: Mock; isLocalGatewayDisabledForUser: Mock;
getPermissions: Mock; getPermissions: Mock;
isInstanceAiSetupPanelEnabled: Mock;
}; };
gatewayService: { findGateway: Mock; applyToolPolicy: Mock }; gatewayService: { findGateway: Mock; applyToolPolicy: Mock };
aiService: { isProxyEnabled: Mock }; aiService: { isProxyEnabled: Mock };
@ -762,6 +761,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
getPromptConfiguration: Mock; getPromptConfiguration: Mock;
setPromptConfiguration: Mock; setPromptConfiguration: Mock;
setBuildMode: Mock; setBuildMode: Mock;
setSetupPanelEnabled: Mock;
setPromptVersion: Mock; setPromptVersion: Mock;
getComputerUseChannels: Mock; getComputerUseChannels: Mock;
}; };
@ -787,7 +787,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
})), })),
isLocalGatewayDisabledForUser: vi.fn(async () => false), isLocalGatewayDisabledForUser: vi.fn(async () => false),
getPermissions: vi.fn(() => ({})), getPermissions: vi.fn(() => ({})),
isInstanceAiSetupPanelEnabled: vi.fn(() => snapshotMode !== 'off'),
}; };
service.gatewayService = { findGateway: vi.fn(() => undefined), applyToolPolicy: vi.fn() }; service.gatewayService = { findGateway: vi.fn(() => undefined), applyToolPolicy: vi.fn() };
service.aiService = { isProxyEnabled: vi.fn(() => false) }; service.aiService = { isProxyEnabled: vi.fn(() => false) };
@ -795,6 +794,8 @@ describe('InstanceAiService — runtime workspace setup', () => {
createContext: vi.fn(() => ({})), createContext: vi.fn(() => ({})),
getNodeDefinitionDirs: vi.fn(() => []), getNodeDefinitionDirs: vi.fn(() => []),
resolveExperimentGates: vi.fn().mockResolvedValue({ resolveExperimentGates: vi.fn().mockResolvedValue({
setupPanelEnabled: snapshotMode !== 'off',
setupPanelVariant: snapshotMode === 'off' ? 'control' : 'variant',
configEvalsEnabled: true, configEvalsEnabled: true,
mcpConnectionsEnabled: false, mcpConnectionsEnabled: false,
conversationHistoryEnabled: false, conversationHistoryEnabled: false,
@ -835,6 +836,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
getPromptConfiguration: vi.fn(), getPromptConfiguration: vi.fn(),
setPromptConfiguration: vi.fn(), setPromptConfiguration: vi.fn(),
setBuildMode: vi.fn(), setBuildMode: vi.fn(),
setSetupPanelEnabled: vi.fn(),
setPromptVersion: vi.fn(), setPromptVersion: vi.fn(),
getComputerUseChannels: vi.fn(() => undefined), getComputerUseChannels: vi.fn(() => undefined),
}; };
@ -884,9 +886,17 @@ describe('InstanceAiService — runtime workspace setup', () => {
expect(environment.instanceContextEnabled).toBe(instanceContextEnabled); expect(environment.instanceContextEnabled).toBe(instanceContextEnabled);
expect(service.adapterService.createContext).toHaveBeenCalledWith( expect(service.adapterService.createContext).toHaveBeenCalledWith(
fakeUser, fakeUser,
expect.objectContaining({ instanceContextEnabled }), expect.objectContaining({
instanceContextEnabled,
configEvalsEnabled: true,
setupPanelVariant: snapshotMode === 'off' ? 'control' : 'variant',
}),
); );
expect(environment.orchestrationContext.setupPanelEnabled).toBe(snapshotMode !== 'off'); expect(environment.orchestrationContext.setupPanelEnabled).toBe(snapshotMode !== 'off');
expect(service.runState.setSetupPanelEnabled).toHaveBeenCalledWith(
'thread-1',
snapshotMode !== 'off',
);
if (snapshotMode === 'off') { if (snapshotMode === 'off') {
expect(service.eventLog.getSetupItemsSnapshots).not.toHaveBeenCalled(); expect(service.eventLog.getSetupItemsSnapshots).not.toHaveBeenCalled();
expect(createSetupItemsEmitter).not.toHaveBeenCalled(); expect(createSetupItemsEmitter).not.toHaveBeenCalled();
@ -1055,7 +1065,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
getSandboxStatus: Mock; getSandboxStatus: Mock;
isLocalGatewayDisabledForUser: Mock; isLocalGatewayDisabledForUser: Mock;
getPermissions: Mock; getPermissions: Mock;
isInstanceAiSetupPanelEnabled: Mock;
}; };
gatewayService: { findGateway: Mock; applyToolPolicy: Mock }; gatewayService: { findGateway: Mock; applyToolPolicy: Mock };
aiService: { isProxyEnabled: Mock }; aiService: { isProxyEnabled: Mock };
@ -1088,6 +1097,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
getPromptConfiguration: Mock; getPromptConfiguration: Mock;
setPromptConfiguration: Mock; setPromptConfiguration: Mock;
setBuildMode: Mock; setBuildMode: Mock;
setSetupPanelEnabled: Mock;
setPromptVersion: Mock; setPromptVersion: Mock;
getComputerUseChannels: Mock; getComputerUseChannels: Mock;
}; };
@ -1113,7 +1123,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
})), })),
isLocalGatewayDisabledForUser: vi.fn(async () => false), isLocalGatewayDisabledForUser: vi.fn(async () => false),
getPermissions: vi.fn(() => ({})), getPermissions: vi.fn(() => ({})),
isInstanceAiSetupPanelEnabled: vi.fn(() => false),
}; };
service.gatewayService = { findGateway: vi.fn(() => undefined), applyToolPolicy: vi.fn() }; service.gatewayService = { findGateway: vi.fn(() => undefined), applyToolPolicy: vi.fn() };
service.aiService = { isProxyEnabled: vi.fn(() => false) }; service.aiService = { isProxyEnabled: vi.fn(() => false) };
@ -1157,6 +1166,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
getPromptConfiguration: vi.fn(), getPromptConfiguration: vi.fn(),
setPromptConfiguration: vi.fn(), setPromptConfiguration: vi.fn(),
setBuildMode: vi.fn(), setBuildMode: vi.fn(),
setSetupPanelEnabled: vi.fn(),
setPromptVersion: vi.fn(), setPromptVersion: vi.fn(),
getComputerUseChannels: vi.fn(() => undefined), getComputerUseChannels: vi.fn(() => undefined),
}; };

View File

@ -95,7 +95,10 @@ describe('WorkflowVerificationObligationService setup panel policy', () => {
async (enabled) => { async (enabled) => {
const getThread = vi.fn<TypeORMAgentMemory['getThread']>(); const getThread = vi.fn<TypeORMAgentMemory['getThread']>();
const memory = mock<TypeORMAgentMemory>({ getThread }); const memory = mock<TypeORMAgentMemory>({ getThread });
const service = new WorkflowVerificationObligationService(memory, () => enabled); const service = new WorkflowVerificationObligationService(
memory,
(threadId) => threadId === 'thread-1' && enabled,
);
const outcome = makeOutcome({ const outcome = makeOutcome({
needsUserInput: true, needsUserInput: true,
nodeSimulationPlan: [], nodeSimulationPlan: [],

View File

@ -1277,11 +1277,6 @@ export class InstanceAiSettingsService {
return this.config.browserUseEnabled; return this.config.browserUseEnabled;
} }
/** Whether the non-blocking setup panel replaces the suspending setup wizard. */
isInstanceAiSetupPanelEnabled(): boolean {
return this.config.instanceAiSetupPanelEnabled;
}
/** Whether this instance is in the activation-capped trial cohort. */ /** Whether this instance is in the activation-capped trial cohort. */
isActivationCapped(): boolean { isActivationCapped(): boolean {
return this.config.activationCapped; return this.config.activationCapped;

View File

@ -16,6 +16,8 @@ import {
INSTANCE_AI_CONVERSATION_HISTORY_FLAG, INSTANCE_AI_CONVERSATION_HISTORY_FLAG,
INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT, INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG, INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
INSTANCE_AI_SETUP_PANEL_FLAG,
INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
} from '@n8n/api-types'; } from '@n8n/api-types';
import type { AiGatewayConfigDto } from '@n8n/api-types'; import type { AiGatewayConfigDto } from '@n8n/api-types';
@ -439,6 +441,7 @@ export class InstanceAiAdapterService {
/** Per-user config-evals gate (via `resolveExperimentGates`). Falsy → /** Per-user config-evals gate (via `resolveExperimentGates`). Falsy →
* eval-config service/tool not wired. */ * eval-config service/tool not wired. */
configEvalsEnabled?: boolean; configEvalsEnabled?: boolean;
setupPanelVariant?: 'control' | 'variant';
/** Per-user MCP registry gate (via `resolveExperimentGates`). Falsy → /** Per-user MCP registry gate (via `resolveExperimentGates`). Falsy →
* mcp service/tool not wired. */ * mcp service/tool not wired. */
mcpConnectionsEnabled?: boolean; mcpConnectionsEnabled?: boolean;
@ -468,6 +471,7 @@ export class InstanceAiAdapterService {
shouldBypassCredentialTest, shouldBypassCredentialTest,
agentId, agentId,
configEvalsEnabled, configEvalsEnabled,
setupPanelVariant,
mcpConnectionsEnabled, mcpConnectionsEnabled,
nodeUsageEnabled, nodeUsageEnabled,
instanceContextEnabled, instanceContextEnabled,
@ -495,6 +499,7 @@ export class InstanceAiAdapterService {
workflowService: this.createWorkflowAdapter(user, threadId, projectId, { workflowService: this.createWorkflowAdapter(user, threadId, projectId, {
nodeUsageGateOpen: nodeUsageEnabled === true, nodeUsageGateOpen: nodeUsageEnabled === true,
folderExploration: folderExplorationEnabled === true, folderExploration: folderExplorationEnabled === true,
setupPanelVariant,
}), }),
executionService: this.createExecutionAdapter(user, pushRef, threadId), executionService: this.createExecutionAdapter(user, pushRef, threadId),
credentialService, credentialService,
@ -590,7 +595,7 @@ export class InstanceAiAdapterService {
} }
} }
/** Resolves user experience flags and the shared activity gate. Unreadable flags stay off. */ /** Resolves experiment assignments. */
async resolveExperimentGates(user: User): Promise<{ async resolveExperimentGates(user: User): Promise<{
/** Config-based evals: never create evals the user can't run. */ /** Config-based evals: never create evals the user can't run. */
configEvalsEnabled: boolean; configEvalsEnabled: boolean;
@ -600,6 +605,8 @@ export class InstanceAiAdapterService {
conversationHistoryEnabled: boolean; conversationHistoryEnabled: boolean;
/** Progressive workflow policy and planning-tool selection. */ /** Progressive workflow policy and planning-tool selection. */
progressiveBuildingEnabled: boolean; progressiveBuildingEnabled: boolean;
setupPanelEnabled: boolean;
setupPanelVariant?: 'control' | 'variant';
/** Node-usage context surface: the `node-usage` action and the `nodeTypes` filter on `list`. */ /** Node-usage context surface: the `node-usage` action and the `nodeTypes` filter on `list`. */
nodeUsageEnabled: boolean; nodeUsageEnabled: boolean;
/** Per-user folder-exploration gate, passed into `createContext`. Fails /** Per-user folder-exploration gate, passed into `createContext`. Fails
@ -622,8 +629,9 @@ export class InstanceAiAdapterService {
if (userFlags.status === 'fulfilled') flags = userFlags.value; if (userFlags.status === 'fulfilled') flags = userFlags.value;
instanceContextEnabled = instanceFlag.status === 'fulfilled' && instanceFlag.value === true; instanceContextEnabled = instanceFlag.status === 'fulfilled' && instanceFlag.value === true;
} catch { } catch {
// Keep the gates closed if the client cannot start an evaluation. // Leave unreadable flags unassigned.
} }
const setupPanelVariant = flags[INSTANCE_AI_SETUP_PANEL_FLAG];
return { return {
configEvalsEnabled: flags[CONFIG_EVALUATIONS_FLAG] === CONFIG_EVALUATIONS_ENABLED_VARIANT, configEvalsEnabled: flags[CONFIG_EVALUATIONS_FLAG] === CONFIG_EVALUATIONS_ENABLED_VARIANT,
mcpConnectionsEnabled: mcpConnectionsEnabled:
@ -635,6 +643,10 @@ export class InstanceAiAdapterService {
progressiveBuildingEnabled: progressiveBuildingEnabled:
flags[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG] === flags[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG] ===
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT, INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
setupPanelEnabled: setupPanelVariant === INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
...(setupPanelVariant === 'control' || setupPanelVariant === 'variant'
? { setupPanelVariant }
: {}),
nodeUsageEnabled: flags[INSTANCE_AI_NODE_USAGE_FLAG] === true, nodeUsageEnabled: flags[INSTANCE_AI_NODE_USAGE_FLAG] === true,
folderExplorationEnabled: folderExplorationEnabled:
flags[INSTANCE_AI_FOLDER_EXPLORATION_FLAG] === flags[INSTANCE_AI_FOLDER_EXPLORATION_FLAG] ===
@ -873,8 +885,18 @@ export class InstanceAiAdapterService {
user: User, user: User,
threadId?: string, threadId?: string,
boundProjectId?: string, boundProjectId?: string,
options: { nodeUsageGateOpen?: boolean; folderExploration?: boolean } = {}, options: {
nodeUsageGateOpen?: boolean;
folderExploration?: boolean;
setupPanelVariant?: 'control' | 'variant';
} = {},
): InstanceAiWorkflowService { ): InstanceAiWorkflowService {
const setupExperimentProperties = options.setupPanelVariant
? {
variant: options.setupPanelVariant,
[`$feature/${INSTANCE_AI_SETUP_PANEL_FLAG}`]: options.setupPanelVariant,
}
: {};
const foldersOn = options.folderExploration === true; const foldersOn = options.folderExploration === true;
// Attribution reveals folder ids, names and paths on every row, so it needs // Attribution reveals folder ids, names and paths on every row, so it needs
// the licence as well as the flag. Resolution stays on the flag alone so an // the licence as well as the flag. Resolution stays on the flag alone so an
@ -1375,6 +1397,7 @@ export class InstanceAiAdapterService {
if (threadId) { if (threadId) {
telemetry.track('Builder published workflow', { telemetry.track('Builder published workflow', {
...setupExperimentProperties,
user_id: user.id, user_id: user.id,
thread_id: threadId, thread_id: threadId,
workflow_id: workflowId, workflow_id: workflowId,
@ -1608,6 +1631,7 @@ export class InstanceAiAdapterService {
if (threadId) { if (threadId) {
telemetry.track('Builder created workflow', { telemetry.track('Builder created workflow', {
...setupExperimentProperties,
user_id: user.id, user_id: user.id,
thread_id: threadId, thread_id: threadId,
workflow_id: updated.id, workflow_id: updated.id,
@ -1699,6 +1723,7 @@ export class InstanceAiAdapterService {
if (threadId) { if (threadId) {
telemetry.track('Builder modified workflow', { telemetry.track('Builder modified workflow', {
...setupExperimentProperties,
user_id: user.id, user_id: user.id,
thread_id: threadId, thread_id: threadId,
workflow_id: workflowId, workflow_id: workflowId,

View File

@ -87,7 +87,6 @@ export class InstanceAiModule implements ModuleInterface {
sandboxUnavailableReason: sandboxStatus.unavailableReason, sandboxUnavailableReason: sandboxStatus.unavailableReason,
runDebugEnabled: globalConfig.instanceAi.runDebugEnabled, runDebugEnabled: globalConfig.instanceAi.runDebugEnabled,
activationCapped: settingsService.isActivationCapped(), activationCapped: settingsService.isActivationCapped(),
instanceAiSetupPanelEnabled: settingsService.isInstanceAiSetupPanelEnabled(),
}; };
} }

View File

@ -772,8 +772,9 @@ export class InstanceAiService {
) { ) {
this.logger = logger.scoped('instance-ai'); this.logger = logger.scoped('instance-ai');
runProbe.registerActiveRunCountProvider(() => this.runState.activeRunCount()); runProbe.registerActiveRunCountProvider(() => this.runState.activeRunCount());
this.workflowObligations = new WorkflowVerificationObligationService(this.agentMemory, () => this.workflowObligations = new WorkflowVerificationObligationService(
this.settingsService.isInstanceAiSetupPanelEnabled(), this.agentMemory,
(threadId) => this.runState.isSetupPanelEnabled(threadId),
); );
this.taskProjector = new WorkflowVerificationTaskProjector( this.taskProjector = new WorkflowVerificationTaskProjector(
this.agentMemory, this.agentMemory,
@ -2417,11 +2418,14 @@ export class InstanceAiService {
mcpConnectionsEnabled, mcpConnectionsEnabled,
conversationHistoryEnabled, conversationHistoryEnabled,
progressiveBuildingEnabled, progressiveBuildingEnabled,
setupPanelEnabled,
setupPanelVariant,
nodeUsageEnabled, nodeUsageEnabled,
folderExplorationEnabled, folderExplorationEnabled,
aiPreferencesEnabled, aiPreferencesEnabled,
instanceContextEnabled, instanceContextEnabled,
} = await this.adapterService.resolveExperimentGates(user); } = await this.adapterService.resolveExperimentGates(user);
this.runState.setSetupPanelEnabled(threadId, setupPanelEnabled);
// One scoped reader backs both the tool and the first-turn hint. // One scoped reader backs both the tool and the first-turn hint.
const conversationHistory = conversationHistoryEnabled const conversationHistory = conversationHistoryEnabled
? this.conversationHistoryService.forContext(user.id, boundProjectId, threadId) ? this.conversationHistoryService.forContext(user.id, boundProjectId, threadId)
@ -2444,6 +2448,7 @@ export class InstanceAiService {
shouldBypassCredentialTest: (credentialId: string) => shouldBypassCredentialTest: (credentialId: string) =>
this.evalCredentialAllowlists.shouldBypassTest(threadId, credentialId), this.evalCredentialAllowlists.shouldBypassTest(threadId, credentialId),
configEvalsEnabled, configEvalsEnabled,
setupPanelVariant,
mcpConnectionsEnabled, mcpConnectionsEnabled,
nodeUsageEnabled, nodeUsageEnabled,
instanceContextEnabled, instanceContextEnabled,
@ -2482,7 +2487,7 @@ export class InstanceAiService {
// Setup panel v2: wire the durable `setup-items` sink only while the flag // Setup panel v2: wire the durable `setup-items` sink only while the flag
// is on — its presence is the package-side gate. Seeded with the thread's // is on — its presence is the package-side gate. Seeded with the thread's
// persisted snapshots so a recomputed, unchanged list publishes nothing. // persisted snapshots so a recomputed, unchanged list publishes nothing.
if (this.settingsService.isInstanceAiSetupPanelEnabled()) { if (setupPanelEnabled) {
context.setupItemsEmitter = createSetupItemsEmitter({ context.setupItemsEmitter = createSetupItemsEmitter({
eventBus: this.eventBus, eventBus: this.eventBus,
threadId, threadId,

View File

@ -26,7 +26,7 @@ export function parseWorkflowBuildOutcome(
export class WorkflowVerificationObligationService { export class WorkflowVerificationObligationService {
constructor( constructor(
private readonly agentMemory: TypeORMAgentMemory, private readonly agentMemory: TypeORMAgentMemory,
private readonly isSetupPanelEnabled: () => boolean = () => false, private readonly isSetupPanelEnabled: (threadId: string) => boolean = () => false,
) {} ) {}
private storage(): WorkflowLoopStorage { private storage(): WorkflowLoopStorage {
@ -56,7 +56,7 @@ export class WorkflowVerificationObligationService {
): WorkflowVerificationObligation { ): WorkflowVerificationObligation {
return deriveWorkflowVerificationObligation(threadId, record, { return deriveWorkflowVerificationObligation(threadId, record, {
...options, ...options,
setupPanelEnabled: this.isSetupPanelEnabled(), setupPanelEnabled: this.isSetupPanelEnabled(threadId),
}); });
} }
@ -97,7 +97,7 @@ export class WorkflowVerificationObligationService {
const options = { const options = {
source: 'planned', source: 'planned',
plannedTaskId: task.id, plannedTaskId: task.id,
setupPanelEnabled: this.isSetupPanelEnabled(), setupPanelEnabled: this.isSetupPanelEnabled(threadId),
} satisfies Parameters<typeof deriveWorkflowVerificationObligation>[2]; } satisfies Parameters<typeof deriveWorkflowVerificationObligation>[2];
const record = await this.storage().getWorkItem(threadId, baseOutcome.workItemId); const record = await this.storage().getWorkItem(threadId, baseOutcome.workItemId);
const outcome = record?.lastBuildOutcome ?? baseOutcome; const outcome = record?.lastBuildOutcome ?? baseOutcome;

View File

@ -15,7 +15,15 @@ the test.
- The receiver runs on `http://127.0.0.1:3456` and answers `POST - The receiver runs on `http://127.0.0.1:3456` and answers `POST
/api/v1/instance-reports` with **201** on success. Any other status counts as /api/v1/instance-reports` with **201** on success. Any other status counts as
a rejection. a rejection.
- The receiver accepts `Authorization: Bearer testing`. - The receiver trusts the license certificate this instance sends. This applies
to the default run; with `N8N_INSTANCE_REPORTING_AUTH_TOKEN` set (scenario
5.4c) no certificate is needed. The
instance sends whatever `N8N_LICENSE_CERT` holds, so either use a real
n8n-issued certificate, or a certificate minted by a development CA that the
receiver is configured to trust (see the receiver's `mock-license` tooling
and `N8N_MONITORING_ADDITIONAL_ISSUER_CERTS`). A development certificate
makes the license SDK log `cert could not be initialized` once at boot; the
instance then runs as community and still reports.
- Keep the receiver's request log visible. You must see the raw body. - Keep the receiver's request log visible. You must see the raw body.
- `sqlite3` is installed. The dev instance uses SQLite at - `sqlite3` is installed. The dev instance uses SQLite at
`~/.n8n/database.sqlite` unless you set another database. `~/.n8n/database.sqlite` unless you set another database.
@ -34,8 +42,8 @@ cd packages/cli
export N8N_ENABLED_MODULES=instance-reporting export N8N_ENABLED_MODULES=instance-reporting
export N8N_INSTANCE_REPORTING_BASE_URL=http://127.0.0.1:3456 export N8N_INSTANCE_REPORTING_BASE_URL=http://127.0.0.1:3456
export N8N_INSTANCE_REPORTING_AUTH_TOKEN=testing
export N8N_INSTANCE_REPORTING_LABEL=local-dev export N8N_INSTANCE_REPORTING_LABEL=local-dev
export N8N_LICENSE_CERT='<certificate the receiver trusts, see prerequisites>'
# Compact insights quickly, so raw rows reach insights_by_period in a minute. # Compact insights quickly, so raw rows reach insights_by_period in a minute.
export N8N_INSIGHTS_COMPACTION_INTERVAL_MINUTES=1 export N8N_INSIGHTS_COMPACTION_INTERVAL_MINUTES=1
@ -49,7 +57,8 @@ pnpm dev
Expected on the first boot: Expected on the first boot:
- No warning about `N8N_INSTANCE_REPORTING_BASE_URL` being unset. - No warning about `N8N_INSTANCE_REPORTING_BASE_URL` being unset, and none
about a missing license certificate.
- The log line `Started the instance reporting timer`. - The log line `Started the instance reporting timer`.
- A `Resolved the instance reporting time` line with a random `HH:mm` at or - A `Resolved the instance reporting time` line with a random `HH:mm` at or
after `03:00`. after `03:00`.
@ -137,7 +146,8 @@ ahead and the tick correctly skips.
**Receiver side.** The receiver must log one `POST /api/v1/instance-reports` **Receiver side.** The receiver must log one `POST /api/v1/instance-reports`
with: with:
- header `Authorization: Bearer testing` - no `Authorization` header (the token is unset in this run)
- `licenseCert`: the exact value of `N8N_LICENSE_CERT`
- `instanceId`: a 64-character hex string, equal to the instance id in - `instanceId`: a 64-character hex string, equal to the instance id in
`~/.n8n/config` `~/.n8n/config`
- `batchId`: a UUID-like id, equal to the row id from the query below - `batchId`: a UUID-like id, equal to the row id from the query below
@ -172,10 +182,12 @@ Run these after step 4. Each is short.
| 5.1 | No second report the same day | Restart the instance | No new request, no new row. `hasDeliveredToday` short-circuits the tick | | 5.1 | No second report the same day | Restart the instance | No new request, no new row. `hasDeliveredToday` short-circuits the tick |
| 5.2 | Retry resends the same measurement | Make the receiver answer 500. Delete today's delivered row, then restart | Request arrives, `lastError` holds `rejected with status 500`, `deliveredAt` NULL, `attempts` grows. Retries land ~5 minutes apart, 3 attempts in total, then `Giving up on the instance report for today`. Every retry carries the **same** `batchId` and the same values — no re-measurement | | 5.2 | Retry resends the same measurement | Make the receiver answer 500. Delete today's delivered row, then restart | Request arrives, `lastError` holds `rejected with status 500`, `deliveredAt` NULL, `attempts` grows. Retries land ~5 minutes apart, 3 attempts in total, then `Giving up on the instance report for today`. Every retry carries the **same** `batchId` and the same values — no re-measurement |
| 5.3 | Recovery keeps the pending row | During 5.2, switch the receiver back to 201 before the third attempt | The next attempt reuses the pending row and marks it delivered. No second row for the day | | 5.3 | Recovery keeps the pending row | During 5.2, switch the receiver back to 201 before the third attempt | The next attempt reuses the pending row and marks it delivered. No second row for the day |
| 5.4 | Wrong token | Set `N8N_INSTANCE_REPORTING_AUTH_TOKEN=wrong`, clear today's row, restart | The receiver answers 401/403, delivery fails, `lastError` names the status. Nothing is marked delivered | | 5.4 | Untrusted certificate | Set `N8N_LICENSE_CERT` to a certificate the receiver does not trust (any well-formed one from another CA), clear today's row, restart | The receiver answers 401, delivery fails, `lastError` names the status. Nothing is marked delivered |
| 5.4b | No certificate | Unset `N8N_LICENSE_CERT`, clear today's row, restart | Warning `no license certificate, so no reports will be sent`; no timer, no request, no row |
| 5.4c | Token set | Set `N8N_INSTANCE_REPORTING_AUTH_TOKEN` to a token the receiver accepts, unset `N8N_LICENSE_CERT`, clear today's row, restart | No certificate warning. The request carries `Authorization: Bearer <token>` and the body has no `licenseCert`. The receiver answers 201 and the row is marked delivered |
| 5.5 | Non-201 success code is a failure | Make the receiver answer 200 | Treated as a rejection: `Instance report was rejected with status 200` | | 5.5 | Non-201 success code is a failure | Make the receiver answer 200 | Treated as a rejection: `Instance report was rejected with status 200` |
| 5.6 | Receiver down | Stop the receiver, clear today's row, restart | Delivery fails with a connection error in `lastError`; the instance stays healthy and keeps serving | | 5.6 | Receiver down | Stop the receiver, clear today's row, restart | Delivery fails with a connection error in `lastError`; the instance stays healthy and keeps serving |
| 5.7 | Redirect is not followed | Make the receiver answer 302 to another local port | The report is rejected with status 302. The second port never sees the bearer token | | 5.7 | Redirect is not followed | Make the receiver answer 302 to another local port | The report is rejected with status 302. The second port never sees the license certificate |
| 5.8 | Base URL unset | Unset `N8N_INSTANCE_REPORTING_BASE_URL`, restart | Warning `enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset`; no timer, no request | | 5.8 | Base URL unset | Unset `N8N_INSTANCE_REPORTING_BASE_URL`, restart | Warning `enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset`; no timer, no request |
| 5.9 | Insights disabled | `N8N_DISABLED_MODULES=insights`, restart | Startup fails with the `UserError` that names both variables | | 5.9 | Insights disabled | `N8N_DISABLED_MODULES=insights`, restart | Startup fails with the `UserError` that names both variables |
| 5.10 | Trailing slash in the base URL | Use `http://127.0.0.1:3456/`, clear today's row, restart | The path is still `/api/v1/instance-reports`, with no double slash | | 5.10 | Trailing slash in the base URL | Use `http://127.0.0.1:3456/`, clear today's row, restart | The path is still `/api/v1/instance-reports`, with no double slash |
@ -202,8 +214,8 @@ sqlite3 "$N8N_DB" "DELETE FROM instance_monitoring_report;"
sqlite3 "$N8N_DB" "DELETE FROM settings WHERE key = 'features.centralInstanceMonitoring';" sqlite3 "$N8N_DB" "DELETE FROM settings WHERE key = 'features.centralInstanceMonitoring';"
``` ```
Unset the `N8N_INSTANCE_REPORTING_*` and `N8N_INSIGHTS_*` variables, or start a Unset the `N8N_INSTANCE_REPORTING_*`, `N8N_LICENSE_CERT` and `N8N_INSIGHTS_*`
new shell. The backdated `insights_by_period` rows stay wrong for the Insights variables, or start a new shell. The backdated `insights_by_period` rows stay wrong for the Insights
UI; drop them, or use a throwaway `N8N_USER_FOLDER` for the whole test if you UI; drop them, or use a throwaway `N8N_USER_FOLDER` for the whole test if you
want your dev data untouched. want your dev data untouched.

View File

@ -37,7 +37,7 @@ without reading this document:
|---|---| |---|---|
| `pending` | Not delivered yet, and attempts remain | | `pending` | Not delivered yet, and attempts remain |
| `delivered` | The receiver answered 201 | | `delivered` | The receiver answered 201 |
| `skipped_after_max_retries` | Three attempts failed, so the instance stopped for that day | | `skipped_after_max_retries` | The instance stopped delivering that day, either after three failed attempts or because the report's own slot passed before it landed |
A skipped report is **not** lost data. Only a delivered report crosses a day A skipped report is **not** lost data. Only a delivered report crosses a day
off, so the days a skipped report covered are measured again and sent by the off, so the days a skipped report covered are measured again and sent by the
@ -50,6 +50,10 @@ five minutes since the last attempt before trying again — otherwise a crash lo
would spend the whole budget in seconds. `InstanceReportingScheduler` keeps no would spend the whole budget in seconds. `InstanceReportingScheduler` keeps no
attempt state of its own. attempt state of its own.
The delivery retry above and the missed-day backfill are two separate
mechanisms. [RETRIES.md](./RETRIES.md) explains each one, how they connect,
and where the logic lives.
## Scheduling ## Scheduling
The daily fire is driven by `InstanceReportingScheduler`, a leader-gated The daily fire is driven by `InstanceReportingScheduler`, a leader-gated
@ -81,13 +85,47 @@ N8N_INSTANCE_REPORTING_BASE_URL=https://monitoring.example.com
The `insights` module must stay enabled, since the daily figure comes from The `insights` module must stay enabled, since the daily figure comes from
there. there.
The instance must hold a license certificate, unless
`N8N_INSTANCE_REPORTING_AUTH_TOKEN` is set. Reporting is for licensed
instances, and the certificate is how the receiver knows that, see
[Authentication](#authentication). Without a credential the module loads but
warns and never sends, exactly as without a receiver.
## Authentication
There are two credentials. The token wins when it is set.
**License certificate (default).** Every report carries the instance's license
certificate, the string `License.loadCertStr()` returns (`N8N_LICENSE_CERT`,
or the persisted certificate of an activated license), as the `licenseCert`
field of the body. The receiver verifies that the certificate was issued by
n8n and then discards it; nothing from it is stored. There is no token to
configure or distribute.
The certificate is read fresh for every report, so a renewed license is sent
as soon as it is stored. An expired certificate is still sent and still
accepted: the receiver checks the issuer, not the validity period.
It travels in the body, not in an `Authorization` header, because a
certificate is several KB and grows with the license, which is more than
common reverse proxies allow per header.
**Bearer token.** When `N8N_INSTANCE_REPORTING_AUTH_TOKEN` is set, every
report carries it as `Authorization: Bearer …` and the body has no
`licenseCert` field. The certificate is not read at all, so an unlicensed
instance can report with a token.
Redirects are never followed, so either credential reaches only the configured
host.
## Configuration ## Configuration
| Env var | Default | Notes | | Env var | Default | Notes |
|---|---|---| |---|---|---|
| `N8N_INSTANCE_REPORTING_BASE_URL` | `''` | Base URL of the receiver. The report is POSTed to `<base>/api/v1/instance-reports`. Left unset, the module loads but warns and never sends: it starts no scheduler and claims no report time. | | `N8N_INSTANCE_REPORTING_BASE_URL` | `''` | Base URL of the receiver. The report is POSTed to `<base>/api/v1/instance-reports`. Left unset, the module loads but warns and never sends: it starts no scheduler and claims no report time. |
| `N8N_INSTANCE_REPORTING_LABEL` | `''` | Sent as `label` in the payload, when set. | | `N8N_INSTANCE_REPORTING_LABEL` | `''` | Sent as `label` in the payload, when set. |
| `N8N_INSTANCE_REPORTING_AUTH_TOKEN` | `''` | Sent as `Authorization: Bearer …`, when set. | | `N8N_INSTANCE_REPORTING_AUTH_TOKEN` | `''` | Sent as `Authorization: Bearer …`, when set. Replaces the license certificate as the credential; `licenseCert` is then omitted from the body. |
| `N8N_LICENSE_CERT` | `''` | Not owned by this module. Its value, or the persisted certificate of an activated license, is sent as `licenseCert` and is the credential the receiver checks, unless a token is set. |
## Report time ## Report time
@ -113,9 +151,11 @@ is enabled on this instance:
} }
``` ```
`enabled` says whether a receiver is configured. Without one the key reads `enabled` says whether a receiver is configured and a credential (an auth
`{ "enabled": false }` and carries no `reportTime`, since no time is claimed. A token or a license certificate) is present. Without either the key reads
missing key means the module is not enabled at all. `{ "enabled": false }` and carries no
`reportTime`, since no time is claimed. A missing key means the module is not
enabled at all.
These settings are built once, during module init, and served from a cache These settings are built once, during module init, and served from a cache
afterwards, so they carry only values that stay the same for the lifetime of afterwards, so they carry only values that stay the same for the lifetime of

View File

@ -0,0 +1,112 @@
# Retries
The module has two retry mechanisms. The code keeps them separate. Both feed
the same state machine on `instance_monitoring_report`: a row goes from
`pending` to `delivered`, or from `pending` to `skipped_after_max_retries`.
- **Type 1, delivery retry.** Resend the pending row until it lands, its budget
runs out, or its own next slot passes. It continues across the UTC midnight
boundary.
- **Type 2, missed-day backfill.** Put every day without a delivered row into
the next report.
Type 2 decides how many days the next `pending` row must cover. Type 1 then
takes over and delivers that row.
## The send decision
`start()` fires the first `tick()` (on `server-started`, or on leader takeover).
Every pass then re-arms the next with `setTimeout`, so the loop drives itself.
Each pass makes one choice, from `findPending()` — the newest row, and only if
that row is still pending:
```mermaid
flowchart TD
S(["start(): server-started / leader takeover"]) --> T
T["tick()"] --> P{"waiting between retries?<br/>(up to 5 min, but not past its slot)"}
P -- yes --> ARM
P -- no --> L["report = findPending()"]
L --> Q1{"is there a report to resume?"}
Q1 -- yes --> Q1b{"too old to still send?"}
Q1b -- no --> R["resend it — retry, even past midnight"]
Q1b -- yes --> SK["mark it skipped"] --> Q2
Q1 -- "no — delivered, skipped, or none" --> Q2{"reached today's report time?"}
Q2 -- no --> N["do nothing — wait"]
Q2 -- yes --> Q3{"already reported today?"}
Q3 -- yes --> N
Q3 -- no --> C["create a new report"]
R --> ARM
C --> ARM
N --> ARM
ARM["scheduleNext() — arms setTimeout(tick)"] -. re-arms .-> T
```
## Type 1: delivery retry
Scope: the newest row while it is `pending`, whose last delivery attempt failed.
It can come from an earlier slot, so the retry crosses midnight. Once its own
next slot has passed, the retry stops: the row is marked
`skipped_after_max_retries` and a new report covers its day instead.
- The row is created once, as `pending`, with its data points already
measured. A retry changes only `attempts`, `lastAttemptAt`, `lastError` and,
in the end, `status`. The data points and the `batchId` stay the same.
- `InstanceReportingService.msUntilRetryAllowed()` reads `lastAttemptAt` from
the row. The scheduler waits `RETRY_DELAY_MS` (5 minutes) between attempts.
- The budget is `MAX_ATTEMPTS` (3). The attempt that spends the last one flips
the row to `skipped_after_max_retries` at once. There is no fourth attempt,
and no new row for today.
- A response of `201` flips the row to `delivered`. A `409` means the receiver
already holds this `batchId`, so the row is marked `delivered` as well.
Type 1 alone decides whether the row lands. Because the budget and the
pacing live on the row, a restart resumes them. A fresh process does not get
three new attempts, and it waits out the rest of the five minutes before it
tries again.
## Type 2: missed-day backfill
Scope: no `pending` row is the newest, but one or more calendar days have no
`delivered` row.
- `InstanceMonitoringReportRepository.findLastCoveredDay()` counts only
`delivered` rows. A day whose row was skipped, or a day the instance was down
for, is still owed.
- `InstanceReportingService.missedDays()` walks back from yesterday to the day
after the last delivered day. It stops at `MAX_BACKFILL_DAYS` (30) because
`insights` buckets a longer range by week, which cannot fill a daily point.
Older days are dropped.
- The next report creates one new row. It carries one `daily` point for every
missed day and one `cumulative` point. The cumulative point is a fresh
lifetime total, not one per missed day.
- That new row is then subject to type 1 if its own delivery fails.
Type 2 is the catch-up layer. It reacts to type 1 giving up, and to any other
gap in delivered rows.
## How the two connect
```mermaid
flowchart TD
subgraph T1["Type 1: delivery retry (max 3 attempts, 5 min apart, crosses midnight)"]
A["Newest row is pending\n+ measured data points"] --> B["POST attempt"]
B -->|"201 / 409"| C["delivered"]
B -->|"failure"| D["attempts++\nlastAttemptAt, lastError"]
D --> E{"attempts >= 3?"}
E -- no --> F["wait 5 min"] --> B
E -- yes --> G["skipped_after_max_retries"]
end
C --> H["Day counts as covered\n(findLastCoveredDay)"]
G --> I["Day stays uncovered\n(not delivered)"]
subgraph T2["Type 2: missed-day backfill"]
J["Next cycle: newest row not pending"] --> K["missedDays() = every day after\nlast DELIVERED day, up to 30"]
K --> L{"any missed days?"}
L -- no --> M["nothing to send"]
L -- yes --> N["new row: one daily point per\nmissed day + 1 cumulative point"]
N --> A
end
I --> K
```

View File

@ -0,0 +1,669 @@
import { createFakeOutboundHttp, type Route } from '@n8n/backend-network/testing';
import {
createTeamProject,
createWorkflow,
mockLogger,
testDb,
testModules,
} from '@n8n/backend-test-utils';
import { Time } from '@n8n/constants';
import { LicenseMetricsRepository, SettingsRepository } from '@n8n/db';
import { Container } from '@n8n/di';
import { DateTime } from 'luxon';
import { InstanceSettings } from 'n8n-core';
import type { MockInstance } from 'vitest';
import { mock } from 'vitest-mock-extended';
import { EventService } from '@/events/event.service';
import type { License } from '@/license';
import { createCompactedInsightsEvent } from '@/modules/insights/database/entities/__tests__/db-utils';
import { InsightsService } from '@/modules/insights/insights.service';
import { OwnershipService } from '@/services/ownership.service';
import { createOwner } from '@test-integration/db/users';
import type { InstanceReportDataPoint } from '../database/entities/instance-monitoring-report';
import { InstanceMonitoringReportRepository } from '../database/repositories/instance-monitoring-report.repository';
import { InstanceReportingScheduler } from '../instance-reporting-scheduler.service';
import { InstanceReportingSettingsService } from '../instance-reporting-settings.service';
import { InstanceReportingConfig } from '../instance-reporting.config';
import {
CENTRAL_INSTANCE_MONITORING_SETTINGS_KEY,
INSTANCE_REPORTS_PATH,
} from '../instance-reporting.constants';
import { InstanceReportingService } from '../instance-reporting.service';
const RECEIVER_URL = 'https://receiver.test';
const REPORT_TIME = '07:42';
/** One minute after the slot, so the first pass reports at once. */
const AFTER_SLOT = '2026-03-26T07:43:00.000Z';
const NEXT_SLOT = '2026-03-27T07:42:00.000Z';
// The retry contract, spelled out rather than imported, so a change to either
// constant in the service fails here instead of moving the test along with it.
const MAX_ATTEMPTS = 3;
const RETRY_DELAY_MS = 5 * Time.minutes.toMilliseconds;
/** Room for the fake clock's drift while `vi.waitFor` polls. */
const CLOCK_TOLERANCE_MS = 10 * Time.seconds.toMilliseconds;
interface ReportPayload {
batchId: string;
dataPoints: InstanceReportDataPoint[];
}
const unreachable = (): Route => ({
method: 'POST',
pathname: INSTANCE_REPORTS_PATH,
networkError: 'ECONNREFUSED',
});
const accepted = (): Route => ({ method: 'POST', pathname: INSTANCE_REPORTS_PATH, status: 201 });
/** The private arming hook, spied on so a test can tell when a pass has finished. */
type SchedulerInternals = { scheduleNext(delayMs: number): void };
interface Harness {
scheduler: InstanceReportingScheduler;
/** Every resolved request the fake receiver saw. */
httpRequest: ReturnType<typeof createFakeOutboundHttp>['httpRequest'];
/**
* Called once a pass has finished its last database write, so its call count
* marks "pass over, next timer armed". Pending fake timers cannot serve as
* that signal: the SQLite pool arms a transient timeout for every query.
*/
scheduleNext: MockInstance<SchedulerInternals['scheduleNext']>;
}
describe('instance reporting retries', () => {
let repository: InstanceMonitoringReportRepository;
let schedulers: InstanceReportingScheduler[] = [];
beforeAll(async () => {
await testModules.loadModules(['insights', 'instance-reporting']);
await testDb.init();
repository = Container.get(InstanceMonitoringReportRepository);
Container.get(InstanceReportingConfig).instanceReportingBaseUrl = RECEIVER_URL;
const instanceSettings = Container.get(InstanceSettings);
expect(instanceSettings.instanceType).toBe('main');
instanceSettings.markAsLeader();
});
beforeEach(async () => {
await testDb.truncate([
'InstanceMonitoringReport',
'Settings',
'User',
'InsightsByPeriod',
'InsightsMetadata',
'WorkflowEntity',
'Project',
]);
await createOwner();
await Container.get(SettingsRepository).upsertByKey(
CENTRAL_INSTANCE_MONITORING_SETTINGS_KEY,
JSON.stringify({ reportTime: REPORT_TIME }),
false,
{},
);
// Only what the scheduler uses. `setImmediate` and `nextTick` stay real so
// the database driver's I/O still completes under the fake clock.
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
vi.setSystemTime(new Date(AFTER_SLOT));
});
afterEach(() => {
for (const scheduler of schedulers) scheduler.shutdown();
schedulers = [];
vi.useRealTimers();
vi.restoreAllMocks();
});
afterAll(async () => {
await testDb.terminate();
});
function makeHarness(routes: Route[]): Harness {
const { outboundHttp, httpRequest } = createFakeOutboundHttp(
routes,
vi.fn as unknown as Parameters<typeof createFakeOutboundHttp>[1],
);
const instanceSettings = Container.get(InstanceSettings);
const eventService = Container.get(EventService);
const service = new InstanceReportingService(
Container.get(InstanceReportingConfig),
repository,
Container.get(InsightsService),
instanceSettings,
Container.get(OwnershipService),
Container.get(LicenseMetricsRepository),
mock<License>({ loadCertStr: async () => 'license-cert' }),
mockLogger(),
eventService,
outboundHttp,
);
const scheduler = new InstanceReportingScheduler(
service,
repository,
Container.get(InstanceReportingSettingsService),
instanceSettings,
eventService,
mockLogger(),
);
schedulers.push(scheduler);
const scheduleNext = vi.spyOn(scheduler as unknown as SchedulerInternals, 'scheduleNext');
return { scheduler, httpRequest, scheduleNext };
}
/** Wait until the scheduler has completed `passes` passes and armed the next one. */
async function armed({ scheduleNext }: Harness, passes: number) {
await vi.waitFor(() => expect(scheduleNext).toHaveBeenCalledTimes(passes), {
timeout: 5_000,
});
}
/** Assert the delay the scheduler last armed lands on `slot`. */
function expectArmedFor({ scheduleNext }: Harness, slot: string) {
const delay = scheduleNext.mock.lastCall?.[0] ?? Number.NaN;
const target = new Date(slot).getTime();
expect(Date.now() + delay).toBeGreaterThanOrEqual(target - CLOCK_TOLERANCE_MS);
expect(Date.now() + delay).toBeLessThanOrEqual(target + CLOCK_TOLERANCE_MS);
}
function sentPayload({ httpRequest }: Harness, index: number): ReportPayload {
return httpRequest.mock.calls[index][0].body as unknown as ReportPayload;
}
function dailyPoints(payload: ReportPayload) {
return payload.dataPoints.flatMap((point) =>
point.kind === 'daily' ? [{ date: point.date, value: point.value }] : [],
);
}
/**
* The database stamps `createdAt` with the real clock, not the fake one, so
* a row from a fake "yesterday" reads as today's. Pin it to the day meant.
*/
async function stampCreatedAt(id: string, createdAt: Date) {
await repository.update({ id }, { createdAt });
}
/** A delivered report that covers `day`, generated the morning after it. */
async function seedDeliveredReport(day: string) {
const report = await repository.createPending([
{ kind: 'cumulative', name: 'billableExecutions', value: 0 },
{ kind: 'daily', name: 'billableExecutions', value: 3, date: day },
]);
await repository.markDelivered(report.id, new Date());
await stampCreatedAt(
report.id,
new Date(new Date(`${day}T07:43:00.000Z`).getTime() + Time.days.toMilliseconds),
);
return report;
}
async function seedDailyExecutions(totalsByDay: Record<string, number>) {
const workflow = await createWorkflow({}, await createTeamProject());
for (const [day, value] of Object.entries(totalsByDay)) {
await createCompactedInsightsEvent(workflow, {
type: 'success',
value,
periodUnit: 'day',
periodStart: DateTime.fromISO(day, { zone: 'utc' }),
});
}
}
async function seedPendingReport(
day: string,
{
attempts,
lastAttemptAt,
createdAt,
}: { attempts: number; lastAttemptAt: Date; createdAt: Date },
) {
const report = await repository.createPending([
{ kind: 'cumulative', name: 'billableExecutions', value: 0 },
{ kind: 'daily', name: 'billableExecutions', value: 5, date: day },
]);
await repository.update(
{ id: report.id },
{ attempts, lastError: 'ECONNREFUSED', lastAttemptAt },
);
await stampCreatedAt(report.id, createdAt);
return report;
}
async function deliveredDailyDates(): Promise<string[]> {
const delivered = await repository.find({ where: { status: 'delivered' } });
return delivered.flatMap((report) =>
report.dataPoints.flatMap((point) => (point.kind === 'daily' ? [point.date] : [])),
);
}
async function setReportTime(reportTime: string) {
await Container.get(SettingsRepository).upsertByKey(
CENTRAL_INSTANCE_MONITORING_SETTINGS_KEY,
JSON.stringify({ reportTime }),
false,
{},
);
}
test('delivers on the third attempt once the receiver is reachable again', async () => {
const harness = makeHarness([unreachable(), unreachable(), accepted()]);
harness.scheduler.start();
await armed(harness, 1);
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
await expect(repository.count()).resolves.toBe(1);
const [report] = await repository.find();
expect(report).toMatchObject({ status: 'pending', attempts: 1 });
expect(report.lastError).toEqual(expect.any(String));
expect(report.lastAttemptAt).toEqual(expect.any(Date));
expect(harness.scheduleNext).toHaveBeenLastCalledWith(RETRY_DELAY_MS);
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 2);
expect(harness.httpRequest).toHaveBeenCalledTimes(2);
await expect(repository.findOneByOrFail({ id: report.id })).resolves.toMatchObject({
status: 'pending',
attempts: 2,
});
expect(harness.scheduleNext).toHaveBeenLastCalledWith(RETRY_DELAY_MS);
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 3);
expect(harness.httpRequest).toHaveBeenCalledTimes(3);
await expect(repository.findOneByOrFail({ id: report.id })).resolves.toMatchObject({
status: 'delivered',
attempts: 3,
deliveredAt: expect.any(Date),
lastError: null,
});
// Every attempt resent the row as measured, under the same batchId.
const payloads = [0, 1, 2].map((index) => sentPayload(harness, index));
for (const payload of payloads) {
expect(payload.batchId).toBe(report.id);
expect(payload.dataPoints).toEqual(payloads[0].dataPoints);
}
// The day is settled, so the scheduler now waits for tomorrow's slot.
expectArmedFor(harness, NEXT_SLOT);
await vi.advanceTimersByTimeAsync(Time.hours.toMilliseconds);
expect(harness.httpRequest).toHaveBeenCalledTimes(3);
expect(harness.scheduleNext).toHaveBeenCalledTimes(3);
await expect(repository.count()).resolves.toBe(1);
});
test('gives up after three failures and covers the day again in the next report', async () => {
// A delivered report for the day before last, so the gap has a start.
await seedDeliveredReport('2026-03-24');
await seedDailyExecutions({ '2026-03-25': 5, '2026-03-26': 7 });
const harness = makeHarness([unreachable(), unreachable(), unreachable(), accepted()]);
harness.scheduler.start();
await armed(harness, 1);
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 2);
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 3);
expect(harness.httpRequest).toHaveBeenCalledTimes(MAX_ATTEMPTS);
const [abandoned] = await repository.find({ where: { status: 'skipped_after_max_retries' } });
expect(abandoned).toMatchObject({ attempts: MAX_ATTEMPTS });
expect(dailyPoints(sentPayload(harness, 0))).toEqual([{ date: '2026-03-25', value: 5 }]);
// Keep the abandoned row on the day it was made, or tomorrow reads as settled too.
await stampCreatedAt(abandoned.id, new Date(AFTER_SLOT));
// The next pass finds the day settled and moves on to tomorrow's slot.
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 4);
expect(harness.httpRequest).toHaveBeenCalledTimes(MAX_ATTEMPTS);
expectArmedFor(harness, NEXT_SLOT);
await vi.advanceTimersByTimeAsync(new Date(NEXT_SLOT).getTime() - Date.now());
await armed(harness, 5);
expect(harness.httpRequest).toHaveBeenCalledTimes(MAX_ATTEMPTS + 1);
const backfill = sentPayload(harness, MAX_ATTEMPTS);
expect(backfill.batchId).not.toBe(abandoned.id);
expect(dailyPoints(backfill)).toEqual([
{ date: '2026-03-25', value: 5 },
{ date: '2026-03-26', value: 7 },
]);
expect(backfill.dataPoints.filter((point) => point.kind === 'cumulative')).toHaveLength(1);
await expect(repository.findOneByOrFail({ id: backfill.batchId })).resolves.toMatchObject({
status: 'delivered',
attempts: 1,
});
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-26');
});
test('carries a daily point for every day of a downtime gap in one report', async () => {
await seedDeliveredReport('2026-03-22');
// Nothing for 03-24: a day with no executions has no insights row at all.
await seedDailyExecutions({ '2026-03-23': 4, '2026-03-25': 9 });
const harness = makeHarness([accepted()]);
harness.scheduler.start();
await armed(harness, 1);
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
const payload = sentPayload(harness, 0);
expect(dailyPoints(payload)).toEqual([
{ date: '2026-03-23', value: 4 },
{ date: '2026-03-24', value: 0 },
{ date: '2026-03-25', value: 9 },
]);
expect(payload.dataPoints.filter((point) => point.kind === 'cumulative')).toHaveLength(1);
await expect(repository.findOneByOrFail({ id: payload.batchId })).resolves.toMatchObject({
status: 'delivered',
attempts: 1,
});
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-25');
});
test('caps a long gap at 30 days and still reads every day on its own', async () => {
await seedDeliveredReport('2026-01-01');
await seedDailyExecutions({ '2026-02-24': 6, '2026-03-25': 8 });
const harness = makeHarness([accepted()]);
harness.scheduler.start();
await armed(harness, 1);
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
const points = dailyPoints(sentPayload(harness, 0));
expect(points).toHaveLength(30);
// Exact days, not weekly buckets: a 30-day range still reads as days.
expect(points.at(0)).toEqual({ date: '2026-02-24', value: 6 });
expect(points.at(-1)).toEqual({ date: '2026-03-25', value: 8 });
expect(points.slice(1, -1).every((point) => point.value === 0)).toBe(true);
});
test('resumes the retry budget and the remaining wait after a restart', async () => {
const before = makeHarness([unreachable()]);
before.scheduler.start();
await armed(before, 1);
expect(before.httpRequest).toHaveBeenCalledTimes(1);
await expect(repository.count()).resolves.toBe(1);
const [report] = await repository.find();
expect(report).toMatchObject({ status: 'pending', attempts: 1 });
// The process dies two minutes into the five-minute wait.
before.scheduler.shutdown();
await vi.advanceTimersByTimeAsync(2 * Time.minutes.toMilliseconds);
const after = makeHarness([accepted()]);
after.scheduler.start();
await armed(after, 1);
// Held back by the row's `lastAttemptAt` rather than attempting at once.
expect(after.httpRequest).not.toHaveBeenCalled();
// Three of the five minutes remain, less the fake clock's drift.
const [remainingWait] = after.scheduleNext.mock.calls[0];
const expectedWait = RETRY_DELAY_MS - 2 * Time.minutes.toMilliseconds;
expect(remainingWait).toBeGreaterThanOrEqual(expectedWait - CLOCK_TOLERANCE_MS);
expect(remainingWait).toBeLessThanOrEqual(expectedWait);
await vi.advanceTimersByTimeAsync(remainingWait);
await armed(after, 2);
expect(after.httpRequest).toHaveBeenCalledTimes(1);
expect(sentPayload(after, 0).batchId).toBe(report.id);
await expect(repository.findOneByOrFail({ id: report.id })).resolves.toMatchObject({
status: 'delivered',
attempts: 2,
});
});
test('resumes a pending report across the UTC midnight boundary', async () => {
await setReportTime('23:56');
vi.setSystemTime(new Date('2026-03-27T00:01:00.000Z'));
// The 23:56 slot on 03-26 made this row. Its first attempt failed.
const seeded = await seedPendingReport('2026-03-25', {
attempts: 1,
lastAttemptAt: new Date('2026-03-26T23:56:00.000Z'),
createdAt: new Date('2026-03-26T23:56:00.000Z'),
});
const harness = makeHarness([accepted()]);
harness.scheduler.start();
await armed(harness, 1);
// The retry after midnight resends the same row. It does not make a new row.
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
const payload = sentPayload(harness, 0);
expect(payload.batchId).toBe(seeded.id);
expect(dailyPoints(payload)).toEqual([{ date: '2026-03-25', value: 5 }]);
await expect(repository.findOneByOrFail({ id: seeded.id })).resolves.toMatchObject({
status: 'delivered',
attempts: 2,
});
await expect(repository.count()).resolves.toBe(1);
expectArmedFor(harness, '2026-03-27T23:56:00.000Z');
});
test('keeps retrying across the boundary, then backfills the day after it is skipped', async () => {
await setReportTime('23:56');
vi.setSystemTime(new Date('2026-03-27T00:01:00.000Z'));
await seedDeliveredReport('2026-03-24');
await seedDailyExecutions({ '2026-03-25': 5, '2026-03-26': 7 });
const seeded = await seedPendingReport('2026-03-25', {
attempts: 1,
lastAttemptAt: new Date('2026-03-26T23:56:00.000Z'),
createdAt: new Date('2026-03-26T23:56:00.000Z'),
});
const harness = makeHarness([unreachable(), unreachable(), accepted()]);
harness.scheduler.start();
await armed(harness, 1);
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 2);
// Two more attempts run after midnight. Then the row stops.
expect(harness.httpRequest).toHaveBeenCalledTimes(2);
await expect(repository.findOneByOrFail({ id: seeded.id })).resolves.toMatchObject({
status: 'skipped_after_max_retries',
attempts: MAX_ATTEMPTS,
});
// The next pass finds the day settled. It waits for the next slot.
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
await armed(harness, 3);
expectArmedFor(harness, '2026-03-27T23:56:00.000Z');
await vi.advanceTimersByTimeAsync(new Date('2026-03-27T23:56:00.000Z').getTime() - Date.now());
await armed(harness, 4);
expect(harness.httpRequest).toHaveBeenCalledTimes(3);
const backfill = sentPayload(harness, 2);
expect(backfill.batchId).not.toBe(seeded.id);
expect(dailyPoints(backfill)).toEqual([
{ date: '2026-03-25', value: 5 },
{ date: '2026-03-26', value: 7 },
]);
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-26');
// The skipped row never lands. So no day reaches the receiver two times.
const dates = await deliveredDailyDates();
expect(new Set(dates).size).toBe(dates.length);
});
test('does not resurrect an orphan already covered by a newer delivered report', async () => {
// A newer delivered report is above an older pending row that never delivered.
await seedDeliveredReport('2026-03-25');
const orphan = await seedPendingReport('2026-03-20', {
attempts: 1,
lastAttemptAt: new Date('2026-03-20T23:56:00.000Z'),
createdAt: new Date('2026-03-20T23:56:00.000Z'),
});
const harness = makeHarness([accepted()]);
harness.scheduler.start();
await armed(harness, 1);
// The latest row is delivered. So the code sends nothing and leaves the orphan.
expect(harness.httpRequest).not.toHaveBeenCalled();
await expect(repository.findOneByOrFail({ id: orphan.id })).resolves.toMatchObject({
status: 'pending',
});
const dates = await deliveredDailyDates();
expect(dates).not.toContain('2026-03-20');
});
test('expires a pending row after days of downtime, even before the slot, then backfills', async () => {
await setReportTime('23:58');
// A delivered report for 03-23, so the gap has a start.
await seedDeliveredReport('2026-03-23');
await seedDailyExecutions({
'2026-03-24': 4,
'2026-03-25': 6,
'2026-03-26': 8,
'2026-03-27': 10,
'2026-03-28': 12,
});
// The 03-25 23:58 slot made this row for 03-24, then delivery failed.
const stale = await seedPendingReport('2026-03-24', {
attempts: 2,
lastAttemptAt: new Date('2026-03-26T00:03:00.000Z'),
createdAt: new Date('2026-03-25T23:58:00.000Z'),
});
// Back up four days later, and before tonight's 23:58 slot.
vi.setSystemTime(new Date('2026-03-29T20:59:00.000Z'));
const harness = makeHarness([accepted()]);
harness.scheduler.start();
await armed(harness, 1);
// Four days is past its own next slot, so it is given up, not resent — even
// though the time of day is still before tonight's slot.
expect(harness.httpRequest).not.toHaveBeenCalled();
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
status: 'skipped_after_max_retries',
});
expectArmedFor(harness, '2026-03-29T23:58:00.000Z');
// Tonight's slot creates one fresh report for the whole gap.
await vi.advanceTimersByTimeAsync(new Date('2026-03-29T23:58:00.000Z').getTime() - Date.now());
await armed(harness, 2);
const backfill = sentPayload(harness, 0);
expect(backfill.batchId).not.toBe(stale.id);
expect(dailyPoints(backfill)).toEqual([
{ date: '2026-03-24', value: 4 },
{ date: '2026-03-25', value: 6 },
{ date: '2026-03-26', value: 8 },
{ date: '2026-03-27', value: 10 },
{ date: '2026-03-28', value: 12 },
]);
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-28');
const dates = await deliveredDailyDates();
expect(new Set(dates).size).toBe(dates.length);
});
test('caps the retry to the slot, so a failure at the end of the day backfills the same night', async () => {
await setReportTime('23:58');
await seedDeliveredReport('2026-03-23');
await seedDailyExecutions({ '2026-03-24': 4, '2026-03-25': 6 });
// The 03-25 23:58 slot made this row for 03-24, its first attempt failed there,
// then the instance was down for almost a day.
const stale = await seedPendingReport('2026-03-24', {
attempts: 1,
lastAttemptAt: new Date('2026-03-25T23:58:00.000Z'),
createdAt: new Date('2026-03-25T23:58:00.000Z'),
});
// Back up two minutes before tonight's slot, at the very end of the UTC day.
vi.setSystemTime(new Date('2026-03-26T23:56:00.000Z'));
const harness = makeHarness([unreachable(), accepted()]);
harness.scheduler.start();
await armed(harness, 1);
// The resume fails; the next attempt is capped to the slot, two minutes away,
// rather than a full five minutes that would cross midnight.
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
expect(sentPayload(harness, 0).batchId).toBe(stale.id);
// Two minutes later, at the slot, the row is given up and a fresh report goes
// out the same night — not deferred to the next day's slot.
await vi.advanceTimersByTimeAsync(2 * Time.minutes.toMilliseconds);
await armed(harness, 2);
expect(harness.httpRequest).toHaveBeenCalledTimes(2);
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
status: 'skipped_after_max_retries',
});
const backfill = sentPayload(harness, 1);
expect(backfill.batchId).not.toBe(stale.id);
expect(dailyPoints(backfill)).toEqual([
{ date: '2026-03-24', value: 4 },
{ date: '2026-03-25', value: 6 },
]);
expectArmedFor(harness, '2026-03-27T23:58:00.000Z');
});
test('skips a stale row and sends a fresh report in the same pass, after the slot', async () => {
await setReportTime('23:58');
await seedDeliveredReport('2026-03-23');
await seedDailyExecutions({ '2026-03-24': 4, '2026-03-25': 6, '2026-03-26': 8 });
// A pending row for 03-24 from the 03-25 slot, left over from downtime.
const stale = await seedPendingReport('2026-03-24', {
attempts: 2,
lastAttemptAt: new Date('2026-03-26T00:03:00.000Z'),
createdAt: new Date('2026-03-25T23:58:00.000Z'),
});
// Back up after tonight's slot, so one pass both skips and sends.
vi.setSystemTime(new Date('2026-03-27T23:59:00.000Z'));
const harness = makeHarness([accepted()]);
harness.scheduler.start();
await armed(harness, 1);
// The first pass gives up the stale row and creates the new report at once.
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
status: 'skipped_after_max_retries',
});
const backfill = sentPayload(harness, 0);
expect(backfill.batchId).not.toBe(stale.id);
expect(dailyPoints(backfill)).toEqual([
{ date: '2026-03-24', value: 4 },
{ date: '2026-03-25', value: 6 },
{ date: '2026-03-26', value: 8 },
]);
const dates = await deliveredDailyDates();
expect(new Set(dates).size).toBe(dates.length);
});
});

View File

@ -8,6 +8,8 @@ import { Container } from '@n8n/di';
import { UserError } from 'n8n-workflow'; import { UserError } from 'n8n-workflow';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import { License } from '@/license';
import { InstanceMonitoringReportRepository } from '../database/repositories/instance-monitoring-report.repository'; import { InstanceMonitoringReportRepository } from '../database/repositories/instance-monitoring-report.repository';
import { InstanceReportingScheduler } from '../instance-reporting-scheduler.service'; import { InstanceReportingScheduler } from '../instance-reporting-scheduler.service';
import { InstanceReportingSettingsService } from '../instance-reporting-settings.service'; import { InstanceReportingSettingsService } from '../instance-reporting-settings.service';
@ -24,13 +26,20 @@ const LAST_DELIVERY = new Date('2026-03-25T07:42:13.000Z');
function setUpContainer({ function setUpContainer({
baseUrl = 'https://example.com', baseUrl = 'https://example.com',
licenseCert = 'base64-license-cert',
authToken = '',
disabledModules = [] as ModuleName[], disabledModules = [] as ModuleName[],
lastDelivery = LAST_DELIVERY as Date | null, lastDelivery = LAST_DELIVERY as Date | null,
} = {}) { } = {}) {
const config = new InstanceReportingConfig(); const config = new InstanceReportingConfig();
config.instanceReportingBaseUrl = baseUrl; config.instanceReportingBaseUrl = baseUrl;
config.instanceReportingAuthToken = authToken;
Container.set(InstanceReportingConfig, config); Container.set(InstanceReportingConfig, config);
const license = mock<License>();
license.loadCertStr.mockResolvedValue(licenseCert);
Container.set(License, license);
Container.set(ModulesConfig, mock<ModulesConfig>({ disabledModules })); Container.set(ModulesConfig, mock<ModulesConfig>({ disabledModules }));
Container.set(Logger, mockLogger()); Container.set(Logger, mockLogger());
@ -67,6 +76,26 @@ describe('InstanceReportingModule', () => {
expect(settingsService.getReportTime).not.toHaveBeenCalled(); expect(settingsService.getReportTime).not.toHaveBeenCalled();
expect(reportRepository.findLastDeliveryTime).not.toHaveBeenCalled(); expect(reportRepository.findLastDeliveryTime).not.toHaveBeenCalled();
}); });
// The certificate is the credential, so without one nothing would ever be
// accepted and no report time is claimed.
it('reports as disabled without a license certificate', async () => {
const { settingsService } = setUpContainer({ licenseCert: '' });
const settings = await new InstanceReportingModule().settings();
expect(settings).toEqual({ enabled: false });
expect(settingsService.getReportTime).not.toHaveBeenCalled();
});
// A token is a credential on its own, so the certificate is not needed.
it('reports as enabled with an auth token but no license certificate', async () => {
setUpContainer({ licenseCert: '', authToken: 'secret-token' });
const settings = await new InstanceReportingModule().settings();
expect(settings).toEqual({ enabled: true, reportTime: REPORT_TIME });
});
}); });
describe('init()', () => { describe('init()', () => {
@ -92,6 +121,22 @@ describe('InstanceReportingModule', () => {
expect(scheduler.init).not.toHaveBeenCalled(); expect(scheduler.init).not.toHaveBeenCalled();
}); });
it('leaves the scheduler alone when the instance has no license certificate', async () => {
const { scheduler } = setUpContainer({ licenseCert: '' });
await new InstanceReportingModule().init();
expect(scheduler.init).not.toHaveBeenCalled();
});
it('starts the scheduler with an auth token but no license certificate', async () => {
const { scheduler } = setUpContainer({ licenseCert: '', authToken: 'secret-token' });
await new InstanceReportingModule().init();
expect(scheduler.init).toHaveBeenCalled();
});
// The route belongs to the loaded module, not to the receiver, so a client // The route belongs to the loaded module, not to the receiver, so a client
// that asks anyway gets an answer instead of a 404. // that asks anyway gets an answer instead of a 404.
it('registers the status route even when no receiver is configured', async () => { it('registers the status route even when no receiver is configured', async () => {

View File

@ -12,6 +12,7 @@ import type { Mocked } from 'vitest';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import type { EventService } from '@/events/event.service'; import type { EventService } from '@/events/event.service';
import type { License } from '@/license';
import type { InsightsService } from '@/modules/insights/insights.service'; import type { InsightsService } from '@/modules/insights/insights.service';
import type { OwnershipService } from '@/services/ownership.service'; import type { OwnershipService } from '@/services/ownership.service';
@ -31,10 +32,13 @@ interface ReportPayload {
label?: string; label?: string;
n8nVersion: string; n8nVersion: string;
dataPoints: Array<{ kind: string; name: string; value: number; date?: string }>; dataPoints: Array<{ kind: string; name: string; value: number; date?: string }>;
licenseCert?: string;
} }
const REPORT_DATE = '2026-03-25'; const REPORT_DATE = '2026-03-25';
const BATCH_ID = 'batch-id-1'; const BATCH_ID = 'batch-id-1';
/** Opaque to this module: whatever `License.loadCertStr()` returns is sent as is. */
const LICENSE_CERT = 'base64-license-cert';
const OWNER_MOCK = mock<User>({ id: 'owner-id' }); const OWNER_MOCK = mock<User>({ id: 'owner-id' });
@ -91,6 +95,7 @@ interface Harness {
service: InstanceReportingService; service: InstanceReportingService;
reportRepository: Mocked<InstanceMonitoringReportRepository>; reportRepository: Mocked<InstanceMonitoringReportRepository>;
insightsService: Mocked<InsightsService>; insightsService: Mocked<InsightsService>;
license: Mocked<License>;
http: HttpRequestClient; http: HttpRequestClient;
eventService: Mocked<EventService>; eventService: Mocked<EventService>;
clientOptions: HttpRequestClientOptions | undefined; clientOptions: HttpRequestClientOptions | undefined;
@ -98,7 +103,7 @@ interface Harness {
function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness { function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness {
const reportRepository = mock<InstanceMonitoringReportRepository>(); const reportRepository = mock<InstanceMonitoringReportRepository>();
reportRepository.findTodaysPending.mockResolvedValue(null); reportRepository.findPending.mockResolvedValue(null);
// A report already covers the day before the one under test, so the default // A report already covers the day before the one under test, so the default
// harness reports exactly one day. // harness reports exactly one day.
reportRepository.findLastCoveredDay.mockResolvedValue('2026-03-24'); reportRepository.findLastCoveredDay.mockResolvedValue('2026-03-24');
@ -115,6 +120,9 @@ function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness {
const licenseMetricsRepository = mock<LicenseMetricsRepository>(); const licenseMetricsRepository = mock<LicenseMetricsRepository>();
licenseMetricsRepository.getLicenseRenewalMetrics.mockResolvedValue(LICENSE_METRICS_MOCK); licenseMetricsRepository.getLicenseRenewalMetrics.mockResolvedValue(LICENSE_METRICS_MOCK);
const license = mock<License>();
license.loadCertStr.mockResolvedValue(LICENSE_CERT);
const http = mock<HttpRequestClient>(); const http = mock<HttpRequestClient>();
vi.mocked(http.request).mockResolvedValue({ statusCode: 201, body: '', headers: {} }); vi.mocked(http.request).mockResolvedValue({ statusCode: 201, body: '', headers: {} });
@ -135,12 +143,13 @@ function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness {
mock<InstanceSettings>({ instanceId: 'abc123' }), mock<InstanceSettings>({ instanceId: 'abc123' }),
ownershipService, ownershipService,
licenseMetricsRepository, licenseMetricsRepository,
license,
mockLogger(), mockLogger(),
eventService, eventService,
outboundHttp, outboundHttp,
); );
return { service, reportRepository, insightsService, http, eventService, clientOptions }; return { service, reportRepository, insightsService, license, http, eventService, clientOptions };
} }
describe('InstanceReportingService', () => { describe('InstanceReportingService', () => {
@ -206,7 +215,7 @@ describe('InstanceReportingService', () => {
expect(clientOptions?.timeout).toBe(30_000); expect(clientOptions?.timeout).toBe(30_000);
}); });
test('does not follow redirects, so the auth token reaches only the configured host', async () => { test('does not follow redirects, so the credential reaches only the configured host', async () => {
const { service, http } = makeHarness(); const { service, http } = makeHarness();
await service.sendReport(); await service.sendReport();
@ -239,19 +248,68 @@ describe('InstanceReportingService', () => {
expect(body(http)).not.toHaveProperty('label'); expect(body(http)).not.toHaveProperty('label');
}); });
test('sends the auth token as a bearer token when configured', () => { test('sends the license certificate in the body as the credential', async () => {
const { clientOptions } = makeHarness( const { service, http, clientOptions } = makeHarness();
makeConfig({ instanceReportingAuthToken: 'secret-token' }),
);
expect(clientHeaders(clientOptions).authorization).toBe('Bearer secret-token'); await service.sendReport();
expect(body(http).licenseCert).toBe(LICENSE_CERT);
// No token: the certificate is the whole credential, and an `undefined`
// default header is dropped before the request goes out.
expect(clientHeaders(clientOptions).authorization).toBeUndefined();
}); });
test('omits the Authorization header when no auth token is configured', () => { test('reads the certificate fresh for every report, so a renewed license is sent', async () => {
const { clientOptions } = makeHarness(); const { service, license, http, reportRepository } = makeHarness();
// An `undefined` default header is dropped before the request goes out. await service.sendReport();
expect(clientHeaders(clientOptions).authorization).toBeUndefined(); license.loadCertStr.mockResolvedValue('renewed-license-cert');
reportRepository.findLastCoveredDay.mockResolvedValue('2026-03-23');
await service.sendReport();
expect(body(http, 1).licenseCert).toBe('renewed-license-cert');
});
test('sends nothing and measures nothing without a license certificate', async () => {
const { service, license, http, reportRepository, insightsService } = makeHarness();
license.loadCertStr.mockResolvedValue('');
// Resolves rather than throws: this is not a delivery failure to retry.
await expect(service.sendReport()).resolves.toBeUndefined();
expect(http.request).not.toHaveBeenCalled();
expect(insightsService.getInsightsByTime).not.toHaveBeenCalled();
expect(reportRepository.createPending).not.toHaveBeenCalled();
});
describe('with an auth token configured', () => {
const tokenConfig = () => makeConfig({ instanceReportingAuthToken: 'secret-token' });
test('sends the token as a bearer token', () => {
const { clientOptions } = makeHarness(tokenConfig());
expect(clientHeaders(clientOptions).authorization).toBe('Bearer secret-token');
});
test('does not send or read the license certificate', async () => {
const { service, http, license } = makeHarness(tokenConfig());
await service.sendReport();
expect(body(http)).not.toHaveProperty('licenseCert');
expect(license.loadCertStr).not.toHaveBeenCalled();
});
// The token is the whole credential, so an unlicensed instance still reports.
test('reports without a license certificate', async () => {
const { service, http, license } = makeHarness(tokenConfig());
license.loadCertStr.mockResolvedValue('');
await service.sendReport();
expect(http.request).toHaveBeenCalledTimes(1);
});
}); });
test('queries the instance owner insights for the reported UTC day', async () => { test('queries the instance owner insights for the reported UTC day', async () => {
@ -366,7 +424,7 @@ describe('InstanceReportingService', () => {
await expect(service.sendReport()).rejects.toThrow(); await expect(service.sendReport()).rejects.toThrow();
// The retry picks up the still-undelivered row the first attempt created. // The retry picks up the still-undelivered row the first attempt created.
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 1 })); reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 1 }));
await service.sendReport(); await service.sendReport();
expect(body(http, 1).batchId).toBe(BATCH_ID); expect(body(http, 1).batchId).toBe(BATCH_ID);
@ -379,7 +437,7 @@ describe('InstanceReportingService', () => {
{ kind: 'cumulative', name: 'billableExecutions', value: 800 }, { kind: 'cumulative', name: 'billableExecutions', value: 800 },
{ kind: 'daily', name: 'billableExecutions', value: 40, date: REPORT_DATE }, { kind: 'daily', name: 'billableExecutions', value: 40, date: REPORT_DATE },
] as InstanceMonitoringReport['dataPoints']; ] as InstanceMonitoringReport['dataPoints'];
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ dataPoints: measured })); reportRepository.findPending.mockResolvedValue(makeReport({ dataPoints: measured }));
await service.sendReport(); await service.sendReport();
@ -405,7 +463,7 @@ describe('InstanceReportingService', () => {
test('leaves the report pending while attempts remain', async () => { test('leaves the report pending while attempts remain', async () => {
const { service, reportRepository, http } = makeHarness(); const { service, reportRepository, http } = makeHarness();
vi.mocked(http.request).mockRejectedValue(new Error('Network error')); vi.mocked(http.request).mockRejectedValue(new Error('Network error'));
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 1 })); reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 1 }));
await expect(service.sendReport()).rejects.toThrow(); await expect(service.sendReport()).rejects.toThrow();
@ -416,7 +474,7 @@ describe('InstanceReportingService', () => {
const { service, reportRepository, http } = makeHarness(); const { service, reportRepository, http } = makeHarness();
vi.mocked(http.request).mockRejectedValue(new Error('Network error')); vi.mocked(http.request).mockRejectedValue(new Error('Network error'));
// Two attempts already recorded on the row, so this one is the last. // Two attempts already recorded on the row, so this one is the last.
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 2 })); reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 2 }));
await expect(service.sendReport()).rejects.toThrow(); await expect(service.sendReport()).rejects.toThrow();
@ -426,7 +484,7 @@ describe('InstanceReportingService', () => {
test('skips an exhausted row without attempting again', async () => { test('skips an exhausted row without attempting again', async () => {
const { service, reportRepository, http } = makeHarness(); const { service, reportRepository, http } = makeHarness();
// A crash between the failure record and the skip leaves this row pending. // A crash between the failure record and the skip leaves this row pending.
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 3 })); reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 3 }));
await expect(service.sendReport()).resolves.toBeUndefined(); await expect(service.sendReport()).resolves.toBeUndefined();
@ -441,21 +499,21 @@ describe('InstanceReportingService', () => {
test('allows an attempt when no report is pending', async () => { test('allows an attempt when no report is pending', async () => {
const { service, reportRepository } = makeHarness(); const { service, reportRepository } = makeHarness();
reportRepository.findTodaysPending.mockResolvedValue(null); reportRepository.findPending.mockResolvedValue(null);
await expect(service.msUntilRetryAllowed(now)).resolves.toBe(0); await expect(service.msUntilRetryAllowed(now)).resolves.toBe(0);
}); });
test('allows the first attempt, which has nothing to wait for', async () => { test('allows the first attempt, which has nothing to wait for', async () => {
const { service, reportRepository } = makeHarness(); const { service, reportRepository } = makeHarness();
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ lastAttemptAt: null })); reportRepository.findPending.mockResolvedValue(makeReport({ lastAttemptAt: null }));
await expect(service.msUntilRetryAllowed(now)).resolves.toBe(0); await expect(service.msUntilRetryAllowed(now)).resolves.toBe(0);
}); });
test('returns the remaining wait when the last attempt was recent', async () => { test('returns the remaining wait when the last attempt was recent', async () => {
const { service, reportRepository } = makeHarness(); const { service, reportRepository } = makeHarness();
reportRepository.findTodaysPending.mockResolvedValue( reportRepository.findPending.mockResolvedValue(
makeReport({ lastAttemptAt: new Date('2026-03-26T07:40:00.000Z') }), makeReport({ lastAttemptAt: new Date('2026-03-26T07:40:00.000Z') }),
); );
@ -465,7 +523,7 @@ describe('InstanceReportingService', () => {
test('allows an attempt once the wait has passed', async () => { test('allows an attempt once the wait has passed', async () => {
const { service, reportRepository } = makeHarness(); const { service, reportRepository } = makeHarness();
reportRepository.findTodaysPending.mockResolvedValue( reportRepository.findPending.mockResolvedValue(
makeReport({ lastAttemptAt: new Date('2026-03-26T07:30:00.000Z') }), makeReport({ lastAttemptAt: new Date('2026-03-26T07:30:00.000Z') }),
); );

View File

@ -42,45 +42,52 @@ describe('InstanceMonitoringReportRepository', () => {
}); });
}); });
describe('findTodaysPending', () => { describe('findPending', () => {
async function createdOn(date: string) {
const report = await repository.createPending(DATA_POINTS);
await repository.update({ id: report.id }, { createdAt: new Date(date) });
return report;
}
test('returns nothing when no report was generated yet', async () => { test('returns nothing when no report was generated yet', async () => {
await expect(repository.findTodaysPending(new Date())).resolves.toBeNull(); await expect(repository.findPending()).resolves.toBeNull();
}); });
test("returns today's undelivered report with the numbers it measured", async () => { test('returns the pending report with the numbers it measured', async () => {
const created = await repository.createPending(DATA_POINTS); const created = await repository.createPending(DATA_POINTS);
const pending = await repository.findTodaysPending(new Date()); const pending = await repository.findPending();
expect(pending?.id).toBe(created.id); expect(pending?.id).toBe(created.id);
expect(pending?.status).toBe('pending');
expect(pending?.dataPoints).toEqual(DATA_POINTS); expect(pending?.dataPoints).toEqual(DATA_POINTS);
}); });
test("returns nothing once today's report was delivered", async () => { test('returns nothing once the newest report is delivered', async () => {
const created = await repository.createPending(DATA_POINTS); const created = await repository.createPending(DATA_POINTS);
await repository.markDelivered(created.id, new Date()); await repository.markDelivered(created.id, new Date());
await expect(repository.findTodaysPending(new Date())).resolves.toBeNull(); await expect(repository.findPending()).resolves.toBeNull();
}); });
test("ignores an earlier day's undelivered report, leaving it untouched", async () => { test('returns nothing when an older pending row sits under a newer delivered one', async () => {
const stale = await repository.createPending(DATA_POINTS); // Earlier versions left a failed report pending for good. The newer report
// already covers that orphan's days, so a resend would report them two
// times. Filtering on `status` in the query would return the orphan here.
await createdOn('2026-03-20T07:42:00.000Z');
const newer = await createdOn('2026-03-25T07:42:00.000Z');
await repository.markDelivered(newer.id, new Date());
const tomorrow = new Date(Date.now() + 24 * 60 * 60 * 1000); await expect(repository.findPending()).resolves.toBeNull();
await expect(repository.findTodaysPending(tomorrow)).resolves.toBeNull();
// The stale report keeps the numbers it measured; only backfill may touch it.
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
dataPoints: DATA_POINTS,
deliveredAt: null,
});
}); });
test('returns nothing once the report ran out of attempts, so it is not resent', async () => { test('returns the newest row when it is pending, ignoring an older delivered one', async () => {
const created = await repository.createPending(DATA_POINTS); const older = await createdOn('2026-03-20T07:42:00.000Z');
await repository.markSkipped(created.id); await repository.markDelivered(older.id, new Date());
const newer = await createdOn('2026-03-25T07:42:00.000Z');
await expect(repository.findTodaysPending(new Date())).resolves.toBeNull(); await expect(repository.findPending()).resolves.toMatchObject({ id: newer.id });
}); });
test('carries the last attempt time, so the wait between attempts survives a restart', async () => { test('carries the last attempt time, so the wait between attempts survives a restart', async () => {
@ -88,7 +95,7 @@ describe('InstanceMonitoringReportRepository', () => {
const failedAt = new Date('2026-03-26T07:42:00.000Z'); const failedAt = new Date('2026-03-26T07:42:00.000Z');
await repository.recordFailure(created.id, 'Network error', failedAt); await repository.recordFailure(created.id, 'Network error', failedAt);
const pending = await repository.findTodaysPending(new Date()); const pending = await repository.findPending();
expect(pending?.lastAttemptAt?.toISOString()).toBe(failedAt.toISOString()); expect(pending?.lastAttemptAt?.toISOString()).toBe(failedAt.toISOString());
}); });

View File

@ -12,24 +12,25 @@ export class InstanceMonitoringReportRepository extends Repository<InstanceMonit
} }
/** /**
* Today's report, if one was already generated and never reached the receiver. * The report a retry may resume, or `null` when there is nothing to resume.
* It is the newest row, and only if that row is still `pending`.
* *
* Resending it — same `batchId`, same data points — is what keeps a retry from * At most one report is genuinely pending: the scheduler settles a stale row
* re-measuring: the cumulative series is only comparable day to day while its * before it creates the next one. But earlier versions left a failed report
* sampling interval stays a fixed 24 hours. * pending for good, so an instance can still hold an old `pending` row below
* a newer `delivered` one. That row is an orphan — the newer report already
* covers its days, and a resend would report those days two times.
* *
* Scoped to `now`'s UTC day on purpose: an older undelivered report measured a * So this reads the newest row of any status and then checks that status. It
* different day, so it must not stand in for today's. It stays as it is, a * must not filter on `status` in the query.
* record of a report that never landed, and its days are covered again by the
* next report.
* *
* A report that ran out of attempts is not pending, so it is never resent. * `id` breaks a `createdAt` tie so the result is stable. A tie needs two
* reports in the same millisecond, which the once-a-day cadence never makes.
*/ */
async findTodaysPending(now: Date): Promise<InstanceMonitoringReport | null> { async findPending(): Promise<InstanceMonitoringReport | null> {
return await this.findOne({ const [latest] = await this.find({ order: { createdAt: 'DESC', id: 'DESC' }, take: 1 });
where: { status: 'pending', createdAt: MoreThanOrEqual(startOfUtcDay(now)) },
order: { createdAt: 'DESC' }, return latest?.status === 'pending' ? latest : null;
});
} }
/** /**

View File

@ -7,6 +7,7 @@ import { strict } from 'node:assert';
import { EventService } from '@/events/event.service'; import { EventService } from '@/events/event.service';
import type { InstanceMonitoringReport } from './database/entities/instance-monitoring-report';
import { InstanceMonitoringReportRepository } from './database/repositories/instance-monitoring-report.repository'; import { InstanceMonitoringReportRepository } from './database/repositories/instance-monitoring-report.repository';
import { InstanceReportingSettingsService } from './instance-reporting-settings.service'; import { InstanceReportingSettingsService } from './instance-reporting-settings.service';
import { InstanceReportingService, RETRY_DELAY_MS } from './instance-reporting.service'; import { InstanceReportingService, RETRY_DELAY_MS } from './instance-reporting.service';
@ -112,9 +113,14 @@ export class InstanceReportingScheduler {
try { try {
const reportTime = await this.settingsService.getReportTime(); const reportTime = await this.settingsService.getReportTime();
// An attempt too soon after the last one arms for the remainder rather // Wait out the gap between retries, but never past the pending row's own
// than falling through, which would sleep until tomorrow and drop the retry. // next slot — at that slot it is skipped, not retried. A wait that reaches
const waitMs = await this.reportingService.msUntilRetryAllowed(new Date()); // the slot collapses to zero, so this pass falls through to reportIfDue,
// which skips the stale row and sends a fresh report.
const waitMs = Math.min(
await this.reportingService.msUntilRetryAllowed(new Date()),
await this.msUntilNextSlot(reportTime),
);
if (waitMs > 0) { if (waitMs > 0) {
this.scheduleNext(waitMs); this.scheduleNext(waitMs);
return; return;
@ -123,7 +129,7 @@ export class InstanceReportingScheduler {
// A failed delivery retries; the row decides when the attempts run out, // A failed delivery retries; the row decides when the attempts run out,
// after which the day reads as settled and this waits for the next slot. // after which the day reads as settled and this waits for the next slot.
if ((await this.reportIfDue(reportTime)) === 'failed') { if ((await this.reportIfDue(reportTime)) === 'failed') {
this.scheduleNext(RETRY_DELAY_MS); this.scheduleNext(Math.min(RETRY_DELAY_MS, await this.msUntilNextSlot(reportTime)));
return; return;
} }
@ -137,16 +143,49 @@ export class InstanceReportingScheduler {
} }
/** /**
* Report when this day's slot has passed and the day is not settled yet. Both * Send the report when it is due.
* conditions are re-checked here rather than inferred from the timer having *
* fired, so an early fire (a backward clock jump) reports nothing and a * A pending report is a retry. Resend it now, also after midnight; it holds
* duplicate fire is a no-op. * frozen data, so the slot does not apply to it. But once its own next slot
* has passed, its cycle is over: stop trying and let a new report cover its
* day. A new report waits for the slot, which makes sure the day is complete
* before the code measures it.
*/ */
private async reportIfDue(reportTime: string): Promise<'sent' | 'skipped' | 'failed'> { private async reportIfDue(reportTime: string): Promise<'sent' | 'skipped' | 'failed'> {
const now = new Date(); const now = new Date();
const pending = await this.reportRepository.findPending();
if (pending) {
if (pendingIsStale(pending, reportTime, now)) {
await this.reportingService.skip(pending.id, pending.attempts, 'slot-passed');
} else {
return await this.trySend();
}
}
// A new report waits for the slot and runs only when the day is not settled.
if (now.getTime() < slotOn(reportTime, now)) return 'skipped'; if (now.getTime() < slotOn(reportTime, now)) return 'skipped';
if (await this.reportRepository.hasSettledToday(now)) return 'skipped'; if (await this.reportRepository.hasSettledToday(now)) return 'skipped';
return await this.trySend();
}
/**
* Milliseconds until the pending row's own next slot, or `Infinity` when no row
* is pending. Capping a retry sleep with this keeps the timer from sleeping past
* the slot, where the row is skipped and a fresh report takes over — otherwise a
* slot near the end of the UTC day would defer the next report by almost a day.
*/
private async msUntilNextSlot(reportTime: string): Promise<number> {
const pending = await this.reportRepository.findPending();
if (!pending) return Number.POSITIVE_INFINITY;
const nextSlot =
slotOn(reportTime, pending.createdAt) + MINUTES_PER_DAY * Time.minutes.toMilliseconds;
return nextSlot - Date.now();
}
private async trySend(): Promise<'sent' | 'failed'> {
try { try {
await this.reportingService.sendReport(); await this.reportingService.sendReport();
return 'sent'; return 'sent';
@ -163,6 +202,17 @@ export class InstanceReportingScheduler {
} }
} }
/**
* Whether a pending row sat unsent past its own next slot. Its cycle is then
* over, so a new report covers its day instead of a late resend.
*/
function pendingIsStale(report: InstanceMonitoringReport, reportTime: string, now: Date): boolean {
const nextSlot =
slotOn(reportTime, report.createdAt) + MINUTES_PER_DAY * Time.minutes.toMilliseconds;
return now.getTime() >= nextSlot;
}
/** Epoch ms of `reportTime` on `now`'s UTC day. */ /** Epoch ms of `reportTime` on `now`'s UTC day. */
function slotOn(reportTime: string, now: Date): number { function slotOn(reportTime: string, now: Date): number {
const [hour, minute] = reportTime.split(':').map(Number); const [hour, minute] = reportTime.split(':').map(Number);

View File

@ -18,7 +18,8 @@ export class InstanceReportingConfig {
instanceReportingBaseUrl: string = ''; instanceReportingBaseUrl: string = '';
/** /**
* Sent as a bearer token; leave unset if the receiver does not require one. * Optional. When set, it is sent as `Authorization: Bearer …` and the license
* certificate is not sent. Leave unset to authenticate with the certificate.
* *
* @beta - breaking changes may still occur * @beta - breaking changes may still occur
*/ */

View File

@ -30,12 +30,19 @@ export class InstanceReportingModule implements ModuleInterface {
); );
} }
if (!(await this.isConfigured())) { const logger = Container.get(Logger).scoped('instance-reporting');
Container.get(Logger)
.scoped('instance-reporting') if (!(await this.hasReceiver())) {
.warn( logger.warn(
'Instance reporting is enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset, so no reports will be sent', 'Instance reporting is enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset, so no reports will be sent',
); );
return;
}
if (!(await this.hasCredential())) {
logger.warn(
'Instance reporting is enabled but this instance has no license certificate, so no reports will be sent. The receiver accepts reports only from licensed instances. Set N8N_LICENSE_CERT, activate a license, or set N8N_INSTANCE_REPORTING_AUTH_TOKEN.',
);
return; return;
} }
@ -50,8 +57,8 @@ export class InstanceReportingModule implements ModuleInterface {
* Settings exposed to the frontend under `/rest/module-settings`. * Settings exposed to the frontend under `/rest/module-settings`.
* *
* Return values: * Return values:
* { enabled: false } - module loaded but no receiver configured * { enabled: false } - module loaded but no receiver configured, or neither an auth token nor a license certificate
* { enabled: true, reportTime: 'HH:mm' } - module loaded and receiver configured * { enabled: true, reportTime: 'HH:mm' } - module loaded, receiver configured, credential present
* *
* Built once at startup and cached for the process lifetime. * Built once at startup and cached for the process lifetime.
**/ **/
@ -75,10 +82,28 @@ export class InstanceReportingModule implements ModuleInterface {
return [InstanceMonitoringReport]; return [InstanceMonitoringReport];
} }
/** Whether a receiver is configured, i.e. whether reports are actually sent. */ /** Whether reports are actually sent: a receiver is configured and there is a credential to authenticate with. */
private async isConfigured(): Promise<boolean> { private async isConfigured(): Promise<boolean> {
return (await this.hasReceiver()) && (await this.hasCredential());
}
private async hasReceiver(): Promise<boolean> {
const { InstanceReportingConfig } = await import('./instance-reporting.config.js'); const { InstanceReportingConfig } = await import('./instance-reporting.config.js');
return Container.get(InstanceReportingConfig).instanceReportingBaseUrl !== ''; return Container.get(InstanceReportingConfig).instanceReportingBaseUrl !== '';
} }
/**
* A configured auth token is a credential on its own. Without one, the
* license certificate is the credential the receiver checks, so an
* unlicensed (community) instance cannot report.
*/
private async hasCredential(): Promise<boolean> {
const { InstanceReportingConfig } = await import('./instance-reporting.config.js');
if (Container.get(InstanceReportingConfig).instanceReportingAuthToken !== '') return true;
const { License } = await import('@/license.js');
return (await Container.get(License).loadCertStr()) !== '';
}
} }

View File

@ -9,6 +9,7 @@ import { OperationalError } from 'n8n-workflow';
import { N8N_VERSION } from '@/constants'; import { N8N_VERSION } from '@/constants';
import { EventService } from '@/events/event.service'; import { EventService } from '@/events/event.service';
import { License } from '@/license';
import { InsightsService } from '@/modules/insights/insights.service'; import { InsightsService } from '@/modules/insights/insights.service';
import { OwnershipService } from '@/services/ownership.service'; import { OwnershipService } from '@/services/ownership.service';
@ -55,6 +56,7 @@ export class InstanceReportingService {
private readonly instanceSettings: InstanceSettings, private readonly instanceSettings: InstanceSettings,
private readonly ownershipService: OwnershipService, private readonly ownershipService: OwnershipService,
private readonly licenseMetricsRepository: LicenseMetricsRepository, private readonly licenseMetricsRepository: LicenseMetricsRepository,
private readonly license: License,
private readonly logger: Logger, private readonly logger: Logger,
private readonly eventService: EventService, private readonly eventService: EventService,
outboundHttp: OutboundHttp, outboundHttp: OutboundHttp,
@ -66,7 +68,7 @@ export class InstanceReportingService {
// collector, so the URL is never user-controlled. // collector, so the URL is never user-controlled.
useDefaultSsrfPolicy: 'unsafe', useDefaultSsrfPolicy: 'unsafe',
baseURL: this.config.instanceReportingBaseUrl.replace(/\/+$/, ''), baseURL: this.config.instanceReportingBaseUrl.replace(/\/+$/, ''),
// An unset token drops the header, so an unauthenticated receiver works. // An unset token drops the header; the license certificate is the credential then.
headers: () => ({ headers: () => ({
authorization: this.config.instanceReportingAuthToken authorization: this.config.instanceReportingAuthToken
? `Bearer ${this.config.instanceReportingAuthToken}` ? `Bearer ${this.config.instanceReportingAuthToken}`
@ -87,23 +89,37 @@ export class InstanceReportingService {
* `batchId`, so a redelivery reuses the row instead of measuring the day again, * `batchId`, so a redelivery reuses the row instead of measuring the day again,
* and only a delivered report crosses its days off. * and only a delivered report crosses its days off.
* *
* A retry resends today's pending report exactly as measured instead of taking * A retry resends the pending report exactly as measured instead of taking
* fresh numbers. The cumulative point is a lifetime total sampled at this * fresh numbers. The cumulative point is a lifetime total sampled at this
* instance's report time, so its day-to-day difference only lines up with the * instance's report time, so its day-to-day difference only lines up with the
* daily point while every sample sits 24 hours apart; re-measuring hours later * daily point while every sample sits 24 hours apart; re-measuring hours later
* would stretch one interval and skew the whole series. * would stretch one interval and skew the whole series.
* *
* The credential is the license certificate, sent in the body, unless a
* bearer token is configured; then the token goes in the header and the
* certificate is not sent at all.
*
* @throws when delivery fails, so the scheduler retries with backoff. * @throws when delivery fails, so the scheduler retries with backoff.
*/ */
async sendReport(): Promise<void> { async sendReport(): Promise<void> {
const licenseCert = this.config.instanceReportingAuthToken
? undefined
: await this.license.loadCertStr();
if (licenseCert === '') {
this.logger.warn(
'Skipping the instance report because this instance has no license certificate.',
);
return;
}
const now = new Date(); const now = new Date();
let report = await this.reportRepository.findTodaysPending(now); let report = await this.reportRepository.findPending();
// A crash between recording a failure and skipping the report leaves an // A crash between recording a failure and skipping the report leaves an
// exhausted row pending, so the budget is re-checked before sending rather // exhausted row pending, so the budget is re-checked before sending rather
// than only after. Settling it here also ends the day for the scheduler. // than only after. Settling it here also ends the day for the scheduler.
if (report && report.attempts >= MAX_ATTEMPTS) { if (report && report.attempts >= MAX_ATTEMPTS) {
await this.skip(report.id, report.attempts); await this.skip(report.id, report.attempts, 'max-retries');
return; return;
} }
@ -126,6 +142,7 @@ export class InstanceReportingService {
...(this.config.instanceReportingLabel ? { label: this.config.instanceReportingLabel } : {}), ...(this.config.instanceReportingLabel ? { label: this.config.instanceReportingLabel } : {}),
n8nVersion: N8N_VERSION, n8nVersion: N8N_VERSION,
dataPoints: report.dataPoints, dataPoints: report.dataPoints,
...(licenseCert ? { licenseCert } : {}),
}; };
try { try {
@ -137,7 +154,7 @@ export class InstanceReportingService {
returnFullResponse: true, returnFullResponse: true,
// Inspect the status here rather than catching a generic request error. // Inspect the status here rather than catching a generic request error.
ignoreHttpStatusErrors: true, ignoreHttpStatusErrors: true,
// A redirect would forward the auth token to whatever host it names. // A redirect would forward the credential to whatever host it names.
disableFollowRedirect: true, disableFollowRedirect: true,
}); });
@ -163,7 +180,7 @@ export class InstanceReportingService {
// `recordFailure` incremented the count, so the in-memory row is one behind. // `recordFailure` incremented the count, so the in-memory row is one behind.
if (report.attempts + 1 >= MAX_ATTEMPTS) { if (report.attempts + 1 >= MAX_ATTEMPTS) {
await this.skip(report.id, report.attempts + 1); await this.skip(report.id, report.attempts + 1, 'max-retries');
} }
throw error; throw error;
@ -174,12 +191,15 @@ export class InstanceReportingService {
} }
/** Stop trying to deliver this report; the next one covers its days again. */ /** Stop trying to deliver this report; the next one covers its days again. */
private async skip(id: string, attempts: number): Promise<void> { async skip(id: string, attempts: number, reason: 'max-retries' | 'slot-passed'): Promise<void> {
await this.reportRepository.markSkipped(id); await this.reportRepository.markSkipped(id);
this.logger.error('Giving up on the instance report after repeated delivery failures', {
batchId: id, const message =
attempts, reason === 'max-retries'
}); ? 'Giving up on the instance report after repeated delivery failures'
: 'Giving up on the instance report because its slot has passed';
this.logger.error(message, { batchId: id, attempts });
} }
/** /**
@ -191,7 +211,7 @@ export class InstanceReportingService {
* seconds. * seconds.
*/ */
async msUntilRetryAllowed(now: Date): Promise<number> { async msUntilRetryAllowed(now: Date): Promise<number> {
const pending = await this.reportRepository.findTodaysPending(now); const pending = await this.reportRepository.findPending();
if (!pending?.lastAttemptAt) return 0; if (!pending?.lastAttemptAt) return 0;
const elapsed = now.getTime() - pending.lastAttemptAt.getTime(); const elapsed = now.getTime() - pending.lastAttemptAt.getTime();

View File

@ -12,72 +12,114 @@ Given OTEL often involves events triggered from elsewhere within the n8n system
### Attributes ### Attributes
All attributes are listed in `otel.constants.ts` All attributes are listed in `otel.constants.ts`
### Global API ownership
`OtelService` starts a `NodeTracerProvider` and hands out tracers through `getTracer()`.
The module's own spans never depend on the global `@opentelemetry/api` registry.
At start, the service checks whether another library already registered a global tracer
provider. Sentry does this in `ErrorReporter.init`. A user SDK loaded with `--require` does
it too.
- The slot is free: the service registers its provider, an `AsyncLocalStorageContextManager`
and the default W3C propagators. A restart swaps only the tracer provider. Shutdown leaves
the globals in place.
- The slot is taken: the service keeps its provider private and logs one info line. It never
disables or replaces the other library's globals.
Consequences:
- Workflow spans go to the module exporter only. To get them into Sentry, point the OTel
settings at Sentry's OTLP endpoint.
- Workflow root spans start on `ROOT_CONTEXT`. They never join a trace that is active in the
process, such as a Sentry request span or an HTTP server span from a user SDK. An inbound
`traceparent` header still sets the parent.
- Persisted trace context and outbound headers always use the W3C `traceparent` and
`tracestate` format, whoever owns the global propagator. `OTEL_PROPAGATORS` has no effect
on them.
- The resource uses the env, process and host detectors. `OTEL_NODE_RESOURCE_DETECTORS` has
no effect.
### Module architecture ### Module architecture
```mermaid ```mermaid
graph TD graph TD
subgraph Module Layer subgraph Module Layer
MOD["OtelModule MOD["OtelModule
@BackendModule"] @BackendModule on main, worker and webhook"]
end end
subgraph Configuration subgraph Configuration
CFG["OtelConfig CFG["OtelConfig
env vars to typed config"] env vars to typed config"]
end SET["OtelSettingsService
DB row merged with config, env vars win"]
CTRL["OtelSettingsController
main only: GET and PUT settings, POST test trace"]
end
subgraph SDK Layer subgraph SDK Layer
SVC["OtelService SVC["OtelService
owns NodeSDK lifecycle"] owns the NodeTracerProvider, getTracer()"]
SDK["OpenTelemetry NodeSDK SDK["NodeTracerProvider
exporter, sampler, resource"] OTLP exporter, sampler, resource"]
end API["Global @opentelemetry/api
registered only when the slot is free"]
end
subgraph Instrumentation Layer subgraph Instrumentation Layer
INST["N8nWorkflowInstrumentation LCH["OtelLifecycleHandler
@OnLifecycleEvent listeners"] @OnLifecycleEvent and @OnPubSubEvent"]
REG["SpanRegistry ELT["ExecutionLevelTracer
Map of executionId to Span"] span maps by execution id, W3C propagator"]
end TCS["TraceContextService
traceparent on the execution row"]
end
subgraph Handler Layer subgraph n8n Core
IFC{{"SpanHandler interface"}} LC(("Lifecycle events
WS["WorkflowStartHandler"] workflowExecuteBefore, Resume, After
WE["WorkflowEndHandler"] nodeExecuteBefore, After"))
end PUB(("Pub/Sub
reload-otel-config"))
REQ["Request helpers
module context injectTraceHeaders"]
AGT["AgentRunTracingService
AgentWorkflowExecutionService"]
end
subgraph n8n Core COL[("OTLP collector")]
LC(("Lifecycle Events
workflowExecuteBefore
workflowExecuteAfter"))
end
MOD -- "1. check enabled" --> CFG MOD -- "1. load settings" --> SET
MOD -- "2. init SDK" --> SVC SET --> CFG
SVC -- "creates" --> SDK MOD -- "2. start provider" --> SVC
MOD -- "3. register listeners" --> INST SVC -- "creates" --> SDK
SVC -. "registers when free" .-> API
MOD -- "3. import handler" --> LCH
MOD -- "4. main only" --> CTRL
CTRL -- "save, then publish" --> PUB
PUB -. "fires" .-> LCH
LCH -- "restart()" --> SVC
LC -. "fires event" .-> INST LC -. "fires" .-> LCH
INST -- "dispatches to" --> IFC LCH -- "start and end workflow and node spans" --> ELT
IFC -. "implemented by" .-> WS LCH -- "persist and read" --> TCS
IFC -. "implemented by" .-> WE ELT -- "getTracer()" --> SVC
REQ -- "outbound traceparent" --> ELT
AGT -- "getActiveContext()" --> ELT
AGT -- "getTracer()" --> SVC
SDK -. "exports spans" .-> COL
WS -- "startSpan, store" --> REG classDef module fill:#4a9eff,color:#fff
WE -- "retrieve, enrich, end" --> REG classDef config fill:#f5a623,color:#fff
REG -. "spans exported via" .-> SDK classDef sdk fill:#7b68ee,color:#fff
classDef inst fill:#50c878,color:#fff
classDef core fill:#888,color:#fff
classDef module fill:#4a9eff,color:#fff class MOD module
classDef config fill:#f5a623,color:#fff class CFG,SET,CTRL config
classDef sdk fill:#7b68ee,color:#fff class SVC,SDK,API sdk
classDef inst fill:#50c878,color:#fff class LCH,ELT,TCS inst
classDef handler fill:#ff6b6b,color:#fff class LC,PUB,REQ,AGT,COL core
classDef core fill:#888,color:#fff
class MOD module
class CFG config
class SVC,SDK sdk
class INST,REG inst
class IFC,WS,WE handler
class LC core
``` ```
#### Manual validation #### Manual validation

View File

@ -1,5 +1,6 @@
import type { Logger } from '@n8n/backend-common'; import type { Logger } from '@n8n/backend-common';
import { SpanStatusCode, trace } from '@opentelemetry/api'; import type { TextMapPropagator } from '@opentelemetry/api';
import { context, propagation, SpanStatusCode, trace } from '@opentelemetry/api';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import { ExecutionLevelTracer } from '../execution-level-tracer'; import { ExecutionLevelTracer } from '../execution-level-tracer';
@ -20,7 +21,7 @@ describe('ExecutionLevelTracer', () => {
}; };
beforeAll(() => { beforeAll(() => {
otel = OtelTestProvider.create(); otel = OtelTestProvider.create({ withContextManager: true });
}); });
afterAll(async () => { afterAll(async () => {
@ -29,7 +30,7 @@ describe('ExecutionLevelTracer', () => {
beforeEach(() => { beforeEach(() => {
otel.reset(); otel.reset();
tracer = new ExecutionLevelTracer(makeOtelSettingsService(), logger); tracer = new ExecutionLevelTracer(otel.asOtelService(), makeOtelSettingsService(), logger);
}); });
const inboundTracingContext = { const inboundTracingContext = {
@ -88,37 +89,52 @@ describe('ExecutionLevelTracer', () => {
expect(span.attributes['n8n.project.custom.team']).toBe('platform'); expect(span.attributes['n8n.project.custom.team']).toBe('platform');
}); });
it('should not attach project custom attributes to node spans', () => { const runSingleNodeExecution = (
tracer.startWorkflow({ executionId: string,
executionId: 'exec-node-no-tags', project?: { id: string; customAttributes?: Record<string, string> },
workflow: defaultWorkflow, ) => {
project: { tracer.startWorkflow({ executionId, workflow: defaultWorkflow, project });
id: 'proj-tags',
customAttributes: { env: 'staging' },
},
});
const node = { id: 'n1', name: 'MyNode', type: 'test', typeVersion: 1 }; const node = { id: 'n1', name: 'MyNode', type: 'test', typeVersion: 1 };
tracer.startNode({ executionId: 'exec-node-no-tags', node }); tracer.startNode({ executionId, node });
tracer.endNode({ tracer.endNode({ executionId, node, inputItemCount: 1, outputItemCount: 1 });
executionId: 'exec-node-no-tags', tracer.endWorkflow({ executionId, status: 'success', mode: 'manual', isRetry: false });
node,
inputItemCount: 1,
outputItemCount: 1,
});
tracer.endWorkflow({
executionId: 'exec-node-no-tags',
status: 'success',
mode: 'manual',
isRetry: false,
});
const spans = otel.getFinishedSpans(); const spans = otel.getFinishedSpans();
const nodeSpan = spans.find((s) => s.name === 'node.execute')!; return {
// No project custom attributes should appear on the node span workflowSpan: spans.find((s) => s.name === 'workflow.execute')!,
const projectCustomKeys = Object.keys(nodeSpan.attributes).filter((k) => nodeSpan: spans.find((s) => s.name === 'node.execute')!,
k.startsWith('n8n.project.custom.'), };
};
const projectCustomKeys = (attributes: Record<string, unknown>) =>
Object.keys(attributes).filter((k) => k.startsWith('n8n.project.custom.'));
it('should attach project id and custom attributes to node spans', () => {
const { workflowSpan, nodeSpan } = runSingleNodeExecution('exec-node-tags', {
id: 'proj-tags',
customAttributes: { env: 'staging' },
});
expect(nodeSpan.attributes['n8n.project.id']).toBe('proj-tags');
expect(nodeSpan.attributes['n8n.project.custom.env']).toBe('staging');
expect(nodeSpan.attributes['n8n.project.id']).toBe(workflowSpan.attributes['n8n.project.id']);
expect(nodeSpan.attributes['n8n.project.custom.env']).toBe(
workflowSpan.attributes['n8n.project.custom.env'],
); );
expect(projectCustomKeys).toHaveLength(0); });
it('should attach only project id to node spans when the project has no custom attributes', () => {
const { nodeSpan } = runSingleNodeExecution('exec-node-no-tags', { id: 'proj-no-tags' });
expect(nodeSpan.attributes['n8n.project.id']).toBe('proj-no-tags');
expect(projectCustomKeys(nodeSpan.attributes)).toHaveLength(0);
});
it('should omit project attributes on node spans when project is not provided', () => {
const { nodeSpan } = runSingleNodeExecution('exec-node-no-project');
expect(nodeSpan.attributes['n8n.project.id']).toBeUndefined();
expect(projectCustomKeys(nodeSpan.attributes)).toHaveLength(0);
}); });
it('should omit project id attribute when project is not provided', () => { it('should omit project id attribute when project is not provided', () => {
@ -783,6 +799,7 @@ describe('ExecutionLevelTracer', () => {
it('should no-op when injectOutbound is false', () => { it('should no-op when injectOutbound is false', () => {
const noInjectTracer = new ExecutionLevelTracer( const noInjectTracer = new ExecutionLevelTracer(
otel.asOtelService(),
makeOtelSettingsService({ injectOutbound: false }), makeOtelSettingsService({ injectOutbound: false }),
logger, logger,
); );
@ -829,6 +846,59 @@ describe('ExecutionLevelTracer', () => {
}); });
}); });
describe('trace context ownership', () => {
const traceparentPattern = /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/;
it('should start a new trace when no tracing context is given, even inside an active foreign span', () => {
const foreignSpan = trace.getTracer('foreign').startSpan('GET /webhook');
context.with(trace.setSpan(context.active(), foreignSpan), () => {
tracer.startWorkflow({ executionId: 'exec-root', workflow: defaultWorkflow });
});
tracer.endWorkflow({
executionId: 'exec-root',
status: 'success',
mode: 'webhook',
isRetry: false,
});
foreignSpan.end();
const workflowSpan = otel.getFinishedSpans().find((s) => s.name === 'workflow.execute')!;
expect(workflowSpan.parentSpanContext).toBeUndefined();
expect(workflowSpan.spanContext().traceId).not.toBe(foreignSpan.spanContext().traceId);
});
it('should emit W3C trace context while another library owns the global propagator', () => {
const foreignPropagator: TextMapPropagator = {
inject: (_ctx, carrier, setter) => setter.set(carrier, 'sentry-trace', 'foreign'),
extract: (ctx) => ctx,
fields: () => ['sentry-trace'],
};
propagation.setGlobalPropagator(foreignPropagator);
try {
const persisted = tracer.startWorkflow({
executionId: 'exec-w3c',
workflow: defaultWorkflow,
});
const headers: Record<string, string> = {};
tracer.injectTraceHeaders('exec-w3c', undefined, headers);
tracer.endWorkflow({
executionId: 'exec-w3c',
status: 'success',
mode: 'webhook',
isRetry: false,
});
expect(persisted.traceparent).toMatch(traceparentPattern);
expect(headers.traceparent).toMatch(traceparentPattern);
expect(headers['sentry-trace']).toBeUndefined();
} finally {
propagation.disable();
}
});
});
describe('getActiveContext', () => { describe('getActiveContext', () => {
it('returns a context carrying the node span when one is active', () => { it('returns a context carrying the node span when one is active', () => {
tracer.startWorkflow({ executionId: 'exec-ctx-node', workflow: defaultWorkflow }); tracer.startWorkflow({ executionId: 'exec-ctx-node', workflow: defaultWorkflow });

View File

@ -1019,14 +1019,13 @@ describe('onReloadOtelConfig', () => {
vi.clearAllMocks(); vi.clearAllMocks();
}); });
it('restarts the OTel SDK and refreshes the tracer', async () => { it('restarts the OTel tracer provider', async () => {
const handler = makeHandler(); const handler = makeHandler();
otelService.restart.mockResolvedValue(undefined); otelService.restart.mockResolvedValue(undefined);
await handler.onReloadOtelConfig(); await handler.onReloadOtelConfig();
expect(otelService.restart).toHaveBeenCalledTimes(1); expect(otelService.restart).toHaveBeenCalledTimes(1);
expect(tracer.refreshTracer).toHaveBeenCalledTimes(1);
}); });
}); });

View File

@ -0,0 +1,225 @@
import type { Logger } from '@n8n/backend-common';
import type { OutboundHttp } from '@n8n/backend-network';
import type { Context } from '@opentelemetry/api';
import {
context,
createContextKey,
propagation,
ProxyTracerProvider,
ROOT_CONTEXT,
trace,
} from '@opentelemetry/api';
import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
import {
InMemorySpanExporter,
NodeTracerProvider,
SimpleSpanProcessor,
} from '@opentelemetry/sdk-trace-node';
import { mock } from 'vitest-mock-extended';
import type { InstanceSettings } from 'n8n-core';
import type { OtelSettingsService } from '../otel-settings.service';
import type { OtelConfig } from '../otel.config';
import { ATTR } from '../otel.constants';
import { OtelService } from '../otel.service';
import { N8N_VERSION } from '@/constants';
const { exportedSpans } = vi.hoisted(() => ({ exportedSpans: [] as ReadableSpan[] }));
vi.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
OTLPTraceExporter: vi.fn().mockImplementation(function () {
return {
export: (spans: ReadableSpan[], resultCallback: (result: { code: number }) => void) => {
exportedSpans.push(...spans);
resultCallback({ code: 0 });
},
shutdown: async () => {},
forceFlush: async () => {},
};
}),
}));
vi.mock('@opentelemetry/sdk-trace-base', async (importActual) => {
const actual = await importActual<typeof import('@opentelemetry/sdk-trace-base')>();
return { ...actual, BatchSpanProcessor: actual.SimpleSpanProcessor };
});
vi.mock('@opentelemetry/resources', async (importActual) => {
const actual = await importActual<typeof import('@opentelemetry/resources')>();
return { ...actual, detectResources: () => actual.emptyResource() };
});
const enabledSettings: OtelConfig = {
enabled: true,
exporterProtocol: 'http/protobuf',
exporterEndpoint: 'http://localhost:4318',
exporterTracingPath: '/v1/traces',
exporterHeaders: '',
exporterServiceName: 'n8n',
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
injectOutbound: true,
productionExecutionsOnly: true,
};
const disabledSettings: OtelConfig = { ...enabledSettings, enabled: false };
const traceparentPattern = /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/;
const markedContext = ROOT_CONTEXT.setValue(createContextKey('marker'), 'marked');
function activeContextInside(ctx: Context): Context {
return context.with(ctx, () => context.active());
}
function globalTracerProviderDelegate() {
const globalProvider = trace.getTracerProvider();
return globalProvider instanceof ProxyTracerProvider
? globalProvider.getDelegate()
: globalProvider;
}
function registerForeignProvider() {
const exporter = new InMemorySpanExporter();
const provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });
provider.register();
return { provider, exporter };
}
function exportedSpanNames() {
return exportedSpans.map((span) => span.name);
}
describe('OtelService tracer provider', () => {
const fetchMock = vi.fn();
const outboundHttp = {
transport: () => ({ asCustomFetch: () => fetchMock }),
} as unknown as OutboundHttp;
let otelSettingsService: ReturnType<typeof mock<OtelSettingsService>>;
let logger: ReturnType<typeof mock<Logger>>;
let service: OtelService;
let foreign: ReturnType<typeof registerForeignProvider> | undefined;
beforeEach(() => {
exportedSpans.length = 0;
fetchMock.mockResolvedValue({ ok: true });
otelSettingsService = mock<OtelSettingsService>();
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
logger = mock<Logger>();
service = new OtelService(
otelSettingsService,
mock<InstanceSettings>({ instanceId: 'inst-1', instanceType: 'main' }),
logger,
outboundHttp,
);
});
afterEach(async () => {
await service.shutdown();
await foreign?.provider.shutdown();
foreign = undefined;
trace.disable();
context.disable();
propagation.disable();
});
describe('when another library owns the global API', () => {
beforeEach(() => {
foreign = registerForeignProvider();
});
it('exports n8n spans through its own provider and leaves the foreign exporter empty', async () => {
await service.init();
service.getTracer('n8n-workflow').startSpan('workflow.execute').end();
expect(exportedSpanNames()).toEqual(['workflow.execute']);
expect(exportedSpans[0].resource.attributes).toMatchObject({
[ATTR.OTEL_SERVICE_NAME]: 'n8n',
[ATTR.OTEL_SERVICE_VERSION]: N8N_VERSION,
[ATTR.INSTANCE_ID]: 'inst-1',
[ATTR.INSTANCE_ROLE]: 'main',
});
expect(foreign?.exporter.getFinishedSpans()).toHaveLength(0);
});
it('keeps exporting after a restart and leaves the foreign globals in place', async () => {
await service.init();
await service.restart();
service.getTracer('n8n-workflow').startSpan('workflow.execute').end();
trace.getTracer('foreign').startSpan('GET /webhook').end();
expect(exportedSpanNames()).toEqual(['workflow.execute']);
expect(foreign?.exporter.getFinishedSpans().map((span) => span.name)).toEqual([
'GET /webhook',
]);
expect(globalTracerProviderDelegate()).toBe(foreign?.provider);
expect(activeContextInside(markedContext)).toBe(markedContext);
});
it('logs the foreign owner once for the lifetime of the service', async () => {
await service.init();
await service.restart();
expect(logger.info).toHaveBeenCalledTimes(1);
expect(logger.info).toHaveBeenCalledWith(
expect.stringContaining('runs on its own tracer provider'),
);
});
});
describe('when the global API is free', () => {
it('registers its provider, so third-party tracers export through the module', async () => {
await service.init();
trace.getTracer('third-party').startSpan('http.request').end();
const headers: Record<string, string> = {};
const span = service.getTracer('n8n-workflow').startSpan('workflow.execute');
propagation.inject(trace.setSpan(context.active(), span), headers);
span.end();
expect(exportedSpanNames()).toEqual(['http.request', 'workflow.execute']);
expect(headers.traceparent).toMatch(traceparentPattern);
expect(activeContextInside(markedContext)).toBe(markedContext);
expect(logger.info).not.toHaveBeenCalled();
});
it('swaps the registered provider on restart and keeps the context manager', async () => {
await service.init();
await service.restart();
trace.getTracer('third-party').startSpan('http.request').end();
expect(exportedSpanNames()).toEqual(['http.request']);
expect(activeContextInside(markedContext)).toBe(markedContext);
});
it('keeps the context manager after shutdown and hands out no-op tracers', async () => {
await service.init();
await service.shutdown();
expect(activeContextInside(markedContext)).toBe(markedContext);
expect(service.getTracer('n8n-workflow').startSpan('workflow.execute').isRecording()).toBe(
false,
);
});
it('registers on the first enabled restart after a disabled boot', async () => {
otelSettingsService.loadSettings
.mockResolvedValueOnce(disabledSettings)
.mockResolvedValueOnce(enabledSettings);
await service.init();
expect(service.getTracer('n8n-workflow').startSpan('workflow.execute').isRecording()).toBe(
false,
);
await service.restart();
service.getTracer('n8n-workflow').startSpan('workflow.execute').end();
trace.getTracer('third-party').startSpan('http.request').end();
expect(exportedSpanNames()).toEqual(['workflow.execute', 'http.request']);
});
});
});

View File

@ -1,9 +1,10 @@
import type { Logger } from '@n8n/backend-common'; import type { Logger } from '@n8n/backend-common';
import type { OutboundHttp } from '@n8n/backend-network'; import type { OutboundHttp } from '@n8n/backend-network';
import { context, diag, metrics, propagation, trace } from '@opentelemetry/api'; import { diag } from '@opentelemetry/api';
import { OTLPTraceExporter as OTLPGrpcTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; import { OTLPTraceExporter as OTLPGrpcTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'; import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base';
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
import { mock } from 'vitest-mock-extended'; import { mock } from 'vitest-mock-extended';
import type { InstanceSettings } from 'n8n-core'; import type { InstanceSettings } from 'n8n-core';
@ -12,8 +13,9 @@ import type { OtelConfig } from '../otel.config';
import { ATTR, OTEL_TEST_SPAN_NAME } from '../otel.constants'; import { ATTR, OTEL_TEST_SPAN_NAME } from '../otel.constants';
import { OtelService } from '../otel.service'; import { OtelService } from '../otel.service';
const start = vi.fn(); const register = vi.fn();
const shutdown = vi.fn(); const shutdown = vi.fn();
const providerGetTracer = vi.fn();
// The connectivity-check fetch, obtained via outboundHttp.transport().asCustomFetch(). // The connectivity-check fetch, obtained via outboundHttp.transport().asCustomFetch().
// checkEndpointReachability ignores the response and only catches network errors. // checkEndpointReachability ignores the response and only catches network errors.
@ -86,15 +88,6 @@ const mockSpanEnd = vi.fn();
const mockStartSpan = vi.fn(); const mockStartSpan = vi.fn();
const mockGetTracer = vi.fn(); const mockGetTracer = vi.fn();
vi.mock('@opentelemetry/sdk-node', () => ({
NodeSDK: vi.fn().mockImplementation(function () {
return {
start,
shutdown,
};
}),
}));
vi.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({ vi.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
OTLPTraceExporter: vi.fn().mockImplementation(function () { OTLPTraceExporter: vi.fn().mockImplementation(function () {
return { return {
@ -122,6 +115,7 @@ vi.mock('@grpc/grpc-js', () => ({
})); }));
vi.mock('@opentelemetry/sdk-trace-base', () => ({ vi.mock('@opentelemetry/sdk-trace-base', () => ({
BatchSpanProcessor: vi.fn(),
BasicTracerProvider: vi.fn().mockImplementation(function (config: { BasicTracerProvider: vi.fn().mockImplementation(function (config: {
spanProcessors?: unknown[]; spanProcessors?: unknown[];
}) { }) {
@ -136,21 +130,24 @@ vi.mock('@opentelemetry/sdk-trace-base', () => ({
})); }));
vi.mock('@opentelemetry/resources', () => ({ vi.mock('@opentelemetry/resources', () => ({
resourceFromAttributes: vi.fn().mockReturnValue({}), resourceFromAttributes: vi.fn().mockReturnValue({ merge: () => ({}) }),
detectResources: vi.fn().mockReturnValue({}),
envDetector: {},
processDetector: {},
hostDetector: {},
})); }));
vi.mock('@opentelemetry/sdk-trace-node', () => ({ vi.mock('@opentelemetry/sdk-trace-node', () => ({
TraceIdRatioBasedSampler: vi.fn().mockImplementation(function () { TraceIdRatioBasedSampler: vi.fn().mockImplementation(function () {
return {}; return {};
}), }),
NodeTracerProvider: vi.fn().mockImplementation(function () {
return { register, shutdown, getTracer: providerGetTracer };
}),
})); }));
vi.mock('@opentelemetry/api', async () => ({ vi.mock('@opentelemetry/api', async () => ({
...(await vi.importActual<typeof import('@opentelemetry/api')>('@opentelemetry/api')), ...(await vi.importActual<typeof import('@opentelemetry/api')>('@opentelemetry/api')),
trace: { disable: vi.fn() },
context: { disable: vi.fn() },
propagation: { disable: vi.fn() },
metrics: { disable: vi.fn() },
DiagLogLevel: { WARN: 'WARN' }, DiagLogLevel: { WARN: 'WARN' },
diag: { setLogger: vi.fn() }, diag: { setLogger: vi.fn() },
})); }));
@ -201,20 +198,20 @@ describe('OtelService', () => {
}); });
describe('init', () => { describe('init', () => {
it('does not start SDK when enabled is false', async () => { it('does not start a tracer provider when enabled is false', async () => {
otelSettingsService.loadSettings.mockResolvedValue(disabledSettings); otelSettingsService.loadSettings.mockResolvedValue(disabledSettings);
await service.init(); await service.init();
expect(start).not.toHaveBeenCalled(); expect(NodeTracerProvider).not.toHaveBeenCalled();
}); });
it('starts SDK when enabled is true', async () => { it('starts a tracer provider when enabled is true', async () => {
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings); otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
await service.init(); await service.init();
expect(start).toHaveBeenCalledTimes(1); expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
}); });
it('builds the HTTP exporter with the traces path and no gRPC exporter', async () => { it('builds the HTTP exporter with the traces path and no gRPC exporter', async () => {
@ -241,7 +238,7 @@ describe('OtelService', () => {
await service.init(); await service.init();
await flushPromises(); await flushPromises();
expect(start).toHaveBeenCalledTimes(1); expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith( expect(logger.error).toHaveBeenCalledWith(
'Failed to connect to OpenTelemetry OTLP endpoint during startup', 'Failed to connect to OpenTelemetry OTLP endpoint during startup',
expect.objectContaining({ endpoint: 'http://localhost:4318/v1/traces' }), expect.objectContaining({ endpoint: 'http://localhost:4318/v1/traces' }),
@ -259,7 +256,7 @@ describe('OtelService', () => {
expect.objectContaining({ url: 'http://collector.example.com:4317' }), expect.objectContaining({ url: 'http://collector.example.com:4317' }),
); );
expect(OTLPTraceExporter).not.toHaveBeenCalled(); expect(OTLPTraceExporter).not.toHaveBeenCalled();
expect(start).toHaveBeenCalledTimes(1); expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
}); });
it('lowercases only the endpoint scheme before it reaches the exporter', async () => { it('lowercases only the endpoint scheme before it reaches the exporter', async () => {
@ -408,7 +405,7 @@ describe('OtelService', () => {
await service.init(); await service.init();
await flushPromises(); await flushPromises();
expect(start).toHaveBeenCalledTimes(1); expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledTimes(1); expect(logger.error).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith( expect(logger.error).toHaveBeenCalledWith(
'Failed to connect to OpenTelemetry OTLP endpoint during startup', 'Failed to connect to OpenTelemetry OTLP endpoint during startup',
@ -427,7 +424,7 @@ describe('OtelService', () => {
await flushPromises(); await flushPromises();
expect(newClient).not.toHaveBeenCalled(); expect(newClient).not.toHaveBeenCalled();
expect(start).toHaveBeenCalledTimes(1); expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith( expect(logger.error).toHaveBeenCalledWith(
'Failed to connect to OpenTelemetry OTLP endpoint during startup', 'Failed to connect to OpenTelemetry OTLP endpoint during startup',
expect.objectContaining({ endpoint: 'not a valid endpoint' }), expect.objectContaining({ endpoint: 'not a valid endpoint' }),
@ -436,7 +433,7 @@ describe('OtelService', () => {
}); });
describe('restart', () => { describe('restart', () => {
it('shuts down existing SDK then reloads settings and starts a new one', async () => { it('shuts down the existing tracer provider then reloads settings and starts a new one', async () => {
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings); otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
await service.init(); await service.init();
@ -447,10 +444,10 @@ describe('OtelService', () => {
expect(shutdown).toHaveBeenCalledTimes(1); expect(shutdown).toHaveBeenCalledTimes(1);
expect(otelSettingsService.loadSettings).toHaveBeenCalledTimes(1); expect(otelSettingsService.loadSettings).toHaveBeenCalledTimes(1);
expect(start).toHaveBeenCalledTimes(1); expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
}); });
it('does not start SDK after restart when reloaded settings have enabled=false', async () => { it('does not start a tracer provider after restart when reloaded settings have enabled=false', async () => {
otelSettingsService.loadSettings.mockResolvedValueOnce(enabledSettings); otelSettingsService.loadSettings.mockResolvedValueOnce(enabledSettings);
otelSettingsService.loadSettings.mockResolvedValueOnce(disabledSettings); otelSettingsService.loadSettings.mockResolvedValueOnce(disabledSettings);
@ -459,11 +456,11 @@ describe('OtelService', () => {
await service.restart(); await service.restart();
expect(start).not.toHaveBeenCalled(); expect(NodeTracerProvider).not.toHaveBeenCalled();
}); });
}); });
describe('SDK startup failure', () => { describe('tracer provider startup failure', () => {
const exporterFailure = () => { const exporterFailure = () => {
vi.mocked(OTLPTraceExporter).mockImplementationOnce(function () { vi.mocked(OTLPTraceExporter).mockImplementationOnce(function () {
throw new Error('exporter unavailable'); throw new Error('exporter unavailable');
@ -476,7 +473,7 @@ describe('OtelService', () => {
await expect(service.init()).resolves.not.toThrow(); await expect(service.init()).resolves.not.toThrow();
expect(start).not.toHaveBeenCalled(); expect(NodeTracerProvider).not.toHaveBeenCalled();
expect(fetchMock).not.toHaveBeenCalled(); expect(fetchMock).not.toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledTimes(1); expect(logger.error).toHaveBeenCalledTimes(1);
expect(logger.error).toHaveBeenCalledWith( expect(logger.error).toHaveBeenCalledWith(
@ -485,9 +482,9 @@ describe('OtelService', () => {
); );
}); });
it('tears down partially installed providers when the SDK start call throws', async () => { it('shuts the provider down when registering it with the global API throws', async () => {
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings); otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
start.mockImplementationOnce(() => { register.mockImplementationOnce(() => {
throw new Error('provider registration failed'); throw new Error('provider registration failed');
}); });
@ -498,7 +495,6 @@ describe('OtelService', () => {
{ error: 'provider registration failed' }, { error: 'provider registration failed' },
); );
expect(shutdown).toHaveBeenCalledTimes(1); expect(shutdown).toHaveBeenCalledTimes(1);
expect(trace.disable).toHaveBeenCalledTimes(1);
}); });
it('leaves the service in a non-exporting state when a restart fails', async () => { it('leaves the service in a non-exporting state when a restart fails', async () => {
@ -509,7 +505,7 @@ describe('OtelService', () => {
await expect(service.restart()).resolves.not.toThrow(); await expect(service.restart()).resolves.not.toThrow();
expect(start).not.toHaveBeenCalled(); expect(NodeTracerProvider).not.toHaveBeenCalled();
expect(shutdown).toHaveBeenCalledTimes(1); expect(shutdown).toHaveBeenCalledTimes(1);
shutdown.mockClear(); shutdown.mockClear();
@ -520,23 +516,20 @@ describe('OtelService', () => {
}); });
describe('shutdown', () => { describe('shutdown', () => {
it('disables all four OTel globals so the next SDK start can re-register providers', async () => { it('shuts the running provider down', async () => {
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings); otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
await service.init(); await service.init();
await service.shutdown(); await service.shutdown();
expect(trace.disable).toHaveBeenCalledTimes(1); expect(shutdown).toHaveBeenCalledTimes(1);
expect(context.disable).toHaveBeenCalledTimes(1);
expect(propagation.disable).toHaveBeenCalledTimes(1);
expect(metrics.disable).toHaveBeenCalledTimes(1);
}); });
it('does not throw when called before init', async () => { it('does not throw when called before init', async () => {
await expect(service.shutdown()).resolves.not.toThrow(); await expect(service.shutdown()).resolves.not.toThrow();
}); });
it('does not throw when the SDK shutdown rejects (e.g. exporter flush failure)', async () => { it('does not throw when the provider shutdown rejects (e.g. exporter flush failure)', async () => {
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings); otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
await service.init(); await service.init();
shutdown.mockRejectedValueOnce(new Error('connect ECONNREFUSED 127.0.0.1:9')); shutdown.mockRejectedValueOnce(new Error('connect ECONNREFUSED 127.0.0.1:9'));
@ -545,6 +538,24 @@ describe('OtelService', () => {
}); });
}); });
describe('getTracer', () => {
it('hands out a no-op tracer before a provider has started', () => {
const span = service.getTracer('n8n-workflow').startSpan('workflow.execute');
expect(span.isRecording()).toBe(false);
});
it("hands out the running provider's tracer", async () => {
const tracer = { startSpan: vi.fn() };
providerGetTracer.mockReturnValue(tracer);
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
await service.init();
expect(service.getTracer('n8n-workflow')).toBe(tracer);
expect(providerGetTracer).toHaveBeenCalledWith('n8n-workflow');
});
});
describe('diagnostics logger', () => { describe('diagnostics logger', () => {
beforeEach(() => { beforeEach(() => {
( (
@ -806,7 +817,7 @@ describe('OtelService', () => {
await service.sendTestTrace(connection); await service.sendTestTrace(connection);
expect(BasicTracerProvider).toHaveBeenCalledTimes(1); expect(BasicTracerProvider).toHaveBeenCalledTimes(1);
expect(start).not.toHaveBeenCalled(); expect(NodeTracerProvider).not.toHaveBeenCalled();
}); });
describe('over gRPC', () => { describe('over gRPC', () => {

View File

@ -23,7 +23,9 @@ import * as utils from '@test-integration/utils';
import { OtelTestProvider } from './otel-test-provider'; import { OtelTestProvider } from './otel-test-provider';
import { TestNodeWithTracing } from './test-node-with-tracing'; import { TestNodeWithTracing } from './test-node-with-tracing';
import { OtelSettingsService } from '../../otel-settings.service';
import { OtelConfig } from '../../otel.config'; import { OtelConfig } from '../../otel.config';
import { OtelService } from '../../otel.service';
const BASE_DIR = path.resolve(__dirname, '../../../../../..'); const BASE_DIR = path.resolve(__dirname, '../../../../../..');
@ -51,6 +53,8 @@ export async function initOtelTestEnvironment() {
await testModules.loadModules(['otel']); await testModules.loadModules(['otel']);
await testDb.init(); await testDb.init();
Container.set(OtelService, otel.asOtelService());
await Container.get(OtelSettingsService).loadSettings();
await Container.get(ModuleRegistry).initModules('main'); await Container.get(ModuleRegistry).initModules('main');
Container.get(LicenseState).setLicenseProvider({ Container.get(LicenseState).setLicenseProvider({
isLicensed: (feature) => feature === LICENSE_FEATURES.OTEL_CUSTOM_SPAN_ATTRIBUTES, isLicensed: (feature) => feature === LICENSE_FEATURES.OTEL_CUSTOM_SPAN_ATTRIBUTES,

View File

@ -1,17 +1,19 @@
import type { Context, ContextManager } from '@opentelemetry/api'; import type { Context, ContextManager } from '@opentelemetry/api';
import { context, propagation, ROOT_CONTEXT, trace } from '@opentelemetry/api'; import { context, ROOT_CONTEXT, trace } from '@opentelemetry/api';
import { W3CTraceContextPropagator } from '@opentelemetry/core';
import { import {
BasicTracerProvider,
InMemorySpanExporter, InMemorySpanExporter,
NodeTracerProvider,
SimpleSpanProcessor, SimpleSpanProcessor,
} from '@opentelemetry/sdk-trace-node'; } from '@opentelemetry/sdk-trace-node';
import { AsyncLocalStorage } from 'node:async_hooks'; import { AsyncLocalStorage } from 'node:async_hooks';
import { mock } from 'vitest-mock-extended';
import type { OtelService } from '../../otel.service';
/** /**
* Minimal stand-in for `@opentelemetry/context-async-hooks`' * Minimal stand-in for `@opentelemetry/context-async-hooks`'
* `AsyncLocalStorageContextManager` (the one `NodeSDK` registers by default in * `AsyncLocalStorageContextManager` (the one `NodeTracerProvider.register()`
* production) — avoids adding that package as a test-only dependency. Without * installs in production) — avoids adding that package as a test-only dependency. Without
* a real context manager registered, `context.with()` is a no-op (the default * a real context manager registered, `context.with()` is a no-op (the default
* `NoopContextManager` ignores the context it's given), so span nesting via * `NoopContextManager` ignores the context it's given), so span nesting via
* ambient context can't be exercised at all. * ambient context can't be exercised at all.
@ -46,11 +48,14 @@ class AsyncLocalStorageTestContextManager implements ContextManager {
} }
/** /**
* Disposable OTel test harness. Sets up an in-memory tracer * Disposable OTel test harness. Sets up an in-memory tracer provider that
* that captures spans for assertion, and tears down cleanly. * captures spans for assertion, registers it as the global tracer provider
* (so the module under test sees the slot taken, like it does next to
* Sentry), and tears down cleanly.
* *
* Usage: * Usage:
* const otel = OtelTestProvider.create(); * const otel = OtelTestProvider.create();
* const tracer = new ExecutionLevelTracer(otel.asOtelService(), ...);
* // ... run code that creates spans ... * // ... run code that creates spans ...
* expect(otel.getFinishedSpans()).toHaveLength(1); * expect(otel.getFinishedSpans()).toHaveLength(1);
* otel.reset(); // between tests * otel.reset(); // between tests
@ -62,24 +67,28 @@ class AsyncLocalStorageTestContextManager implements ContextManager {
*/ */
export class OtelTestProvider { export class OtelTestProvider {
private constructor( private constructor(
private readonly provider: BasicTracerProvider, readonly provider: NodeTracerProvider,
private readonly exporter: InMemorySpanExporter, private readonly exporter: InMemorySpanExporter,
private readonly contextManagerEnabled: boolean, private readonly contextManagerEnabled: boolean,
) {} ) {}
static create(options: { withContextManager?: boolean } = {}): OtelTestProvider { static create(options: { withContextManager?: boolean } = {}): OtelTestProvider {
const exporter = new InMemorySpanExporter(); const exporter = new InMemorySpanExporter();
const provider = new BasicTracerProvider({ const provider = new NodeTracerProvider({
spanProcessors: [new SimpleSpanProcessor(exporter)], spanProcessors: [new SimpleSpanProcessor(exporter)],
}); });
trace.setGlobalTracerProvider(provider); trace.setGlobalTracerProvider(provider);
propagation.setGlobalPropagator(new W3CTraceContextPropagator());
if (options.withContextManager) { if (options.withContextManager) {
context.setGlobalContextManager(new AsyncLocalStorageTestContextManager()); context.setGlobalContextManager(new AsyncLocalStorageTestContextManager());
} }
return new OtelTestProvider(provider, exporter, options.withContextManager ?? false); return new OtelTestProvider(provider, exporter, options.withContextManager ?? false);
} }
/** An `OtelService` stand-in that hands out this provider's tracers. */
asOtelService(): OtelService {
return mock<OtelService>({ getTracer: (name: string) => this.provider.getTracer(name) });
}
getFinishedSpans() { getFinishedSpans() {
return this.exporter.getFinishedSpans(); return this.exporter.getFinishedSpans();
} }
@ -92,7 +101,6 @@ export class OtelTestProvider {
this.exporter.reset(); this.exporter.reset();
await this.provider.shutdown(); await this.provider.shutdown();
trace.disable(); trace.disable();
propagation.disable();
if (this.contextManagerEnabled) context.disable(); if (this.contextManagerEnabled) context.disable();
} }
} }

View File

@ -1,7 +1,15 @@
import { Logger } from '@n8n/backend-common'; import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di'; import { Service } from '@n8n/di';
import type { Context, Exception, Span } from '@opentelemetry/api'; import type { Context, Exception, Span } from '@opentelemetry/api';
import { context, propagation, SpanStatusCode, trace } from '@opentelemetry/api'; import {
context,
defaultTextMapGetter,
defaultTextMapSetter,
ROOT_CONTEXT,
SpanStatusCode,
trace,
} from '@opentelemetry/api';
import { W3CTraceContextPropagator } from '@opentelemetry/core';
import type { ExecutionStatus } from 'n8n-workflow'; import type { ExecutionStatus } from 'n8n-workflow';
import { import {
@ -13,9 +21,11 @@ import {
} from './execution-level-tracer.types'; } from './execution-level-tracer.types';
import { OtelSettingsService } from './otel-settings.service'; import { OtelSettingsService } from './otel-settings.service';
import { ATTR } from './otel.constants'; import { ATTR } from './otel.constants';
import { OtelService } from './otel.service';
import type { TracingContext } from './tracing-context'; import type { TracingContext } from './tracing-context';
const TRACER_NAME = 'n8n-workflow'; const TRACER_NAME = 'n8n-workflow';
const propagator = new W3CTraceContextPropagator();
const UNKNOWN_ERROR_TYPE = 'UnknownError'; const UNKNOWN_ERROR_TYPE = 'UnknownError';
const OBJECT_ERROR_TYPE = 'Object'; const OBJECT_ERROR_TYPE = 'Object';
@ -24,31 +34,28 @@ function isError(status: ExecutionStatus): boolean {
} }
type TrackedSpan = { span: Span }; type TrackedSpan = { span: Span };
type TrackedWorkflowSpan = TrackedSpan & { projectAttributes: Record<string, string> };
@Service() @Service()
export class ExecutionLevelTracer { export class ExecutionLevelTracer {
private readonly activeWorkflowSpans = new Map<string, TrackedSpan>(); private readonly activeWorkflowSpans = new Map<string, TrackedWorkflowSpan>();
private readonly activeNodeSpansByExecutionId = new Map<string, Map<string, TrackedSpan>>(); private readonly activeNodeSpansByExecutionId = new Map<string, Map<string, TrackedSpan>>();
private tracer = trace.getTracer(TRACER_NAME);
/**
* Called by OtelService after a SDK restart so this instance picks up the
* new NodeTracerProvider. Without this, the cached NodeTracer stays bound
* to the old (shutdown) provider and all spans are silently dropped.
*/
refreshTracer(): void {
this.tracer = trace.getTracer(TRACER_NAME);
}
constructor( constructor(
private readonly otelService: OtelService,
private readonly otelSettingsService: OtelSettingsService, private readonly otelSettingsService: OtelSettingsService,
private readonly logger: Logger, private readonly logger: Logger,
) {} ) {}
private get tracer() {
return this.otelService.getTracer(TRACER_NAME);
}
startWorkflow(params: StartWorkflowParams) { startWorkflow(params: StartWorkflowParams) {
try { try {
const parentCtx = this.parseTraceParentHeaders(params.tracingContext); const parentCtx = this.parseTraceParentHeaders(params.tracingContext);
const links = this.buildContinuationLinks(params.linkTo); const links = this.buildContinuationLinks(params.linkTo);
const projectAttributes = buildProjectAttributes(params.project);
const span = this.tracer.startSpan( const span = this.tracer.startSpan(
'workflow.execute', 'workflow.execute',
@ -59,21 +66,18 @@ export class ExecutionLevelTracer {
[ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '', [ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '',
[ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount, [ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount,
[ATTR.EXECUTION_ID]: params.executionId, [ATTR.EXECUTION_ID]: params.executionId,
...(params.project?.id && { [ATTR.PROJECT_ID]: params.project.id }),
...buildCustomAttributes( ...buildCustomAttributes(
ATTR.WORKFLOW_CUSTOM_PREFIX, ATTR.WORKFLOW_CUSTOM_PREFIX,
params.workflow?.customAttributes, params.workflow?.customAttributes,
), ),
...buildCustomAttributes(ATTR.PROJECT_CUSTOM_PREFIX, params.project?.customAttributes), ...projectAttributes,
}, },
links, links,
}, },
parentCtx, parentCtx,
); );
this.activeWorkflowSpans.set(params.executionId, { this.activeWorkflowSpans.set(params.executionId, { span, projectAttributes });
span,
});
return toTracingParentContext(span); return toTracingParentContext(span);
} catch (error) { } catch (error) {
this.logger.warn('Failed to start workflow span', { this.logger.warn('Failed to start workflow span', {
@ -122,10 +126,10 @@ export class ExecutionLevelTracer {
startNode(params: StartNodeParams): void { startNode(params: StartNodeParams): void {
try { try {
// We should always have the node running in a workflow so parentCtx should never be null // We should always have the node running in a workflow so the tracked span should never be missing
const parentCtx = this.findWorkflowSpanContext(params.executionId); const tracked = this.activeWorkflowSpans.get(params.executionId);
if (!parentCtx) { if (!tracked) {
this.logger.warn( this.logger.warn(
'Trying to start a node without a pre-existing parent workflow trace - ignoring', 'Trying to start a node without a pre-existing parent workflow trace - ignoring',
); );
@ -140,9 +144,10 @@ export class ExecutionLevelTracer {
[ATTR.NODE_NAME]: params.node.name, [ATTR.NODE_NAME]: params.node.name,
[ATTR.NODE_TYPE]: params.node.type, [ATTR.NODE_TYPE]: params.node.type,
[ATTR.NODE_TYPE_VERSION]: params.node.typeVersion, [ATTR.NODE_TYPE_VERSION]: params.node.typeVersion,
...tracked.projectAttributes,
}, },
}, },
parentCtx, trace.setSpan(context.active(), tracked.span),
); );
let executionNodes = this.activeNodeSpansByExecutionId.get(params.executionId); let executionNodes = this.activeNodeSpansByExecutionId.get(params.executionId);
@ -220,7 +225,7 @@ export class ExecutionLevelTracer {
const span = this.findMostSpecificSpan(executionId, nodeName); const span = this.findMostSpecificSpan(executionId, nodeName);
if (!span) return; if (!span) return;
propagation.inject(trace.setSpan(context.active(), span), headers); propagator.inject(trace.setSpan(ROOT_CONTEXT, span), headers, defaultTextMapSetter);
} catch (error) { } catch (error) {
this.logger.warn('Failed to inject trace headers', { this.logger.warn('Failed to inject trace headers', {
executionId, executionId,
@ -230,15 +235,14 @@ export class ExecutionLevelTracer {
} }
} }
private parseTraceParentHeaders(tracingContext?: TracingContext) { private parseTraceParentHeaders(tracingContext?: TracingContext): Context {
return tracingContext if (!tracingContext) return ROOT_CONTEXT;
? propagation.extract(context.active(), tracingContext) return propagator.extract(ROOT_CONTEXT, tracingContext, defaultTextMapGetter);
: context.active();
} }
private buildContinuationLinks(linkTo?: TracingContext) { private buildContinuationLinks(linkTo?: TracingContext) {
if (!linkTo) return undefined; if (!linkTo) return undefined;
const extracted = propagation.extract(context.active(), linkTo); const extracted = propagator.extract(ROOT_CONTEXT, linkTo, defaultTextMapGetter);
const spanContext = trace.getSpanContext(extracted); const spanContext = trace.getSpanContext(extracted);
if (!spanContext) return undefined; if (!spanContext) return undefined;
return [ return [
@ -249,11 +253,6 @@ export class ExecutionLevelTracer {
]; ];
} }
private findWorkflowSpanContext(executionId: string) {
const tracked = this.activeWorkflowSpans.get(executionId);
return tracked ? trace.setSpan(context.active(), tracked.span) : undefined;
}
private findMostSpecificSpan(executionId: string, nodeName?: string): Span | undefined { private findMostSpecificSpan(executionId: string, nodeName?: string): Span | undefined {
return ( return (
(nodeName (nodeName
@ -286,6 +285,14 @@ function buildCustomAttributes(
return result; return result;
} }
function buildProjectAttributes(project: StartWorkflowParams['project']): Record<string, string> {
if (!project) return {};
return {
[ATTR.PROJECT_ID]: project.id,
...buildCustomAttributes(ATTR.PROJECT_CUSTOM_PREFIX, project.customAttributes),
};
}
function buildNodeEndAttributes(params: EndNodeParams): Record<string, string | number> { function buildNodeEndAttributes(params: EndNodeParams): Record<string, string | number> {
const attrs: Record<string, string | number> = { const attrs: Record<string, string | number> = {
[ATTR.NODE_ITEMS_INPUT]: params.inputItemCount, [ATTR.NODE_ITEMS_INPUT]: params.inputItemCount,
@ -297,7 +304,7 @@ function buildNodeEndAttributes(params: EndNodeParams): Record<string, string |
function toTracingParentContext(span: Span): TracingContext { function toTracingParentContext(span: Span): TracingContext {
const carrier: Record<string, string> = {}; const carrier: Record<string, string> = {};
propagation.inject(trace.setSpan(context.active(), span), carrier); propagator.inject(trace.setSpan(ROOT_CONTEXT, span), carrier, defaultTextMapSetter);
return { traceparent: carrier.traceparent, tracestate: carrier.tracestate }; return { traceparent: carrier.traceparent, tracestate: carrier.tracestate };
} }

View File

@ -51,7 +51,6 @@ export class OtelLifecycleHandler {
@OnPubSubEvent('reload-otel-config') @OnPubSubEvent('reload-otel-config')
async onReloadOtelConfig(): Promise<void> { async onReloadOtelConfig(): Promise<void> {
await this.otelService.restart(); await this.otelService.restart();
this.tracer.refreshTracer();
} }
private shouldTrace(ctx: { type: string; mode: WorkflowExecuteMode }): boolean { private shouldTrace(ctx: { type: string; mode: WorkflowExecuteMode }): boolean {

View File

@ -2,18 +2,24 @@ import type { Metadata } from '@grpc/grpc-js';
import { Logger } from '@n8n/backend-common'; import { Logger } from '@n8n/backend-common';
import { OutboundHttp } from '@n8n/backend-network'; import { OutboundHttp } from '@n8n/backend-network';
import { Service } from '@n8n/di'; import { Service } from '@n8n/di';
import type { DiagLogger } from '@opentelemetry/api'; import type { DiagLogger, Tracer, TracerProvider } from '@opentelemetry/api';
import { DiagLogLevel, diag, context, metrics, propagation, trace } from '@opentelemetry/api'; import { DiagLogLevel, ProxyTracerProvider, diag, trace } from '@opentelemetry/api';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
import { resourceFromAttributes } from '@opentelemetry/resources'; import {
import { NodeSDK } from '@opentelemetry/sdk-node'; detectResources,
envDetector,
hostDetector,
processDetector,
resourceFromAttributes,
} from '@opentelemetry/resources';
import { import {
BasicTracerProvider, BasicTracerProvider,
BatchSpanProcessor,
type ReadableSpan, type ReadableSpan,
type SpanExporter, type SpanExporter,
type SpanProcessor, type SpanProcessor,
} from '@opentelemetry/sdk-trace-base'; } from '@opentelemetry/sdk-trace-base';
import { TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-node'; import { NodeTracerProvider, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-node';
import { InstanceSettings } from 'n8n-core'; import { InstanceSettings } from 'n8n-core';
import { OperationalError } from 'n8n-workflow'; import { OperationalError } from 'n8n-workflow';
@ -29,10 +35,22 @@ export type OtelTestTraceResult = { success: true } | { success: false; error: s
const stripEmptyResolutionNote = (message: string) => const stripEmptyResolutionNote = (message: string) =>
message.replace(/\s*Resolution note:\s*$/, ''); message.replace(/\s*Resolution note:\s*$/, '');
// Deprecated in `@opentelemetry/api` 1.x, but the only public source of a no-op tracer provider.
const noopTracerProvider = new ProxyTracerProvider();
function registeredGlobalTracerProvider(): TracerProvider | undefined {
const globalProvider = trace.getTracerProvider();
const delegate =
globalProvider instanceof ProxyTracerProvider ? globalProvider.getDelegate() : globalProvider;
return delegate === noopTracerProvider.getDelegate() ? undefined : delegate;
}
@Service() @Service()
export class OtelService { export class OtelService {
private static isDiagnosticsLoggerConfigured = false; private static isDiagnosticsLoggerConfigured = false;
private sdk?: NodeSDK; private provider?: NodeTracerProvider;
private ownsGlobalApi = false;
private hasLoggedForeignGlobalApiOwner = false;
constructor( constructor(
private readonly otelSettingsService: OtelSettingsService, private readonly otelSettingsService: OtelSettingsService,
@ -52,9 +70,13 @@ export class OtelService {
await this.start(settings); await this.start(settings);
} }
getTracer(name: string): Tracer {
return (this.provider ?? noopTracerProvider).getTracer(name);
}
/** /**
* Sends a single `n8n.test_trace` span to the given OTLP endpoint and waits * Sends a single `n8n.test_trace` span to the given OTLP endpoint and waits
* for the exporter's result. Unlike the long-running SDK (which batches spans * for the exporter's result. Unlike the long-running provider (which batches spans
* fire-and-forget), this uses a throwaway provider/exporter so the collector's * fire-and-forget), this uses a throwaway provider/exporter so the collector's
* response — success, a rejection (HTTP status or gRPC status code), or a * response — success, a rejection (HTTP status or gRPC status code), or a
* network error — can be reported back to the caller. Runs independently of * network error — can be reported back to the caller. Runs independently of
@ -111,9 +133,9 @@ export class OtelService {
if (!settings.enabled) return; if (!settings.enabled) return;
this.configureDiagnosticsLogger(); this.configureDiagnosticsLogger();
let sdk: NodeSDK; let provider: NodeTracerProvider;
try { try {
sdk = await this.startSdk(settings); provider = await this.startProvider(settings);
} catch (error) { } catch (error) {
this.logger.error('Failed to start OpenTelemetry tracing, so tracing stays off', { this.logger.error('Failed to start OpenTelemetry tracing, so tracing stays off', {
error: error instanceof Error ? error.message : String(error), error: error instanceof Error ? error.message : String(error),
@ -122,47 +144,67 @@ export class OtelService {
return; return;
} }
void this.checkEndpointReachability(settings, sdk); void this.checkEndpointReachability(settings, provider);
} }
async shutdown(): Promise<void> { async shutdown(): Promise<void> {
// Cleared before the flush, so a probe that fails meanwhile sees that its SDK is gone. // Cleared before the flush, so a probe that fails meanwhile sees that its provider is gone.
const sdk = this.sdk; const provider = this.provider;
this.sdk = undefined; this.provider = undefined;
try { try {
await sdk?.shutdown(); await provider?.shutdown();
} catch (error) { } catch (error) {
this.logger.warn( this.logger.warn(
'Failed to cleanly shut down OpenTelemetry SDK (exporter flush may have failed)', 'Failed to cleanly shut down OpenTelemetry tracer provider (exporter flush may have failed)',
{ error: error instanceof Error ? error.message : String(error) }, { error: error instanceof Error ? error.message : String(error) },
); );
} finally {
// Unregister the global providers so the next NodeSDK.start() can register
// new ones. Without this, OTel's allowOverride=false guard blocks
// re-registration and the restart silently fails.
trace?.disable();
context?.disable();
propagation?.disable();
metrics?.disable();
} }
} }
private async startSdk(settings: OtelConfig): Promise<NodeSDK> { private async startProvider(settings: OtelConfig): Promise<NodeTracerProvider> {
const traceExporter = await this.createTraceExporter(settings); const traceExporter = await this.createTraceExporter(settings);
this.sdk = new NodeSDK({ this.provider = new NodeTracerProvider({
resource: resourceFromAttributes({ resource: this.buildResource(settings.exporterServiceName),
[ATTR.OTEL_SERVICE_NAME]: settings.exporterServiceName,
[ATTR.OTEL_SERVICE_VERSION]: N8N_VERSION,
[ATTR.INSTANCE_ID]: this.instanceSettings.instanceId,
[ATTR.INSTANCE_ROLE]: this.instanceSettings.instanceType,
}),
traceExporter,
sampler: new TraceIdRatioBasedSampler(settings.tracesSampleRate), sampler: new TraceIdRatioBasedSampler(settings.tracesSampleRate),
spanProcessors: [new BatchSpanProcessor(traceExporter)],
}); });
this.registerGlobalApi(this.provider);
return this.provider;
}
this.sdk.start(); private buildResource(serviceName: string) {
return this.sdk; return resourceFromAttributes({
[ATTR.OTEL_SERVICE_NAME]: serviceName,
[ATTR.OTEL_SERVICE_VERSION]: N8N_VERSION,
[ATTR.INSTANCE_ID]: this.instanceSettings.instanceId,
[ATTR.INSTANCE_ROLE]: this.instanceSettings.instanceType,
}).merge(detectResources({ detectors: [envDetector, processDetector, hostDetector] }));
}
private registerGlobalApi(provider: NodeTracerProvider): void {
if (this.ownsGlobalApi) {
trace.disable();
this.ownsGlobalApi = trace.setGlobalTracerProvider(provider);
return;
}
if (registeredGlobalTracerProvider()) {
this.logForeignGlobalApiOwner();
return;
}
provider.register();
this.ownsGlobalApi = registeredGlobalTracerProvider() === provider;
}
private logForeignGlobalApiOwner(): void {
if (this.hasLoggedForeignGlobalApiOwner) return;
this.hasLoggedForeignGlobalApiOwner = true;
this.logger.info(
'Another library owns the global OpenTelemetry API, so n8n workflow tracing runs on its own tracer provider',
);
} }
private async createTraceExporter( private async createTraceExporter(
@ -284,7 +326,7 @@ export class OtelService {
private async checkEndpointReachability( private async checkEndpointReachability(
settings: OtelConfig, settings: OtelConfig,
startedSdk: NodeSDK, startedProvider: NodeTracerProvider,
): Promise<void> { ): Promise<void> {
const url = this.resolveExporterUrl(settings); const url = this.resolveExporterUrl(settings);
const timeoutMs = settings.startupConnectivityTimeoutMs; const timeoutMs = settings.startupConnectivityTimeoutMs;
@ -296,8 +338,8 @@ export class OtelService {
await this.probeHttpEndpoint(url, timeoutMs); await this.probeHttpEndpoint(url, timeoutMs);
} }
} catch (error) { } catch (error) {
// A restart or shutdown replaced the SDK this probe belongs to. // A restart or shutdown replaced the provider this probe belongs to.
if (this.sdk !== startedSdk) return; if (this.provider !== startedProvider) return;
this.logger.error('Failed to connect to OpenTelemetry OTLP endpoint during startup', { this.logger.error('Failed to connect to OpenTelemetry OTLP endpoint during startup', {
endpoint: url, endpoint: url,

View File

@ -1,3 +1,5 @@
import { Time } from '@n8n/constants';
import type { PubSub } from './pubsub/pubsub.types'; import type { PubSub } from './pubsub/pubsub.types';
export const JOB_TYPE_NAME = 'job'; export const JOB_TYPE_NAME = 'job';
@ -11,6 +13,12 @@ export const WORKER_RESPONSE_PUBSUB_CHANNEL = 'n8n.worker-response';
/** Pubsub channel for MCP relay messages between main instances in multi-main queue mode. */ /** Pubsub channel for MCP relay messages between main instances in multi-main queue mode. */
export const MCP_RELAY_PUBSUB_CHANNEL = 'n8n.mcp-relay'; export const MCP_RELAY_PUBSUB_CHANNEL = 'n8n.mcp-relay';
/** How often to re-issue SUBSCRIBE on the pubsub subscriber connection. Short enough to keep idle NAT/IPVS flows alive. */
export const SUBSCRIBER_LIVENESS_INTERVAL_MS = 30 * Time.seconds.toMilliseconds;
/** How long to wait for the SUBSCRIBE reply before dropping the subscriber connection so ioredis reconnects. */
export const SUBSCRIBER_LIVENESS_TIMEOUT_MS = 10 * Time.seconds.toMilliseconds;
/** /**
* Max allowed size in bytes of a message relayed over the pubsub channel. Events * Max allowed size in bytes of a message relayed over the pubsub channel. Events
* exceeding this are skipped (or trimmed) rather than bloating the channel. * exceeding this are skipped (or trimmed) rather than bloating the channel.

View File

@ -20,6 +20,12 @@ describe('Subscriber', () => {
const executionsConfig = mockInstance(ExecutionsConfig, { mode: 'queue' }); const executionsConfig = mockInstance(ExecutionsConfig, { mode: 'queue' });
const globalConfig = mockInstance(GlobalConfig, { redis: { prefix: 'n8n' } }); const globalConfig = mockInstance(GlobalConfig, { redis: { prefix: 'n8n' } });
function getHandler(event: string) {
const call = client.on.mock.calls.find(([e]) => e === event);
expect(call).toBeDefined();
return call![1] as () => void;
}
describe('constructor', () => { describe('constructor', () => {
it('should init Redis client in scaling mode', () => { it('should init Redis client in scaling mode', () => {
const subscriber = new Subscriber( const subscriber = new Subscriber(
@ -82,6 +88,149 @@ describe('Subscriber', () => {
}); });
}); });
describe('reconnect', () => {
beforeEach(() => {
client.on.mockClear();
client.subscribe.mockClear();
});
it('should resubscribe to all channels when the client recovers from a lost connection', async () => {
const subscriber = new Subscriber(
mockLogger(),
mock(),
mock(),
redisClientService,
executionsConfig,
globalConfig,
);
await subscriber.subscribe(subscriber.getCommandChannel());
await subscriber.subscribe(subscriber.getWorkerResponseChannel());
client.subscribe.mockClear();
getHandler('close')();
getHandler('ready')();
await vi.waitFor(() => expect(client.subscribe).toHaveBeenCalledTimes(2));
expect(client.subscribe).toHaveBeenCalledWith('n8n:n8n.commands', expect.any(Function));
expect(client.subscribe).toHaveBeenCalledWith(
'n8n:n8n.worker-response',
expect.any(Function),
);
});
it('should retry on the next ready when resubscribing fails', async () => {
const scopedLogger = mock<Logger>();
const logger = mock<Logger>({ scoped: vi.fn().mockReturnValue(scopedLogger) });
const subscriber = new Subscriber(
logger,
mock(),
mock(),
redisClientService,
executionsConfig,
globalConfig,
);
await subscriber.subscribe(subscriber.getCommandChannel());
client.subscribe.mockClear();
client.subscribe.mockRejectedValueOnce(new Error('write ECONNRESET'));
getHandler('close')();
getHandler('ready')();
await vi.waitFor(() =>
expect(scopedLogger.error).toHaveBeenCalledWith(
'Failed to resubscribe to pubsub channels after Redis reconnect',
expect.any(Object),
),
);
getHandler('ready')();
await vi.waitFor(() => expect(client.subscribe).toHaveBeenCalledTimes(2));
});
it('should not resubscribe on the initial ready', async () => {
const subscriber = new Subscriber(
mockLogger(),
mock(),
mock(),
redisClientService,
executionsConfig,
globalConfig,
);
await subscriber.subscribe(subscriber.getCommandChannel());
client.subscribe.mockClear();
getHandler('ready')();
await Promise.resolve();
expect(client.subscribe).not.toHaveBeenCalled();
});
});
describe('liveness', () => {
beforeEach(() => {
vi.useFakeTimers();
client.subscribe.mockReset();
client.disconnect.mockClear();
});
afterEach(() => {
vi.useRealTimers();
});
async function subscribedSubscriber() {
const subscriber = new Subscriber(
mockLogger(),
mock(),
mock(),
redisClientService,
executionsConfig,
globalConfig,
);
await subscriber.subscribe(subscriber.getCommandChannel());
await subscriber.subscribe(subscriber.getWorkerResponseChannel());
client.subscribe.mockReset();
return subscriber;
}
it('should re-issue SUBSCRIBE for all channels periodically', async () => {
const subscriber = await subscribedSubscriber();
client.subscribe.mockResolvedValue(2);
await vi.advanceTimersByTimeAsync(30_000 + 10_000);
expect(client.subscribe).toHaveBeenCalledTimes(1);
expect(client.subscribe).toHaveBeenCalledWith('n8n:n8n.commands', 'n8n:n8n.worker-response');
expect(client.disconnect).not.toHaveBeenCalled();
subscriber.shutdown();
});
it('should drop the connection when SUBSCRIBE gets no reply in time', async () => {
const subscriber = await subscribedSubscriber();
client.subscribe.mockReturnValue(new Promise(() => {}));
await vi.advanceTimersByTimeAsync(30_000 + 10_000);
expect(client.subscribe).toHaveBeenCalledTimes(1);
expect(client.disconnect).toHaveBeenCalledWith(true);
subscriber.shutdown();
});
it('should not check before any channel was requested', async () => {
const subscriber = new Subscriber(
mockLogger(),
mock(),
mock(),
redisClientService,
executionsConfig,
globalConfig,
);
await vi.advanceTimersByTimeAsync(60_000);
expect(client.subscribe).not.toHaveBeenCalled();
subscriber.shutdown();
});
});
describe('prefix isolation', () => { describe('prefix isolation', () => {
it('should apply configured prefix when subscribing to channels', async () => { it('should apply configured prefix when subscribing to channels', async () => {
const customConfig = mockInstance(GlobalConfig, { redis: { prefix: 'n8n-instance-1' } }); const customConfig = mockInstance(GlobalConfig, { redis: { prefix: 'n8n-instance-1' } });

View File

@ -15,6 +15,8 @@ import {
COMMAND_PUBSUB_CHANNEL, COMMAND_PUBSUB_CHANNEL,
WORKER_RESPONSE_PUBSUB_CHANNEL, WORKER_RESPONSE_PUBSUB_CHANNEL,
MCP_RELAY_PUBSUB_CHANNEL, MCP_RELAY_PUBSUB_CHANNEL,
SUBSCRIBER_LIVENESS_INTERVAL_MS,
SUBSCRIBER_LIVENESS_TIMEOUT_MS,
} from '../constants'; } from '../constants';
/** /**
@ -45,6 +47,12 @@ export class Subscriber {
private readonly debouncedHandlers = new Map<string, ReturnType<typeof debounce>>(); private readonly debouncedHandlers = new Map<string, ReturnType<typeof debounce>>();
private readonly channels = new Set<string>();
private lostConnection = false;
private livenessTimer?: NodeJS.Timeout;
constructor( constructor(
private readonly logger: Logger, private readonly logger: Logger,
private readonly instanceSettings: InstanceSettings, private readonly instanceSettings: InstanceSettings,
@ -66,6 +74,21 @@ export class Subscriber {
this.client = this.redisClientService.createClient({ type: 'subscriber(n8n)' }); this.client = this.redisClientService.createClient({ type: 'subscriber(n8n)' });
// ioredis replays SUBSCRIBE on reconnect without awaiting it, so a failed replay leaves
// the connection ready with zero subscriptions. Re-issue our own on every recovery.
this.client.on('close', () => {
this.lostConnection = true;
});
this.client.on('ready', () => {
if (!this.lostConnection) return;
this.lostConnection = false;
void this.resubscribe();
});
this.livenessTimer = setInterval(async () => {
await this.checkLiveness();
}, SUBSCRIBER_LIVENESS_INTERVAL_MS).unref();
const handlerFn = (msg: PubSub.Command | PubSub.WorkerResponse) => { const handlerFn = (msg: PubSub.Command | PubSub.WorkerResponse) => {
this.pubsubEventBus.emit(this.eventNameFrom(msg), msg.payload); this.pubsubEventBus.emit(this.eventNameFrom(msg), msg.payload);
}; };
@ -145,11 +168,13 @@ export class Subscriber {
// @TODO: Use `@OnShutdown()` decorator // @TODO: Use `@OnShutdown()` decorator
shutdown() { shutdown() {
clearInterval(this.livenessTimer);
for (const handler of this.debouncedHandlers.values()) handler.cancel(); for (const handler of this.debouncedHandlers.values()) handler.cancel();
this.client.disconnect(); this.client.disconnect();
} }
async subscribe(channel: string) { async subscribe(channel: string) {
this.channels.add(channel);
await this.client.subscribe(channel, (error) => { await this.client.subscribe(channel, (error) => {
if (error) { if (error) {
this.logger.error(`Failed to subscribe to channel ${channel}`, { error }); this.logger.error(`Failed to subscribe to channel ${channel}`, { error });
@ -160,6 +185,43 @@ export class Subscriber {
}); });
} }
private async resubscribe() {
if (this.channels.size === 0) return;
try {
for (const channel of this.channels) await this.subscribe(channel);
this.logger.info('Resubscribed to pubsub channels after Redis reconnect', {
channels: [...this.channels],
});
} catch (error) {
this.lostConnection = true;
this.logger.error('Failed to resubscribe to pubsub channels after Redis reconnect', {
error,
});
}
}
/**
* A subscriber connection is idle by nature, so a half-open socket (peer gone, no RST
* received) is never detected by ioredis. Re-issuing SUBSCRIBE is idempotent, travels over
* the subscriber connection in both single-node and cluster mode (PING would not), and
* confirms the subscriptions still exist. No reply in time drops the socket so ioredis
* reconnects and `resubscribe` runs.
*/
private async checkLiveness() {
if (this.channels.size === 0) return;
const timeout = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error('timeout')), SUBSCRIBER_LIVENESS_TIMEOUT_MS).unref();
});
try {
await Promise.race([this.client.subscribe(...this.channels), timeout]);
} catch (error) {
this.logger.warn('Pubsub subscriber connection is unresponsive, reconnecting', { error });
this.client.disconnect(true);
}
}
private eventNameFrom(msg: PubSub.Command | PubSub.WorkerResponse) { private eventNameFrom(msg: PubSub.Command | PubSub.WorkerResponse) {
return 'command' in msg ? msg.command : msg.response; return 'command' in msg ? msg.command : msg.response;
} }

View File

@ -1,6 +1,6 @@
{ {
"name": "n8n-core", "name": "n8n-core",
"version": "2.41.0", "version": "2.41.2",
"description": "Core functionality of n8n", "description": "Core functionality of n8n",
"main": "dist/index", "main": "dist/index",
"types": "dist/index.d.ts", "types": "dist/index.d.ts",

View File

@ -1,6 +1,6 @@
{ {
"name": "@n8n/chat", "name": "@n8n/chat",
"version": "1.40.0", "version": "1.40.1",
"scripts": { "scripts": {
"clean": "rimraf dist .turbo", "clean": "rimraf dist .turbo",
"dev": "pnpm run --dir=../storybook dev --initial-path=/docs/chat-chat--docs", "dev": "pnpm run --dir=../storybook dev --initial-path=/docs/chat-chat--docs",

Some files were not shown because too many files have changed in this diff Show More