mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-18 14:56:50 +02:00
Compare commits
13 Commits
master
...
n8n@2.41.3
| Author | SHA1 | Date | |
|---|---|---|---|
|
7f7a8ac25b | ||
|
|
29afe517b3 | ||
|
|
2db9ab8dce | ||
|
|
56f9e97b8e | ||
|
|
2de7d47c2e | ||
|
|
b4e38b3cc7 | ||
|
|
665bcd9af7 | ||
|
|
07f2d3675a | ||
|
|
4c5ecc22fc | ||
|
|
397e4f2eb7 | ||
|
|
d8c49e2a3e | ||
|
|
69022dc863 | ||
|
|
b3196fb520 |
36
CHANGELOG.md
36
CHANGELOG.md
|
|
@ -1,3 +1,39 @@
|
|||
## [2.41.3](https://github.com/n8n-io/n8n/compare/n8n@2.41.2...n8n@2.41.3) (2026-09-25)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **core:** Propagate project span attributes to node spans ([#39456](https://github.com/n8n-io/n8n/issues/39456)) ([29afe51](https://github.com/n8n-io/n8n/commit/29afe517b3a6954a6070c13e26dde9e9d648bfe3))
|
||||
|
||||
|
||||
## [2.41.2](https://github.com/n8n-io/n8n/compare/n8n@2.41.1...n8n@2.41.2) (2026-09-24)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **core:** Keep OpenTelemetry export working after a restart when Sentry is enabled ([#39367](https://github.com/n8n-io/n8n/issues/39367)) ([2de7d47](https://github.com/n8n-io/n8n/commit/2de7d47c2e079bd4474984b11e038877ac5fd7cf))
|
||||
* **core:** Record live n8n Assistant test runs as verification evidence ([#39383](https://github.com/n8n-io/n8n/issues/39383)) ([07f2d36](https://github.com/n8n-io/n8n/commit/07f2d3675a31fbdc2b82d3e5c205669102d9bb2c))
|
||||
* **core:** Retry instance reports that cross the UTC midnight boundary ([#39433](https://github.com/n8n-io/n8n/issues/39433)) ([56f9e97](https://github.com/n8n-io/n8n/commit/56f9e97b8e026d072e98c0eea6ce17f3c6076b4c))
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **core:** Authenticate instance reports with the license certificate ([#39380](https://github.com/n8n-io/n8n/issues/39380)) ([4c5ecc2](https://github.com/n8n-io/n8n/commit/4c5ecc22fce7990ae4ff9ba44bfb99f3c202b61d))
|
||||
|
||||
|
||||
## [2.41.1](https://github.com/n8n-io/n8n/compare/n8n@2.41.0...n8n@2.41.1) (2026-09-23)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **core:** Restore pubsub subscriptions after Redis reconnect and detect half-open subscriber connections ([#39241](https://github.com/n8n-io/n8n/issues/39241)) ([b3196fb](https://github.com/n8n-io/n8n/commit/b3196fb520d4011ed66fdfcceb8da5da2b2e819d))
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **core:** Enable Agents by default ([#39328](https://github.com/n8n-io/n8n/issues/39328)) ([d8c49e2](https://github.com/n8n-io/n8n/commit/d8c49e2a3e85344e3d651bcf26dea90fa3707d2c))
|
||||
|
||||
|
||||
# [2.41.0](https://github.com/n8n-io/n8n/compare/n8n@2.40.0...n8n@2.41.0) (2026-09-22)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "n8n-monorepo",
|
||||
"version": "2.41.0",
|
||||
"version": "2.41.3",
|
||||
"private": true,
|
||||
"engines": {
|
||||
"node": ">=24.0.0",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/agents",
|
||||
"version": "0.26.0",
|
||||
"version": "0.26.2",
|
||||
"description": "AI agent SDK for n8n's code-first execution engine",
|
||||
"engines": {
|
||||
"node": ">=24.0.0"
|
||||
|
|
|
|||
|
|
@ -67,29 +67,31 @@
|
|||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqshPA7QoqLUtLRhcA\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Krds9KGve8gJNTNa9jn6N7\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"engineering-status\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1045,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCd2zmwW8c3i9MDr6fG\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"I'll load the engineering-status skill first, then use it to format the status token.\"},{\"type\":\"tool_use\",\"id\":\"toolu_019R6LssCYfGrz8ddVcPvYQC\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"engineering-status\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1045,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":76,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:04Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:55Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:04Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:55Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:04Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:55Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:04Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:55Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f485dd8bc7b-PRG",
|
||||
"cf-ray": "a3f01ad7ea6cbbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 10 Sep 2026 07:12:04 GMT",
|
||||
"request-id": "req_011CeuLqs6fxp1AkSuMSRWYb",
|
||||
"date": "Tue, 22 Sep 2026 08:57:55 GMT",
|
||||
"request-id": "req_011CfJCd2nsYW1GzkhCoPFhw",
|
||||
"server": "cloudflare",
|
||||
"server-timing": "x-originResponse;dur=667",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-77c37e49f4c62f7623d8cc66c4f3a26a-6bf7325e3f3f69d3-01",
|
||||
"traceresponse": "00-234e70eda24ff35d6c847bca1695098f-c9010c0bb2f634e5-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
@ -105,7 +107,7 @@
|
|||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually.\n\n</active_skills>"
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
|
|
@ -121,9 +123,13 @@
|
|||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I'll load the engineering-status skill first, then use it to format the status token."
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01Krds9KGve8gJNTNa9jn6N7",
|
||||
"id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
|
||||
"name": "load_skill",
|
||||
"input": {
|
||||
"skillId": "engineering-status"
|
||||
|
|
@ -139,8 +145,17 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01Krds9KGve8gJNTNa9jn6N7",
|
||||
"content": "{\"skillId\":\"engineering-status\",\"active\":true}"
|
||||
"tool_use_id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"skillId\":\"engineering-status\",\"active\":true}"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -189,29 +204,30 @@
|
|||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqxFjGLPD3QE1zh7Ef\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Now I'll format the status token:\"},{\"type\":\"tool_use\",\"id\":\"toolu_01CKRsNPiekmX9TWf3ooQdHU\",\"name\":\"format_status\",\"input\":{\"status\":\"waiting_on_vendor_info\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1244,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":68,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCd6hmu4KWXr78SxvJK\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Now I'll format the status token using the format_status tool:\"},{\"type\":\"tool_use\",\"id\":\"toolu_011UF3b8wMRddXy2WYPGRYZe\",\"name\":\"format_status\",\"input\":{\"status\":\"waiting_on_vendor_info\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1230,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":74,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:05Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:57Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:05Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:57Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:05Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:56Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:05Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:57Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f4f38babc7b-PRG",
|
||||
"cf-ray": "a3f01add2e2bbbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 10 Sep 2026 07:12:06 GMT",
|
||||
"request-id": "req_011CeuLqwpw3fPBk6xt6nrrm",
|
||||
"date": "Tue, 22 Sep 2026 08:57:57 GMT",
|
||||
"request-id": "req_011CfJCd6Vcovu2SLcUWAPC4",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-d17795404368021a8e502b8755cc4b35-c750063766d5eb87-01",
|
||||
"traceresponse": "00-e98096358e475d79d338f7d1cf690dbb-aa6a8d0521eacc76-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
@ -227,7 +243,7 @@
|
|||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually.\n\n</active_skills>"
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Engineering status formatter\"\n description: \"Use for formatting engineering workflow status identifiers.\"\n id: \"engineering-status\"\n category: \"engineering\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
|
|
@ -243,9 +259,13 @@
|
|||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "I'll load the engineering-status skill first, then use it to format the status token."
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01Krds9KGve8gJNTNa9jn6N7",
|
||||
"id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
|
||||
"name": "load_skill",
|
||||
"input": {
|
||||
"skillId": "engineering-status"
|
||||
|
|
@ -261,8 +281,17 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01Krds9KGve8gJNTNa9jn6N7",
|
||||
"content": "{\"skillId\":\"engineering-status\",\"active\":true}"
|
||||
"tool_use_id": "toolu_019R6LssCYfGrz8ddVcPvYQC",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"skillId\":\"engineering-status\",\"active\":true}"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "[Skill: \"Engineering status formatter\"]\n[Skill category: \"engineering\"]\n[Skill hash: \"f62e1e0bd274\"]\n\nFor status-formatting requests, call the format_status tool with the exact status token from the user. Use the returned formatted value verbatim in the final answer and include STATUS_FORMATTED_BRAVO. Do not format the status manually."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -271,11 +300,11 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Now I'll format the status token:"
|
||||
"text": "Now I'll format the status token using the format_status tool:"
|
||||
},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01CKRsNPiekmX9TWf3ooQdHU",
|
||||
"id": "toolu_011UF3b8wMRddXy2WYPGRYZe",
|
||||
"name": "format_status",
|
||||
"input": {
|
||||
"status": "waiting_on_vendor_info"
|
||||
|
|
@ -291,7 +320,7 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01CKRsNPiekmX9TWf3ooQdHU",
|
||||
"tool_use_id": "toolu_011UF3b8wMRddXy2WYPGRYZe",
|
||||
"content": "{\"formatted\":\"STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO\"}"
|
||||
}
|
||||
]
|
||||
|
|
@ -341,29 +370,30 @@
|
|||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLr3AQ92UTrYgeFRerd\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The status token has been formatted successfully:\\n\\n**STATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO**\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1346,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":32,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCdCtqry5T2NJGAMCjV\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Perfect! The engineering-status skill has been loaded and the status token has been formatted.\\n\\n**Formatted Status:**\\nSTATUS_FORMATTED_BRAVO: WAITING_ON_VENDOR_INFO\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1338,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":46,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:06Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:57Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:06Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:58Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:06Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:57Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:06Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:57Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f561a95bc7b-PRG",
|
||||
"cf-ray": "a3f01ae61be4bbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 10 Sep 2026 07:12:07 GMT",
|
||||
"request-id": "req_011CeuLr2Whz42pkAm7AzGhs",
|
||||
"date": "Tue, 22 Sep 2026 08:57:58 GMT",
|
||||
"request-id": "req_011CfJCdCcii1dW1VhpqXjuj",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-80319e2d05f81fe007810b9d0f60ed01-c7956b0a3a60554e-01",
|
||||
"traceresponse": "00-4d1157be05f9b0ab551736730e6ba0e2-b17f94e742ef7628-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -51,29 +51,30 @@
|
|||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqj5DrEFutdjSD8ETs\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Eq5aFLJhGriLQF4UBg7tyT\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"summary-marker\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":957,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCcuKuoVBeB7Ake3FNv\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Rhg9DwH7n3MzSh9iz6TFzh\",\"name\":\"load_skill\",\"input\":{\"skillId\":\"summary-marker\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":957,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:02Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:53Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:02Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:54Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:02Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:53Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:02Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:53Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f3bd9d2bc7b-PRG",
|
||||
"cf-ray": "a3f01acbda36bbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 10 Sep 2026 07:12:02 GMT",
|
||||
"request-id": "req_011CeuLqiWV7tDxhVGbXmda8",
|
||||
"date": "Tue, 22 Sep 2026 08:57:54 GMT",
|
||||
"request-id": "req_011CfJCcteyXnhh5BmvQdDTp",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-7cf7d7c3cfa9378a22214da3c62a1ea0-1ad2b8e7d42a1569-01",
|
||||
"traceresponse": "00-66ff12c75cd9a23f50952244bb3cba2c-133f511c645be506-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
@ -89,7 +90,7 @@
|
|||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged.\n\n</active_skills>"
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
|
|
@ -107,7 +108,7 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01Eq5aFLJhGriLQF4UBg7tyT",
|
||||
"id": "toolu_01Rhg9DwH7n3MzSh9iz6TFzh",
|
||||
"name": "load_skill",
|
||||
"input": {
|
||||
"skillId": "summary-marker"
|
||||
|
|
@ -123,8 +124,17 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01Eq5aFLJhGriLQF4UBg7tyT",
|
||||
"content": "{\"skillId\":\"summary-marker\",\"active\":true}"
|
||||
"tool_use_id": "toolu_01Rhg9DwH7n3MzSh9iz6TFzh",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"skillId\":\"summary-marker\",\"active\":true}"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -157,29 +167,30 @@
|
|||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeuLqo1c45tLcGPFM5dEA\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"**Summary:** n8n agents maintain efficient workflow operations by keeping context compact, reducing overhead and improving performance.\\n\\nSKILL_MARKER_ALPHA\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1139,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":34,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfJCcxfLB1kLZMwT9hmcx\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"Based on the Summary marker skill instructions, here's the summary:\\n\\n**Summary:** n8n agents maintain workflow context efficiency by keeping data structures and variables compact, reducing memory overhead and improving performance during workflow execution.\\n\\nSKILL_MARKER_ALPHA\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":1106,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":55,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:03Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:54Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:03Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:55Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:02Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:54Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:03Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:54Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f417db9bc7b-PRG",
|
||||
"cf-ray": "a3f01ad17e0abbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 10 Sep 2026 07:12:03 GMT",
|
||||
"request-id": "req_011CeuLqnRP3vkdg9opi7gNC",
|
||||
"date": "Tue, 22 Sep 2026 08:57:55 GMT",
|
||||
"request-id": "req_011CfJCcxUAmJjy1fhrHrwCP",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-86beb3374d817d1d427aabf5c903ca34-f5a5fa52f1cb2789-01",
|
||||
"traceresponse": "00-7063877d53ab0eae4686a22ce533a4ec-69260e5b962e00c7-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -53,32 +53,34 @@
|
|||
},
|
||||
"status": 200,
|
||||
"response": [
|
||||
"1f8b0800000000000013c5554d8bdb3010bde757189d1348c2ba21be9585d2427ae852e8a12942b1a6896259f2ea23ec12f4df3b72e2f823d96d2f253e18346fa479ef69d0c00194cb9212ac655ba0d631e3469c39962547e25e2b2019e98164dcac497624a5e620312597cc7398ec9828fce461924ee6d3793a9b4e67982e783cc36ee974367b04bf328ba765fa8def3fa63f5e162bf3b9c09c7e250c182d6380592bb0ac8a6573ad1c9225d9cf5fa705130a0cc99497724cacd31535c0ac56bd9085670f2a875e90036e96b689f9468e509577d4e9021482cbf4010bb17c0734c7939dd08af633a60d8e307f0b6bf6c60250eda004c3244dcbebfc169ded86681813ed5d3734ff806ac01c440ed489e8048956716678745dfd061385d32d68849476941d5035dba0b3212449188de074fb6767e946eabcb8dd033752ea221c5e4e52bb78547adee7b496d45b68fa20ae3d76c222cfbf2f3f6ded53517eddad9ee71b2b166a85598a95719fd468a82d84947519d48d878668a99451eaa500170672570b3a7fadae4aa8ed4048426290bca51dbbd9b1f7b5d7293deda7484be974737becc44b72857e09bcd77ddd9e84843ed1bb9038ae892561e8d95da8c47bfec2d7b52f77a69225688b2fb1a1eecba364e67582bf8284ebc6be0b23306b125a36ef528aafecdf9e105d7508b50735d3e6a6aa1e483a2a7acf7ff7d5f9e721309828e1ff8d85c13b9e2ec2c0cb76e05ebbd8c5c8e512fe00c6a9d291c7070000"
|
||||
"1f8b0800000000000013cd544d6fd34010bdf757587376243b6a5a756f10a440c501845490085a4dbdd364f17ad7ec47441bf9bfa3b5ebc63629f482826f3b6fc6f3dedbd9a11d69cf928a9cc30d71e7d1fa33811e59b2077f5f1330188190f667607ba88c20050c0a8541d06c8bb20cb3f3d96236cfe68b3ccb7248418af80fb7e1599e2fefae9762e5de5cac1e56e2cb03bef5b95c423ae9042958a362009d93cea38e6d0ba33d690fecebb7ee80529305a6835229386f6a6e099dd1a390a31f817441a3a0208f52b93e167a3952d7c1736f4ad20ed8d5e23c85028b2df1c2127a69341f67643d6e09c573585f1b1b50bda58a2c2abea87ecf3fa0f9768a362998e087a1f9450a8eec4e16c4bd8c4e40b44aa015d1757d47360ae71b32c0401bcf718752e1ad22689a2469cecea8bbfd4767f9ad3245797c068ea4b44d04fdeca40ef1a8f4b1ce1ba37870d4cf413c079ee5d757bbf2e6f2755d7d78f5193fdd6cdf7f9ccf439c048d55ac53060577a554aa6d53070f6cdf444b958a529f1a0869a9f0ada0f63ba8aaa5de4c64241083f09c7241cae39f95b72923e55de440a8bbb7efcee8a7e41aad97a8da2030809eeb98ef49b8ecd7d0b9dcfc0764de8935b0640dae65736232a1aad0decf0e9775623e15da92ec1a9a178d4f5c737f7bc3a61e509afcaddff947c58d401888192de1e1db7ff12a9eecf5e6df2de7c9365d5c0e6d3de2c431478718b445bf00ccc6474350070000"
|
||||
],
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:07Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:58Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:07Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:58Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:07Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:58Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:07Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:58Z",
|
||||
"cache-control": "no-cache",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f5c5a57bc7b-PRG",
|
||||
"cf-ray": "a3f01aebf8c4bbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-encoding": "gzip",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"date": "Thu, 10 Sep 2026 07:12:08 GMT",
|
||||
"request-id": "req_011CeuLr6og4ahrpyfnPec45",
|
||||
"date": "Tue, 22 Sep 2026 08:57:59 GMT",
|
||||
"request-id": "req_011CfJCdGeJvd8FiNh3bikfH",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-db154d560f530266545ecc9baf7015e2-c0470cb196446bc0-01",
|
||||
"traceresponse": "00-5b8f08c4391de2116ab063ddaa4cdd99-24fa357e98008a81-01",
|
||||
"transfer-encoding": "chunked",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
@ -94,7 +96,7 @@
|
|||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies.\n\n<active_skills>\n\nUse these current skill versions when relevant to the task. Earlier tool results may describe older versions.\n\n[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged.\n\n</active_skills>"
|
||||
"text": "Skill loading protocol:\nSkills are optional instruction packs, not execution tools. Use them to get extra guidance only when they are relevant to the user's current request.\n\nAvailable skills:\n- name: \"Summary marker\"\n description: \"Use for summarizing notes, updates, or short text while preserving a marker.\"\n id: \"summary-marker\"\n category: \"productivity\"\n\nWhen deciding whether to load a skill:\n- Match the user's request against the skill name and description.\n- If one skill clearly matches, call load_skill once with `{ \"skillId\": \"<id>\" }`, then follow the returned instructions.\n- If a loaded skill references a supporting file, call load_skill with `{ \"skillId\": \"<id>\", \"filePath\": \"<relative path>\" }`.\n- If the relevant skill was already loaded for this request, do not call load_skill again.\n- If no skill clearly matches, do not call load_skill.\n- Do not load a skill just because it is listed here.\n\nYou are testing runtime skill loading. Use the skill catalog in your instructions to pick a matching skillId. Call load_skill with the exact matching skillId before answering when a skill applies. After loading a skill, strictly follow its returned instructions. Do not answer from memory when a relevant skill applies."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
|
|
@ -112,7 +114,7 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_017ccT9FgsRkmMhLq2bsi7nL",
|
||||
"id": "toolu_01J9vkV7BpmPAWaSVhLQ22uC",
|
||||
"name": "load_skill",
|
||||
"input": {
|
||||
"skillId": "summary-marker"
|
||||
|
|
@ -128,8 +130,17 @@
|
|||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_017ccT9FgsRkmMhLq2bsi7nL",
|
||||
"content": "{\"skillId\":\"summary-marker\",\"active\":true}"
|
||||
"tool_use_id": "toolu_01J9vkV7BpmPAWaSVhLQ22uC",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "{\"skillId\":\"summary-marker\",\"active\":true}"
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "[Skill: \"Summary marker\"]\n[Skill category: \"productivity\"]\n[Skill hash: \"e3433b77c9e3\"]\n\nEvery final answer after loading this skill must include the exact marker SKILL_MARKER_ALPHA. For summaries, be concise and keep the marker unchanged."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -164,32 +175,34 @@
|
|||
},
|
||||
"status": 200,
|
||||
"response": [
|
||||
"1f8b0800000000000013c555cb8e1a3110bcf315968f082266b3ec616e681569a32551b428a76c6419bb99b1f0d8133f106885948fc817e64bd21e98c0f0489443122e305d6557755334b00213725281f7bc00e60377a12779e03979a1615303cd6907a483f699e62fb4b212345284e651c2b0e46a1987b7c3f1f0667433ce46a30ce94aa63b7cc14659760f71eaee65b40f73fbf1bd288a30dddc21a7ab840567752a70ef15ca9a242bac096896e69f3eef1eb832e0686ea2d603ea83ad9903eeade9943c7c896004748a12f0b0f66d2db6ed2853c7c0825d824130cb5edfa112172530815707650deb52462d8eb0bc86b5679302d42554e0b866e3ea9c7f40b3f214dd0ea88da1e30fbb01b7520258506912348d4a7227d3d4cd025c6a9c156011323630bec2aef91c27bbdd1242b6bd1eecbefefd68d95c5bb1bc1c820b944645c27ad7ea319e3add9f0bb04ec4e62da734e936af8378ad4c71a246682ad26b06317381ffda6043e918dc55bac67ed2f6f6fafdbdc17fae3c8b55c5dd26eff7c92c605e2aec9f602a4df0042a1548b05693c693fffef51b274bd8e0af5694dc285f918575043f4add1cf31b234a678d8d9e28f4e6b848f9f30419a47676a564e2a18c1e0655015900c839174b22a34bc84e6c0d2236b9fd4f3379f56c9ecdecf1ed74cade4d9e1edf3cb1c9f4c3c3e490a1b3289d86d4d6bf8bb1ad8f6c5db9b5dd7f171bed80f4a8b1ce42a260240bd119fa076be964c76dffe2a23ad92cb7d9f5191ffe0dcea77b8cd1c3c91f64c432c465060000"
|
||||
"1f8b0800000000000013c598514fdb3010c7dff914d6bd4c4201356c4c53de109a36a09b26d8db98a2233e1aab8e9dda97b608e5bb4f4e5b9ab4c0b60792bc44b9bbe47ef78fedf84273329c8882bcc709a59ed1f18144c6443c023f940409749c106dae217984c24ad29040a6b1927494a39a56471f8e4e8f4e4627a7f1681443044a8667f8493a8ae3f3fbcb73399ebae5a74b9fcf5c7c15cfe4178876324104ceea6040ef956734216d660d9361487efd5e5da032e4203195d61178b665ea08bd351d93a7594526a38e5112a3d27e63ab36e52853569cb29d92f190c4f1e87d04196639a5992364654dda0d196dfc8e50bee4dbdc1b32509953410e757a5aecc76fbd71beebad23b01577f822f0e4e62aa3945550028254129d0caa9b7b72a1f0744216123096539ca3d278a709ea5aac8ffae080566360ad6f7aa76d367d7e243c13d2a492b45cd5dbf68772d7f7312d4360734a00ea76da5299c94e1e01c1082fa149d28cafa335211db495a58bf414b606fb6e1750bf26ca9b651617efb416a5b37325a961e81f81f355e6f6d13f85af8a02dd835828cebb388328221ccd2ae548b658fa0729d04dc925b7e6d61c1e6e487ac7b859bd9a648b30941e37ec080b6526022764d80b2a140bb6568b86c41f0f358f7fe6e4879ac0e868b0b1b1d6bd3d30fa6798902187bc9dacfd23dc3dac87e47008b60c22341fd0810894197a81f04f0b44d8a146629193a36681189a4c9939d4fde7b559b305f502cd80b3c391af34fbb05289cc1645655416e6ab400f75ff3852f9cc11ef6d7b865a3d8543cec909ced18805aad6241e84eadebaf09a4abdafd100122d29ab9a36aa1e8ae1b8dbb9f49435ecfa6eae2ec6e3f4dbd9f5d5e7ebf46cfce3ebd9bf7c6943affbb716ce962da01777ff9b5f00cf16d97142aba84e4f0e6464ca9533f01f9df94e9b5fbf61afbed35c7f3cdd5b25b7ff41f6656dfb605fc63f07b52da767110000"
|
||||
],
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-10T07:12:08Z",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-22T08:57:59Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-10T07:12:08Z",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-22T08:57:59Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-10T07:12:08Z",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-22T08:57:59Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-10T07:12:08Z",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-22T08:57:59Z",
|
||||
"cache-control": "no-cache",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a38c9f63ce57bc7b-PRG",
|
||||
"cf-ray": "a3f01af1acffbbdd-FRA",
|
||||
"connection": "keep-alive",
|
||||
"content-encoding": "gzip",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"date": "Thu, 10 Sep 2026 07:12:08 GMT",
|
||||
"request-id": "req_011CeuLrBuGK4E7uxpMM6NMq",
|
||||
"date": "Tue, 22 Sep 2026 08:57:59 GMT",
|
||||
"request-id": "req_011CfJCdLVz3EfLeiVHjxihN",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-ae497abd3e1ae1a20e06ccaeba57c754-f8de5c4ed0f0dac4-01",
|
||||
"traceresponse": "00-0dcb0fbb57a4e0ed3374476fe9b454e3-795c47a4b1e4f26c-01",
|
||||
"transfer-encoding": "chunked",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -5,6 +5,19 @@ import { AgentMessageList } from '../model/message-list';
|
|||
import { ActiveSkills } from '../skills/active-skills';
|
||||
|
||||
const scope = { threadId: 'thread-1', resourceId: 'user-1' };
|
||||
|
||||
/**
|
||||
* Mark the list as observation-masked, so recovered skills render in the
|
||||
* `<active_skills>` system block. Without a mask the block stays empty and
|
||||
* skills ride on their still-visible activating tool result.
|
||||
*/
|
||||
function mask(list: AgentMessageList): AgentMessageList {
|
||||
list.maskObservedMessages({
|
||||
lastObservedAt: new Date(Date.now() + 60_000),
|
||||
lastObservedMessageId: 'observation-cursor',
|
||||
});
|
||||
return list;
|
||||
}
|
||||
const source = createRuntimeSkillSource([
|
||||
{
|
||||
id: 'planning',
|
||||
|
|
@ -87,12 +100,46 @@ describe('active skills', () => {
|
|||
|
||||
await active.restore(list, scope);
|
||||
|
||||
expect(active.instructions()).toContain('Build one workflow.');
|
||||
// The load_skill result is still visible, so the body rides on it and the
|
||||
// system block stays empty — no cached-prefix rewrite.
|
||||
expect(active.instructions()).toBeUndefined();
|
||||
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).toContain(
|
||||
'Build one workflow.',
|
||||
);
|
||||
await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([
|
||||
'builder',
|
||||
]);
|
||||
});
|
||||
|
||||
it('recovers a skill into the system block once observation masks its load', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
const list = new AgentMessageList();
|
||||
list.addHistory([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'load_skill',
|
||||
toolCallId: 'old-load',
|
||||
input: { skillId: 'builder' },
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'Build one workflow.' }] },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
await active.restore(list, scope);
|
||||
|
||||
// Visible load → block empty.
|
||||
expect(active.instructions()).toBeUndefined();
|
||||
|
||||
// Observation hides the load result → the skill moves into the block.
|
||||
mask(list);
|
||||
expect(active.instructions()).toContain('Build one workflow.');
|
||||
});
|
||||
|
||||
it('persists checkpoint skills when they differ from stored state', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['planning']);
|
||||
|
|
@ -120,7 +167,15 @@ describe('active skills', () => {
|
|||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'load_skill',
|
||||
toolCallId: 'old-load',
|
||||
toolCallId: 'builder-load',
|
||||
input: { skillId: 'builder' },
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'Build one workflow.' }] },
|
||||
},
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'load_skill',
|
||||
toolCallId: 'planning-load',
|
||||
input: { skillId: 'planning' },
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'Create a task plan.' }] },
|
||||
|
|
@ -135,16 +190,132 @@ describe('active skills', () => {
|
|||
);
|
||||
await active.restore(list, scope);
|
||||
|
||||
expect(active.instructions()).toContain('Build one workflow.');
|
||||
expect(active.instructions()).not.toContain('Create a task plan.');
|
||||
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).not.toContain(
|
||||
'Create a task plan.',
|
||||
);
|
||||
// Both loads are still visible, so the block stays empty; the enabled skill
|
||||
// rides on its result while the disabled one is collapsed out of it.
|
||||
expect(active.instructions()).toBeUndefined();
|
||||
const rendered = JSON.stringify(active.modelMessages(list.forLlm('').messages, list));
|
||||
expect(rendered).toContain('Build one workflow.');
|
||||
expect(rendered).not.toContain('Create a task plan.');
|
||||
await expect(memory.skillState.load({ ...scope, agentName: 'assistant' })).resolves.toEqual([
|
||||
'builder',
|
||||
]);
|
||||
});
|
||||
|
||||
it('re-anchors a stamped tool activation on a later turn instead of the block', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['builder']);
|
||||
const list = new AgentMessageList();
|
||||
// Turn 1 activated the skill from a tool and stamped its result. On the
|
||||
// next turn the body must ride on that result again, not the system block.
|
||||
list.addHistory([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'build_workflow',
|
||||
toolCallId: 'build-1',
|
||||
input: {},
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
|
||||
activatedSkillIds: ['builder'],
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
await active.restore(list, scope);
|
||||
|
||||
expect(active.instructions()).toBeUndefined();
|
||||
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).toContain(
|
||||
'Build one workflow.',
|
||||
);
|
||||
});
|
||||
|
||||
it('persists a tool activation stamp across serialization', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
const list = new AgentMessageList();
|
||||
list.addResponse([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'build_workflow',
|
||||
toolCallId: 'build-1',
|
||||
input: {},
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await active.restore(list, scope);
|
||||
await active.load('builder', { toolCallId: 'build-1' });
|
||||
|
||||
const roundTripped = AgentMessageList.deserialize(structuredClone(list.serialize()));
|
||||
const stamped = roundTripped
|
||||
.messages()
|
||||
.flatMap((message) => ('content' in message ? message.content : []))
|
||||
.find((part) => part.type === 'tool-call' && part.toolCallId === 'build-1');
|
||||
expect(stamped).toMatchObject({ activatedSkillIds: ['builder'] });
|
||||
});
|
||||
|
||||
it('recovers a restored programmatic skill through the block when no stamp survives', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
await memory.skillState.save({ ...scope, agentName: 'assistant' }, ['builder']);
|
||||
const list = new AgentMessageList();
|
||||
// Legacy data: a prior programmatic activation left no stamp and no
|
||||
// `load_skill` record. The skill must still reach the model via the block.
|
||||
list.addHistory([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'build_workflow',
|
||||
toolCallId: 'build-1',
|
||||
input: {},
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
await active.restore(list, scope);
|
||||
|
||||
expect(active.instructions()).toContain('Build one workflow.');
|
||||
});
|
||||
|
||||
it('recovers a skill through the block when its activating tool call failed', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
const list = new AgentMessageList();
|
||||
list.addResponse([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'build_workflow',
|
||||
toolCallId: 'build-fail',
|
||||
input: {},
|
||||
state: 'rejected',
|
||||
error: 'boom',
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await active.restore(list, scope);
|
||||
// The tool activated the skill, then failed — its error result cannot
|
||||
// carry the body, so the skill belongs in the block, not silently dropped.
|
||||
await active.load('builder', { toolCallId: 'build-fail' });
|
||||
|
||||
expect(active.instructions()).toContain('Build one workflow.');
|
||||
});
|
||||
|
||||
it('keeps threads, resources, and agents isolated', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
|
|
@ -165,6 +336,40 @@ describe('active skills', () => {
|
|||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('keeps the system block empty for an anchored activation until observation masks it', async () => {
|
||||
const memory = new InMemoryMemory();
|
||||
const active = new ActiveSkills(source, 'assistant', memory.skillState);
|
||||
const list = new AgentMessageList();
|
||||
list.addResponse([
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolName: 'build_workflow',
|
||||
toolCallId: 'build-1',
|
||||
input: {},
|
||||
state: 'resolved',
|
||||
output: { type: 'content', value: [{ type: 'text', text: 'built' }] },
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await active.restore(list, scope);
|
||||
|
||||
// A programmatic activation anchored to the calling tool result stays out
|
||||
// of the system prompt — the body rides on that visible result instead.
|
||||
await active.load('builder', { toolCallId: 'build-1' });
|
||||
expect(active.instructions()).toBeUndefined();
|
||||
expect(JSON.stringify(active.modelMessages(list.forLlm('').messages, list))).toContain(
|
||||
'Build one workflow.',
|
||||
);
|
||||
|
||||
// Only once observation hides the anchor does the skill move into the block.
|
||||
mask(list);
|
||||
expect(active.instructions()).toContain('Build one workflow.');
|
||||
});
|
||||
|
||||
it('does not activate a missing skill or a failed load', async () => {
|
||||
const missing: RuntimeSkillSource = { ...source, loadSkill: vi.fn().mockResolvedValue(null) };
|
||||
const active = new ActiveSkills(missing, 'assistant');
|
||||
|
|
|
|||
|
|
@ -295,6 +295,24 @@ export class AgentMessageList {
|
|||
block.suspension = suspension;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that a tool programmatically activated a skill on its own result, so
|
||||
* `ActiveSkills` can re-anchor the skill body there on a later turn instead
|
||||
* of falling back to the `<active_skills>` system prompt. The stamp rides on
|
||||
* the persisted message; it is metadata only and never reaches the model
|
||||
* (`toAiMessages` ignores it). No-op when the tool call is unknown.
|
||||
*/
|
||||
stampActivatedSkill(toolCallId: string, skillId: string): void {
|
||||
const host = this.findToolCallHost(toolCallId);
|
||||
if (!host) return;
|
||||
const block = this.findToolCallBlock(host, toolCallId);
|
||||
if (!block) return;
|
||||
const current = block.activatedSkillIds ?? [];
|
||||
if (current.includes(skillId)) return;
|
||||
block.activatedSkillIds = [...current, skillId];
|
||||
this.responseSet.add(host);
|
||||
}
|
||||
|
||||
private findToolCallHost(toolCallId: string): AgentDbMessage | undefined {
|
||||
// Start from the last message and go backwards to find the host message
|
||||
for (let i = this.all.length - 1; i >= 0; i--) {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { ModelMessage } from 'ai';
|
||||
import type { ModelMessage, ToolResultPart } from 'ai';
|
||||
|
||||
import { formatActiveSkill } from '../../skills/tools';
|
||||
import {
|
||||
|
|
@ -11,7 +11,30 @@ import {
|
|||
import type { AgentPersistenceOptions } from '../../types/sdk/agent';
|
||||
import type { AgentMessageList } from '../model/message-list';
|
||||
|
||||
/** Keeps trusted skill instructions outside the conversation's compaction window. */
|
||||
/**
|
||||
* Keeps trusted skill instructions available without invalidating the cached
|
||||
* prompt prefix.
|
||||
*
|
||||
* A skill body rides on the tool result that activated it (the recorded result
|
||||
* is collapsed first, so an obsolete persisted body is never replayed). The
|
||||
* top-level system prompt does not mention the skill, so activating and
|
||||
* carrying a skill never rewrites the cached prefix — within a run or across
|
||||
* runs.
|
||||
*
|
||||
* The `<active_skills>` block of the system prompt is a recovery path only.
|
||||
* Anthropic renders the prompt as `tools → system → messages`, so any edit to
|
||||
* `system` invalidates the tool block and every message after it. The block
|
||||
* therefore holds only the skills the anchored path cannot deliver on this
|
||||
* call — their result masked by observation memory, their activating call
|
||||
* failed, or the activation predates this window with no `load_skill` record:
|
||||
*
|
||||
* - Anchor still deliverable: the skill rides on its resolved tool result, so
|
||||
* `system` is unchanged and the cached prefix stays warm.
|
||||
* - Anchor gone or unusable: the skill moves into the block so its guidance is
|
||||
* never lost. When observation memory replaced the history it already rewrote
|
||||
* the prefix, so the move is free; otherwise it costs one rewrite, the price
|
||||
* of not dropping an active skill.
|
||||
*/
|
||||
export class ActiveSkills {
|
||||
private readonly loaded = new Map<string, RuntimeSkillContent>();
|
||||
private list?: AgentMessageList;
|
||||
|
|
@ -38,7 +61,11 @@ export class ActiveSkills {
|
|||
list.activeSkillIds === undefined && this.scope
|
||||
? await this.store?.load(this.scope)
|
||||
: undefined;
|
||||
const ids = list.activeSkillIds ?? stored ?? [...this.recordedLoads(list).values()];
|
||||
const ids = list.activeSkillIds ??
|
||||
stored ?? [
|
||||
...this.recordedLoads(list).values(),
|
||||
...this.stampedActivations(list).map(([, skillId]) => skillId),
|
||||
];
|
||||
const registered = new Set(this.source.registry.skills.map(({ id }) => id));
|
||||
for (const id of new Set(ids)) {
|
||||
if (!registered.has(id)) continue;
|
||||
|
|
@ -53,7 +80,10 @@ export class ActiveSkills {
|
|||
if (!storeIsCurrent && (ids.length > 0 || stored !== undefined)) await this.persist();
|
||||
}
|
||||
|
||||
async load(skillId: string): Promise<RuntimeSkillContent | null> {
|
||||
async load(
|
||||
skillId: string,
|
||||
anchor?: { toolCallId: string },
|
||||
): Promise<RuntimeSkillContent | null> {
|
||||
if (!this.source.registry.skills.some(({ id }) => id === skillId)) return null;
|
||||
const skill = this.loaded.get(skillId) ?? (await this.source.loadSkill(skillId));
|
||||
if (!skill) return null;
|
||||
|
|
@ -61,6 +91,9 @@ export class ActiveSkills {
|
|||
if (!this.loaded.has(skillId)) {
|
||||
this.loaded.set(skillId, skill);
|
||||
this.list.activeSkillIds = [...this.loaded.keys()];
|
||||
// Stamp the activating tool result so the body re-anchors there on a
|
||||
// later turn instead of falling back into the system prompt.
|
||||
if (anchor) this.list.stampActivatedSkill(anchor.toolCallId, skillId);
|
||||
await this.persist();
|
||||
}
|
||||
return skill;
|
||||
|
|
@ -72,12 +105,19 @@ export class ActiveSkills {
|
|||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The active skills the anchored path cannot deliver on this call, as one
|
||||
* block for the top-level system prompt. A skill enters the block only when
|
||||
* it has no successfully resolved, visible tool result to ride on, so a
|
||||
* still-anchored skill never edits `system` or breaks a warm cache.
|
||||
*/
|
||||
instructions(): string | undefined {
|
||||
if (this.loaded.size === 0) return undefined;
|
||||
const sections = [...this.loaded].flatMap(([id, skill]) => {
|
||||
const entry = this.source.registry.skills.find((candidate) => candidate.id === id);
|
||||
return entry ? [formatActiveSkill(skill, entry)] : [];
|
||||
const { inBlock } = this.placement();
|
||||
const sections = inBlock.flatMap((id) => {
|
||||
const text = this.formatSkill(id);
|
||||
return text ? [text] : [];
|
||||
});
|
||||
if (sections.length === 0) return undefined;
|
||||
return [
|
||||
'<active_skills>',
|
||||
'Use these current skill versions when relevant to the task. Earlier tool results may describe older versions.',
|
||||
|
|
@ -86,9 +126,23 @@ export class ActiveSkills {
|
|||
].join('\n\n');
|
||||
}
|
||||
|
||||
/** Old conversations can contain full skill bodies. Do not replay an obsolete version. */
|
||||
/**
|
||||
* Per-call view of the conversation for the model.
|
||||
*
|
||||
* Collapses recorded `load_skill` results to `{ skillId, active }`: old
|
||||
* conversations can contain full skill bodies, and an obsolete version must
|
||||
* not be replayed. Skills anchored to a visible tool call then get their
|
||||
* current body appended to that result, keeping them out of the system
|
||||
* block for as long as observational memory keeps the result visible.
|
||||
*/
|
||||
modelMessages(messages: ModelMessage[], list: AgentMessageList): ModelMessage[] {
|
||||
const loads = this.recordedLoads(list);
|
||||
const { anchors } = this.placement(loads);
|
||||
const appendices = new Map<string, string[]>();
|
||||
for (const [skillId, toolCallId] of anchors) {
|
||||
const text = this.formatSkill(skillId);
|
||||
if (text) appendices.set(toolCallId, [...(appendices.get(toolCallId) ?? []), text]);
|
||||
}
|
||||
return messages.map((message) => {
|
||||
if (message.role !== 'tool') return message;
|
||||
return {
|
||||
|
|
@ -96,19 +150,105 @@ export class ActiveSkills {
|
|||
content: message.content.map((part) => {
|
||||
if (part.type !== 'tool-result') return part;
|
||||
const skillId = loads.get(part.toolCallId);
|
||||
if (!skillId) return part;
|
||||
return {
|
||||
...part,
|
||||
output: {
|
||||
type: 'json' as const,
|
||||
value: { skillId, active: this.loaded.has(skillId) },
|
||||
},
|
||||
};
|
||||
const collapsed: ToolResultPart = skillId
|
||||
? {
|
||||
...part,
|
||||
output: {
|
||||
type: 'json' as const,
|
||||
value: { skillId, active: this.loaded.has(skillId) },
|
||||
},
|
||||
}
|
||||
: part;
|
||||
const texts = appendices.get(part.toolCallId);
|
||||
return texts ? appendToolResultText(collapsed, texts) : collapsed;
|
||||
}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Where each active skill is delivered on this call: anchored to a visible
|
||||
* tool result, or in the system block. The anchor is the first visible
|
||||
* recorded `load_skill` call for the skill, else this run's activating
|
||||
* call. `instructions()` and `modelMessages()` share this so a skill is
|
||||
* always delivered exactly once.
|
||||
*
|
||||
* An anchor must be a **successfully resolved** tool result: the skill body
|
||||
* is appended to it (see `appendToolResultText`, which skips error outputs),
|
||||
* so a pending, canceled, or failed call cannot deliver it. A skill with no
|
||||
* such anchor — its result masked by observation memory, or its activating
|
||||
* call failed, or the activation predates this window with no `load_skill`
|
||||
* record — falls back to the `<active_skills>` block, which never drops it.
|
||||
* The block only edits `system` when the anchored path cannot deliver, so an
|
||||
* unchanged, still-anchored skill never rewrites the cached prefix.
|
||||
*/
|
||||
private placement(loads?: Map<string, string>): {
|
||||
anchors: Map<string, string>;
|
||||
inBlock: string[];
|
||||
} {
|
||||
const anchors = new Map<string, string>();
|
||||
if (this.list) {
|
||||
const deliverable = this.deliverableToolResults();
|
||||
// `load_skill` results and tool results that stamped a programmatic
|
||||
// activation are both valid anchors; the stamp is what survives a turn.
|
||||
const sources: Array<[string, string]> = [
|
||||
...(loads ?? this.recordedLoads(this.list)),
|
||||
...this.stampedActivations(this.list),
|
||||
];
|
||||
for (const [toolCallId, skillId] of sources) {
|
||||
if (!anchors.has(skillId) && this.loaded.has(skillId) && deliverable.has(toolCallId)) {
|
||||
anchors.set(skillId, toolCallId);
|
||||
}
|
||||
}
|
||||
}
|
||||
const inBlock = [...this.loaded.keys()].filter((id) => !anchors.has(id));
|
||||
return { anchors, inBlock };
|
||||
}
|
||||
|
||||
/**
|
||||
* `[toolCallId, skillId]` pairs for tool results that stamped a programmatic
|
||||
* skill activation (see `stampActivatedSkill`). Unlike `load_skill` records,
|
||||
* these carry activations a tool started itself, so a post-build skill
|
||||
* re-anchors to its tool result on the next turn instead of the system block.
|
||||
*/
|
||||
private stampedActivations(list: AgentMessageList): Array<[string, string]> {
|
||||
const pairs: Array<[string, string]> = [];
|
||||
for (const message of list.messages()) {
|
||||
if (!('content' in message)) continue;
|
||||
for (const part of message.content) {
|
||||
if (part.type === 'tool-call' && Array.isArray(part.activatedSkillIds)) {
|
||||
for (const skillId of part.activatedSkillIds) pairs.push([part.toolCallId, skillId]);
|
||||
}
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tool-call ids whose result is visible and successfully resolved, so a skill
|
||||
* body can ride on it. Pending, canceled, and failed calls are excluded — a
|
||||
* skill anchored there would silently vanish, so it belongs in the block.
|
||||
*/
|
||||
private deliverableToolResults(): Set<string> {
|
||||
const ids = new Set<string>();
|
||||
if (!this.list) return ids;
|
||||
for (const message of this.list.llmVisibleMessages()) {
|
||||
if (!('content' in message)) continue;
|
||||
for (const part of message.content) {
|
||||
if (part.type === 'tool-call' && part.state === 'resolved' && !part.canceled) {
|
||||
ids.add(part.toolCallId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
private formatSkill(skillId: string): string | undefined {
|
||||
const skill = this.loaded.get(skillId);
|
||||
const entry = this.source.registry.skills.find((candidate) => candidate.id === skillId);
|
||||
return skill && entry ? formatActiveSkill(skill, entry) : undefined;
|
||||
}
|
||||
|
||||
private recordedLoads(list: AgentMessageList): Map<string, string> {
|
||||
const loads = new Map<string, string>();
|
||||
for (const message of list.messages()) {
|
||||
|
|
@ -154,3 +294,32 @@ export class ActiveSkills {
|
|||
await save;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Append trusted text parts after a tool result's own output. Text and JSON
|
||||
* outputs become content parts so the skill text is a separate block from the
|
||||
* (possibly untrusted-wrapped) tool output. Error outputs are left alone.
|
||||
*/
|
||||
function appendToolResultText(part: ToolResultPart, texts: string[]): ToolResultPart {
|
||||
const extra = texts.map((text) => ({ type: 'text' as const, text }));
|
||||
const { output } = part;
|
||||
switch (output.type) {
|
||||
case 'content':
|
||||
return { ...part, output: { type: 'content', value: [...output.value, ...extra] } };
|
||||
case 'text':
|
||||
return {
|
||||
...part,
|
||||
output: { type: 'content', value: [{ type: 'text', text: output.value }, ...extra] },
|
||||
};
|
||||
case 'json':
|
||||
return {
|
||||
...part,
|
||||
output: {
|
||||
type: 'content',
|
||||
value: [{ type: 'text', text: JSON.stringify(output.value) }, ...extra],
|
||||
},
|
||||
};
|
||||
default:
|
||||
return part;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,6 +117,15 @@ export async function executeTool(
|
|||
throw new Error(`No handler found for tool "${builtTool.name}"`);
|
||||
}
|
||||
|
||||
// Anchor programmatic skill activations to the calling tool's result by
|
||||
// default, so the skill body rides on that result (kept out of `system`)
|
||||
// instead of forcing a full prompt-prefix rewrite. An explicit anchor wins.
|
||||
const baseLoadSkill = executionContext.loadSkill;
|
||||
const loadSkill: ToolExecutionContext['loadSkill'] = baseLoadSkill
|
||||
? async (skillId, anchor) =>
|
||||
await baseLoadSkill(skillId, anchor ?? (toolCallId ? { toolCallId } : undefined))
|
||||
: undefined;
|
||||
|
||||
if (builtTool.suspendSchema) {
|
||||
const isCancelled = isCancellation(resumeData);
|
||||
const ctx: InterruptibleToolContext = {
|
||||
|
|
@ -139,7 +148,7 @@ export async function executeTool(
|
|||
toolCallId,
|
||||
toolName: builtTool.name,
|
||||
runId: executionContext.runId,
|
||||
...(executionContext.loadSkill ? { loadSkill: executionContext.loadSkill } : {}),
|
||||
...(loadSkill ? { loadSkill } : {}),
|
||||
persistence: executionContext.persistence,
|
||||
emitEvent: executionContext.emitEvent,
|
||||
abortSignal: executionContext.abortSignal,
|
||||
|
|
@ -156,7 +165,7 @@ export async function executeTool(
|
|||
toolCallId,
|
||||
toolName: builtTool.name,
|
||||
runId: executionContext.runId,
|
||||
...(executionContext.loadSkill ? { loadSkill: executionContext.loadSkill } : {}),
|
||||
...(loadSkill ? { loadSkill } : {}),
|
||||
persistence: executionContext.persistence,
|
||||
emitEvent: executionContext.emitEvent,
|
||||
abortSignal: executionContext.abortSignal,
|
||||
|
|
|
|||
|
|
@ -137,7 +137,15 @@ export interface RuntimeSkillContent extends RuntimeSkillIndexEntry {
|
|||
linkedFiles?: RuntimeSkillLinkedFiles;
|
||||
}
|
||||
|
||||
export type RuntimeSkillLoader = (skillId: string) => Promise<RuntimeSkillContent | null>;
|
||||
export type RuntimeSkillLoader = (
|
||||
skillId: string,
|
||||
/**
|
||||
* Tool result the activation rides on. The skill body is appended to this
|
||||
* result so the top-level system prompt stays byte-identical (no cache
|
||||
* invalidation). Defaults to the calling tool's own result when omitted.
|
||||
*/
|
||||
anchor?: { toolCallId: string },
|
||||
) => Promise<RuntimeSkillContent | null>;
|
||||
|
||||
export interface RuntimeSkillFileContent {
|
||||
skillId: string;
|
||||
|
|
|
|||
|
|
@ -133,6 +133,13 @@ export type ContentToolCall = ContentMetadata & {
|
|||
input: JSONValue;
|
||||
|
||||
providerExecuted?: boolean;
|
||||
|
||||
/**
|
||||
* Skills this tool activated programmatically (via `ctx.loadSkill`), so the
|
||||
* skill body can re-anchor to this result on a later turn instead of falling
|
||||
* back to the system prompt. Metadata only — never sent to the model.
|
||||
*/
|
||||
activatedSkillIds?: string[];
|
||||
} & (
|
||||
| { state: 'pending'; suspension?: ToolCallSuspensionInfo }
|
||||
| { state: 'resolved'; output: JSONValue; canceled?: boolean }
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/ai-node-sdk",
|
||||
"version": "0.31.0",
|
||||
"version": "0.31.2",
|
||||
"description": "SDK for building AI nodes in n8n",
|
||||
"types": "dist/esm/index.d.ts",
|
||||
"module": "dist/esm/index.js",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/ai-utilities",
|
||||
"version": "0.34.0",
|
||||
"version": "0.34.2",
|
||||
"description": "Utilities for building AI nodes in n8n",
|
||||
"types": "dist/esm/index.d.ts",
|
||||
"module": "dist/esm/index.js",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/ai-workflow-builder",
|
||||
"version": "1.41.0",
|
||||
"version": "1.41.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"typecheck": "tsc --noEmit",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/api-types",
|
||||
"version": "1.41.0",
|
||||
"version": "1.41.1",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -391,8 +391,6 @@ export type FrontendModuleSettings = {
|
|||
runDebugEnabled: boolean;
|
||||
/** Whether this instance is in the activation-capped trial cohort (`N8N_INSTANCE_AI_ACTIVATION_CAPPED`). Optional. */
|
||||
activationCapped?: boolean;
|
||||
/** Whether the non-blocking setup panel replaces the suspending setup wizard (`N8N_INSTANCE_AI_SETUP_PANEL_ENABLED`). */
|
||||
instanceAiSetupPanelEnabled?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -425,6 +425,8 @@ export {
|
|||
INSTANCE_AI_CONVERSATION_HISTORY_FLAG,
|
||||
INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
|
||||
INSTANCE_AI_SETUP_PANEL_FLAG,
|
||||
INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
|
||||
INSTANCE_AI_NODE_USAGE_FLAG,
|
||||
INSTANCE_AI_FOLDER_EXPLORATION_FLAG,
|
||||
|
|
|
|||
|
|
@ -2556,6 +2556,9 @@ export const INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT = 'variant';
|
|||
export const INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG = '111_instance_ai_progressive_building';
|
||||
export const INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT = 'variant';
|
||||
|
||||
export const INSTANCE_AI_SETUP_PANEL_FLAG = '118_instance_ai_setup_overhaul';
|
||||
export const INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT = 'variant';
|
||||
|
||||
/** Enables the node-usage context surface for Instance AI: the `node-usage`
|
||||
|
||||
* action and the `nodeTypes` filter on `workflows(action="list")`. */
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/backend-common",
|
||||
"version": "1.41.0",
|
||||
"version": "1.41.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ describe('getModuleEntryUrl', () => {
|
|||
describe('eligibleModules', () => {
|
||||
it('should not include opt-in modules by default', () => {
|
||||
const eligible = Container.get(ModuleRegistry).eligibleModules;
|
||||
expect(eligible).not.toContain('agents');
|
||||
expect(eligible).not.toContain('type-availability-policies');
|
||||
});
|
||||
|
||||
|
|
@ -43,14 +42,17 @@ describe('eligibleModules', () => {
|
|||
expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('policy-infrastructure');
|
||||
});
|
||||
|
||||
it('should include instance-ai by default', () => {
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toContain('instance-ai');
|
||||
it.each(['instance-ai', 'agents'])('should include %s by default', (moduleName) => {
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toContain(moduleName);
|
||||
});
|
||||
|
||||
it('should allow opting out of a default module via env var', () => {
|
||||
process.env.N8N_DISABLED_MODULES = 'instance-ai';
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).not.toContain('instance-ai');
|
||||
});
|
||||
it.each(['instance-ai', 'agents'])(
|
||||
'should allow opting out of the default %s module via env var',
|
||||
(moduleName) => {
|
||||
process.env.N8N_DISABLED_MODULES = moduleName;
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).not.toContain(moduleName);
|
||||
},
|
||||
);
|
||||
|
||||
it('should consider a module ineligible if it was disabled via env var', () => {
|
||||
process.env.N8N_DISABLED_MODULES = 'insights';
|
||||
|
|
@ -85,11 +87,12 @@ describe('eligibleModules', () => {
|
|||
'mcp-registry',
|
||||
'workflow-reviews',
|
||||
'instance-ai',
|
||||
'agents',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should consider a module eligible if it was enabled via env var', () => {
|
||||
process.env.N8N_ENABLED_MODULES = 'agents';
|
||||
process.env.N8N_ENABLED_MODULES = 'type-availability-policies';
|
||||
expect(Container.get(ModuleRegistry).eligibleModules).toEqual([
|
||||
'policy-infrastructure',
|
||||
'insights',
|
||||
|
|
@ -123,6 +126,7 @@ describe('eligibleModules', () => {
|
|||
'workflow-reviews',
|
||||
'instance-ai',
|
||||
'agents',
|
||||
'type-availability-policies',
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ export class ModuleRegistry {
|
|||
'mcp-registry',
|
||||
'workflow-reviews',
|
||||
'instance-ai',
|
||||
'agents',
|
||||
];
|
||||
|
||||
private readonly activeModules: string[] = [];
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/backend-network",
|
||||
"version": "1.15.0",
|
||||
"version": "1.15.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/backend-test-utils",
|
||||
"version": "1.41.0",
|
||||
"version": "1.41.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/blob-storage",
|
||||
"version": "1.9.0",
|
||||
"version": "1.9.2",
|
||||
"description": "Blob storage layer for n8n, with filesystem, S3, and Azure Blob backends",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/chat-hub",
|
||||
"version": "1.34.0",
|
||||
"version": "1.34.1",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/client-oauth2",
|
||||
"version": "1.23.0",
|
||||
"version": "1.23.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/config",
|
||||
"version": "2.39.0",
|
||||
"version": "2.39.1",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -206,15 +206,6 @@ export class InstanceAiConfig {
|
|||
@Env('N8N_INSTANCE_AI_NODE_CONTEXT_ENABLED')
|
||||
canvasNodeContextEnabled: boolean = false;
|
||||
|
||||
/**
|
||||
* Non-blocking setup panel (setup panel v2): the persistent checklist above
|
||||
* the chat input replaces the suspending setup wizard. Env-settable so eval
|
||||
* lanes can exercise both paths; a managed rollout flag may layer on top
|
||||
* later behind the same accessors.
|
||||
*/
|
||||
@Env('N8N_INSTANCE_AI_SETUP_PANEL_ENABLED')
|
||||
instanceAiSetupPanelEnabled: boolean = false;
|
||||
|
||||
/**
|
||||
* Force-enable the node-usage context surface for Instance AI — the `node-usage` action and
|
||||
* the `nodeTypes` filter on `workflows(action="list")`.
|
||||
|
|
|
|||
|
|
@ -379,7 +379,6 @@ describe('GlobalConfig', () => {
|
|||
thinkingEnabled: true,
|
||||
mcpConnectionsEnabled: false,
|
||||
canvasNodeContextEnabled: false,
|
||||
instanceAiSetupPanelEnabled: false,
|
||||
nodeUsageEnabled: false,
|
||||
folderExplorationEnabled: false,
|
||||
activationCapped: false,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/create-node",
|
||||
"version": "0.49.0",
|
||||
"version": "0.49.2",
|
||||
"description": "Official CLI to create new community nodes for n8n",
|
||||
"bin": {
|
||||
"create-node": "bin/create-node.cjs"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/db",
|
||||
"version": "1.41.0",
|
||||
"version": "1.41.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo .tmp-schema-docs",
|
||||
"gen:migration-index": "node scripts/generate-migration-index.mjs",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/decorators",
|
||||
"version": "1.41.0",
|
||||
"version": "1.41.1",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/engine",
|
||||
"version": "0.21.0",
|
||||
"version": "0.21.1",
|
||||
"description": "n8n workflow execution engine (v2)",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo compiled",
|
||||
|
|
|
|||
|
|
@ -47,10 +47,16 @@ For built-in providers, the setup service recognizes `ANTHROPIC_API_KEY`,
|
|||
| `N8N_INSTANCE_AI_NODE_CONTEXT_ENABLED` | boolean | `false` | Force-enable canvas node context. `false` falls back to the PostHog flag. |
|
||||
| `N8N_INSTANCE_AI_FOLDER_EXPLORATION_ENABLED` | boolean | `false` | Force-enable folder exploration (folder attribution and folder scoping on `workflows(action="list")`). `false` falls back to the PostHog flag. |
|
||||
| `N8N_INSTANCE_AI_BROWSER_USE_ENABLED` | boolean | `true` | Computer Use browser tooling, used for credential setup. |
|
||||
| `N8N_INSTANCE_AI_SETUP_PANEL_ENABLED` | boolean | `false` | Non-blocking setup panel (setup panel v2) instead of the suspending setup wizard. |
|
||||
| `N8N_INSTANCE_AI_ACTIVATION_CAPPED` | boolean | `false` | Activation capping. |
|
||||
| `N8N_INSTANCE_AI_ACTIVATION_LOCK_MESSAGE_THRESHOLD` | number | `1` | Assistant messages that must be sent, in addition to instance activation, before an activation lock applies. |
|
||||
|
||||
The PostHog flag `118_instance_ai_setup_overhaul` selects the setup flow.
|
||||
`variant` enables the async panel. `control` or an unavailable assignment uses
|
||||
the wizard. For local testing, set
|
||||
`N8N_FEATURE_FLAG_OVERRIDES='{"118_instance_ai_setup_overhaul":"variant"}'`
|
||||
on the n8n server, then restart it and reload the editor. Use `control` to test
|
||||
the wizard.
|
||||
|
||||
### Debugging
|
||||
|
||||
| Variable | Type | Default | Description |
|
||||
|
|
|
|||
|
|
@ -474,7 +474,7 @@ confirmation card.
|
|||
`nodesStillNeedingSetup` is what nobody has configured yet, `skippedByUser` what the user
|
||||
actively dismissed and the agent must not re-open (see `reopenSkipped`).
|
||||
|
||||
**Setup panel** (`N8N_INSTANCE_AI_SETUP_PANEL_ENABLED`): the normal setup call
|
||||
**Setup panel** (`118_instance_ai_setup_overhaul: variant`): the normal setup call
|
||||
analyzes the whole workflow, including bound slots. It publishes the `setup-items`
|
||||
snapshot and confirms that it reached storage. It then saves the build's setup
|
||||
routing marker. Only after both steps succeed does it return
|
||||
|
|
@ -609,7 +609,15 @@ Default timeout: 5 minutes; max: 10 minutes. On timeout, execution is cancelled.
|
|||
| `timeout` | number | no | 300000 | Max wait time in ms (max 600000) |
|
||||
| `triggerNodeName` | string | no | — | Trigger node to use when a workflow has more than one trigger |
|
||||
|
||||
**Returns**: `{ executionId, status, data?, error?, startedAt?, finishedAt? }`
|
||||
**Returns**: `{ executionId, status, data?, error?, startedAt?, finishedAt?, verificationClaim? }`
|
||||
|
||||
**Live test evidence**: `verify-built-workflow` always simulates destructive
|
||||
nodes, so a live test runs through this action. When a successful run reaches
|
||||
every planned node of the latest build, with no saved pins and no injected
|
||||
trigger input, the run is recorded as a `verified` claim on the build outcome.
|
||||
The publish gate then reads that claim. The result carries it as
|
||||
`verificationClaim`. Other runs leave the stored claim unchanged: a live run
|
||||
can raise the verdict but never lower it.
|
||||
|
||||
**Type-aware pin data**: Constructs proper pin data per trigger type:
|
||||
- **Chat trigger**: `{ chatInput, sessionId, action }`
|
||||
|
|
@ -831,7 +839,7 @@ a service. When `needsBrowserSetup=true`, the orchestrator should load the
|
|||
directly, then call `credentials(action="setup")` again to select the created
|
||||
credential.
|
||||
|
||||
**Setup panel** (`N8N_INSTANCE_AI_SETUP_PANEL_ENABLED`): when the call belongs
|
||||
**Setup panel** (`118_instance_ai_setup_overhaul: variant`): when the call belongs
|
||||
to a workflow (`workflowId`, or the workflow this run last saved) and the stage
|
||||
is not `finalize`, the tool does not suspend. It merges the credential types
|
||||
into the workflow's durable `setup-items` snapshot and returns
|
||||
|
|
|
|||
|
|
@ -310,12 +310,13 @@ The harness remaps that ID and sends the normal chat request with the workflow
|
|||
attachment and structured handoff context. The transcript records the Execute
|
||||
action so process expectations can check the response.
|
||||
|
||||
Start each panel eval instance with `N8N_INSTANCE_AI_SETUP_PANEL_ENABLED=true`.
|
||||
Start each panel eval instance with
|
||||
`N8N_FEATURE_FLAG_OVERRIDES='{"118_instance_ai_setup_overhaul":"variant"}'`.
|
||||
Set this variable on the n8n server process or in the lane's environment file.
|
||||
Setting it only on the eval client does not enable the server feature.
|
||||
Run the normal PR tier with the flag on and off. For panel cases, load the
|
||||
Run the normal PR tier with `control` and `variant`. For panel cases, load the
|
||||
external suite with `--source langtracer --suite <suite-id>`, or stage a local
|
||||
case and select it with `--filter <case-slug>`. Run those cases with the flag on.
|
||||
case and select it with `--filter <case-slug>`. Run those cases with `variant`.
|
||||
The repository does not include a `setup-panel-v2` tier.
|
||||
|
||||
Remote Execute cases require LangTracer to preserve `attach.source` when it
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/instance-ai",
|
||||
"version": "1.26.0",
|
||||
"version": "1.26.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"typecheck": "tsc --noEmit && tsc -p evaluations/tsconfig.json",
|
||||
|
|
|
|||
|
|
@ -1024,7 +1024,11 @@ describe('RunStateRegistry', () => {
|
|||
createdAt: Date.now(),
|
||||
});
|
||||
|
||||
registry.setSetupPanelEnabled('thread-1', true);
|
||||
expect(registry.isSetupPanelEnabled('thread-1')).toBe(true);
|
||||
expect(registry.isSetupPanelEnabled('thread-2')).toBe(false);
|
||||
const result = registry.clearThread('thread-1');
|
||||
expect(registry.isSetupPanelEnabled('thread-1')).toBe(false);
|
||||
|
||||
// Confirmations resolved
|
||||
expect(resolve).toHaveBeenCalledWith({ approved: false });
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ export class RunStateRegistry<TUser = unknown> {
|
|||
|
||||
/** Build mode captured at user-run entry and reused by follow-up runs. */
|
||||
private readonly threadBuildModes = new Map<string, InstanceAiBuildMode>();
|
||||
private readonly threadSetupPanelEnabled = new Map<string, boolean>();
|
||||
private readonly threadPromptSelections = new Map<
|
||||
string,
|
||||
{ version: string; metadata?: InstanceAiPromptConfiguration }
|
||||
|
|
@ -521,6 +522,14 @@ export class RunStateRegistry<TUser = unknown> {
|
|||
return this.threadBuildModes.get(threadId);
|
||||
}
|
||||
|
||||
setSetupPanelEnabled(threadId: string, enabled: boolean): void {
|
||||
this.threadSetupPanelEnabled.set(threadId, enabled);
|
||||
}
|
||||
|
||||
isSetupPanelEnabled(threadId: string): boolean {
|
||||
return this.threadSetupPanelEnabled.get(threadId) === true;
|
||||
}
|
||||
|
||||
setPromptVersion(threadId: string, version: string | undefined): void {
|
||||
if (version === undefined) this.threadPromptSelections.delete(threadId);
|
||||
else this.threadPromptSelections.set(threadId, { version });
|
||||
|
|
@ -690,6 +699,7 @@ export class RunStateRegistry<TUser = unknown> {
|
|||
this.threadTimeZones.delete(threadId);
|
||||
this.threadComputerUseChannels.delete(threadId);
|
||||
this.threadBuildModes.delete(threadId);
|
||||
this.threadSetupPanelEnabled.delete(threadId);
|
||||
this.threadPromptSelections.delete(threadId);
|
||||
|
||||
const groupId = this.threadMessageGroupId.get(threadId);
|
||||
|
|
@ -741,6 +751,7 @@ export class RunStateRegistry<TUser = unknown> {
|
|||
this.threadTimeZones.clear();
|
||||
this.threadComputerUseChannels.clear();
|
||||
this.threadBuildModes.clear();
|
||||
this.threadSetupPanelEnabled.clear();
|
||||
this.threadPromptSelections.clear();
|
||||
this.threadMessageGroupId.clear();
|
||||
this.runIdsByMessageGroup.clear();
|
||||
|
|
|
|||
|
|
@ -4,7 +4,13 @@ import type { z } from 'zod';
|
|||
|
||||
import { executeTool } from '../../__tests__/tool-test-utils';
|
||||
import type { InstanceAiContext, ExecutionResult } from '../../types';
|
||||
import type { VerificationClaim } from '../../workflow-loop/workflow-loop-state';
|
||||
import { createExecutionsTool } from '../executions.tool';
|
||||
import { recordLiveRunVerification } from '../orchestration/verification/record-live-run';
|
||||
|
||||
vi.mock('../orchestration/verification/record-live-run', () => ({
|
||||
recordLiveRunVerification: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
// ── Mock helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -461,6 +467,63 @@ describe('executions tool', () => {
|
|||
);
|
||||
});
|
||||
|
||||
describe('live run verification', () => {
|
||||
const runResult = { executionId: 'exec-1', status: 'success' as const };
|
||||
|
||||
function createAllowedContext() {
|
||||
const context = createMockContext({
|
||||
permissions: { runWorkflow: 'always_allow' },
|
||||
aiCreatedWorkflowIds: new Set(['wf-1']),
|
||||
});
|
||||
(context.executionService.run as Mock).mockResolvedValue(runResult);
|
||||
return context;
|
||||
}
|
||||
|
||||
it('returns the recorded claim with the run result', async () => {
|
||||
const context = createAllowedContext();
|
||||
const claim: VerificationClaim = {
|
||||
level: 'verified',
|
||||
plannedNodeCount: 1,
|
||||
reachedNodeCount: 1,
|
||||
nodesNotReached: [],
|
||||
simulatedNodes: [],
|
||||
pinnedNodes: [],
|
||||
unprovenTargets: [],
|
||||
publishReady: true,
|
||||
liveTestRecommended: false,
|
||||
};
|
||||
vi.mocked(recordLiveRunVerification).mockResolvedValueOnce(claim);
|
||||
|
||||
const tool = createExecutionsTool(context);
|
||||
const result = await executeTool(
|
||||
tool,
|
||||
{ action: 'run' as const, workflowId: 'wf-1', triggerNodeName: 'Every Morning' },
|
||||
createAgentCtx() as never,
|
||||
);
|
||||
|
||||
expect(result).toEqual({ ...runResult, verificationClaim: claim });
|
||||
expect(recordLiveRunVerification).toHaveBeenCalledWith({
|
||||
context,
|
||||
workflowId: 'wf-1',
|
||||
triggerNodeName: 'Every Morning',
|
||||
result: runResult,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns the plain run result when no claim was recorded', async () => {
|
||||
const context = createAllowedContext();
|
||||
|
||||
const tool = createExecutionsTool(context);
|
||||
const result = await executeTool(
|
||||
tool,
|
||||
{ action: 'run' as const, workflowId: 'wf-1' },
|
||||
createAgentCtx() as never,
|
||||
);
|
||||
|
||||
expect(result).toEqual(runResult);
|
||||
});
|
||||
});
|
||||
|
||||
describe('session grant (always allow)', () => {
|
||||
it('runs without HITL when the workflow has a session grant', async () => {
|
||||
const context = createMockContext({
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import { z } from 'zod';
|
|||
import { sanitizeInputSchema } from '../agent/sanitize-mcp-schemas';
|
||||
import type { InstanceAiContext } from '../types';
|
||||
import { approvalSummarySchema, formatApprovalMessage } from './approval-copy';
|
||||
import { recordLiveRunVerification } from './orchestration/verification/record-live-run';
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -416,11 +417,20 @@ async function handleRun(
|
|||
}
|
||||
|
||||
// Approved or always_allow — execute
|
||||
return await context.executionService.run(workflowId, input.inputData, {
|
||||
const result = await context.executionService.run(workflowId, input.inputData, {
|
||||
timeout: input.timeout,
|
||||
triggerNodeName: input.triggerNodeName,
|
||||
abortSignal,
|
||||
});
|
||||
// A live test is the evidence verification cannot produce itself. Record it
|
||||
// so the publish gate stops disclosing simulations that this run replaced.
|
||||
const verificationClaim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId,
|
||||
triggerNodeName: input.triggerNodeName,
|
||||
result,
|
||||
});
|
||||
return verificationClaim ? { ...result, verificationClaim } : result;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,223 @@
|
|||
import type { InstanceAiContext } from '../../../../types';
|
||||
import type { WorkflowBuildOutcome } from '../../../../workflow-loop/workflow-loop-state';
|
||||
import { recordLiveRunVerification } from '../record-live-run';
|
||||
import type { ExecutionRunResult } from '../types';
|
||||
|
||||
function makeOutcome(overrides: Partial<WorkflowBuildOutcome> = {}): WorkflowBuildOutcome {
|
||||
return {
|
||||
workItemId: 'wi_1',
|
||||
taskId: 'task_1',
|
||||
workflowId: 'wf_1',
|
||||
submitted: true,
|
||||
triggerType: 'trigger_only',
|
||||
needsUserInput: false,
|
||||
summary: 'Built',
|
||||
nodeSimulationPlan: [
|
||||
{
|
||||
nodeName: 'Send Telegram Message',
|
||||
verdict: 'simulate',
|
||||
reason: 'Sends a message to a Telegram chat.',
|
||||
confidence: 'high',
|
||||
source: 'deterministic',
|
||||
},
|
||||
],
|
||||
verification: {
|
||||
attempted: true,
|
||||
success: true,
|
||||
status: 'success',
|
||||
claim: {
|
||||
level: 'partial',
|
||||
plannedNodeCount: 1,
|
||||
reachedNodeCount: 1,
|
||||
nodesNotReached: [],
|
||||
simulatedNodes: [
|
||||
{ nodeName: 'Send Telegram Message', reason: 'Sends a message to a Telegram chat.' },
|
||||
],
|
||||
pinnedNodes: [],
|
||||
unprovenTargets: [],
|
||||
publishReady: false,
|
||||
liveTestRecommended: true,
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeRunResult(overrides: Partial<ExecutionRunResult> = {}): ExecutionRunResult {
|
||||
return {
|
||||
executionId: 'exec-2',
|
||||
status: 'success',
|
||||
data: { 'Every Morning': [{}], 'Send Telegram Message': [{ ok: true }] },
|
||||
executedNodeNames: ['Every Morning', 'Send Telegram Message'],
|
||||
workflowVersionId: 'version-2',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createContext({
|
||||
outcome = makeOutcome(),
|
||||
}: { outcome?: WorkflowBuildOutcome | null } = {}) {
|
||||
const workflowTaskService = {
|
||||
getLatestBuildOutcomeForWorkflow: vi.fn().mockResolvedValue(outcome ?? undefined),
|
||||
recordVerification: vi.fn(
|
||||
async (_workItemId: string, verification: { claim: unknown }) =>
|
||||
await Promise.resolve(verification.claim),
|
||||
),
|
||||
};
|
||||
const context = {
|
||||
userId: 'user-1',
|
||||
runId: 'run-2',
|
||||
workflowService: {
|
||||
getWorkflowHead: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ versionId: 'version-2', activeVersionId: null, updatedAt: 0 }),
|
||||
},
|
||||
logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
workflowBuildContext: {
|
||||
threadId: 'thread-1',
|
||||
runId: 'run-2',
|
||||
taskId: 'task_1',
|
||||
workItemId: 'wi_1',
|
||||
workflowTaskService,
|
||||
},
|
||||
} as unknown as InstanceAiContext;
|
||||
return { context, workflowTaskService };
|
||||
}
|
||||
|
||||
describe('recordLiveRunVerification', () => {
|
||||
it('records a verified claim for the executed version when the run covers every planned node', async () => {
|
||||
const { context, workflowTaskService } = createContext();
|
||||
|
||||
const claim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId: 'wf_1',
|
||||
result: makeRunResult(),
|
||||
});
|
||||
|
||||
expect(claim).toMatchObject({
|
||||
level: 'verified',
|
||||
publishReady: true,
|
||||
simulatedNodes: [],
|
||||
verifiedVersionId: 'version-2',
|
||||
liveState: 'unpublished',
|
||||
});
|
||||
expect(workflowTaskService.recordVerification).toHaveBeenCalledWith(
|
||||
'wi_1',
|
||||
expect.objectContaining({
|
||||
attempted: true,
|
||||
success: true,
|
||||
executionId: 'exec-2',
|
||||
status: 'success',
|
||||
claim: expect.objectContaining({ level: 'verified' }),
|
||||
evidence: expect.objectContaining({
|
||||
nodesExecuted: ['Every Morning', 'Send Telegram Message'],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['used injected trigger input', { injectedTriggerNodeName: 'Every Morning' }],
|
||||
['read saved pinned data', { workflowPinnedNodeNames: ['Send Telegram Message'] }],
|
||||
[
|
||||
'did not reach a planned node',
|
||||
{ executedNodeNames: ['Every Morning'], data: { 'Every Morning': [{}] } },
|
||||
],
|
||||
['failed', { status: 'error' as const, error: 'Bad Request: chat not found' }],
|
||||
[
|
||||
'reported a node error',
|
||||
{ nodeErrors: [{ nodeName: 'Send Telegram Message', message: 'Bad Request' }] },
|
||||
],
|
||||
['ran an unknown version', { workflowVersionId: undefined }],
|
||||
])('keeps the stored claim when the run %s', async (_label, overrides) => {
|
||||
const { context, workflowTaskService } = createContext();
|
||||
|
||||
const claim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId: 'wf_1',
|
||||
result: makeRunResult(overrides),
|
||||
});
|
||||
|
||||
expect(claim).toBeUndefined();
|
||||
expect(workflowTaskService.recordVerification).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['there is no build outcome', null],
|
||||
['the build outcome has no simulation plan', makeOutcome({ nodeSimulationPlan: undefined })],
|
||||
[
|
||||
'a verification is still running',
|
||||
makeOutcome({ verification: { attempted: false, success: false, status: 'running' } }),
|
||||
],
|
||||
])('does not record when %s', async (_label, outcome) => {
|
||||
const { context, workflowTaskService } = createContext({ outcome });
|
||||
|
||||
const claim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId: 'wf_1',
|
||||
result: makeRunResult(),
|
||||
});
|
||||
|
||||
expect(claim).toBeUndefined();
|
||||
expect(workflowTaskService.recordVerification).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not record when the draft was saved during the run', async () => {
|
||||
const { context, workflowTaskService } = createContext();
|
||||
vi.mocked(context.workflowService.getWorkflowHead).mockResolvedValueOnce({
|
||||
versionId: 'version-3',
|
||||
activeVersionId: null,
|
||||
updatedAt: 0,
|
||||
});
|
||||
|
||||
const claim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId: 'wf_1',
|
||||
result: makeRunResult(),
|
||||
});
|
||||
|
||||
expect(claim).toBeUndefined();
|
||||
expect(workflowTaskService.recordVerification).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not record outside a workflow build context', async () => {
|
||||
const { context } = createContext();
|
||||
context.workflowBuildContext = undefined;
|
||||
|
||||
const claim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId: 'wf_1',
|
||||
result: makeRunResult(),
|
||||
});
|
||||
|
||||
expect(claim).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['reading the build outcome', 'outcome'],
|
||||
['reading the workflow head', 'head'],
|
||||
['recording the claim', 'record'],
|
||||
] as const)('logs and returns undefined when %s fails', async (_label, failingCall) => {
|
||||
const { context, workflowTaskService } = createContext();
|
||||
const error = new Error('storage down');
|
||||
if (failingCall === 'outcome') {
|
||||
workflowTaskService.getLatestBuildOutcomeForWorkflow.mockRejectedValueOnce(error);
|
||||
} else if (failingCall === 'head') {
|
||||
vi.mocked(context.workflowService.getWorkflowHead).mockRejectedValueOnce(error);
|
||||
} else {
|
||||
workflowTaskService.recordVerification.mockRejectedValueOnce(error);
|
||||
}
|
||||
|
||||
const claim = await recordLiveRunVerification({
|
||||
context,
|
||||
workflowId: 'wf_1',
|
||||
result: makeRunResult(),
|
||||
});
|
||||
|
||||
expect(claim).toBeUndefined();
|
||||
expect(context.logger.warn).toHaveBeenCalledWith(
|
||||
'Failed to record a live run as verification evidence',
|
||||
{ workflowId: 'wf_1', error: 'storage down' },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import type { VerificationPublishState } from './claim';
|
||||
import type { Logger } from '../../../logger';
|
||||
import type { InstanceAiWorkflowService } from '../../../types';
|
||||
|
||||
/**
|
||||
* Version pair behind `claim.liveState`. The executed version has to come from
|
||||
* the execution record: the workflow head moves when anybody saves, so
|
||||
* substituting it would let the claim describe a version this run never ran.
|
||||
* Without that record there is no publish state — an unknown run version must
|
||||
* not become `live-current`, which reads as "production is proven".
|
||||
*/
|
||||
export async function resolvePublishState(args: {
|
||||
workflowService: InstanceAiWorkflowService;
|
||||
workflowId: string;
|
||||
executedVersionId: string | null | undefined;
|
||||
logger: Logger;
|
||||
}): Promise<VerificationPublishState | undefined> {
|
||||
const { workflowService, workflowId, executedVersionId, logger } = args;
|
||||
|
||||
if (!executedVersionId) return undefined;
|
||||
|
||||
try {
|
||||
const head = await workflowService.getWorkflowHead(workflowId);
|
||||
return { activeVersionId: head.activeVersionId, draftVersionId: executedVersionId };
|
||||
} catch (error) {
|
||||
logger.warn('Failed to read publish state for the verification claim', {
|
||||
workflowId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/**
|
||||
* Live runs as verification evidence.
|
||||
*
|
||||
* `verify-built-workflow` always simulates nodes with side effects, so the live
|
||||
* test it recommends runs through `executions(action="run")`. Without this, the
|
||||
* stored claim keeps the simulated verdict and the publish gate warns about a
|
||||
* path the user just saw succeed.
|
||||
*/
|
||||
|
||||
import { analyzeVerificationResult, countProducedOutputRows } from './analyze-result';
|
||||
import { deriveVerificationClaim } from './claim';
|
||||
import type { ExecutionRunResult } from './types';
|
||||
import type { InstanceAiContext } from '../../../types';
|
||||
import type { VerificationClaim } from '../../../workflow-loop/workflow-loop-state';
|
||||
|
||||
/**
|
||||
* Record a live run on the workflow's latest build outcome when it proves the
|
||||
* whole workflow. Returns the stored claim, or undefined when nothing changed.
|
||||
*
|
||||
* Only a `verified` run is recorded: a live run can raise the verdict but never
|
||||
* lower it. A narrow or failed ad hoc run (one branch, injected trigger input,
|
||||
* saved pins) says less than the verification it would replace.
|
||||
*/
|
||||
export async function recordLiveRunVerification(args: {
|
||||
context: InstanceAiContext;
|
||||
workflowId: string;
|
||||
triggerNodeName?: string;
|
||||
result: ExecutionRunResult;
|
||||
}): Promise<VerificationClaim | undefined> {
|
||||
const { context, workflowId, triggerNodeName, result } = args;
|
||||
const buildContext = context.workflowBuildContext;
|
||||
const workflowTaskService = buildContext?.workflowTaskService;
|
||||
if (!buildContext || !workflowTaskService || result.status !== 'success') return undefined;
|
||||
|
||||
try {
|
||||
const outcome = await workflowTaskService.getLatestBuildOutcomeForWorkflow(workflowId);
|
||||
const plan = outcome?.nodeSimulationPlan;
|
||||
// A verification in flight owns the record; do not race its write.
|
||||
if (!outcome || !plan || outcome.verification?.status === 'running') return undefined;
|
||||
|
||||
// A save during the run moves the draft past the version that ran, and
|
||||
// this run proves nothing about the newer draft.
|
||||
const executedVersionId = result.workflowVersionId;
|
||||
const head = await context.workflowService.getWorkflowHead(workflowId);
|
||||
if (!executedVersionId || head.versionId !== executedVersionId) return undefined;
|
||||
|
||||
// No planned simulations: this run used no verification pin data. Saved
|
||||
// pins and injected trigger input still count as simulated in the analysis.
|
||||
const analysis = analyzeVerificationResult({
|
||||
result,
|
||||
buildOutcome: outcome,
|
||||
simulatedNodes: [],
|
||||
stateBefore: undefined,
|
||||
runId: context.runId ?? buildContext.runId,
|
||||
triggerNodeName,
|
||||
});
|
||||
const claim = deriveVerificationClaim({
|
||||
analysis,
|
||||
plannedNodeCount: plan.length,
|
||||
publishState: { activeVersionId: head.activeVersionId, draftVersionId: executedVersionId },
|
||||
});
|
||||
if (claim.level !== 'verified') return undefined;
|
||||
|
||||
return await workflowTaskService.recordVerification(outcome.workItemId, {
|
||||
attempted: true,
|
||||
success: true,
|
||||
executionId: result.executionId || undefined,
|
||||
status: result.status,
|
||||
claim,
|
||||
evidence: {
|
||||
nodesExecuted: analysis.nodesExecuted,
|
||||
producedOutputRows: countProducedOutputRows(result.data),
|
||||
},
|
||||
verifiedAt: new Date().toISOString(),
|
||||
});
|
||||
} catch (error) {
|
||||
// Advisory: the run itself succeeded, and the stored verdict stays as it was.
|
||||
context.logger.warn('Failed to record a live run as verification evidence', {
|
||||
workflowId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
|
@ -10,16 +10,16 @@ import { Tool } from '@n8n/agents';
|
|||
import { isTriggerNodeType } from 'n8n-workflow';
|
||||
import { z } from 'zod';
|
||||
|
||||
import type { InstanceAiWorkflowService, OrchestrationContext } from '../../types';
|
||||
import type { OrchestrationContext } from '../../types';
|
||||
import { analyzeVerificationResult, buildNodePreviews } from './verification/analyze-result';
|
||||
import { deriveVerificationClaim } from './verification/claim';
|
||||
import type { VerificationPublishState } from './verification/claim';
|
||||
import {
|
||||
handleMissingSimulationPlan,
|
||||
handleBlockedVerification,
|
||||
persistVerificationOutcome,
|
||||
} from './verification/finalize-result';
|
||||
import { prepareVerificationRun } from './verification/prepare-run';
|
||||
import { resolvePublishState } from './verification/publish-state';
|
||||
import { reconcileStaleCredentialPlan } from './verification/reconcile-plan';
|
||||
import { resolveVerificationTarget } from './verification/resolve-target';
|
||||
import {
|
||||
|
|
@ -64,35 +64,6 @@ function formatLiveStateNote(claim: VerificationClaim | undefined): string | und
|
|||
: fact;
|
||||
}
|
||||
|
||||
/**
|
||||
* Version pair behind `claim.liveState`. The executed version has to come from
|
||||
* the execution record: the workflow head moves when anybody saves, so
|
||||
* substituting it would let the claim describe a version this run never ran.
|
||||
* Without that record there is no publish state — an unknown run version must
|
||||
* not become `live-current`, which reads as "production is proven".
|
||||
*/
|
||||
async function resolvePublishState(args: {
|
||||
workflowService: InstanceAiWorkflowService;
|
||||
workflowId: string;
|
||||
executedVersionId: string | null | undefined;
|
||||
logger: OrchestrationContext['logger'];
|
||||
}): Promise<VerificationPublishState | undefined> {
|
||||
const { workflowService, workflowId, executedVersionId, logger } = args;
|
||||
|
||||
if (!executedVersionId) return undefined;
|
||||
|
||||
try {
|
||||
const head = await workflowService.getWorkflowHead(workflowId);
|
||||
return { activeVersionId: head.activeVersionId, draftVersionId: executedVersionId };
|
||||
} catch (error) {
|
||||
logger.warn('Failed to read publish state for the verification claim', {
|
||||
workflowId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export const verifyBuiltWorkflowInputSchema = z.object({
|
||||
workItemId: z
|
||||
.string()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/mcp-apps",
|
||||
"version": "0.18.0",
|
||||
"version": "0.18.1",
|
||||
"description": "MCP Apps UI resources and server helpers for n8n",
|
||||
"main": "dist/server/index.js",
|
||||
"types": "dist/server/index.d.ts",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/node-cli",
|
||||
"version": "0.50.0",
|
||||
"version": "0.50.2",
|
||||
"description": "Official CLI for developing community nodes for n8n",
|
||||
"bin": {
|
||||
"n8n-node": "bin/n8n-node.mjs"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/node-engine-compatibility",
|
||||
"version": "0.9.0",
|
||||
"version": "0.9.2",
|
||||
"description": "Adapts v1 nodes and workflows to the Engine 2.0 execution model",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/n8n-nodes-langchain",
|
||||
"version": "2.41.0",
|
||||
"version": "2.41.2",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"exports": {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/task-runner",
|
||||
"version": "2.41.0",
|
||||
"version": "2.41.2",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"start": "node dist/start.js",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/telemetry",
|
||||
"version": "1.9.0",
|
||||
"version": "1.9.1",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm watch",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { z } from 'zod/v4';
|
||||
|
||||
import { defineTelemetryEvents } from '../define';
|
||||
import { setupTelemetryProperties } from '../setup-properties';
|
||||
|
||||
const descriptionProperties = {
|
||||
has_description: z.boolean().describe('Whether the saved credential has a nonempty description.'),
|
||||
|
|
@ -62,6 +63,7 @@ export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'The user started a connection attempt or selected an existing credential in the setup panel. Includes retries.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
source: z.literal('instance_ai_setup_panel'),
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
|
|
@ -74,6 +76,7 @@ export const CREDENTIALS_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'A setup-panel credential was selected or connected and its workflow binding was applied or queued. This does not report workflow validation.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
source: z.literal('instance_ai_setup_panel'),
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
import { z } from 'zod/v4';
|
||||
|
||||
import { defineTelemetryEvents } from '../define';
|
||||
import { setupItemProperties, setupTelemetryProperties } from '../setup-properties';
|
||||
|
||||
/**
|
||||
* How each n8n Assistant setup component is configured. Source (who set it) and
|
||||
|
|
@ -57,6 +58,7 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'The setup panel observed a new requirement snapshot. Also fires when no setup is needed, including workflows whose credentials were already connected.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
credential_count: z.number(),
|
||||
|
|
@ -70,11 +72,13 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'A setup checklist row became visible. Reopening the same row within the mounted panel does not emit another event.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
kind: z.enum(['credential', 'parameters', 'details']),
|
||||
credential_type: z.string().optional(),
|
||||
parameter_count: z.number(),
|
||||
items: z.array(z.object(setupItemProperties)).optional(),
|
||||
}),
|
||||
},
|
||||
SETUP_PANEL_DISMISSED: {
|
||||
|
|
@ -82,6 +86,7 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'A visible setup panel closed. The reason separates navigation and removed requirements from a finished execution or an explicit dismissal.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
reason: z.enum([
|
||||
|
|
@ -421,6 +426,10 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'The user sent a message to the n8n Assistant. Fires once per message on the optimistic send, before the request is admitted, so a refused send still counts as an attempt. Carries who wrote the text: a pre-fill is an opener n8n composed (a failed execution, a credential modal, a template card, a suggestion chip) that the user accepted or edited, so pre-fill share must be read from prefill_type rather than matched against the message body.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
workflow_id: z.string().optional(),
|
||||
pending_credential_count: z.number().optional(),
|
||||
pending_parameter_count: z.number().optional(),
|
||||
thread_id: z.string(),
|
||||
instance_id: z.string(),
|
||||
is_first_message: z
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { z } from 'zod/v4';
|
||||
|
||||
import { defineTelemetryEvents } from '../define';
|
||||
import { setupItemProperties, setupTelemetryProperties } from '../setup-properties';
|
||||
|
||||
export const WORKFLOW_TELEMETRY = defineTelemetryEvents({
|
||||
USER_REQUESTED_WORKFLOW_TEST: {
|
||||
|
|
@ -8,9 +9,40 @@ export const WORKFLOW_TELEMETRY = defineTelemetryEvents({
|
|||
description:
|
||||
'The user started a manual workflow test from the AI Assistant setup panel. This event does not report execution success.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
source: z.literal('instance_ai_setup_panel'),
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string().optional(),
|
||||
test_request_id: z.string().optional(),
|
||||
}),
|
||||
},
|
||||
SETUP_TEST_FINISHED: {
|
||||
name: 'AI Assistant setup test finished',
|
||||
description:
|
||||
'The setup panel observed a terminal test result or a failed run request. A failed request does not prove that execution did not start. Leaving setup before the result can omit this event.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
source: z.literal('instance_ai_setup_panel'),
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
test_request_id: z.string(),
|
||||
execution_id: z.string().optional(),
|
||||
status: z.enum(['success', 'error', 'crashed', 'canceled', 'request_failed']),
|
||||
}),
|
||||
},
|
||||
SETUP_SAVED: {
|
||||
name: 'AI Assistant workflow setup saved',
|
||||
description:
|
||||
'A setup surface observed saved requirements after applying changes. Completion covers the observed setup requirements, not credential validity or edits made while setup is closed.',
|
||||
properties: z.object({
|
||||
...setupTelemetryProperties,
|
||||
instance_id: z.string(),
|
||||
workflow_id: z.string(),
|
||||
thread_id: z.string(),
|
||||
source: z.enum(['instance_ai_setup_wizard', 'instance_ai_setup_panel']),
|
||||
request_id: z.string().optional(),
|
||||
items: z.array(z.object({ ...setupItemProperties, completed: z.boolean() })),
|
||||
setup_complete: z.boolean(),
|
||||
}),
|
||||
},
|
||||
MULTIPLE_NODES_SELECTED: {
|
||||
|
|
|
|||
18
packages/@n8n/telemetry/src/setup-properties.ts
Normal file
18
packages/@n8n/telemetry/src/setup-properties.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { z } from 'zod/v4';
|
||||
|
||||
const setupVariant = z.enum(['control', 'variant']).optional();
|
||||
|
||||
export const setupTelemetryProperties = {
|
||||
session_id: z.string().optional(),
|
||||
variant: setupVariant,
|
||||
'$feature/118_instance_ai_setup_overhaul': setupVariant,
|
||||
};
|
||||
|
||||
// A requirement is identified by its node, kind, and credential type or parameter name.
|
||||
export const setupItemProperties = {
|
||||
node_id: z.string(),
|
||||
node_type: z.string().optional(),
|
||||
kind: z.enum(['credential', 'parameter']),
|
||||
credential_type: z.string().optional(),
|
||||
parameter_name: z.string().optional(),
|
||||
};
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "n8n",
|
||||
"version": "2.41.0",
|
||||
"version": "2.41.3",
|
||||
"description": "n8n Workflow Automation Tool",
|
||||
"main": "dist/index",
|
||||
"types": "dist/index.d.ts",
|
||||
|
|
@ -189,7 +189,6 @@
|
|||
"@opentelemetry/exporter-trace-otlp-grpc": "^0.221.0",
|
||||
"@opentelemetry/exporter-trace-otlp-proto": "^0.221.0",
|
||||
"@opentelemetry/resources": "^2.10.0",
|
||||
"@opentelemetry/sdk-node": "^0.221.0",
|
||||
"@opentelemetry/sdk-trace-base": "^2.10.0",
|
||||
"@opentelemetry/sdk-trace-node": "^2.10.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.43.0",
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import { License } from '@/license';
|
|||
import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
|
||||
import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config';
|
||||
import { CommunityPackagesService } from '@/modules/community-packages/community-packages.service';
|
||||
import { OtelService } from '@/modules/otel/otel.service';
|
||||
import { NodeTypes } from '@/node-types';
|
||||
import { PostHogClient } from '@/posthog';
|
||||
import { PollJobProvider } from '@/scheduling/poll-trigger-node/poll-job-provider';
|
||||
|
|
@ -72,6 +73,7 @@ mockInstance(NodeTypes);
|
|||
const shutdownService = mockInstance(ShutdownService);
|
||||
shutdownService.validate.mockReturnValue(undefined);
|
||||
mockInstance(PostHogClient);
|
||||
mockInstance(OtelService);
|
||||
mockInstance(TelemetryEventRelay);
|
||||
mockInstance(ActivityEventRelay);
|
||||
mockInstance(WorkflowFailureNotificationEventRelay);
|
||||
|
|
@ -126,6 +128,7 @@ describe('Start - AuthRolesService initialization', () => {
|
|||
Container.set(MultiMainSetup, multiMainSetup);
|
||||
Container.set(AuthHandlerRegistry, authHandlerRegistry);
|
||||
Container.set(PostHogClient, mockInstance(PostHogClient));
|
||||
Container.set(OtelService, mockInstance(OtelService));
|
||||
Container.set(TelemetryEventRelay, mockInstance(TelemetryEventRelay));
|
||||
Container.set(ActivityEventRelay, mockInstance(ActivityEventRelay));
|
||||
Container.set(
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import { WorkflowFailureNotificationEventRelay } from '@/events/relays/workflow-
|
|||
import type { ExecutionPersistence } from '@/executions/execution-persistence';
|
||||
import { LoadNodesAndCredentials } from '@/load-nodes-and-credentials';
|
||||
import { CommunityPackagesConfig } from '@/modules/community-packages/community-packages.config';
|
||||
import { OtelService } from '@/modules/otel/otel.service';
|
||||
import { NodeTypes } from '@/node-types';
|
||||
import { PostHogClient } from '@/posthog';
|
||||
import { PubSubRegistry } from '@/scaling/pubsub/pubsub.registry';
|
||||
|
|
@ -61,6 +62,7 @@ mockInstance(NodeTypes);
|
|||
mockInstance(ShutdownService);
|
||||
mockInstance(MessageEventBus);
|
||||
mockInstance(PostHogClient);
|
||||
mockInstance(OtelService);
|
||||
mockInstance(TelemetryEventRelay);
|
||||
mockInstance(ActivityEventRelay);
|
||||
mockInstance(WorkflowFailureNotificationEventRelay);
|
||||
|
|
|
|||
|
|
@ -4,13 +4,22 @@
|
|||
import type { Logger } from '@n8n/backend-common';
|
||||
import { ExpressionEngineConfig, type GlobalConfig } from '@n8n/config';
|
||||
import { EXPRESSION_METRICS } from '@n8n/expression-runtime';
|
||||
import { trace } from '@opentelemetry/api';
|
||||
import type { Tracer } from '@opentelemetry/api';
|
||||
import promClient from 'prom-client';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { ExpressionObservabilityProvider } from '../expression-observability.provider';
|
||||
|
||||
import type { OtelService } from '@/modules/otel/otel.service';
|
||||
|
||||
const scopedLogger = mock<Logger>();
|
||||
const otelService = mock<OtelService>();
|
||||
const startSpanMock = vi.fn().mockReturnValue({
|
||||
setStatus: vi.fn(),
|
||||
setAttribute: vi.fn(),
|
||||
recordException: vi.fn(),
|
||||
end: vi.fn(),
|
||||
});
|
||||
|
||||
function buildConfig(overrides: Partial<ExpressionEngineConfig> = {}): ExpressionEngineConfig {
|
||||
const config = new ExpressionEngineConfig();
|
||||
|
|
@ -32,6 +41,7 @@ function buildGlobalConfig(prefix = 'n8n_'): GlobalConfig {
|
|||
describe('ExpressionObservabilityProvider', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
otelService.getTracer.mockReturnValue({ startSpan: startSpanMock } as unknown as Tracer);
|
||||
promClient.register.clear();
|
||||
});
|
||||
|
||||
|
|
@ -41,6 +51,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig({ observabilityEnabled: false }),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
|
||||
expect(() => {
|
||||
|
|
@ -57,6 +68,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig({ engine: 'legacy' }),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
|
||||
provider.metrics.counter(EXPRESSION_METRICS.poolAcquired.name, 1);
|
||||
|
|
@ -72,6 +84,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.counter(EXPRESSION_METRICS.poolAcquired.name, 2);
|
||||
|
||||
|
|
@ -86,6 +99,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.gauge(EXPRESSION_METRICS.codeCacheSize.name, 42);
|
||||
|
||||
|
|
@ -98,6 +112,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 3, {
|
||||
status: 'success',
|
||||
|
|
@ -116,6 +131,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
|
||||
promClient.register.clear();
|
||||
|
|
@ -140,6 +156,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
|
||||
promClient.register.clear();
|
||||
|
|
@ -153,25 +170,12 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
});
|
||||
|
||||
describe('tail sampling', () => {
|
||||
const startSpanMock = vi.fn().mockReturnValue({
|
||||
setStatus: vi.fn(),
|
||||
setAttribute: vi.fn(),
|
||||
recordException: vi.fn(),
|
||||
end: vi.fn(),
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
startSpanMock.mockClear();
|
||||
vi.spyOn(trace, 'getTracer').mockReturnValue({
|
||||
startSpan: startSpanMock,
|
||||
} as unknown as ReturnType<typeof trace.getTracer>);
|
||||
});
|
||||
|
||||
it('drops healthy spans under the slow threshold', () => {
|
||||
const provider = new ExpressionObservabilityProvider(
|
||||
buildConfig({ slowEvaluationThresholdMs: 50, tracesSampleRate: 0 }),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.01, {
|
||||
status: 'success',
|
||||
|
|
@ -185,6 +189,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig({ slowEvaluationThresholdMs: 50 }),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.1, {
|
||||
status: 'success',
|
||||
|
|
@ -206,6 +211,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig({ slowEvaluationThresholdMs: 50 }),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.005, {
|
||||
status: 'error',
|
||||
|
|
@ -227,6 +233,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig({ tracesEnabled: false, slowEvaluationThresholdMs: 10 }),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.5, {
|
||||
status: 'error',
|
||||
|
|
@ -242,6 +249,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
expect(() => {
|
||||
provider.metrics.counter('test.unknown', 1);
|
||||
|
|
@ -256,6 +264,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
|
||||
provider.metrics.histogram(EXPRESSION_METRICS.evaluationDuration.name, 0.01, {
|
||||
|
|
@ -285,6 +294,7 @@ describe('ExpressionObservabilityProvider', () => {
|
|||
buildConfig(),
|
||||
buildLogger(),
|
||||
buildGlobalConfig(),
|
||||
otelService,
|
||||
);
|
||||
provider.logs.info('hello', { k: 'v' });
|
||||
provider.logs.warn('warn', { k: 'v' });
|
||||
|
|
|
|||
|
|
@ -10,8 +10,7 @@ import type {
|
|||
TracesAPI,
|
||||
} from '@n8n/expression-runtime';
|
||||
import { EXPRESSION_METRICS, NoOpProvider } from '@n8n/expression-runtime';
|
||||
import { SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import type { Tracer } from '@opentelemetry/api';
|
||||
import { SpanStatusCode } from '@opentelemetry/api';
|
||||
import { UnexpectedError } from 'n8n-workflow';
|
||||
import promClient, { type Counter, type Gauge, type Histogram } from 'prom-client';
|
||||
|
||||
|
|
@ -26,6 +25,8 @@ import {
|
|||
toPromName,
|
||||
} from './expression-observability.formatters';
|
||||
|
||||
import { OtelService } from '@/modules/otel/otel.service';
|
||||
|
||||
type TailSampleDecision = 'drop' | 'keep';
|
||||
|
||||
@Service()
|
||||
|
|
@ -40,8 +41,6 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
|
|||
|
||||
private readonly prefix!: string;
|
||||
|
||||
private tracer?: Tracer;
|
||||
|
||||
private readonly metricDefs = new Map<string, MetricDef>(
|
||||
(Object.values(EXPRESSION_METRICS) as MetricDef[]).map((def) => [def.name, def]),
|
||||
);
|
||||
|
|
@ -50,6 +49,7 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
|
|||
private readonly config: ExpressionEngineConfig,
|
||||
private readonly logger: Logger,
|
||||
globalConfig: GlobalConfig,
|
||||
private readonly otelService: OtelService,
|
||||
) {
|
||||
this.scopedLogger = this.logger.scoped('expression-engine');
|
||||
|
||||
|
|
@ -167,7 +167,7 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
|
|||
? 'slow'
|
||||
: 'healthy';
|
||||
|
||||
const tracer = this.getTracer();
|
||||
const tracer = this.otelService.getTracer(TRACER_NAME);
|
||||
const errorType = tags?.type && tags.type !== 'none' ? tags.type : undefined;
|
||||
const span = tracer.startSpan('expression.evaluate', {
|
||||
attributes: {
|
||||
|
|
@ -194,7 +194,7 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
|
|||
private startSpan(name: string, attributes?: Record<string, unknown>): Span {
|
||||
if (!this.config.tracesEnabled) return NoOpProvider.traces.startSpan(name, attributes);
|
||||
|
||||
const tracer = this.getTracer();
|
||||
const tracer = this.otelService.getTracer(TRACER_NAME);
|
||||
const otelSpan = tracer.startSpan(name, {
|
||||
attributes: normalizeAttributes(attributes),
|
||||
});
|
||||
|
|
@ -212,9 +212,4 @@ export class ExpressionObservabilityProvider implements ObservabilityProvider {
|
|||
end: () => otelSpan.end(),
|
||||
};
|
||||
}
|
||||
|
||||
private getTracer(): Tracer {
|
||||
this.tracer ??= trace.getTracer(TRACER_NAME);
|
||||
return this.tracer;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,22 @@
|
|||
import type { Logger } from '@n8n/backend-common';
|
||||
import type { OutboundHttp } from '@n8n/backend-network';
|
||||
import type { AgentsConfig } from '@n8n/config';
|
||||
import { context, type Tracer } from '@opentelemetry/api';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
import type { InstanceSettings } from 'n8n-core';
|
||||
|
||||
import { OtelTestProvider } from '@/modules/otel/__tests__/support/otel-test-provider';
|
||||
import { ExecutionLevelTracer } from '@/modules/otel/execution-level-tracer';
|
||||
import type { OtelSettingsService } from '@/modules/otel/otel-settings.service';
|
||||
import { OtelService } from '@/modules/otel/otel.service';
|
||||
|
||||
import { AgentRunTracingService } from '../agent-run-tracing.service';
|
||||
|
||||
// Deliberately does not mock '@opentelemetry/api': agent-run-tracing.service.test.ts
|
||||
// Deliberately uses a real tracer provider: agent-run-tracing.service.test.ts
|
||||
// already covers the built-metadata/branching logic against a stubbed tracer. This
|
||||
// file proves the real OTel path end-to-end — that a registered provider actually
|
||||
// file proves the real OTel path end-to-end — that the module's provider actually
|
||||
// records the span the service's tracer produces, and that a disabled OTel module
|
||||
// (no provider registered) or disabled agent tracing both emit nothing.
|
||||
// (no provider started) or disabled agent tracing both emit nothing.
|
||||
describe('AgentRunTracingService (real OTel provider)', () => {
|
||||
const baseMetadata = {
|
||||
agentId: 'agent-1',
|
||||
|
|
@ -26,7 +29,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
|
|||
const otel = OtelTestProvider.create();
|
||||
try {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otel.asOtelService());
|
||||
|
||||
const built = await service.build(baseMetadata);
|
||||
expect(built).toBeDefined();
|
||||
|
|
@ -57,6 +60,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
|
|||
const otel = OtelTestProvider.create({ withContextManager: true });
|
||||
try {
|
||||
const executionLevelTracer = new ExecutionLevelTracer(
|
||||
otel.asOtelService(),
|
||||
mock<OtelSettingsService>({ getSettings: () => ({ injectOutbound: false }) as never }),
|
||||
mock<Logger>(),
|
||||
);
|
||||
|
|
@ -70,7 +74,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
|
|||
executionLevelTracer.startNode({ executionId, node });
|
||||
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const agentRunTracingService = new AgentRunTracingService(agentsConfig);
|
||||
const agentRunTracingService = new AgentRunTracingService(agentsConfig, otel.asOtelService());
|
||||
|
||||
const parentCtx = executionLevelTracer.getActiveContext(executionId, node.name);
|
||||
expect(parentCtx).toBeDefined();
|
||||
|
|
@ -118,15 +122,21 @@ describe('AgentRunTracingService (real OTel provider)', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('emits no spans when the OTel module has no provider registered', async () => {
|
||||
it('emits no spans when the OTel module has not started a provider', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const otelService = new OtelService(
|
||||
mock<OtelSettingsService>(),
|
||||
mock<InstanceSettings>(),
|
||||
mock<Logger>(),
|
||||
mock<OutboundHttp>(),
|
||||
);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const built = await service.build(baseMetadata);
|
||||
expect(built).toBeDefined();
|
||||
|
||||
// No provider registered — the global tracer falls back to OTel's
|
||||
// no-op implementation, so starting a span records nothing anywhere.
|
||||
// No provider started — the module hands out OTel's no-op tracer, so
|
||||
// starting a span records nothing anywhere.
|
||||
(built?.tracer as Tracer).startActiveSpan('test-span', (span) => {
|
||||
expect(span.isRecording()).toBe(false);
|
||||
span.end();
|
||||
|
|
@ -137,7 +147,7 @@ describe('AgentRunTracingService (real OTel provider)', () => {
|
|||
const otel = OtelTestProvider.create();
|
||||
try {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: false });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otel.asOtelService());
|
||||
|
||||
const built = await service.build(baseMetadata);
|
||||
expect(built).toBeUndefined();
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
import type { AgentsConfig } from '@n8n/config';
|
||||
import { trace } from '@opentelemetry/api';
|
||||
import type { Tracer } from '@opentelemetry/api';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import type { OtelService } from '@/modules/otel/otel.service';
|
||||
|
||||
import { AgentRunTracingService, modelIdFromSnapshot } from '../agent-run-tracing.service';
|
||||
|
||||
vi.mock('@opentelemetry/api', () => ({
|
||||
trace: { getTracer: vi.fn(() => ({ startSpan: vi.fn(), startActiveSpan: vi.fn() })) },
|
||||
}));
|
||||
const otelService = mock<OtelService>();
|
||||
|
||||
describe('AgentRunTracingService', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
otelService.getTracer.mockReturnValue(mock<Tracer>());
|
||||
});
|
||||
|
||||
const baseMetadata = {
|
||||
|
|
@ -21,38 +22,39 @@ describe('AgentRunTracingService', () => {
|
|||
};
|
||||
|
||||
it('exposes whether tracing is enabled via the `enabled` getter', () => {
|
||||
expect(new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: true })).enabled).toBe(
|
||||
true,
|
||||
);
|
||||
expect(new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: false })).enabled).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: true }), otelService).enabled,
|
||||
).toBe(true);
|
||||
expect(
|
||||
new AgentRunTracingService(mock<AgentsConfig>({ tracingEnabled: false }), otelService)
|
||||
.enabled,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('returns undefined when agent tracing is disabled, without touching the tracer', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: false });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const built = await service.build(baseMetadata);
|
||||
|
||||
expect(built).toBeUndefined();
|
||||
expect(trace.getTracer).not.toHaveBeenCalled();
|
||||
expect(otelService.getTracer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('builds telemetry from a fresh tracer fetch when tracing is enabled', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
await service.build(baseMetadata);
|
||||
await service.build(baseMetadata);
|
||||
|
||||
expect(trace.getTracer).toHaveBeenCalledTimes(2);
|
||||
expect(trace.getTracer).toHaveBeenCalledWith('@n8n/agents');
|
||||
expect(otelService.getTracer).toHaveBeenCalledTimes(2);
|
||||
expect(otelService.getTracer).toHaveBeenCalledWith('@n8n/agents');
|
||||
});
|
||||
|
||||
it('never marks the built telemetry as LangSmith, so root spans stay generic', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const built = await service.build(baseMetadata);
|
||||
|
||||
|
|
@ -62,7 +64,7 @@ describe('AgentRunTracingService', () => {
|
|||
|
||||
it('includes execution_id/workflow_id/node_id only when source is "workflow"', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const workflowBuilt = await service.build({
|
||||
...baseMetadata,
|
||||
|
|
@ -95,7 +97,7 @@ describe('AgentRunTracingService', () => {
|
|||
|
||||
it('omits execution_id/workflow_id/node_id individually when not provided, even for a workflow source', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const noneProvided = await service.build({
|
||||
...baseMetadata,
|
||||
|
|
@ -150,7 +152,7 @@ describe('AgentRunTracingService', () => {
|
|||
|
||||
it('rejects workflow-only fields at compile time for a non-workflow source', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
// @ts-expect-error executionId only exists on the 'workflow' branch of
|
||||
// the AgentRunTracingMetadata union — a non-workflow source can't carry
|
||||
|
|
@ -160,7 +162,7 @@ describe('AgentRunTracingService', () => {
|
|||
|
||||
it('marks workflow-sourced runs as rootAnchored: false only when a parent context was found, and leaves other sources anchored', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const workflowBuilt = await service.build({
|
||||
...baseMetadata,
|
||||
|
|
@ -184,7 +186,7 @@ describe('AgentRunTracingService', () => {
|
|||
|
||||
it('includes user_id and model_id only when provided', async () => {
|
||||
const agentsConfig = mock<AgentsConfig>({ tracingEnabled: true });
|
||||
const service = new AgentRunTracingService(agentsConfig);
|
||||
const service = new AgentRunTracingService(agentsConfig, otelService);
|
||||
|
||||
const built = await service.build({
|
||||
...baseMetadata,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import type { AttributeValue, BuiltTelemetry } from '@n8n/agents';
|
|||
import { Telemetry } from '@n8n/agents';
|
||||
import { AgentsConfig } from '@n8n/config';
|
||||
import { Service } from '@n8n/di';
|
||||
import { trace } from '@opentelemetry/api';
|
||||
|
||||
import { OtelService } from '@/modules/otel/otel.service';
|
||||
|
||||
const AGENTS_TRACER_NAME = '@n8n/agents';
|
||||
|
||||
|
|
@ -49,13 +50,16 @@ function isWorkflowTracingMetadata(
|
|||
}
|
||||
|
||||
/**
|
||||
* Builds the per-run OTel telemetry handed to an agent run, riding along with
|
||||
* the workflow OTel module's already-registered global tracer provider — this
|
||||
* service never creates, registers, or shuts down a provider itself.
|
||||
* Builds the per-run OTel telemetry handed to an agent run on the workflow OTel
|
||||
* module's tracer provider — this service never creates, registers, or shuts
|
||||
* down a provider itself.
|
||||
*/
|
||||
@Service()
|
||||
export class AgentRunTracingService {
|
||||
constructor(private readonly agentsConfig: AgentsConfig) {}
|
||||
constructor(
|
||||
private readonly agentsConfig: AgentsConfig,
|
||||
private readonly otelService: OtelService,
|
||||
) {}
|
||||
|
||||
/** Whether agent tracing is enabled — lets callers skip tracing-only work (e.g. a DB lookup) upfront. */
|
||||
get enabled(): boolean {
|
||||
|
|
@ -67,9 +71,8 @@ export class AgentRunTracingService {
|
|||
* `ExecutionOptions.telemetry` entirely in that case, indistinguishable
|
||||
* from "OTel module off" to the SDK (both are no-ops).
|
||||
*
|
||||
* Fetches the tracer fresh on every call rather than caching it: a tracer
|
||||
* obtained before any provider is registered would otherwise go stale
|
||||
* across an OTel module restart. Fetching fresh sidesteps that entirely.
|
||||
* Fetches the tracer fresh on every call rather than caching it, so a
|
||||
* tracer never goes stale across an OTel module restart.
|
||||
*/
|
||||
async build(metadata: AgentRunTracingMetadata): Promise<BuiltTelemetry | undefined> {
|
||||
if (!this.agentsConfig.tracingEnabled) return undefined;
|
||||
|
|
@ -91,7 +94,7 @@ export class AgentRunTracingService {
|
|||
};
|
||||
|
||||
const built = await new Telemetry()
|
||||
.tracer(trace.getTracer(AGENTS_TRACER_NAME))
|
||||
.tracer(this.otelService.getTracer(AGENTS_TRACER_NAME))
|
||||
.metadata(attributes)
|
||||
.recordInputs(this.agentsConfig.tracingRecordInputs)
|
||||
.recordOutputs(this.agentsConfig.tracingRecordOutputs)
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ function createService() {
|
|||
mcpConnectionsEnabled: false,
|
||||
conversationHistoryEnabled: false,
|
||||
progressiveBuildingEnabled: false,
|
||||
setupPanelEnabled: false,
|
||||
nodeUsageEnabled: false,
|
||||
folderExplorationEnabled: false,
|
||||
aiPreferencesEnabled: false,
|
||||
|
|
|
|||
|
|
@ -72,6 +72,8 @@ import {
|
|||
INSTANCE_ACTIVITY_CONTEXT_FLAG,
|
||||
INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
|
||||
INSTANCE_AI_SETUP_PANEL_FLAG,
|
||||
INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
|
||||
CONFIG_EVALUATIONS_ENABLED_VARIANT,
|
||||
INSTANCE_AI_MCP_CONNECTIONS_FLAG,
|
||||
|
|
@ -2287,6 +2289,7 @@ describe('createDataTableAdapter', () => {
|
|||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createWorkflowAdapterForTests(overrides?: {
|
||||
setupPanelVariant?: 'control' | 'variant';
|
||||
namedVersionsLicensed?: boolean;
|
||||
foldersLicensed?: boolean;
|
||||
branchReadOnly?: boolean;
|
||||
|
|
@ -2465,6 +2468,7 @@ function createWorkflowAdapterForTests(overrides?: {
|
|||
threadId: 'thread-1',
|
||||
projectId: boundProjectId,
|
||||
folderExplorationEnabled: overrides?.folderExploration ?? false,
|
||||
setupPanelVariant: overrides?.setupPanelVariant,
|
||||
});
|
||||
const adapter = context.workflowService;
|
||||
|
||||
|
|
@ -3584,6 +3588,38 @@ describe('createWorkflowAdapter', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each(['control', 'variant', undefined] as const)(
|
||||
'carries setup assignment %s on workflow creation, updates, and publishing',
|
||||
async (setupPanelVariant) => {
|
||||
const { adapter, mockTelemetry } = createWorkflowAdapterForTests({ setupPanelVariant });
|
||||
await adapter.createFromWorkflowJSON(minimalWorkflowJSON);
|
||||
await adapter.updateFromWorkflowJSON('wf-new', minimalWorkflowJSON);
|
||||
await adapter.publish('wf-new');
|
||||
|
||||
for (const event of [
|
||||
'Builder created workflow',
|
||||
'Builder modified workflow',
|
||||
'Builder published workflow',
|
||||
]) {
|
||||
const properties = mockTelemetry.track.mock.calls.find(([name]) => name === event)?.[1];
|
||||
expect(properties).toMatchObject({
|
||||
user_id: 'user-1',
|
||||
thread_id: 'thread-1',
|
||||
workflow_id: 'wf-new',
|
||||
});
|
||||
if (setupPanelVariant) {
|
||||
expect(properties).toMatchObject({
|
||||
variant: setupPanelVariant,
|
||||
[`$feature/${INSTANCE_AI_SETUP_PANEL_FLAG}`]: setupPanelVariant,
|
||||
});
|
||||
} else {
|
||||
expect(properties).not.toHaveProperty('variant');
|
||||
expect(properties).not.toHaveProperty(`$feature/${INSTANCE_AI_SETUP_PANEL_FLAG}`);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('marks the workflow as AI-builder temporary when markAsAiTemporary is true', async () => {
|
||||
const {
|
||||
adapter,
|
||||
|
|
@ -6040,6 +6076,7 @@ describe('resolveExperimentGates', () => {
|
|||
[INSTANCE_AI_MCP_CONNECTIONS_FLAG]: INSTANCE_AI_MCP_CONNECTIONS_ENABLED_VARIANT,
|
||||
[INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
|
||||
[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
|
||||
[INSTANCE_AI_SETUP_PANEL_FLAG]: INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
|
||||
[INSTANCE_AI_NODE_USAGE_FLAG]: true,
|
||||
[INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: INSTANCE_AI_FOLDER_EXPLORATION_ENABLED_VARIANT,
|
||||
[CONTEXT_PREFERENCES_FLAG]: CONTEXT_PREFERENCES_ENABLED_VARIANT,
|
||||
|
|
@ -6053,6 +6090,8 @@ describe('resolveExperimentGates', () => {
|
|||
mcpConnectionsEnabled: true,
|
||||
conversationHistoryEnabled: true,
|
||||
progressiveBuildingEnabled: true,
|
||||
setupPanelEnabled: true,
|
||||
setupPanelVariant: 'variant',
|
||||
nodeUsageEnabled: true,
|
||||
folderExplorationEnabled: true,
|
||||
aiPreferencesEnabled: true,
|
||||
|
|
@ -6119,6 +6158,7 @@ describe('resolveExperimentGates', () => {
|
|||
[INSTANCE_AI_MCP_CONNECTIONS_FLAG]: 'control',
|
||||
[INSTANCE_AI_CONVERSATION_HISTORY_FLAG]: 'control',
|
||||
[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG]: 'control',
|
||||
[INSTANCE_AI_SETUP_PANEL_FLAG]: 'control',
|
||||
[INSTANCE_AI_NODE_USAGE_FLAG]: false,
|
||||
[INSTANCE_AI_FOLDER_EXPLORATION_FLAG]: 'control',
|
||||
[CONTEXT_PREFERENCES_FLAG]: CONTEXT_PREFERENCES_CONTROL_VARIANT,
|
||||
|
|
@ -6129,6 +6169,8 @@ describe('resolveExperimentGates', () => {
|
|||
mcpConnectionsEnabled: false,
|
||||
conversationHistoryEnabled: false,
|
||||
progressiveBuildingEnabled: false,
|
||||
setupPanelEnabled: false,
|
||||
setupPanelVariant: 'control',
|
||||
nodeUsageEnabled: false,
|
||||
folderExplorationEnabled: false,
|
||||
aiPreferencesEnabled: false,
|
||||
|
|
@ -6136,6 +6178,18 @@ describe('resolveExperimentGates', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each([true, 'unexpected'])(
|
||||
'keeps setup disabled for invalid assignment %s',
|
||||
async (assignment) => {
|
||||
stubContainer({ [INSTANCE_AI_SETUP_PANEL_FLAG]: assignment });
|
||||
|
||||
const gates = await createAdapter().resolveExperimentGates(user);
|
||||
|
||||
expect(gates.setupPanelEnabled).toBe(false);
|
||||
expect(gates).not.toHaveProperty('setupPanelVariant');
|
||||
},
|
||||
);
|
||||
|
||||
// Regression guard for the shipped bug: the flag is multivariate, so a
|
||||
// boolean `true` is not a value PostHog can return for it. Reading it as one
|
||||
// left the gate shut at every rollout percentage.
|
||||
|
|
@ -6165,6 +6219,7 @@ describe('resolveExperimentGates', () => {
|
|||
mcpConnectionsEnabled: false,
|
||||
conversationHistoryEnabled: false,
|
||||
progressiveBuildingEnabled: false,
|
||||
setupPanelEnabled: false,
|
||||
nodeUsageEnabled: false,
|
||||
folderExplorationEnabled: false,
|
||||
aiPreferencesEnabled: false,
|
||||
|
|
@ -6181,6 +6236,7 @@ describe('resolveExperimentGates', () => {
|
|||
mcpConnectionsEnabled: false,
|
||||
conversationHistoryEnabled: false,
|
||||
progressiveBuildingEnabled: false,
|
||||
setupPanelEnabled: false,
|
||||
nodeUsageEnabled: false,
|
||||
folderExplorationEnabled: false,
|
||||
aiPreferencesEnabled: false,
|
||||
|
|
|
|||
|
|
@ -729,7 +729,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
getSandboxStatus: Mock;
|
||||
isLocalGatewayDisabledForUser: Mock;
|
||||
getPermissions: Mock;
|
||||
isInstanceAiSetupPanelEnabled: Mock;
|
||||
};
|
||||
gatewayService: { findGateway: Mock; applyToolPolicy: Mock };
|
||||
aiService: { isProxyEnabled: Mock };
|
||||
|
|
@ -762,6 +761,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
getPromptConfiguration: Mock;
|
||||
setPromptConfiguration: Mock;
|
||||
setBuildMode: Mock;
|
||||
setSetupPanelEnabled: Mock;
|
||||
setPromptVersion: Mock;
|
||||
getComputerUseChannels: Mock;
|
||||
};
|
||||
|
|
@ -787,7 +787,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
})),
|
||||
isLocalGatewayDisabledForUser: vi.fn(async () => false),
|
||||
getPermissions: vi.fn(() => ({})),
|
||||
isInstanceAiSetupPanelEnabled: vi.fn(() => snapshotMode !== 'off'),
|
||||
};
|
||||
service.gatewayService = { findGateway: vi.fn(() => undefined), applyToolPolicy: vi.fn() };
|
||||
service.aiService = { isProxyEnabled: vi.fn(() => false) };
|
||||
|
|
@ -795,6 +794,8 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
createContext: vi.fn(() => ({})),
|
||||
getNodeDefinitionDirs: vi.fn(() => []),
|
||||
resolveExperimentGates: vi.fn().mockResolvedValue({
|
||||
setupPanelEnabled: snapshotMode !== 'off',
|
||||
setupPanelVariant: snapshotMode === 'off' ? 'control' : 'variant',
|
||||
configEvalsEnabled: true,
|
||||
mcpConnectionsEnabled: false,
|
||||
conversationHistoryEnabled: false,
|
||||
|
|
@ -835,6 +836,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
getPromptConfiguration: vi.fn(),
|
||||
setPromptConfiguration: vi.fn(),
|
||||
setBuildMode: vi.fn(),
|
||||
setSetupPanelEnabled: vi.fn(),
|
||||
setPromptVersion: vi.fn(),
|
||||
getComputerUseChannels: vi.fn(() => undefined),
|
||||
};
|
||||
|
|
@ -884,9 +886,17 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
expect(environment.instanceContextEnabled).toBe(instanceContextEnabled);
|
||||
expect(service.adapterService.createContext).toHaveBeenCalledWith(
|
||||
fakeUser,
|
||||
expect.objectContaining({ instanceContextEnabled }),
|
||||
expect.objectContaining({
|
||||
instanceContextEnabled,
|
||||
configEvalsEnabled: true,
|
||||
setupPanelVariant: snapshotMode === 'off' ? 'control' : 'variant',
|
||||
}),
|
||||
);
|
||||
expect(environment.orchestrationContext.setupPanelEnabled).toBe(snapshotMode !== 'off');
|
||||
expect(service.runState.setSetupPanelEnabled).toHaveBeenCalledWith(
|
||||
'thread-1',
|
||||
snapshotMode !== 'off',
|
||||
);
|
||||
if (snapshotMode === 'off') {
|
||||
expect(service.eventLog.getSetupItemsSnapshots).not.toHaveBeenCalled();
|
||||
expect(createSetupItemsEmitter).not.toHaveBeenCalled();
|
||||
|
|
@ -1055,7 +1065,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
getSandboxStatus: Mock;
|
||||
isLocalGatewayDisabledForUser: Mock;
|
||||
getPermissions: Mock;
|
||||
isInstanceAiSetupPanelEnabled: Mock;
|
||||
};
|
||||
gatewayService: { findGateway: Mock; applyToolPolicy: Mock };
|
||||
aiService: { isProxyEnabled: Mock };
|
||||
|
|
@ -1088,6 +1097,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
getPromptConfiguration: Mock;
|
||||
setPromptConfiguration: Mock;
|
||||
setBuildMode: Mock;
|
||||
setSetupPanelEnabled: Mock;
|
||||
setPromptVersion: Mock;
|
||||
getComputerUseChannels: Mock;
|
||||
};
|
||||
|
|
@ -1113,7 +1123,6 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
})),
|
||||
isLocalGatewayDisabledForUser: vi.fn(async () => false),
|
||||
getPermissions: vi.fn(() => ({})),
|
||||
isInstanceAiSetupPanelEnabled: vi.fn(() => false),
|
||||
};
|
||||
service.gatewayService = { findGateway: vi.fn(() => undefined), applyToolPolicy: vi.fn() };
|
||||
service.aiService = { isProxyEnabled: vi.fn(() => false) };
|
||||
|
|
@ -1157,6 +1166,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
|||
getPromptConfiguration: vi.fn(),
|
||||
setPromptConfiguration: vi.fn(),
|
||||
setBuildMode: vi.fn(),
|
||||
setSetupPanelEnabled: vi.fn(),
|
||||
setPromptVersion: vi.fn(),
|
||||
getComputerUseChannels: vi.fn(() => undefined),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -95,7 +95,10 @@ describe('WorkflowVerificationObligationService setup panel policy', () => {
|
|||
async (enabled) => {
|
||||
const getThread = vi.fn<TypeORMAgentMemory['getThread']>();
|
||||
const memory = mock<TypeORMAgentMemory>({ getThread });
|
||||
const service = new WorkflowVerificationObligationService(memory, () => enabled);
|
||||
const service = new WorkflowVerificationObligationService(
|
||||
memory,
|
||||
(threadId) => threadId === 'thread-1' && enabled,
|
||||
);
|
||||
const outcome = makeOutcome({
|
||||
needsUserInput: true,
|
||||
nodeSimulationPlan: [],
|
||||
|
|
|
|||
|
|
@ -1277,11 +1277,6 @@ export class InstanceAiSettingsService {
|
|||
return this.config.browserUseEnabled;
|
||||
}
|
||||
|
||||
/** Whether the non-blocking setup panel replaces the suspending setup wizard. */
|
||||
isInstanceAiSetupPanelEnabled(): boolean {
|
||||
return this.config.instanceAiSetupPanelEnabled;
|
||||
}
|
||||
|
||||
/** Whether this instance is in the activation-capped trial cohort. */
|
||||
isActivationCapped(): boolean {
|
||||
return this.config.activationCapped;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ import {
|
|||
INSTANCE_AI_CONVERSATION_HISTORY_FLAG,
|
||||
INSTANCE_AI_CONVERSATION_HISTORY_ENABLED_VARIANT,
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
|
||||
INSTANCE_AI_SETUP_PANEL_FLAG,
|
||||
INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
|
||||
} from '@n8n/api-types';
|
||||
import type { AiGatewayConfigDto } from '@n8n/api-types';
|
||||
|
|
@ -439,6 +441,7 @@ export class InstanceAiAdapterService {
|
|||
/** Per-user config-evals gate (via `resolveExperimentGates`). Falsy →
|
||||
* eval-config service/tool not wired. */
|
||||
configEvalsEnabled?: boolean;
|
||||
setupPanelVariant?: 'control' | 'variant';
|
||||
/** Per-user MCP registry gate (via `resolveExperimentGates`). Falsy →
|
||||
* mcp service/tool not wired. */
|
||||
mcpConnectionsEnabled?: boolean;
|
||||
|
|
@ -468,6 +471,7 @@ export class InstanceAiAdapterService {
|
|||
shouldBypassCredentialTest,
|
||||
agentId,
|
||||
configEvalsEnabled,
|
||||
setupPanelVariant,
|
||||
mcpConnectionsEnabled,
|
||||
nodeUsageEnabled,
|
||||
instanceContextEnabled,
|
||||
|
|
@ -495,6 +499,7 @@ export class InstanceAiAdapterService {
|
|||
workflowService: this.createWorkflowAdapter(user, threadId, projectId, {
|
||||
nodeUsageGateOpen: nodeUsageEnabled === true,
|
||||
folderExploration: folderExplorationEnabled === true,
|
||||
setupPanelVariant,
|
||||
}),
|
||||
executionService: this.createExecutionAdapter(user, pushRef, threadId),
|
||||
credentialService,
|
||||
|
|
@ -590,7 +595,7 @@ export class InstanceAiAdapterService {
|
|||
}
|
||||
}
|
||||
|
||||
/** Resolves user experience flags and the shared activity gate. Unreadable flags stay off. */
|
||||
/** Resolves experiment assignments. */
|
||||
async resolveExperimentGates(user: User): Promise<{
|
||||
/** Config-based evals: never create evals the user can't run. */
|
||||
configEvalsEnabled: boolean;
|
||||
|
|
@ -600,6 +605,8 @@ export class InstanceAiAdapterService {
|
|||
conversationHistoryEnabled: boolean;
|
||||
/** Progressive workflow policy and planning-tool selection. */
|
||||
progressiveBuildingEnabled: boolean;
|
||||
setupPanelEnabled: boolean;
|
||||
setupPanelVariant?: 'control' | 'variant';
|
||||
/** Node-usage context surface: the `node-usage` action and the `nodeTypes` filter on `list`. */
|
||||
nodeUsageEnabled: boolean;
|
||||
/** Per-user folder-exploration gate, passed into `createContext`. Fails
|
||||
|
|
@ -622,8 +629,9 @@ export class InstanceAiAdapterService {
|
|||
if (userFlags.status === 'fulfilled') flags = userFlags.value;
|
||||
instanceContextEnabled = instanceFlag.status === 'fulfilled' && instanceFlag.value === true;
|
||||
} catch {
|
||||
// Keep the gates closed if the client cannot start an evaluation.
|
||||
// Leave unreadable flags unassigned.
|
||||
}
|
||||
const setupPanelVariant = flags[INSTANCE_AI_SETUP_PANEL_FLAG];
|
||||
return {
|
||||
configEvalsEnabled: flags[CONFIG_EVALUATIONS_FLAG] === CONFIG_EVALUATIONS_ENABLED_VARIANT,
|
||||
mcpConnectionsEnabled:
|
||||
|
|
@ -635,6 +643,10 @@ export class InstanceAiAdapterService {
|
|||
progressiveBuildingEnabled:
|
||||
flags[INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG] ===
|
||||
INSTANCE_AI_PROGRESSIVE_BUILDING_ENABLED_VARIANT,
|
||||
setupPanelEnabled: setupPanelVariant === INSTANCE_AI_SETUP_PANEL_ENABLED_VARIANT,
|
||||
...(setupPanelVariant === 'control' || setupPanelVariant === 'variant'
|
||||
? { setupPanelVariant }
|
||||
: {}),
|
||||
nodeUsageEnabled: flags[INSTANCE_AI_NODE_USAGE_FLAG] === true,
|
||||
folderExplorationEnabled:
|
||||
flags[INSTANCE_AI_FOLDER_EXPLORATION_FLAG] ===
|
||||
|
|
@ -873,8 +885,18 @@ export class InstanceAiAdapterService {
|
|||
user: User,
|
||||
threadId?: string,
|
||||
boundProjectId?: string,
|
||||
options: { nodeUsageGateOpen?: boolean; folderExploration?: boolean } = {},
|
||||
options: {
|
||||
nodeUsageGateOpen?: boolean;
|
||||
folderExploration?: boolean;
|
||||
setupPanelVariant?: 'control' | 'variant';
|
||||
} = {},
|
||||
): InstanceAiWorkflowService {
|
||||
const setupExperimentProperties = options.setupPanelVariant
|
||||
? {
|
||||
variant: options.setupPanelVariant,
|
||||
[`$feature/${INSTANCE_AI_SETUP_PANEL_FLAG}`]: options.setupPanelVariant,
|
||||
}
|
||||
: {};
|
||||
const foldersOn = options.folderExploration === true;
|
||||
// Attribution reveals folder ids, names and paths on every row, so it needs
|
||||
// the licence as well as the flag. Resolution stays on the flag alone so an
|
||||
|
|
@ -1375,6 +1397,7 @@ export class InstanceAiAdapterService {
|
|||
|
||||
if (threadId) {
|
||||
telemetry.track('Builder published workflow', {
|
||||
...setupExperimentProperties,
|
||||
user_id: user.id,
|
||||
thread_id: threadId,
|
||||
workflow_id: workflowId,
|
||||
|
|
@ -1608,6 +1631,7 @@ export class InstanceAiAdapterService {
|
|||
|
||||
if (threadId) {
|
||||
telemetry.track('Builder created workflow', {
|
||||
...setupExperimentProperties,
|
||||
user_id: user.id,
|
||||
thread_id: threadId,
|
||||
workflow_id: updated.id,
|
||||
|
|
@ -1699,6 +1723,7 @@ export class InstanceAiAdapterService {
|
|||
|
||||
if (threadId) {
|
||||
telemetry.track('Builder modified workflow', {
|
||||
...setupExperimentProperties,
|
||||
user_id: user.id,
|
||||
thread_id: threadId,
|
||||
workflow_id: workflowId,
|
||||
|
|
|
|||
|
|
@ -87,7 +87,6 @@ export class InstanceAiModule implements ModuleInterface {
|
|||
sandboxUnavailableReason: sandboxStatus.unavailableReason,
|
||||
runDebugEnabled: globalConfig.instanceAi.runDebugEnabled,
|
||||
activationCapped: settingsService.isActivationCapped(),
|
||||
instanceAiSetupPanelEnabled: settingsService.isInstanceAiSetupPanelEnabled(),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -772,8 +772,9 @@ export class InstanceAiService {
|
|||
) {
|
||||
this.logger = logger.scoped('instance-ai');
|
||||
runProbe.registerActiveRunCountProvider(() => this.runState.activeRunCount());
|
||||
this.workflowObligations = new WorkflowVerificationObligationService(this.agentMemory, () =>
|
||||
this.settingsService.isInstanceAiSetupPanelEnabled(),
|
||||
this.workflowObligations = new WorkflowVerificationObligationService(
|
||||
this.agentMemory,
|
||||
(threadId) => this.runState.isSetupPanelEnabled(threadId),
|
||||
);
|
||||
this.taskProjector = new WorkflowVerificationTaskProjector(
|
||||
this.agentMemory,
|
||||
|
|
@ -2417,11 +2418,14 @@ export class InstanceAiService {
|
|||
mcpConnectionsEnabled,
|
||||
conversationHistoryEnabled,
|
||||
progressiveBuildingEnabled,
|
||||
setupPanelEnabled,
|
||||
setupPanelVariant,
|
||||
nodeUsageEnabled,
|
||||
folderExplorationEnabled,
|
||||
aiPreferencesEnabled,
|
||||
instanceContextEnabled,
|
||||
} = await this.adapterService.resolveExperimentGates(user);
|
||||
this.runState.setSetupPanelEnabled(threadId, setupPanelEnabled);
|
||||
// One scoped reader backs both the tool and the first-turn hint.
|
||||
const conversationHistory = conversationHistoryEnabled
|
||||
? this.conversationHistoryService.forContext(user.id, boundProjectId, threadId)
|
||||
|
|
@ -2444,6 +2448,7 @@ export class InstanceAiService {
|
|||
shouldBypassCredentialTest: (credentialId: string) =>
|
||||
this.evalCredentialAllowlists.shouldBypassTest(threadId, credentialId),
|
||||
configEvalsEnabled,
|
||||
setupPanelVariant,
|
||||
mcpConnectionsEnabled,
|
||||
nodeUsageEnabled,
|
||||
instanceContextEnabled,
|
||||
|
|
@ -2482,7 +2487,7 @@ export class InstanceAiService {
|
|||
// Setup panel v2: wire the durable `setup-items` sink only while the flag
|
||||
// is on — its presence is the package-side gate. Seeded with the thread's
|
||||
// persisted snapshots so a recomputed, unchanged list publishes nothing.
|
||||
if (this.settingsService.isInstanceAiSetupPanelEnabled()) {
|
||||
if (setupPanelEnabled) {
|
||||
context.setupItemsEmitter = createSetupItemsEmitter({
|
||||
eventBus: this.eventBus,
|
||||
threadId,
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ export function parseWorkflowBuildOutcome(
|
|||
export class WorkflowVerificationObligationService {
|
||||
constructor(
|
||||
private readonly agentMemory: TypeORMAgentMemory,
|
||||
private readonly isSetupPanelEnabled: () => boolean = () => false,
|
||||
private readonly isSetupPanelEnabled: (threadId: string) => boolean = () => false,
|
||||
) {}
|
||||
|
||||
private storage(): WorkflowLoopStorage {
|
||||
|
|
@ -56,7 +56,7 @@ export class WorkflowVerificationObligationService {
|
|||
): WorkflowVerificationObligation {
|
||||
return deriveWorkflowVerificationObligation(threadId, record, {
|
||||
...options,
|
||||
setupPanelEnabled: this.isSetupPanelEnabled(),
|
||||
setupPanelEnabled: this.isSetupPanelEnabled(threadId),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -97,7 +97,7 @@ export class WorkflowVerificationObligationService {
|
|||
const options = {
|
||||
source: 'planned',
|
||||
plannedTaskId: task.id,
|
||||
setupPanelEnabled: this.isSetupPanelEnabled(),
|
||||
setupPanelEnabled: this.isSetupPanelEnabled(threadId),
|
||||
} satisfies Parameters<typeof deriveWorkflowVerificationObligation>[2];
|
||||
const record = await this.storage().getWorkItem(threadId, baseOutcome.workItemId);
|
||||
const outcome = record?.lastBuildOutcome ?? baseOutcome;
|
||||
|
|
|
|||
|
|
@ -15,7 +15,15 @@ the test.
|
|||
- The receiver runs on `http://127.0.0.1:3456` and answers `POST
|
||||
/api/v1/instance-reports` with **201** on success. Any other status counts as
|
||||
a rejection.
|
||||
- The receiver accepts `Authorization: Bearer testing`.
|
||||
- The receiver trusts the license certificate this instance sends. This applies
|
||||
to the default run; with `N8N_INSTANCE_REPORTING_AUTH_TOKEN` set (scenario
|
||||
5.4c) no certificate is needed. The
|
||||
instance sends whatever `N8N_LICENSE_CERT` holds, so either use a real
|
||||
n8n-issued certificate, or a certificate minted by a development CA that the
|
||||
receiver is configured to trust (see the receiver's `mock-license` tooling
|
||||
and `N8N_MONITORING_ADDITIONAL_ISSUER_CERTS`). A development certificate
|
||||
makes the license SDK log `cert could not be initialized` once at boot; the
|
||||
instance then runs as community and still reports.
|
||||
- Keep the receiver's request log visible. You must see the raw body.
|
||||
- `sqlite3` is installed. The dev instance uses SQLite at
|
||||
`~/.n8n/database.sqlite` unless you set another database.
|
||||
|
|
@ -34,8 +42,8 @@ cd packages/cli
|
|||
|
||||
export N8N_ENABLED_MODULES=instance-reporting
|
||||
export N8N_INSTANCE_REPORTING_BASE_URL=http://127.0.0.1:3456
|
||||
export N8N_INSTANCE_REPORTING_AUTH_TOKEN=testing
|
||||
export N8N_INSTANCE_REPORTING_LABEL=local-dev
|
||||
export N8N_LICENSE_CERT='<certificate the receiver trusts, see prerequisites>'
|
||||
|
||||
# Compact insights quickly, so raw rows reach insights_by_period in a minute.
|
||||
export N8N_INSIGHTS_COMPACTION_INTERVAL_MINUTES=1
|
||||
|
|
@ -49,7 +57,8 @@ pnpm dev
|
|||
|
||||
Expected on the first boot:
|
||||
|
||||
- No warning about `N8N_INSTANCE_REPORTING_BASE_URL` being unset.
|
||||
- No warning about `N8N_INSTANCE_REPORTING_BASE_URL` being unset, and none
|
||||
about a missing license certificate.
|
||||
- The log line `Started the instance reporting timer`.
|
||||
- A `Resolved the instance reporting time` line with a random `HH:mm` at or
|
||||
after `03:00`.
|
||||
|
|
@ -137,7 +146,8 @@ ahead and the tick correctly skips.
|
|||
**Receiver side.** The receiver must log one `POST /api/v1/instance-reports`
|
||||
with:
|
||||
|
||||
- header `Authorization: Bearer testing`
|
||||
- no `Authorization` header (the token is unset in this run)
|
||||
- `licenseCert`: the exact value of `N8N_LICENSE_CERT`
|
||||
- `instanceId`: a 64-character hex string, equal to the instance id in
|
||||
`~/.n8n/config`
|
||||
- `batchId`: a UUID-like id, equal to the row id from the query below
|
||||
|
|
@ -172,10 +182,12 @@ Run these after step 4. Each is short.
|
|||
| 5.1 | No second report the same day | Restart the instance | No new request, no new row. `hasDeliveredToday` short-circuits the tick |
|
||||
| 5.2 | Retry resends the same measurement | Make the receiver answer 500. Delete today's delivered row, then restart | Request arrives, `lastError` holds `rejected with status 500`, `deliveredAt` NULL, `attempts` grows. Retries land ~5 minutes apart, 3 attempts in total, then `Giving up on the instance report for today`. Every retry carries the **same** `batchId` and the same values — no re-measurement |
|
||||
| 5.3 | Recovery keeps the pending row | During 5.2, switch the receiver back to 201 before the third attempt | The next attempt reuses the pending row and marks it delivered. No second row for the day |
|
||||
| 5.4 | Wrong token | Set `N8N_INSTANCE_REPORTING_AUTH_TOKEN=wrong`, clear today's row, restart | The receiver answers 401/403, delivery fails, `lastError` names the status. Nothing is marked delivered |
|
||||
| 5.4 | Untrusted certificate | Set `N8N_LICENSE_CERT` to a certificate the receiver does not trust (any well-formed one from another CA), clear today's row, restart | The receiver answers 401, delivery fails, `lastError` names the status. Nothing is marked delivered |
|
||||
| 5.4b | No certificate | Unset `N8N_LICENSE_CERT`, clear today's row, restart | Warning `no license certificate, so no reports will be sent`; no timer, no request, no row |
|
||||
| 5.4c | Token set | Set `N8N_INSTANCE_REPORTING_AUTH_TOKEN` to a token the receiver accepts, unset `N8N_LICENSE_CERT`, clear today's row, restart | No certificate warning. The request carries `Authorization: Bearer <token>` and the body has no `licenseCert`. The receiver answers 201 and the row is marked delivered |
|
||||
| 5.5 | Non-201 success code is a failure | Make the receiver answer 200 | Treated as a rejection: `Instance report was rejected with status 200` |
|
||||
| 5.6 | Receiver down | Stop the receiver, clear today's row, restart | Delivery fails with a connection error in `lastError`; the instance stays healthy and keeps serving |
|
||||
| 5.7 | Redirect is not followed | Make the receiver answer 302 to another local port | The report is rejected with status 302. The second port never sees the bearer token |
|
||||
| 5.7 | Redirect is not followed | Make the receiver answer 302 to another local port | The report is rejected with status 302. The second port never sees the license certificate |
|
||||
| 5.8 | Base URL unset | Unset `N8N_INSTANCE_REPORTING_BASE_URL`, restart | Warning `enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset`; no timer, no request |
|
||||
| 5.9 | Insights disabled | `N8N_DISABLED_MODULES=insights`, restart | Startup fails with the `UserError` that names both variables |
|
||||
| 5.10 | Trailing slash in the base URL | Use `http://127.0.0.1:3456/`, clear today's row, restart | The path is still `/api/v1/instance-reports`, with no double slash |
|
||||
|
|
@ -202,8 +214,8 @@ sqlite3 "$N8N_DB" "DELETE FROM instance_monitoring_report;"
|
|||
sqlite3 "$N8N_DB" "DELETE FROM settings WHERE key = 'features.centralInstanceMonitoring';"
|
||||
```
|
||||
|
||||
Unset the `N8N_INSTANCE_REPORTING_*` and `N8N_INSIGHTS_*` variables, or start a
|
||||
new shell. The backdated `insights_by_period` rows stay wrong for the Insights
|
||||
Unset the `N8N_INSTANCE_REPORTING_*`, `N8N_LICENSE_CERT` and `N8N_INSIGHTS_*`
|
||||
variables, or start a new shell. The backdated `insights_by_period` rows stay wrong for the Insights
|
||||
UI; drop them, or use a throwaway `N8N_USER_FOLDER` for the whole test if you
|
||||
want your dev data untouched.
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ without reading this document:
|
|||
|---|---|
|
||||
| `pending` | Not delivered yet, and attempts remain |
|
||||
| `delivered` | The receiver answered 201 |
|
||||
| `skipped_after_max_retries` | Three attempts failed, so the instance stopped for that day |
|
||||
| `skipped_after_max_retries` | The instance stopped delivering that day, either after three failed attempts or because the report's own slot passed before it landed |
|
||||
|
||||
A skipped report is **not** lost data. Only a delivered report crosses a day
|
||||
off, so the days a skipped report covered are measured again and sent by the
|
||||
|
|
@ -50,6 +50,10 @@ five minutes since the last attempt before trying again — otherwise a crash lo
|
|||
would spend the whole budget in seconds. `InstanceReportingScheduler` keeps no
|
||||
attempt state of its own.
|
||||
|
||||
The delivery retry above and the missed-day backfill are two separate
|
||||
mechanisms. [RETRIES.md](./RETRIES.md) explains each one, how they connect,
|
||||
and where the logic lives.
|
||||
|
||||
## Scheduling
|
||||
|
||||
The daily fire is driven by `InstanceReportingScheduler`, a leader-gated
|
||||
|
|
@ -81,13 +85,47 @@ N8N_INSTANCE_REPORTING_BASE_URL=https://monitoring.example.com
|
|||
The `insights` module must stay enabled, since the daily figure comes from
|
||||
there.
|
||||
|
||||
The instance must hold a license certificate, unless
|
||||
`N8N_INSTANCE_REPORTING_AUTH_TOKEN` is set. Reporting is for licensed
|
||||
instances, and the certificate is how the receiver knows that, see
|
||||
[Authentication](#authentication). Without a credential the module loads but
|
||||
warns and never sends, exactly as without a receiver.
|
||||
|
||||
## Authentication
|
||||
|
||||
There are two credentials. The token wins when it is set.
|
||||
|
||||
**License certificate (default).** Every report carries the instance's license
|
||||
certificate, the string `License.loadCertStr()` returns (`N8N_LICENSE_CERT`,
|
||||
or the persisted certificate of an activated license), as the `licenseCert`
|
||||
field of the body. The receiver verifies that the certificate was issued by
|
||||
n8n and then discards it; nothing from it is stored. There is no token to
|
||||
configure or distribute.
|
||||
|
||||
The certificate is read fresh for every report, so a renewed license is sent
|
||||
as soon as it is stored. An expired certificate is still sent and still
|
||||
accepted: the receiver checks the issuer, not the validity period.
|
||||
|
||||
It travels in the body, not in an `Authorization` header, because a
|
||||
certificate is several KB and grows with the license, which is more than
|
||||
common reverse proxies allow per header.
|
||||
|
||||
**Bearer token.** When `N8N_INSTANCE_REPORTING_AUTH_TOKEN` is set, every
|
||||
report carries it as `Authorization: Bearer …` and the body has no
|
||||
`licenseCert` field. The certificate is not read at all, so an unlicensed
|
||||
instance can report with a token.
|
||||
|
||||
Redirects are never followed, so either credential reaches only the configured
|
||||
host.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Env var | Default | Notes |
|
||||
|---|---|---|
|
||||
| `N8N_INSTANCE_REPORTING_BASE_URL` | `''` | Base URL of the receiver. The report is POSTed to `<base>/api/v1/instance-reports`. Left unset, the module loads but warns and never sends: it starts no scheduler and claims no report time. |
|
||||
| `N8N_INSTANCE_REPORTING_LABEL` | `''` | Sent as `label` in the payload, when set. |
|
||||
| `N8N_INSTANCE_REPORTING_AUTH_TOKEN` | `''` | Sent as `Authorization: Bearer …`, when set. |
|
||||
| `N8N_INSTANCE_REPORTING_AUTH_TOKEN` | `''` | Sent as `Authorization: Bearer …`, when set. Replaces the license certificate as the credential; `licenseCert` is then omitted from the body. |
|
||||
| `N8N_LICENSE_CERT` | `''` | Not owned by this module. Its value, or the persisted certificate of an activated license, is sent as `licenseCert` and is the credential the receiver checks, unless a token is set. |
|
||||
|
||||
## Report time
|
||||
|
||||
|
|
@ -113,9 +151,11 @@ is enabled on this instance:
|
|||
}
|
||||
```
|
||||
|
||||
`enabled` says whether a receiver is configured. Without one the key reads
|
||||
`{ "enabled": false }` and carries no `reportTime`, since no time is claimed. A
|
||||
missing key means the module is not enabled at all.
|
||||
`enabled` says whether a receiver is configured and a credential (an auth
|
||||
token or a license certificate) is present. Without either the key reads
|
||||
`{ "enabled": false }` and carries no
|
||||
`reportTime`, since no time is claimed. A missing key means the module is not
|
||||
enabled at all.
|
||||
|
||||
These settings are built once, during module init, and served from a cache
|
||||
afterwards, so they carry only values that stay the same for the lifetime of
|
||||
|
|
|
|||
112
packages/cli/src/modules/instance-reporting.ee/RETRIES.md
Normal file
112
packages/cli/src/modules/instance-reporting.ee/RETRIES.md
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
# Retries
|
||||
|
||||
The module has two retry mechanisms. The code keeps them separate. Both feed
|
||||
the same state machine on `instance_monitoring_report`: a row goes from
|
||||
`pending` to `delivered`, or from `pending` to `skipped_after_max_retries`.
|
||||
|
||||
- **Type 1, delivery retry.** Resend the pending row until it lands, its budget
|
||||
runs out, or its own next slot passes. It continues across the UTC midnight
|
||||
boundary.
|
||||
- **Type 2, missed-day backfill.** Put every day without a delivered row into
|
||||
the next report.
|
||||
|
||||
Type 2 decides how many days the next `pending` row must cover. Type 1 then
|
||||
takes over and delivers that row.
|
||||
|
||||
## The send decision
|
||||
|
||||
`start()` fires the first `tick()` (on `server-started`, or on leader takeover).
|
||||
Every pass then re-arms the next with `setTimeout`, so the loop drives itself.
|
||||
Each pass makes one choice, from `findPending()` — the newest row, and only if
|
||||
that row is still pending:
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
S(["start(): server-started / leader takeover"]) --> T
|
||||
T["tick()"] --> P{"waiting between retries?<br/>(up to 5 min, but not past its slot)"}
|
||||
P -- yes --> ARM
|
||||
P -- no --> L["report = findPending()"]
|
||||
L --> Q1{"is there a report to resume?"}
|
||||
Q1 -- yes --> Q1b{"too old to still send?"}
|
||||
Q1b -- no --> R["resend it — retry, even past midnight"]
|
||||
Q1b -- yes --> SK["mark it skipped"] --> Q2
|
||||
Q1 -- "no — delivered, skipped, or none" --> Q2{"reached today's report time?"}
|
||||
Q2 -- no --> N["do nothing — wait"]
|
||||
Q2 -- yes --> Q3{"already reported today?"}
|
||||
Q3 -- yes --> N
|
||||
Q3 -- no --> C["create a new report"]
|
||||
R --> ARM
|
||||
C --> ARM
|
||||
N --> ARM
|
||||
ARM["scheduleNext() — arms setTimeout(tick)"] -. re-arms .-> T
|
||||
```
|
||||
|
||||
## Type 1: delivery retry
|
||||
|
||||
Scope: the newest row while it is `pending`, whose last delivery attempt failed.
|
||||
It can come from an earlier slot, so the retry crosses midnight. Once its own
|
||||
next slot has passed, the retry stops: the row is marked
|
||||
`skipped_after_max_retries` and a new report covers its day instead.
|
||||
|
||||
- The row is created once, as `pending`, with its data points already
|
||||
measured. A retry changes only `attempts`, `lastAttemptAt`, `lastError` and,
|
||||
in the end, `status`. The data points and the `batchId` stay the same.
|
||||
- `InstanceReportingService.msUntilRetryAllowed()` reads `lastAttemptAt` from
|
||||
the row. The scheduler waits `RETRY_DELAY_MS` (5 minutes) between attempts.
|
||||
- The budget is `MAX_ATTEMPTS` (3). The attempt that spends the last one flips
|
||||
the row to `skipped_after_max_retries` at once. There is no fourth attempt,
|
||||
and no new row for today.
|
||||
- A response of `201` flips the row to `delivered`. A `409` means the receiver
|
||||
already holds this `batchId`, so the row is marked `delivered` as well.
|
||||
|
||||
Type 1 alone decides whether the row lands. Because the budget and the
|
||||
pacing live on the row, a restart resumes them. A fresh process does not get
|
||||
three new attempts, and it waits out the rest of the five minutes before it
|
||||
tries again.
|
||||
|
||||
## Type 2: missed-day backfill
|
||||
|
||||
Scope: no `pending` row is the newest, but one or more calendar days have no
|
||||
`delivered` row.
|
||||
|
||||
- `InstanceMonitoringReportRepository.findLastCoveredDay()` counts only
|
||||
`delivered` rows. A day whose row was skipped, or a day the instance was down
|
||||
for, is still owed.
|
||||
- `InstanceReportingService.missedDays()` walks back from yesterday to the day
|
||||
after the last delivered day. It stops at `MAX_BACKFILL_DAYS` (30) because
|
||||
`insights` buckets a longer range by week, which cannot fill a daily point.
|
||||
Older days are dropped.
|
||||
- The next report creates one new row. It carries one `daily` point for every
|
||||
missed day and one `cumulative` point. The cumulative point is a fresh
|
||||
lifetime total, not one per missed day.
|
||||
- That new row is then subject to type 1 if its own delivery fails.
|
||||
|
||||
Type 2 is the catch-up layer. It reacts to type 1 giving up, and to any other
|
||||
gap in delivered rows.
|
||||
|
||||
## How the two connect
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
subgraph T1["Type 1: delivery retry (max 3 attempts, 5 min apart, crosses midnight)"]
|
||||
A["Newest row is pending\n+ measured data points"] --> B["POST attempt"]
|
||||
B -->|"201 / 409"| C["delivered"]
|
||||
B -->|"failure"| D["attempts++\nlastAttemptAt, lastError"]
|
||||
D --> E{"attempts >= 3?"}
|
||||
E -- no --> F["wait 5 min"] --> B
|
||||
E -- yes --> G["skipped_after_max_retries"]
|
||||
end
|
||||
|
||||
C --> H["Day counts as covered\n(findLastCoveredDay)"]
|
||||
G --> I["Day stays uncovered\n(not delivered)"]
|
||||
|
||||
subgraph T2["Type 2: missed-day backfill"]
|
||||
J["Next cycle: newest row not pending"] --> K["missedDays() = every day after\nlast DELIVERED day, up to 30"]
|
||||
K --> L{"any missed days?"}
|
||||
L -- no --> M["nothing to send"]
|
||||
L -- yes --> N["new row: one daily point per\nmissed day + 1 cumulative point"]
|
||||
N --> A
|
||||
end
|
||||
|
||||
I --> K
|
||||
```
|
||||
|
|
@ -0,0 +1,669 @@
|
|||
import { createFakeOutboundHttp, type Route } from '@n8n/backend-network/testing';
|
||||
import {
|
||||
createTeamProject,
|
||||
createWorkflow,
|
||||
mockLogger,
|
||||
testDb,
|
||||
testModules,
|
||||
} from '@n8n/backend-test-utils';
|
||||
import { Time } from '@n8n/constants';
|
||||
import { LicenseMetricsRepository, SettingsRepository } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import { DateTime } from 'luxon';
|
||||
import { InstanceSettings } from 'n8n-core';
|
||||
import type { MockInstance } from 'vitest';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { EventService } from '@/events/event.service';
|
||||
import type { License } from '@/license';
|
||||
import { createCompactedInsightsEvent } from '@/modules/insights/database/entities/__tests__/db-utils';
|
||||
import { InsightsService } from '@/modules/insights/insights.service';
|
||||
import { OwnershipService } from '@/services/ownership.service';
|
||||
import { createOwner } from '@test-integration/db/users';
|
||||
|
||||
import type { InstanceReportDataPoint } from '../database/entities/instance-monitoring-report';
|
||||
import { InstanceMonitoringReportRepository } from '../database/repositories/instance-monitoring-report.repository';
|
||||
import { InstanceReportingScheduler } from '../instance-reporting-scheduler.service';
|
||||
import { InstanceReportingSettingsService } from '../instance-reporting-settings.service';
|
||||
import { InstanceReportingConfig } from '../instance-reporting.config';
|
||||
import {
|
||||
CENTRAL_INSTANCE_MONITORING_SETTINGS_KEY,
|
||||
INSTANCE_REPORTS_PATH,
|
||||
} from '../instance-reporting.constants';
|
||||
import { InstanceReportingService } from '../instance-reporting.service';
|
||||
|
||||
const RECEIVER_URL = 'https://receiver.test';
|
||||
|
||||
const REPORT_TIME = '07:42';
|
||||
|
||||
/** One minute after the slot, so the first pass reports at once. */
|
||||
const AFTER_SLOT = '2026-03-26T07:43:00.000Z';
|
||||
|
||||
const NEXT_SLOT = '2026-03-27T07:42:00.000Z';
|
||||
|
||||
// The retry contract, spelled out rather than imported, so a change to either
|
||||
// constant in the service fails here instead of moving the test along with it.
|
||||
const MAX_ATTEMPTS = 3;
|
||||
const RETRY_DELAY_MS = 5 * Time.minutes.toMilliseconds;
|
||||
|
||||
/** Room for the fake clock's drift while `vi.waitFor` polls. */
|
||||
const CLOCK_TOLERANCE_MS = 10 * Time.seconds.toMilliseconds;
|
||||
|
||||
interface ReportPayload {
|
||||
batchId: string;
|
||||
dataPoints: InstanceReportDataPoint[];
|
||||
}
|
||||
|
||||
const unreachable = (): Route => ({
|
||||
method: 'POST',
|
||||
pathname: INSTANCE_REPORTS_PATH,
|
||||
networkError: 'ECONNREFUSED',
|
||||
});
|
||||
|
||||
const accepted = (): Route => ({ method: 'POST', pathname: INSTANCE_REPORTS_PATH, status: 201 });
|
||||
|
||||
/** The private arming hook, spied on so a test can tell when a pass has finished. */
|
||||
type SchedulerInternals = { scheduleNext(delayMs: number): void };
|
||||
|
||||
interface Harness {
|
||||
scheduler: InstanceReportingScheduler;
|
||||
/** Every resolved request the fake receiver saw. */
|
||||
httpRequest: ReturnType<typeof createFakeOutboundHttp>['httpRequest'];
|
||||
/**
|
||||
* Called once a pass has finished its last database write, so its call count
|
||||
* marks "pass over, next timer armed". Pending fake timers cannot serve as
|
||||
* that signal: the SQLite pool arms a transient timeout for every query.
|
||||
*/
|
||||
scheduleNext: MockInstance<SchedulerInternals['scheduleNext']>;
|
||||
}
|
||||
|
||||
describe('instance reporting retries', () => {
|
||||
let repository: InstanceMonitoringReportRepository;
|
||||
let schedulers: InstanceReportingScheduler[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
await testModules.loadModules(['insights', 'instance-reporting']);
|
||||
await testDb.init();
|
||||
|
||||
repository = Container.get(InstanceMonitoringReportRepository);
|
||||
Container.get(InstanceReportingConfig).instanceReportingBaseUrl = RECEIVER_URL;
|
||||
|
||||
const instanceSettings = Container.get(InstanceSettings);
|
||||
expect(instanceSettings.instanceType).toBe('main');
|
||||
instanceSettings.markAsLeader();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await testDb.truncate([
|
||||
'InstanceMonitoringReport',
|
||||
'Settings',
|
||||
'User',
|
||||
'InsightsByPeriod',
|
||||
'InsightsMetadata',
|
||||
'WorkflowEntity',
|
||||
'Project',
|
||||
]);
|
||||
await createOwner();
|
||||
await Container.get(SettingsRepository).upsertByKey(
|
||||
CENTRAL_INSTANCE_MONITORING_SETTINGS_KEY,
|
||||
JSON.stringify({ reportTime: REPORT_TIME }),
|
||||
false,
|
||||
{},
|
||||
);
|
||||
|
||||
// Only what the scheduler uses. `setImmediate` and `nextTick` stay real so
|
||||
// the database driver's I/O still completes under the fake clock.
|
||||
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
|
||||
vi.setSystemTime(new Date(AFTER_SLOT));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const scheduler of schedulers) scheduler.shutdown();
|
||||
schedulers = [];
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await testDb.terminate();
|
||||
});
|
||||
|
||||
function makeHarness(routes: Route[]): Harness {
|
||||
const { outboundHttp, httpRequest } = createFakeOutboundHttp(
|
||||
routes,
|
||||
vi.fn as unknown as Parameters<typeof createFakeOutboundHttp>[1],
|
||||
);
|
||||
const instanceSettings = Container.get(InstanceSettings);
|
||||
const eventService = Container.get(EventService);
|
||||
|
||||
const service = new InstanceReportingService(
|
||||
Container.get(InstanceReportingConfig),
|
||||
repository,
|
||||
Container.get(InsightsService),
|
||||
instanceSettings,
|
||||
Container.get(OwnershipService),
|
||||
Container.get(LicenseMetricsRepository),
|
||||
mock<License>({ loadCertStr: async () => 'license-cert' }),
|
||||
mockLogger(),
|
||||
eventService,
|
||||
outboundHttp,
|
||||
);
|
||||
const scheduler = new InstanceReportingScheduler(
|
||||
service,
|
||||
repository,
|
||||
Container.get(InstanceReportingSettingsService),
|
||||
instanceSettings,
|
||||
eventService,
|
||||
mockLogger(),
|
||||
);
|
||||
schedulers.push(scheduler);
|
||||
|
||||
const scheduleNext = vi.spyOn(scheduler as unknown as SchedulerInternals, 'scheduleNext');
|
||||
|
||||
return { scheduler, httpRequest, scheduleNext };
|
||||
}
|
||||
|
||||
/** Wait until the scheduler has completed `passes` passes and armed the next one. */
|
||||
async function armed({ scheduleNext }: Harness, passes: number) {
|
||||
await vi.waitFor(() => expect(scheduleNext).toHaveBeenCalledTimes(passes), {
|
||||
timeout: 5_000,
|
||||
});
|
||||
}
|
||||
|
||||
/** Assert the delay the scheduler last armed lands on `slot`. */
|
||||
function expectArmedFor({ scheduleNext }: Harness, slot: string) {
|
||||
const delay = scheduleNext.mock.lastCall?.[0] ?? Number.NaN;
|
||||
const target = new Date(slot).getTime();
|
||||
expect(Date.now() + delay).toBeGreaterThanOrEqual(target - CLOCK_TOLERANCE_MS);
|
||||
expect(Date.now() + delay).toBeLessThanOrEqual(target + CLOCK_TOLERANCE_MS);
|
||||
}
|
||||
|
||||
function sentPayload({ httpRequest }: Harness, index: number): ReportPayload {
|
||||
return httpRequest.mock.calls[index][0].body as unknown as ReportPayload;
|
||||
}
|
||||
|
||||
function dailyPoints(payload: ReportPayload) {
|
||||
return payload.dataPoints.flatMap((point) =>
|
||||
point.kind === 'daily' ? [{ date: point.date, value: point.value }] : [],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The database stamps `createdAt` with the real clock, not the fake one, so
|
||||
* a row from a fake "yesterday" reads as today's. Pin it to the day meant.
|
||||
*/
|
||||
async function stampCreatedAt(id: string, createdAt: Date) {
|
||||
await repository.update({ id }, { createdAt });
|
||||
}
|
||||
|
||||
/** A delivered report that covers `day`, generated the morning after it. */
|
||||
async function seedDeliveredReport(day: string) {
|
||||
const report = await repository.createPending([
|
||||
{ kind: 'cumulative', name: 'billableExecutions', value: 0 },
|
||||
{ kind: 'daily', name: 'billableExecutions', value: 3, date: day },
|
||||
]);
|
||||
await repository.markDelivered(report.id, new Date());
|
||||
await stampCreatedAt(
|
||||
report.id,
|
||||
new Date(new Date(`${day}T07:43:00.000Z`).getTime() + Time.days.toMilliseconds),
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
async function seedDailyExecutions(totalsByDay: Record<string, number>) {
|
||||
const workflow = await createWorkflow({}, await createTeamProject());
|
||||
for (const [day, value] of Object.entries(totalsByDay)) {
|
||||
await createCompactedInsightsEvent(workflow, {
|
||||
type: 'success',
|
||||
value,
|
||||
periodUnit: 'day',
|
||||
periodStart: DateTime.fromISO(day, { zone: 'utc' }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function seedPendingReport(
|
||||
day: string,
|
||||
{
|
||||
attempts,
|
||||
lastAttemptAt,
|
||||
createdAt,
|
||||
}: { attempts: number; lastAttemptAt: Date; createdAt: Date },
|
||||
) {
|
||||
const report = await repository.createPending([
|
||||
{ kind: 'cumulative', name: 'billableExecutions', value: 0 },
|
||||
{ kind: 'daily', name: 'billableExecutions', value: 5, date: day },
|
||||
]);
|
||||
await repository.update(
|
||||
{ id: report.id },
|
||||
{ attempts, lastError: 'ECONNREFUSED', lastAttemptAt },
|
||||
);
|
||||
await stampCreatedAt(report.id, createdAt);
|
||||
return report;
|
||||
}
|
||||
|
||||
async function deliveredDailyDates(): Promise<string[]> {
|
||||
const delivered = await repository.find({ where: { status: 'delivered' } });
|
||||
return delivered.flatMap((report) =>
|
||||
report.dataPoints.flatMap((point) => (point.kind === 'daily' ? [point.date] : [])),
|
||||
);
|
||||
}
|
||||
|
||||
async function setReportTime(reportTime: string) {
|
||||
await Container.get(SettingsRepository).upsertByKey(
|
||||
CENTRAL_INSTANCE_MONITORING_SETTINGS_KEY,
|
||||
JSON.stringify({ reportTime }),
|
||||
false,
|
||||
{},
|
||||
);
|
||||
}
|
||||
|
||||
test('delivers on the third attempt once the receiver is reachable again', async () => {
|
||||
const harness = makeHarness([unreachable(), unreachable(), accepted()]);
|
||||
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
|
||||
await expect(repository.count()).resolves.toBe(1);
|
||||
const [report] = await repository.find();
|
||||
expect(report).toMatchObject({ status: 'pending', attempts: 1 });
|
||||
expect(report.lastError).toEqual(expect.any(String));
|
||||
expect(report.lastAttemptAt).toEqual(expect.any(Date));
|
||||
expect(harness.scheduleNext).toHaveBeenLastCalledWith(RETRY_DELAY_MS);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 2);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(2);
|
||||
await expect(repository.findOneByOrFail({ id: report.id })).resolves.toMatchObject({
|
||||
status: 'pending',
|
||||
attempts: 2,
|
||||
});
|
||||
expect(harness.scheduleNext).toHaveBeenLastCalledWith(RETRY_DELAY_MS);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 3);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(3);
|
||||
await expect(repository.findOneByOrFail({ id: report.id })).resolves.toMatchObject({
|
||||
status: 'delivered',
|
||||
attempts: 3,
|
||||
deliveredAt: expect.any(Date),
|
||||
lastError: null,
|
||||
});
|
||||
|
||||
// Every attempt resent the row as measured, under the same batchId.
|
||||
const payloads = [0, 1, 2].map((index) => sentPayload(harness, index));
|
||||
for (const payload of payloads) {
|
||||
expect(payload.batchId).toBe(report.id);
|
||||
expect(payload.dataPoints).toEqual(payloads[0].dataPoints);
|
||||
}
|
||||
|
||||
// The day is settled, so the scheduler now waits for tomorrow's slot.
|
||||
expectArmedFor(harness, NEXT_SLOT);
|
||||
await vi.advanceTimersByTimeAsync(Time.hours.toMilliseconds);
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(3);
|
||||
expect(harness.scheduleNext).toHaveBeenCalledTimes(3);
|
||||
await expect(repository.count()).resolves.toBe(1);
|
||||
});
|
||||
|
||||
test('gives up after three failures and covers the day again in the next report', async () => {
|
||||
// A delivered report for the day before last, so the gap has a start.
|
||||
await seedDeliveredReport('2026-03-24');
|
||||
await seedDailyExecutions({ '2026-03-25': 5, '2026-03-26': 7 });
|
||||
|
||||
const harness = makeHarness([unreachable(), unreachable(), unreachable(), accepted()]);
|
||||
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 2);
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 3);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(MAX_ATTEMPTS);
|
||||
const [abandoned] = await repository.find({ where: { status: 'skipped_after_max_retries' } });
|
||||
expect(abandoned).toMatchObject({ attempts: MAX_ATTEMPTS });
|
||||
expect(dailyPoints(sentPayload(harness, 0))).toEqual([{ date: '2026-03-25', value: 5 }]);
|
||||
// Keep the abandoned row on the day it was made, or tomorrow reads as settled too.
|
||||
await stampCreatedAt(abandoned.id, new Date(AFTER_SLOT));
|
||||
|
||||
// The next pass finds the day settled and moves on to tomorrow's slot.
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 4);
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(MAX_ATTEMPTS);
|
||||
expectArmedFor(harness, NEXT_SLOT);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(new Date(NEXT_SLOT).getTime() - Date.now());
|
||||
await armed(harness, 5);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(MAX_ATTEMPTS + 1);
|
||||
const backfill = sentPayload(harness, MAX_ATTEMPTS);
|
||||
expect(backfill.batchId).not.toBe(abandoned.id);
|
||||
expect(dailyPoints(backfill)).toEqual([
|
||||
{ date: '2026-03-25', value: 5 },
|
||||
{ date: '2026-03-26', value: 7 },
|
||||
]);
|
||||
expect(backfill.dataPoints.filter((point) => point.kind === 'cumulative')).toHaveLength(1);
|
||||
|
||||
await expect(repository.findOneByOrFail({ id: backfill.batchId })).resolves.toMatchObject({
|
||||
status: 'delivered',
|
||||
attempts: 1,
|
||||
});
|
||||
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-26');
|
||||
});
|
||||
|
||||
test('carries a daily point for every day of a downtime gap in one report', async () => {
|
||||
await seedDeliveredReport('2026-03-22');
|
||||
// Nothing for 03-24: a day with no executions has no insights row at all.
|
||||
await seedDailyExecutions({ '2026-03-23': 4, '2026-03-25': 9 });
|
||||
|
||||
const harness = makeHarness([accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
|
||||
const payload = sentPayload(harness, 0);
|
||||
expect(dailyPoints(payload)).toEqual([
|
||||
{ date: '2026-03-23', value: 4 },
|
||||
{ date: '2026-03-24', value: 0 },
|
||||
{ date: '2026-03-25', value: 9 },
|
||||
]);
|
||||
expect(payload.dataPoints.filter((point) => point.kind === 'cumulative')).toHaveLength(1);
|
||||
|
||||
await expect(repository.findOneByOrFail({ id: payload.batchId })).resolves.toMatchObject({
|
||||
status: 'delivered',
|
||||
attempts: 1,
|
||||
});
|
||||
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-25');
|
||||
});
|
||||
|
||||
test('caps a long gap at 30 days and still reads every day on its own', async () => {
|
||||
await seedDeliveredReport('2026-01-01');
|
||||
await seedDailyExecutions({ '2026-02-24': 6, '2026-03-25': 8 });
|
||||
|
||||
const harness = makeHarness([accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
|
||||
const points = dailyPoints(sentPayload(harness, 0));
|
||||
expect(points).toHaveLength(30);
|
||||
// Exact days, not weekly buckets: a 30-day range still reads as days.
|
||||
expect(points.at(0)).toEqual({ date: '2026-02-24', value: 6 });
|
||||
expect(points.at(-1)).toEqual({ date: '2026-03-25', value: 8 });
|
||||
expect(points.slice(1, -1).every((point) => point.value === 0)).toBe(true);
|
||||
});
|
||||
|
||||
test('resumes the retry budget and the remaining wait after a restart', async () => {
|
||||
const before = makeHarness([unreachable()]);
|
||||
|
||||
before.scheduler.start();
|
||||
await armed(before, 1);
|
||||
|
||||
expect(before.httpRequest).toHaveBeenCalledTimes(1);
|
||||
await expect(repository.count()).resolves.toBe(1);
|
||||
const [report] = await repository.find();
|
||||
expect(report).toMatchObject({ status: 'pending', attempts: 1 });
|
||||
|
||||
// The process dies two minutes into the five-minute wait.
|
||||
before.scheduler.shutdown();
|
||||
await vi.advanceTimersByTimeAsync(2 * Time.minutes.toMilliseconds);
|
||||
|
||||
const after = makeHarness([accepted()]);
|
||||
after.scheduler.start();
|
||||
await armed(after, 1);
|
||||
|
||||
// Held back by the row's `lastAttemptAt` rather than attempting at once.
|
||||
expect(after.httpRequest).not.toHaveBeenCalled();
|
||||
// Three of the five minutes remain, less the fake clock's drift.
|
||||
const [remainingWait] = after.scheduleNext.mock.calls[0];
|
||||
const expectedWait = RETRY_DELAY_MS - 2 * Time.minutes.toMilliseconds;
|
||||
expect(remainingWait).toBeGreaterThanOrEqual(expectedWait - CLOCK_TOLERANCE_MS);
|
||||
expect(remainingWait).toBeLessThanOrEqual(expectedWait);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(remainingWait);
|
||||
await armed(after, 2);
|
||||
|
||||
expect(after.httpRequest).toHaveBeenCalledTimes(1);
|
||||
expect(sentPayload(after, 0).batchId).toBe(report.id);
|
||||
await expect(repository.findOneByOrFail({ id: report.id })).resolves.toMatchObject({
|
||||
status: 'delivered',
|
||||
attempts: 2,
|
||||
});
|
||||
});
|
||||
|
||||
test('resumes a pending report across the UTC midnight boundary', async () => {
|
||||
await setReportTime('23:56');
|
||||
vi.setSystemTime(new Date('2026-03-27T00:01:00.000Z'));
|
||||
|
||||
// The 23:56 slot on 03-26 made this row. Its first attempt failed.
|
||||
const seeded = await seedPendingReport('2026-03-25', {
|
||||
attempts: 1,
|
||||
lastAttemptAt: new Date('2026-03-26T23:56:00.000Z'),
|
||||
createdAt: new Date('2026-03-26T23:56:00.000Z'),
|
||||
});
|
||||
|
||||
const harness = makeHarness([accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
// The retry after midnight resends the same row. It does not make a new row.
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
|
||||
const payload = sentPayload(harness, 0);
|
||||
expect(payload.batchId).toBe(seeded.id);
|
||||
expect(dailyPoints(payload)).toEqual([{ date: '2026-03-25', value: 5 }]);
|
||||
|
||||
await expect(repository.findOneByOrFail({ id: seeded.id })).resolves.toMatchObject({
|
||||
status: 'delivered',
|
||||
attempts: 2,
|
||||
});
|
||||
await expect(repository.count()).resolves.toBe(1);
|
||||
expectArmedFor(harness, '2026-03-27T23:56:00.000Z');
|
||||
});
|
||||
|
||||
test('keeps retrying across the boundary, then backfills the day after it is skipped', async () => {
|
||||
await setReportTime('23:56');
|
||||
vi.setSystemTime(new Date('2026-03-27T00:01:00.000Z'));
|
||||
|
||||
await seedDeliveredReport('2026-03-24');
|
||||
await seedDailyExecutions({ '2026-03-25': 5, '2026-03-26': 7 });
|
||||
const seeded = await seedPendingReport('2026-03-25', {
|
||||
attempts: 1,
|
||||
lastAttemptAt: new Date('2026-03-26T23:56:00.000Z'),
|
||||
createdAt: new Date('2026-03-26T23:56:00.000Z'),
|
||||
});
|
||||
|
||||
const harness = makeHarness([unreachable(), unreachable(), accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 2);
|
||||
|
||||
// Two more attempts run after midnight. Then the row stops.
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(2);
|
||||
await expect(repository.findOneByOrFail({ id: seeded.id })).resolves.toMatchObject({
|
||||
status: 'skipped_after_max_retries',
|
||||
attempts: MAX_ATTEMPTS,
|
||||
});
|
||||
|
||||
// The next pass finds the day settled. It waits for the next slot.
|
||||
await vi.advanceTimersByTimeAsync(RETRY_DELAY_MS);
|
||||
await armed(harness, 3);
|
||||
expectArmedFor(harness, '2026-03-27T23:56:00.000Z');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(new Date('2026-03-27T23:56:00.000Z').getTime() - Date.now());
|
||||
await armed(harness, 4);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(3);
|
||||
const backfill = sentPayload(harness, 2);
|
||||
expect(backfill.batchId).not.toBe(seeded.id);
|
||||
expect(dailyPoints(backfill)).toEqual([
|
||||
{ date: '2026-03-25', value: 5 },
|
||||
{ date: '2026-03-26', value: 7 },
|
||||
]);
|
||||
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-26');
|
||||
|
||||
// The skipped row never lands. So no day reaches the receiver two times.
|
||||
const dates = await deliveredDailyDates();
|
||||
expect(new Set(dates).size).toBe(dates.length);
|
||||
});
|
||||
|
||||
test('does not resurrect an orphan already covered by a newer delivered report', async () => {
|
||||
// A newer delivered report is above an older pending row that never delivered.
|
||||
await seedDeliveredReport('2026-03-25');
|
||||
const orphan = await seedPendingReport('2026-03-20', {
|
||||
attempts: 1,
|
||||
lastAttemptAt: new Date('2026-03-20T23:56:00.000Z'),
|
||||
createdAt: new Date('2026-03-20T23:56:00.000Z'),
|
||||
});
|
||||
|
||||
const harness = makeHarness([accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
// The latest row is delivered. So the code sends nothing and leaves the orphan.
|
||||
expect(harness.httpRequest).not.toHaveBeenCalled();
|
||||
await expect(repository.findOneByOrFail({ id: orphan.id })).resolves.toMatchObject({
|
||||
status: 'pending',
|
||||
});
|
||||
const dates = await deliveredDailyDates();
|
||||
expect(dates).not.toContain('2026-03-20');
|
||||
});
|
||||
|
||||
test('expires a pending row after days of downtime, even before the slot, then backfills', async () => {
|
||||
await setReportTime('23:58');
|
||||
|
||||
// A delivered report for 03-23, so the gap has a start.
|
||||
await seedDeliveredReport('2026-03-23');
|
||||
await seedDailyExecutions({
|
||||
'2026-03-24': 4,
|
||||
'2026-03-25': 6,
|
||||
'2026-03-26': 8,
|
||||
'2026-03-27': 10,
|
||||
'2026-03-28': 12,
|
||||
});
|
||||
|
||||
// The 03-25 23:58 slot made this row for 03-24, then delivery failed.
|
||||
const stale = await seedPendingReport('2026-03-24', {
|
||||
attempts: 2,
|
||||
lastAttemptAt: new Date('2026-03-26T00:03:00.000Z'),
|
||||
createdAt: new Date('2026-03-25T23:58:00.000Z'),
|
||||
});
|
||||
|
||||
// Back up four days later, and before tonight's 23:58 slot.
|
||||
vi.setSystemTime(new Date('2026-03-29T20:59:00.000Z'));
|
||||
|
||||
const harness = makeHarness([accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
// Four days is past its own next slot, so it is given up, not resent — even
|
||||
// though the time of day is still before tonight's slot.
|
||||
expect(harness.httpRequest).not.toHaveBeenCalled();
|
||||
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
|
||||
status: 'skipped_after_max_retries',
|
||||
});
|
||||
expectArmedFor(harness, '2026-03-29T23:58:00.000Z');
|
||||
|
||||
// Tonight's slot creates one fresh report for the whole gap.
|
||||
await vi.advanceTimersByTimeAsync(new Date('2026-03-29T23:58:00.000Z').getTime() - Date.now());
|
||||
await armed(harness, 2);
|
||||
|
||||
const backfill = sentPayload(harness, 0);
|
||||
expect(backfill.batchId).not.toBe(stale.id);
|
||||
expect(dailyPoints(backfill)).toEqual([
|
||||
{ date: '2026-03-24', value: 4 },
|
||||
{ date: '2026-03-25', value: 6 },
|
||||
{ date: '2026-03-26', value: 8 },
|
||||
{ date: '2026-03-27', value: 10 },
|
||||
{ date: '2026-03-28', value: 12 },
|
||||
]);
|
||||
await expect(repository.findLastCoveredDay()).resolves.toBe('2026-03-28');
|
||||
|
||||
const dates = await deliveredDailyDates();
|
||||
expect(new Set(dates).size).toBe(dates.length);
|
||||
});
|
||||
|
||||
test('caps the retry to the slot, so a failure at the end of the day backfills the same night', async () => {
|
||||
await setReportTime('23:58');
|
||||
|
||||
await seedDeliveredReport('2026-03-23');
|
||||
await seedDailyExecutions({ '2026-03-24': 4, '2026-03-25': 6 });
|
||||
|
||||
// The 03-25 23:58 slot made this row for 03-24, its first attempt failed there,
|
||||
// then the instance was down for almost a day.
|
||||
const stale = await seedPendingReport('2026-03-24', {
|
||||
attempts: 1,
|
||||
lastAttemptAt: new Date('2026-03-25T23:58:00.000Z'),
|
||||
createdAt: new Date('2026-03-25T23:58:00.000Z'),
|
||||
});
|
||||
|
||||
// Back up two minutes before tonight's slot, at the very end of the UTC day.
|
||||
vi.setSystemTime(new Date('2026-03-26T23:56:00.000Z'));
|
||||
|
||||
const harness = makeHarness([unreachable(), accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
// The resume fails; the next attempt is capped to the slot, two minutes away,
|
||||
// rather than a full five minutes that would cross midnight.
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
|
||||
expect(sentPayload(harness, 0).batchId).toBe(stale.id);
|
||||
|
||||
// Two minutes later, at the slot, the row is given up and a fresh report goes
|
||||
// out the same night — not deferred to the next day's slot.
|
||||
await vi.advanceTimersByTimeAsync(2 * Time.minutes.toMilliseconds);
|
||||
await armed(harness, 2);
|
||||
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(2);
|
||||
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
|
||||
status: 'skipped_after_max_retries',
|
||||
});
|
||||
const backfill = sentPayload(harness, 1);
|
||||
expect(backfill.batchId).not.toBe(stale.id);
|
||||
expect(dailyPoints(backfill)).toEqual([
|
||||
{ date: '2026-03-24', value: 4 },
|
||||
{ date: '2026-03-25', value: 6 },
|
||||
]);
|
||||
expectArmedFor(harness, '2026-03-27T23:58:00.000Z');
|
||||
});
|
||||
|
||||
test('skips a stale row and sends a fresh report in the same pass, after the slot', async () => {
|
||||
await setReportTime('23:58');
|
||||
|
||||
await seedDeliveredReport('2026-03-23');
|
||||
await seedDailyExecutions({ '2026-03-24': 4, '2026-03-25': 6, '2026-03-26': 8 });
|
||||
|
||||
// A pending row for 03-24 from the 03-25 slot, left over from downtime.
|
||||
const stale = await seedPendingReport('2026-03-24', {
|
||||
attempts: 2,
|
||||
lastAttemptAt: new Date('2026-03-26T00:03:00.000Z'),
|
||||
createdAt: new Date('2026-03-25T23:58:00.000Z'),
|
||||
});
|
||||
|
||||
// Back up after tonight's slot, so one pass both skips and sends.
|
||||
vi.setSystemTime(new Date('2026-03-27T23:59:00.000Z'));
|
||||
|
||||
const harness = makeHarness([accepted()]);
|
||||
harness.scheduler.start();
|
||||
await armed(harness, 1);
|
||||
|
||||
// The first pass gives up the stale row and creates the new report at once.
|
||||
expect(harness.httpRequest).toHaveBeenCalledTimes(1);
|
||||
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
|
||||
status: 'skipped_after_max_retries',
|
||||
});
|
||||
const backfill = sentPayload(harness, 0);
|
||||
expect(backfill.batchId).not.toBe(stale.id);
|
||||
expect(dailyPoints(backfill)).toEqual([
|
||||
{ date: '2026-03-24', value: 4 },
|
||||
{ date: '2026-03-25', value: 6 },
|
||||
{ date: '2026-03-26', value: 8 },
|
||||
]);
|
||||
|
||||
const dates = await deliveredDailyDates();
|
||||
expect(new Set(dates).size).toBe(dates.length);
|
||||
});
|
||||
});
|
||||
|
|
@ -8,6 +8,8 @@ import { Container } from '@n8n/di';
|
|||
import { UserError } from 'n8n-workflow';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { License } from '@/license';
|
||||
|
||||
import { InstanceMonitoringReportRepository } from '../database/repositories/instance-monitoring-report.repository';
|
||||
import { InstanceReportingScheduler } from '../instance-reporting-scheduler.service';
|
||||
import { InstanceReportingSettingsService } from '../instance-reporting-settings.service';
|
||||
|
|
@ -24,13 +26,20 @@ const LAST_DELIVERY = new Date('2026-03-25T07:42:13.000Z');
|
|||
|
||||
function setUpContainer({
|
||||
baseUrl = 'https://example.com',
|
||||
licenseCert = 'base64-license-cert',
|
||||
authToken = '',
|
||||
disabledModules = [] as ModuleName[],
|
||||
lastDelivery = LAST_DELIVERY as Date | null,
|
||||
} = {}) {
|
||||
const config = new InstanceReportingConfig();
|
||||
config.instanceReportingBaseUrl = baseUrl;
|
||||
config.instanceReportingAuthToken = authToken;
|
||||
Container.set(InstanceReportingConfig, config);
|
||||
|
||||
const license = mock<License>();
|
||||
license.loadCertStr.mockResolvedValue(licenseCert);
|
||||
Container.set(License, license);
|
||||
|
||||
Container.set(ModulesConfig, mock<ModulesConfig>({ disabledModules }));
|
||||
Container.set(Logger, mockLogger());
|
||||
|
||||
|
|
@ -67,6 +76,26 @@ describe('InstanceReportingModule', () => {
|
|||
expect(settingsService.getReportTime).not.toHaveBeenCalled();
|
||||
expect(reportRepository.findLastDeliveryTime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// The certificate is the credential, so without one nothing would ever be
|
||||
// accepted and no report time is claimed.
|
||||
it('reports as disabled without a license certificate', async () => {
|
||||
const { settingsService } = setUpContainer({ licenseCert: '' });
|
||||
|
||||
const settings = await new InstanceReportingModule().settings();
|
||||
|
||||
expect(settings).toEqual({ enabled: false });
|
||||
expect(settingsService.getReportTime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// A token is a credential on its own, so the certificate is not needed.
|
||||
it('reports as enabled with an auth token but no license certificate', async () => {
|
||||
setUpContainer({ licenseCert: '', authToken: 'secret-token' });
|
||||
|
||||
const settings = await new InstanceReportingModule().settings();
|
||||
|
||||
expect(settings).toEqual({ enabled: true, reportTime: REPORT_TIME });
|
||||
});
|
||||
});
|
||||
|
||||
describe('init()', () => {
|
||||
|
|
@ -92,6 +121,22 @@ describe('InstanceReportingModule', () => {
|
|||
expect(scheduler.init).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves the scheduler alone when the instance has no license certificate', async () => {
|
||||
const { scheduler } = setUpContainer({ licenseCert: '' });
|
||||
|
||||
await new InstanceReportingModule().init();
|
||||
|
||||
expect(scheduler.init).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('starts the scheduler with an auth token but no license certificate', async () => {
|
||||
const { scheduler } = setUpContainer({ licenseCert: '', authToken: 'secret-token' });
|
||||
|
||||
await new InstanceReportingModule().init();
|
||||
|
||||
expect(scheduler.init).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// The route belongs to the loaded module, not to the receiver, so a client
|
||||
// that asks anyway gets an answer instead of a 404.
|
||||
it('registers the status route even when no receiver is configured', async () => {
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import type { Mocked } from 'vitest';
|
|||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import type { EventService } from '@/events/event.service';
|
||||
import type { License } from '@/license';
|
||||
import type { InsightsService } from '@/modules/insights/insights.service';
|
||||
import type { OwnershipService } from '@/services/ownership.service';
|
||||
|
||||
|
|
@ -31,10 +32,13 @@ interface ReportPayload {
|
|||
label?: string;
|
||||
n8nVersion: string;
|
||||
dataPoints: Array<{ kind: string; name: string; value: number; date?: string }>;
|
||||
licenseCert?: string;
|
||||
}
|
||||
|
||||
const REPORT_DATE = '2026-03-25';
|
||||
const BATCH_ID = 'batch-id-1';
|
||||
/** Opaque to this module: whatever `License.loadCertStr()` returns is sent as is. */
|
||||
const LICENSE_CERT = 'base64-license-cert';
|
||||
|
||||
const OWNER_MOCK = mock<User>({ id: 'owner-id' });
|
||||
|
||||
|
|
@ -91,6 +95,7 @@ interface Harness {
|
|||
service: InstanceReportingService;
|
||||
reportRepository: Mocked<InstanceMonitoringReportRepository>;
|
||||
insightsService: Mocked<InsightsService>;
|
||||
license: Mocked<License>;
|
||||
http: HttpRequestClient;
|
||||
eventService: Mocked<EventService>;
|
||||
clientOptions: HttpRequestClientOptions | undefined;
|
||||
|
|
@ -98,7 +103,7 @@ interface Harness {
|
|||
|
||||
function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness {
|
||||
const reportRepository = mock<InstanceMonitoringReportRepository>();
|
||||
reportRepository.findTodaysPending.mockResolvedValue(null);
|
||||
reportRepository.findPending.mockResolvedValue(null);
|
||||
// A report already covers the day before the one under test, so the default
|
||||
// harness reports exactly one day.
|
||||
reportRepository.findLastCoveredDay.mockResolvedValue('2026-03-24');
|
||||
|
|
@ -115,6 +120,9 @@ function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness {
|
|||
const licenseMetricsRepository = mock<LicenseMetricsRepository>();
|
||||
licenseMetricsRepository.getLicenseRenewalMetrics.mockResolvedValue(LICENSE_METRICS_MOCK);
|
||||
|
||||
const license = mock<License>();
|
||||
license.loadCertStr.mockResolvedValue(LICENSE_CERT);
|
||||
|
||||
const http = mock<HttpRequestClient>();
|
||||
vi.mocked(http.request).mockResolvedValue({ statusCode: 201, body: '', headers: {} });
|
||||
|
||||
|
|
@ -135,12 +143,13 @@ function makeHarness(config: InstanceReportingConfig = makeConfig()): Harness {
|
|||
mock<InstanceSettings>({ instanceId: 'abc123' }),
|
||||
ownershipService,
|
||||
licenseMetricsRepository,
|
||||
license,
|
||||
mockLogger(),
|
||||
eventService,
|
||||
outboundHttp,
|
||||
);
|
||||
|
||||
return { service, reportRepository, insightsService, http, eventService, clientOptions };
|
||||
return { service, reportRepository, insightsService, license, http, eventService, clientOptions };
|
||||
}
|
||||
|
||||
describe('InstanceReportingService', () => {
|
||||
|
|
@ -206,7 +215,7 @@ describe('InstanceReportingService', () => {
|
|||
expect(clientOptions?.timeout).toBe(30_000);
|
||||
});
|
||||
|
||||
test('does not follow redirects, so the auth token reaches only the configured host', async () => {
|
||||
test('does not follow redirects, so the credential reaches only the configured host', async () => {
|
||||
const { service, http } = makeHarness();
|
||||
|
||||
await service.sendReport();
|
||||
|
|
@ -239,19 +248,68 @@ describe('InstanceReportingService', () => {
|
|||
expect(body(http)).not.toHaveProperty('label');
|
||||
});
|
||||
|
||||
test('sends the auth token as a bearer token when configured', () => {
|
||||
const { clientOptions } = makeHarness(
|
||||
makeConfig({ instanceReportingAuthToken: 'secret-token' }),
|
||||
);
|
||||
test('sends the license certificate in the body as the credential', async () => {
|
||||
const { service, http, clientOptions } = makeHarness();
|
||||
|
||||
expect(clientHeaders(clientOptions).authorization).toBe('Bearer secret-token');
|
||||
await service.sendReport();
|
||||
|
||||
expect(body(http).licenseCert).toBe(LICENSE_CERT);
|
||||
// No token: the certificate is the whole credential, and an `undefined`
|
||||
// default header is dropped before the request goes out.
|
||||
expect(clientHeaders(clientOptions).authorization).toBeUndefined();
|
||||
});
|
||||
|
||||
test('omits the Authorization header when no auth token is configured', () => {
|
||||
const { clientOptions } = makeHarness();
|
||||
test('reads the certificate fresh for every report, so a renewed license is sent', async () => {
|
||||
const { service, license, http, reportRepository } = makeHarness();
|
||||
|
||||
// An `undefined` default header is dropped before the request goes out.
|
||||
expect(clientHeaders(clientOptions).authorization).toBeUndefined();
|
||||
await service.sendReport();
|
||||
license.loadCertStr.mockResolvedValue('renewed-license-cert');
|
||||
reportRepository.findLastCoveredDay.mockResolvedValue('2026-03-23');
|
||||
|
||||
await service.sendReport();
|
||||
|
||||
expect(body(http, 1).licenseCert).toBe('renewed-license-cert');
|
||||
});
|
||||
|
||||
test('sends nothing and measures nothing without a license certificate', async () => {
|
||||
const { service, license, http, reportRepository, insightsService } = makeHarness();
|
||||
license.loadCertStr.mockResolvedValue('');
|
||||
|
||||
// Resolves rather than throws: this is not a delivery failure to retry.
|
||||
await expect(service.sendReport()).resolves.toBeUndefined();
|
||||
|
||||
expect(http.request).not.toHaveBeenCalled();
|
||||
expect(insightsService.getInsightsByTime).not.toHaveBeenCalled();
|
||||
expect(reportRepository.createPending).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('with an auth token configured', () => {
|
||||
const tokenConfig = () => makeConfig({ instanceReportingAuthToken: 'secret-token' });
|
||||
|
||||
test('sends the token as a bearer token', () => {
|
||||
const { clientOptions } = makeHarness(tokenConfig());
|
||||
|
||||
expect(clientHeaders(clientOptions).authorization).toBe('Bearer secret-token');
|
||||
});
|
||||
|
||||
test('does not send or read the license certificate', async () => {
|
||||
const { service, http, license } = makeHarness(tokenConfig());
|
||||
|
||||
await service.sendReport();
|
||||
|
||||
expect(body(http)).not.toHaveProperty('licenseCert');
|
||||
expect(license.loadCertStr).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// The token is the whole credential, so an unlicensed instance still reports.
|
||||
test('reports without a license certificate', async () => {
|
||||
const { service, http, license } = makeHarness(tokenConfig());
|
||||
license.loadCertStr.mockResolvedValue('');
|
||||
|
||||
await service.sendReport();
|
||||
|
||||
expect(http.request).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
test('queries the instance owner insights for the reported UTC day', async () => {
|
||||
|
|
@ -366,7 +424,7 @@ describe('InstanceReportingService', () => {
|
|||
|
||||
await expect(service.sendReport()).rejects.toThrow();
|
||||
// The retry picks up the still-undelivered row the first attempt created.
|
||||
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 1 }));
|
||||
reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 1 }));
|
||||
await service.sendReport();
|
||||
|
||||
expect(body(http, 1).batchId).toBe(BATCH_ID);
|
||||
|
|
@ -379,7 +437,7 @@ describe('InstanceReportingService', () => {
|
|||
{ kind: 'cumulative', name: 'billableExecutions', value: 800 },
|
||||
{ kind: 'daily', name: 'billableExecutions', value: 40, date: REPORT_DATE },
|
||||
] as InstanceMonitoringReport['dataPoints'];
|
||||
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ dataPoints: measured }));
|
||||
reportRepository.findPending.mockResolvedValue(makeReport({ dataPoints: measured }));
|
||||
|
||||
await service.sendReport();
|
||||
|
||||
|
|
@ -405,7 +463,7 @@ describe('InstanceReportingService', () => {
|
|||
test('leaves the report pending while attempts remain', async () => {
|
||||
const { service, reportRepository, http } = makeHarness();
|
||||
vi.mocked(http.request).mockRejectedValue(new Error('Network error'));
|
||||
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 1 }));
|
||||
reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 1 }));
|
||||
|
||||
await expect(service.sendReport()).rejects.toThrow();
|
||||
|
||||
|
|
@ -416,7 +474,7 @@ describe('InstanceReportingService', () => {
|
|||
const { service, reportRepository, http } = makeHarness();
|
||||
vi.mocked(http.request).mockRejectedValue(new Error('Network error'));
|
||||
// Two attempts already recorded on the row, so this one is the last.
|
||||
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 2 }));
|
||||
reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 2 }));
|
||||
|
||||
await expect(service.sendReport()).rejects.toThrow();
|
||||
|
||||
|
|
@ -426,7 +484,7 @@ describe('InstanceReportingService', () => {
|
|||
test('skips an exhausted row without attempting again', async () => {
|
||||
const { service, reportRepository, http } = makeHarness();
|
||||
// A crash between the failure record and the skip leaves this row pending.
|
||||
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ attempts: 3 }));
|
||||
reportRepository.findPending.mockResolvedValue(makeReport({ attempts: 3 }));
|
||||
|
||||
await expect(service.sendReport()).resolves.toBeUndefined();
|
||||
|
||||
|
|
@ -441,21 +499,21 @@ describe('InstanceReportingService', () => {
|
|||
|
||||
test('allows an attempt when no report is pending', async () => {
|
||||
const { service, reportRepository } = makeHarness();
|
||||
reportRepository.findTodaysPending.mockResolvedValue(null);
|
||||
reportRepository.findPending.mockResolvedValue(null);
|
||||
|
||||
await expect(service.msUntilRetryAllowed(now)).resolves.toBe(0);
|
||||
});
|
||||
|
||||
test('allows the first attempt, which has nothing to wait for', async () => {
|
||||
const { service, reportRepository } = makeHarness();
|
||||
reportRepository.findTodaysPending.mockResolvedValue(makeReport({ lastAttemptAt: null }));
|
||||
reportRepository.findPending.mockResolvedValue(makeReport({ lastAttemptAt: null }));
|
||||
|
||||
await expect(service.msUntilRetryAllowed(now)).resolves.toBe(0);
|
||||
});
|
||||
|
||||
test('returns the remaining wait when the last attempt was recent', async () => {
|
||||
const { service, reportRepository } = makeHarness();
|
||||
reportRepository.findTodaysPending.mockResolvedValue(
|
||||
reportRepository.findPending.mockResolvedValue(
|
||||
makeReport({ lastAttemptAt: new Date('2026-03-26T07:40:00.000Z') }),
|
||||
);
|
||||
|
||||
|
|
@ -465,7 +523,7 @@ describe('InstanceReportingService', () => {
|
|||
|
||||
test('allows an attempt once the wait has passed', async () => {
|
||||
const { service, reportRepository } = makeHarness();
|
||||
reportRepository.findTodaysPending.mockResolvedValue(
|
||||
reportRepository.findPending.mockResolvedValue(
|
||||
makeReport({ lastAttemptAt: new Date('2026-03-26T07:30:00.000Z') }),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -42,45 +42,52 @@ describe('InstanceMonitoringReportRepository', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('findTodaysPending', () => {
|
||||
describe('findPending', () => {
|
||||
async function createdOn(date: string) {
|
||||
const report = await repository.createPending(DATA_POINTS);
|
||||
await repository.update({ id: report.id }, { createdAt: new Date(date) });
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
test('returns nothing when no report was generated yet', async () => {
|
||||
await expect(repository.findTodaysPending(new Date())).resolves.toBeNull();
|
||||
await expect(repository.findPending()).resolves.toBeNull();
|
||||
});
|
||||
|
||||
test("returns today's undelivered report with the numbers it measured", async () => {
|
||||
test('returns the pending report with the numbers it measured', async () => {
|
||||
const created = await repository.createPending(DATA_POINTS);
|
||||
|
||||
const pending = await repository.findTodaysPending(new Date());
|
||||
const pending = await repository.findPending();
|
||||
|
||||
expect(pending?.id).toBe(created.id);
|
||||
expect(pending?.status).toBe('pending');
|
||||
expect(pending?.dataPoints).toEqual(DATA_POINTS);
|
||||
});
|
||||
|
||||
test("returns nothing once today's report was delivered", async () => {
|
||||
test('returns nothing once the newest report is delivered', async () => {
|
||||
const created = await repository.createPending(DATA_POINTS);
|
||||
await repository.markDelivered(created.id, new Date());
|
||||
|
||||
await expect(repository.findTodaysPending(new Date())).resolves.toBeNull();
|
||||
await expect(repository.findPending()).resolves.toBeNull();
|
||||
});
|
||||
|
||||
test("ignores an earlier day's undelivered report, leaving it untouched", async () => {
|
||||
const stale = await repository.createPending(DATA_POINTS);
|
||||
test('returns nothing when an older pending row sits under a newer delivered one', async () => {
|
||||
// Earlier versions left a failed report pending for good. The newer report
|
||||
// already covers that orphan's days, so a resend would report them two
|
||||
// times. Filtering on `status` in the query would return the orphan here.
|
||||
await createdOn('2026-03-20T07:42:00.000Z');
|
||||
const newer = await createdOn('2026-03-25T07:42:00.000Z');
|
||||
await repository.markDelivered(newer.id, new Date());
|
||||
|
||||
const tomorrow = new Date(Date.now() + 24 * 60 * 60 * 1000);
|
||||
|
||||
await expect(repository.findTodaysPending(tomorrow)).resolves.toBeNull();
|
||||
// The stale report keeps the numbers it measured; only backfill may touch it.
|
||||
await expect(repository.findOneByOrFail({ id: stale.id })).resolves.toMatchObject({
|
||||
dataPoints: DATA_POINTS,
|
||||
deliveredAt: null,
|
||||
});
|
||||
await expect(repository.findPending()).resolves.toBeNull();
|
||||
});
|
||||
|
||||
test('returns nothing once the report ran out of attempts, so it is not resent', async () => {
|
||||
const created = await repository.createPending(DATA_POINTS);
|
||||
await repository.markSkipped(created.id);
|
||||
test('returns the newest row when it is pending, ignoring an older delivered one', async () => {
|
||||
const older = await createdOn('2026-03-20T07:42:00.000Z');
|
||||
await repository.markDelivered(older.id, new Date());
|
||||
const newer = await createdOn('2026-03-25T07:42:00.000Z');
|
||||
|
||||
await expect(repository.findTodaysPending(new Date())).resolves.toBeNull();
|
||||
await expect(repository.findPending()).resolves.toMatchObject({ id: newer.id });
|
||||
});
|
||||
|
||||
test('carries the last attempt time, so the wait between attempts survives a restart', async () => {
|
||||
|
|
@ -88,7 +95,7 @@ describe('InstanceMonitoringReportRepository', () => {
|
|||
const failedAt = new Date('2026-03-26T07:42:00.000Z');
|
||||
await repository.recordFailure(created.id, 'Network error', failedAt);
|
||||
|
||||
const pending = await repository.findTodaysPending(new Date());
|
||||
const pending = await repository.findPending();
|
||||
|
||||
expect(pending?.lastAttemptAt?.toISOString()).toBe(failedAt.toISOString());
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,24 +12,25 @@ export class InstanceMonitoringReportRepository extends Repository<InstanceMonit
|
|||
}
|
||||
|
||||
/**
|
||||
* Today's report, if one was already generated and never reached the receiver.
|
||||
* The report a retry may resume, or `null` when there is nothing to resume.
|
||||
* It is the newest row, and only if that row is still `pending`.
|
||||
*
|
||||
* Resending it — same `batchId`, same data points — is what keeps a retry from
|
||||
* re-measuring: the cumulative series is only comparable day to day while its
|
||||
* sampling interval stays a fixed 24 hours.
|
||||
* At most one report is genuinely pending: the scheduler settles a stale row
|
||||
* before it creates the next one. But earlier versions left a failed report
|
||||
* pending for good, so an instance can still hold an old `pending` row below
|
||||
* a newer `delivered` one. That row is an orphan — the newer report already
|
||||
* covers its days, and a resend would report those days two times.
|
||||
*
|
||||
* Scoped to `now`'s UTC day on purpose: an older undelivered report measured a
|
||||
* different day, so it must not stand in for today's. It stays as it is, a
|
||||
* record of a report that never landed, and its days are covered again by the
|
||||
* next report.
|
||||
* So this reads the newest row of any status and then checks that status. It
|
||||
* must not filter on `status` in the query.
|
||||
*
|
||||
* A report that ran out of attempts is not pending, so it is never resent.
|
||||
* `id` breaks a `createdAt` tie so the result is stable. A tie needs two
|
||||
* reports in the same millisecond, which the once-a-day cadence never makes.
|
||||
*/
|
||||
async findTodaysPending(now: Date): Promise<InstanceMonitoringReport | null> {
|
||||
return await this.findOne({
|
||||
where: { status: 'pending', createdAt: MoreThanOrEqual(startOfUtcDay(now)) },
|
||||
order: { createdAt: 'DESC' },
|
||||
});
|
||||
async findPending(): Promise<InstanceMonitoringReport | null> {
|
||||
const [latest] = await this.find({ order: { createdAt: 'DESC', id: 'DESC' }, take: 1 });
|
||||
|
||||
return latest?.status === 'pending' ? latest : null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { strict } from 'node:assert';
|
|||
|
||||
import { EventService } from '@/events/event.service';
|
||||
|
||||
import type { InstanceMonitoringReport } from './database/entities/instance-monitoring-report';
|
||||
import { InstanceMonitoringReportRepository } from './database/repositories/instance-monitoring-report.repository';
|
||||
import { InstanceReportingSettingsService } from './instance-reporting-settings.service';
|
||||
import { InstanceReportingService, RETRY_DELAY_MS } from './instance-reporting.service';
|
||||
|
|
@ -112,9 +113,14 @@ export class InstanceReportingScheduler {
|
|||
try {
|
||||
const reportTime = await this.settingsService.getReportTime();
|
||||
|
||||
// An attempt too soon after the last one arms for the remainder rather
|
||||
// than falling through, which would sleep until tomorrow and drop the retry.
|
||||
const waitMs = await this.reportingService.msUntilRetryAllowed(new Date());
|
||||
// Wait out the gap between retries, but never past the pending row's own
|
||||
// next slot — at that slot it is skipped, not retried. A wait that reaches
|
||||
// the slot collapses to zero, so this pass falls through to reportIfDue,
|
||||
// which skips the stale row and sends a fresh report.
|
||||
const waitMs = Math.min(
|
||||
await this.reportingService.msUntilRetryAllowed(new Date()),
|
||||
await this.msUntilNextSlot(reportTime),
|
||||
);
|
||||
if (waitMs > 0) {
|
||||
this.scheduleNext(waitMs);
|
||||
return;
|
||||
|
|
@ -123,7 +129,7 @@ export class InstanceReportingScheduler {
|
|||
// A failed delivery retries; the row decides when the attempts run out,
|
||||
// after which the day reads as settled and this waits for the next slot.
|
||||
if ((await this.reportIfDue(reportTime)) === 'failed') {
|
||||
this.scheduleNext(RETRY_DELAY_MS);
|
||||
this.scheduleNext(Math.min(RETRY_DELAY_MS, await this.msUntilNextSlot(reportTime)));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -137,16 +143,49 @@ export class InstanceReportingScheduler {
|
|||
}
|
||||
|
||||
/**
|
||||
* Report when this day's slot has passed and the day is not settled yet. Both
|
||||
* conditions are re-checked here rather than inferred from the timer having
|
||||
* fired, so an early fire (a backward clock jump) reports nothing and a
|
||||
* duplicate fire is a no-op.
|
||||
* Send the report when it is due.
|
||||
*
|
||||
* A pending report is a retry. Resend it now, also after midnight; it holds
|
||||
* frozen data, so the slot does not apply to it. But once its own next slot
|
||||
* has passed, its cycle is over: stop trying and let a new report cover its
|
||||
* day. A new report waits for the slot, which makes sure the day is complete
|
||||
* before the code measures it.
|
||||
*/
|
||||
private async reportIfDue(reportTime: string): Promise<'sent' | 'skipped' | 'failed'> {
|
||||
const now = new Date();
|
||||
const pending = await this.reportRepository.findPending();
|
||||
|
||||
if (pending) {
|
||||
if (pendingIsStale(pending, reportTime, now)) {
|
||||
await this.reportingService.skip(pending.id, pending.attempts, 'slot-passed');
|
||||
} else {
|
||||
return await this.trySend();
|
||||
}
|
||||
}
|
||||
|
||||
// A new report waits for the slot and runs only when the day is not settled.
|
||||
if (now.getTime() < slotOn(reportTime, now)) return 'skipped';
|
||||
if (await this.reportRepository.hasSettledToday(now)) return 'skipped';
|
||||
return await this.trySend();
|
||||
}
|
||||
|
||||
/**
|
||||
* Milliseconds until the pending row's own next slot, or `Infinity` when no row
|
||||
* is pending. Capping a retry sleep with this keeps the timer from sleeping past
|
||||
* the slot, where the row is skipped and a fresh report takes over — otherwise a
|
||||
* slot near the end of the UTC day would defer the next report by almost a day.
|
||||
*/
|
||||
private async msUntilNextSlot(reportTime: string): Promise<number> {
|
||||
const pending = await this.reportRepository.findPending();
|
||||
if (!pending) return Number.POSITIVE_INFINITY;
|
||||
|
||||
const nextSlot =
|
||||
slotOn(reportTime, pending.createdAt) + MINUTES_PER_DAY * Time.minutes.toMilliseconds;
|
||||
|
||||
return nextSlot - Date.now();
|
||||
}
|
||||
|
||||
private async trySend(): Promise<'sent' | 'failed'> {
|
||||
try {
|
||||
await this.reportingService.sendReport();
|
||||
return 'sent';
|
||||
|
|
@ -163,6 +202,17 @@ export class InstanceReportingScheduler {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a pending row sat unsent past its own next slot. Its cycle is then
|
||||
* over, so a new report covers its day instead of a late resend.
|
||||
*/
|
||||
function pendingIsStale(report: InstanceMonitoringReport, reportTime: string, now: Date): boolean {
|
||||
const nextSlot =
|
||||
slotOn(reportTime, report.createdAt) + MINUTES_PER_DAY * Time.minutes.toMilliseconds;
|
||||
|
||||
return now.getTime() >= nextSlot;
|
||||
}
|
||||
|
||||
/** Epoch ms of `reportTime` on `now`'s UTC day. */
|
||||
function slotOn(reportTime: string, now: Date): number {
|
||||
const [hour, minute] = reportTime.split(':').map(Number);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ export class InstanceReportingConfig {
|
|||
instanceReportingBaseUrl: string = '';
|
||||
|
||||
/**
|
||||
* Sent as a bearer token; leave unset if the receiver does not require one.
|
||||
* Optional. When set, it is sent as `Authorization: Bearer …` and the license
|
||||
* certificate is not sent. Leave unset to authenticate with the certificate.
|
||||
*
|
||||
* @beta - breaking changes may still occur
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -30,12 +30,19 @@ export class InstanceReportingModule implements ModuleInterface {
|
|||
);
|
||||
}
|
||||
|
||||
if (!(await this.isConfigured())) {
|
||||
Container.get(Logger)
|
||||
.scoped('instance-reporting')
|
||||
.warn(
|
||||
'Instance reporting is enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset, so no reports will be sent',
|
||||
);
|
||||
const logger = Container.get(Logger).scoped('instance-reporting');
|
||||
|
||||
if (!(await this.hasReceiver())) {
|
||||
logger.warn(
|
||||
'Instance reporting is enabled but N8N_INSTANCE_REPORTING_BASE_URL is unset, so no reports will be sent',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!(await this.hasCredential())) {
|
||||
logger.warn(
|
||||
'Instance reporting is enabled but this instance has no license certificate, so no reports will be sent. The receiver accepts reports only from licensed instances. Set N8N_LICENSE_CERT, activate a license, or set N8N_INSTANCE_REPORTING_AUTH_TOKEN.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -50,8 +57,8 @@ export class InstanceReportingModule implements ModuleInterface {
|
|||
* Settings exposed to the frontend under `/rest/module-settings`.
|
||||
*
|
||||
* Return values:
|
||||
* { enabled: false } - module loaded but no receiver configured
|
||||
* { enabled: true, reportTime: 'HH:mm' } - module loaded and receiver configured
|
||||
* { enabled: false } - module loaded but no receiver configured, or neither an auth token nor a license certificate
|
||||
* { enabled: true, reportTime: 'HH:mm' } - module loaded, receiver configured, credential present
|
||||
*
|
||||
* Built once at startup and cached for the process lifetime.
|
||||
**/
|
||||
|
|
@ -75,10 +82,28 @@ export class InstanceReportingModule implements ModuleInterface {
|
|||
return [InstanceMonitoringReport];
|
||||
}
|
||||
|
||||
/** Whether a receiver is configured, i.e. whether reports are actually sent. */
|
||||
/** Whether reports are actually sent: a receiver is configured and there is a credential to authenticate with. */
|
||||
private async isConfigured(): Promise<boolean> {
|
||||
return (await this.hasReceiver()) && (await this.hasCredential());
|
||||
}
|
||||
|
||||
private async hasReceiver(): Promise<boolean> {
|
||||
const { InstanceReportingConfig } = await import('./instance-reporting.config.js');
|
||||
|
||||
return Container.get(InstanceReportingConfig).instanceReportingBaseUrl !== '';
|
||||
}
|
||||
|
||||
/**
|
||||
* A configured auth token is a credential on its own. Without one, the
|
||||
* license certificate is the credential the receiver checks, so an
|
||||
* unlicensed (community) instance cannot report.
|
||||
*/
|
||||
private async hasCredential(): Promise<boolean> {
|
||||
const { InstanceReportingConfig } = await import('./instance-reporting.config.js');
|
||||
if (Container.get(InstanceReportingConfig).instanceReportingAuthToken !== '') return true;
|
||||
|
||||
const { License } = await import('@/license.js');
|
||||
|
||||
return (await Container.get(License).loadCertStr()) !== '';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { OperationalError } from 'n8n-workflow';
|
|||
|
||||
import { N8N_VERSION } from '@/constants';
|
||||
import { EventService } from '@/events/event.service';
|
||||
import { License } from '@/license';
|
||||
import { InsightsService } from '@/modules/insights/insights.service';
|
||||
import { OwnershipService } from '@/services/ownership.service';
|
||||
|
||||
|
|
@ -55,6 +56,7 @@ export class InstanceReportingService {
|
|||
private readonly instanceSettings: InstanceSettings,
|
||||
private readonly ownershipService: OwnershipService,
|
||||
private readonly licenseMetricsRepository: LicenseMetricsRepository,
|
||||
private readonly license: License,
|
||||
private readonly logger: Logger,
|
||||
private readonly eventService: EventService,
|
||||
outboundHttp: OutboundHttp,
|
||||
|
|
@ -66,7 +68,7 @@ export class InstanceReportingService {
|
|||
// collector, so the URL is never user-controlled.
|
||||
useDefaultSsrfPolicy: 'unsafe',
|
||||
baseURL: this.config.instanceReportingBaseUrl.replace(/\/+$/, ''),
|
||||
// An unset token drops the header, so an unauthenticated receiver works.
|
||||
// An unset token drops the header; the license certificate is the credential then.
|
||||
headers: () => ({
|
||||
authorization: this.config.instanceReportingAuthToken
|
||||
? `Bearer ${this.config.instanceReportingAuthToken}`
|
||||
|
|
@ -87,23 +89,37 @@ export class InstanceReportingService {
|
|||
* `batchId`, so a redelivery reuses the row instead of measuring the day again,
|
||||
* and only a delivered report crosses its days off.
|
||||
*
|
||||
* A retry resends today's pending report exactly as measured instead of taking
|
||||
* A retry resends the pending report exactly as measured instead of taking
|
||||
* fresh numbers. The cumulative point is a lifetime total sampled at this
|
||||
* instance's report time, so its day-to-day difference only lines up with the
|
||||
* daily point while every sample sits 24 hours apart; re-measuring hours later
|
||||
* would stretch one interval and skew the whole series.
|
||||
*
|
||||
* The credential is the license certificate, sent in the body, unless a
|
||||
* bearer token is configured; then the token goes in the header and the
|
||||
* certificate is not sent at all.
|
||||
*
|
||||
* @throws when delivery fails, so the scheduler retries with backoff.
|
||||
*/
|
||||
async sendReport(): Promise<void> {
|
||||
const licenseCert = this.config.instanceReportingAuthToken
|
||||
? undefined
|
||||
: await this.license.loadCertStr();
|
||||
if (licenseCert === '') {
|
||||
this.logger.warn(
|
||||
'Skipping the instance report because this instance has no license certificate.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
let report = await this.reportRepository.findTodaysPending(now);
|
||||
let report = await this.reportRepository.findPending();
|
||||
|
||||
// A crash between recording a failure and skipping the report leaves an
|
||||
// exhausted row pending, so the budget is re-checked before sending rather
|
||||
// than only after. Settling it here also ends the day for the scheduler.
|
||||
if (report && report.attempts >= MAX_ATTEMPTS) {
|
||||
await this.skip(report.id, report.attempts);
|
||||
await this.skip(report.id, report.attempts, 'max-retries');
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -126,6 +142,7 @@ export class InstanceReportingService {
|
|||
...(this.config.instanceReportingLabel ? { label: this.config.instanceReportingLabel } : {}),
|
||||
n8nVersion: N8N_VERSION,
|
||||
dataPoints: report.dataPoints,
|
||||
...(licenseCert ? { licenseCert } : {}),
|
||||
};
|
||||
|
||||
try {
|
||||
|
|
@ -137,7 +154,7 @@ export class InstanceReportingService {
|
|||
returnFullResponse: true,
|
||||
// Inspect the status here rather than catching a generic request error.
|
||||
ignoreHttpStatusErrors: true,
|
||||
// A redirect would forward the auth token to whatever host it names.
|
||||
// A redirect would forward the credential to whatever host it names.
|
||||
disableFollowRedirect: true,
|
||||
});
|
||||
|
||||
|
|
@ -163,7 +180,7 @@ export class InstanceReportingService {
|
|||
|
||||
// `recordFailure` incremented the count, so the in-memory row is one behind.
|
||||
if (report.attempts + 1 >= MAX_ATTEMPTS) {
|
||||
await this.skip(report.id, report.attempts + 1);
|
||||
await this.skip(report.id, report.attempts + 1, 'max-retries');
|
||||
}
|
||||
|
||||
throw error;
|
||||
|
|
@ -174,12 +191,15 @@ export class InstanceReportingService {
|
|||
}
|
||||
|
||||
/** Stop trying to deliver this report; the next one covers its days again. */
|
||||
private async skip(id: string, attempts: number): Promise<void> {
|
||||
async skip(id: string, attempts: number, reason: 'max-retries' | 'slot-passed'): Promise<void> {
|
||||
await this.reportRepository.markSkipped(id);
|
||||
this.logger.error('Giving up on the instance report after repeated delivery failures', {
|
||||
batchId: id,
|
||||
attempts,
|
||||
});
|
||||
|
||||
const message =
|
||||
reason === 'max-retries'
|
||||
? 'Giving up on the instance report after repeated delivery failures'
|
||||
: 'Giving up on the instance report because its slot has passed';
|
||||
|
||||
this.logger.error(message, { batchId: id, attempts });
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -191,7 +211,7 @@ export class InstanceReportingService {
|
|||
* seconds.
|
||||
*/
|
||||
async msUntilRetryAllowed(now: Date): Promise<number> {
|
||||
const pending = await this.reportRepository.findTodaysPending(now);
|
||||
const pending = await this.reportRepository.findPending();
|
||||
if (!pending?.lastAttemptAt) return 0;
|
||||
|
||||
const elapsed = now.getTime() - pending.lastAttemptAt.getTime();
|
||||
|
|
|
|||
|
|
@ -12,72 +12,114 @@ Given OTEL often involves events triggered from elsewhere within the n8n system
|
|||
### Attributes
|
||||
All attributes are listed in `otel.constants.ts`
|
||||
|
||||
### Global API ownership
|
||||
|
||||
`OtelService` starts a `NodeTracerProvider` and hands out tracers through `getTracer()`.
|
||||
The module's own spans never depend on the global `@opentelemetry/api` registry.
|
||||
|
||||
At start, the service checks whether another library already registered a global tracer
|
||||
provider. Sentry does this in `ErrorReporter.init`. A user SDK loaded with `--require` does
|
||||
it too.
|
||||
|
||||
- The slot is free: the service registers its provider, an `AsyncLocalStorageContextManager`
|
||||
and the default W3C propagators. A restart swaps only the tracer provider. Shutdown leaves
|
||||
the globals in place.
|
||||
- The slot is taken: the service keeps its provider private and logs one info line. It never
|
||||
disables or replaces the other library's globals.
|
||||
|
||||
Consequences:
|
||||
|
||||
- Workflow spans go to the module exporter only. To get them into Sentry, point the OTel
|
||||
settings at Sentry's OTLP endpoint.
|
||||
- Workflow root spans start on `ROOT_CONTEXT`. They never join a trace that is active in the
|
||||
process, such as a Sentry request span or an HTTP server span from a user SDK. An inbound
|
||||
`traceparent` header still sets the parent.
|
||||
- Persisted trace context and outbound headers always use the W3C `traceparent` and
|
||||
`tracestate` format, whoever owns the global propagator. `OTEL_PROPAGATORS` has no effect
|
||||
on them.
|
||||
- The resource uses the env, process and host detectors. `OTEL_NODE_RESOURCE_DETECTORS` has
|
||||
no effect.
|
||||
|
||||
### Module architecture
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Module Layer
|
||||
MOD["OtelModule
|
||||
@BackendModule"]
|
||||
end
|
||||
graph TD
|
||||
subgraph Module Layer
|
||||
MOD["OtelModule
|
||||
@BackendModule on main, worker and webhook"]
|
||||
end
|
||||
|
||||
subgraph Configuration
|
||||
CFG["OtelConfig
|
||||
env vars to typed config"]
|
||||
end
|
||||
subgraph Configuration
|
||||
CFG["OtelConfig
|
||||
env vars to typed config"]
|
||||
SET["OtelSettingsService
|
||||
DB row merged with config, env vars win"]
|
||||
CTRL["OtelSettingsController
|
||||
main only: GET and PUT settings, POST test trace"]
|
||||
end
|
||||
|
||||
subgraph SDK Layer
|
||||
SVC["OtelService
|
||||
owns NodeSDK lifecycle"]
|
||||
SDK["OpenTelemetry NodeSDK
|
||||
exporter, sampler, resource"]
|
||||
end
|
||||
subgraph SDK Layer
|
||||
SVC["OtelService
|
||||
owns the NodeTracerProvider, getTracer()"]
|
||||
SDK["NodeTracerProvider
|
||||
OTLP exporter, sampler, resource"]
|
||||
API["Global @opentelemetry/api
|
||||
registered only when the slot is free"]
|
||||
end
|
||||
|
||||
subgraph Instrumentation Layer
|
||||
INST["N8nWorkflowInstrumentation
|
||||
@OnLifecycleEvent listeners"]
|
||||
REG["SpanRegistry
|
||||
Map of executionId to Span"]
|
||||
end
|
||||
subgraph Instrumentation Layer
|
||||
LCH["OtelLifecycleHandler
|
||||
@OnLifecycleEvent and @OnPubSubEvent"]
|
||||
ELT["ExecutionLevelTracer
|
||||
span maps by execution id, W3C propagator"]
|
||||
TCS["TraceContextService
|
||||
traceparent on the execution row"]
|
||||
end
|
||||
|
||||
subgraph Handler Layer
|
||||
IFC{{"SpanHandler interface"}}
|
||||
WS["WorkflowStartHandler"]
|
||||
WE["WorkflowEndHandler"]
|
||||
end
|
||||
subgraph n8n Core
|
||||
LC(("Lifecycle events
|
||||
workflowExecuteBefore, Resume, After
|
||||
nodeExecuteBefore, After"))
|
||||
PUB(("Pub/Sub
|
||||
reload-otel-config"))
|
||||
REQ["Request helpers
|
||||
module context injectTraceHeaders"]
|
||||
AGT["AgentRunTracingService
|
||||
AgentWorkflowExecutionService"]
|
||||
end
|
||||
|
||||
subgraph n8n Core
|
||||
LC(("Lifecycle Events
|
||||
workflowExecuteBefore
|
||||
workflowExecuteAfter"))
|
||||
end
|
||||
COL[("OTLP collector")]
|
||||
|
||||
MOD -- "1. check enabled" --> CFG
|
||||
MOD -- "2. init SDK" --> SVC
|
||||
SVC -- "creates" --> SDK
|
||||
MOD -- "3. register listeners" --> INST
|
||||
MOD -- "1. load settings" --> SET
|
||||
SET --> CFG
|
||||
MOD -- "2. start provider" --> SVC
|
||||
SVC -- "creates" --> SDK
|
||||
SVC -. "registers when free" .-> API
|
||||
MOD -- "3. import handler" --> LCH
|
||||
MOD -- "4. main only" --> CTRL
|
||||
CTRL -- "save, then publish" --> PUB
|
||||
PUB -. "fires" .-> LCH
|
||||
LCH -- "restart()" --> SVC
|
||||
|
||||
LC -. "fires event" .-> INST
|
||||
INST -- "dispatches to" --> IFC
|
||||
IFC -. "implemented by" .-> WS
|
||||
IFC -. "implemented by" .-> WE
|
||||
LC -. "fires" .-> LCH
|
||||
LCH -- "start and end workflow and node spans" --> ELT
|
||||
LCH -- "persist and read" --> TCS
|
||||
ELT -- "getTracer()" --> SVC
|
||||
REQ -- "outbound traceparent" --> ELT
|
||||
AGT -- "getActiveContext()" --> ELT
|
||||
AGT -- "getTracer()" --> SVC
|
||||
SDK -. "exports spans" .-> COL
|
||||
|
||||
WS -- "startSpan, store" --> REG
|
||||
WE -- "retrieve, enrich, end" --> REG
|
||||
REG -. "spans exported via" .-> SDK
|
||||
classDef module fill:#4a9eff,color:#fff
|
||||
classDef config fill:#f5a623,color:#fff
|
||||
classDef sdk fill:#7b68ee,color:#fff
|
||||
classDef inst fill:#50c878,color:#fff
|
||||
classDef core fill:#888,color:#fff
|
||||
|
||||
classDef module fill:#4a9eff,color:#fff
|
||||
classDef config fill:#f5a623,color:#fff
|
||||
classDef sdk fill:#7b68ee,color:#fff
|
||||
classDef inst fill:#50c878,color:#fff
|
||||
classDef handler fill:#ff6b6b,color:#fff
|
||||
classDef core fill:#888,color:#fff
|
||||
|
||||
class MOD module
|
||||
class CFG config
|
||||
class SVC,SDK sdk
|
||||
class INST,REG inst
|
||||
class IFC,WS,WE handler
|
||||
class LC core
|
||||
class MOD module
|
||||
class CFG,SET,CTRL config
|
||||
class SVC,SDK,API sdk
|
||||
class LCH,ELT,TCS inst
|
||||
class LC,PUB,REQ,AGT,COL core
|
||||
```
|
||||
|
||||
#### Manual validation
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { Logger } from '@n8n/backend-common';
|
||||
import { SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import type { TextMapPropagator } from '@opentelemetry/api';
|
||||
import { context, propagation, SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { ExecutionLevelTracer } from '../execution-level-tracer';
|
||||
|
|
@ -20,7 +21,7 @@ describe('ExecutionLevelTracer', () => {
|
|||
};
|
||||
|
||||
beforeAll(() => {
|
||||
otel = OtelTestProvider.create();
|
||||
otel = OtelTestProvider.create({ withContextManager: true });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
|
@ -29,7 +30,7 @@ describe('ExecutionLevelTracer', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
otel.reset();
|
||||
tracer = new ExecutionLevelTracer(makeOtelSettingsService(), logger);
|
||||
tracer = new ExecutionLevelTracer(otel.asOtelService(), makeOtelSettingsService(), logger);
|
||||
});
|
||||
|
||||
const inboundTracingContext = {
|
||||
|
|
@ -88,37 +89,52 @@ describe('ExecutionLevelTracer', () => {
|
|||
expect(span.attributes['n8n.project.custom.team']).toBe('platform');
|
||||
});
|
||||
|
||||
it('should not attach project custom attributes to node spans', () => {
|
||||
tracer.startWorkflow({
|
||||
executionId: 'exec-node-no-tags',
|
||||
workflow: defaultWorkflow,
|
||||
project: {
|
||||
id: 'proj-tags',
|
||||
customAttributes: { env: 'staging' },
|
||||
},
|
||||
});
|
||||
const runSingleNodeExecution = (
|
||||
executionId: string,
|
||||
project?: { id: string; customAttributes?: Record<string, string> },
|
||||
) => {
|
||||
tracer.startWorkflow({ executionId, workflow: defaultWorkflow, project });
|
||||
const node = { id: 'n1', name: 'MyNode', type: 'test', typeVersion: 1 };
|
||||
tracer.startNode({ executionId: 'exec-node-no-tags', node });
|
||||
tracer.endNode({
|
||||
executionId: 'exec-node-no-tags',
|
||||
node,
|
||||
inputItemCount: 1,
|
||||
outputItemCount: 1,
|
||||
});
|
||||
tracer.endWorkflow({
|
||||
executionId: 'exec-node-no-tags',
|
||||
status: 'success',
|
||||
mode: 'manual',
|
||||
isRetry: false,
|
||||
});
|
||||
tracer.startNode({ executionId, node });
|
||||
tracer.endNode({ executionId, node, inputItemCount: 1, outputItemCount: 1 });
|
||||
tracer.endWorkflow({ executionId, status: 'success', mode: 'manual', isRetry: false });
|
||||
|
||||
const spans = otel.getFinishedSpans();
|
||||
const nodeSpan = spans.find((s) => s.name === 'node.execute')!;
|
||||
// No project custom attributes should appear on the node span
|
||||
const projectCustomKeys = Object.keys(nodeSpan.attributes).filter((k) =>
|
||||
k.startsWith('n8n.project.custom.'),
|
||||
return {
|
||||
workflowSpan: spans.find((s) => s.name === 'workflow.execute')!,
|
||||
nodeSpan: spans.find((s) => s.name === 'node.execute')!,
|
||||
};
|
||||
};
|
||||
|
||||
const projectCustomKeys = (attributes: Record<string, unknown>) =>
|
||||
Object.keys(attributes).filter((k) => k.startsWith('n8n.project.custom.'));
|
||||
|
||||
it('should attach project id and custom attributes to node spans', () => {
|
||||
const { workflowSpan, nodeSpan } = runSingleNodeExecution('exec-node-tags', {
|
||||
id: 'proj-tags',
|
||||
customAttributes: { env: 'staging' },
|
||||
});
|
||||
|
||||
expect(nodeSpan.attributes['n8n.project.id']).toBe('proj-tags');
|
||||
expect(nodeSpan.attributes['n8n.project.custom.env']).toBe('staging');
|
||||
expect(nodeSpan.attributes['n8n.project.id']).toBe(workflowSpan.attributes['n8n.project.id']);
|
||||
expect(nodeSpan.attributes['n8n.project.custom.env']).toBe(
|
||||
workflowSpan.attributes['n8n.project.custom.env'],
|
||||
);
|
||||
expect(projectCustomKeys).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should attach only project id to node spans when the project has no custom attributes', () => {
|
||||
const { nodeSpan } = runSingleNodeExecution('exec-node-no-tags', { id: 'proj-no-tags' });
|
||||
|
||||
expect(nodeSpan.attributes['n8n.project.id']).toBe('proj-no-tags');
|
||||
expect(projectCustomKeys(nodeSpan.attributes)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should omit project attributes on node spans when project is not provided', () => {
|
||||
const { nodeSpan } = runSingleNodeExecution('exec-node-no-project');
|
||||
|
||||
expect(nodeSpan.attributes['n8n.project.id']).toBeUndefined();
|
||||
expect(projectCustomKeys(nodeSpan.attributes)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should omit project id attribute when project is not provided', () => {
|
||||
|
|
@ -783,6 +799,7 @@ describe('ExecutionLevelTracer', () => {
|
|||
|
||||
it('should no-op when injectOutbound is false', () => {
|
||||
const noInjectTracer = new ExecutionLevelTracer(
|
||||
otel.asOtelService(),
|
||||
makeOtelSettingsService({ injectOutbound: false }),
|
||||
logger,
|
||||
);
|
||||
|
|
@ -829,6 +846,59 @@ describe('ExecutionLevelTracer', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('trace context ownership', () => {
|
||||
const traceparentPattern = /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/;
|
||||
|
||||
it('should start a new trace when no tracing context is given, even inside an active foreign span', () => {
|
||||
const foreignSpan = trace.getTracer('foreign').startSpan('GET /webhook');
|
||||
|
||||
context.with(trace.setSpan(context.active(), foreignSpan), () => {
|
||||
tracer.startWorkflow({ executionId: 'exec-root', workflow: defaultWorkflow });
|
||||
});
|
||||
tracer.endWorkflow({
|
||||
executionId: 'exec-root',
|
||||
status: 'success',
|
||||
mode: 'webhook',
|
||||
isRetry: false,
|
||||
});
|
||||
foreignSpan.end();
|
||||
|
||||
const workflowSpan = otel.getFinishedSpans().find((s) => s.name === 'workflow.execute')!;
|
||||
expect(workflowSpan.parentSpanContext).toBeUndefined();
|
||||
expect(workflowSpan.spanContext().traceId).not.toBe(foreignSpan.spanContext().traceId);
|
||||
});
|
||||
|
||||
it('should emit W3C trace context while another library owns the global propagator', () => {
|
||||
const foreignPropagator: TextMapPropagator = {
|
||||
inject: (_ctx, carrier, setter) => setter.set(carrier, 'sentry-trace', 'foreign'),
|
||||
extract: (ctx) => ctx,
|
||||
fields: () => ['sentry-trace'],
|
||||
};
|
||||
propagation.setGlobalPropagator(foreignPropagator);
|
||||
|
||||
try {
|
||||
const persisted = tracer.startWorkflow({
|
||||
executionId: 'exec-w3c',
|
||||
workflow: defaultWorkflow,
|
||||
});
|
||||
const headers: Record<string, string> = {};
|
||||
tracer.injectTraceHeaders('exec-w3c', undefined, headers);
|
||||
tracer.endWorkflow({
|
||||
executionId: 'exec-w3c',
|
||||
status: 'success',
|
||||
mode: 'webhook',
|
||||
isRetry: false,
|
||||
});
|
||||
|
||||
expect(persisted.traceparent).toMatch(traceparentPattern);
|
||||
expect(headers.traceparent).toMatch(traceparentPattern);
|
||||
expect(headers['sentry-trace']).toBeUndefined();
|
||||
} finally {
|
||||
propagation.disable();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('getActiveContext', () => {
|
||||
it('returns a context carrying the node span when one is active', () => {
|
||||
tracer.startWorkflow({ executionId: 'exec-ctx-node', workflow: defaultWorkflow });
|
||||
|
|
|
|||
|
|
@ -1019,14 +1019,13 @@ describe('onReloadOtelConfig', () => {
|
|||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('restarts the OTel SDK and refreshes the tracer', async () => {
|
||||
it('restarts the OTel tracer provider', async () => {
|
||||
const handler = makeHandler();
|
||||
otelService.restart.mockResolvedValue(undefined);
|
||||
|
||||
await handler.onReloadOtelConfig();
|
||||
|
||||
expect(otelService.restart).toHaveBeenCalledTimes(1);
|
||||
expect(tracer.refreshTracer).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,225 @@
|
|||
import type { Logger } from '@n8n/backend-common';
|
||||
import type { OutboundHttp } from '@n8n/backend-network';
|
||||
import type { Context } from '@opentelemetry/api';
|
||||
import {
|
||||
context,
|
||||
createContextKey,
|
||||
propagation,
|
||||
ProxyTracerProvider,
|
||||
ROOT_CONTEXT,
|
||||
trace,
|
||||
} from '@opentelemetry/api';
|
||||
import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
|
||||
import {
|
||||
InMemorySpanExporter,
|
||||
NodeTracerProvider,
|
||||
SimpleSpanProcessor,
|
||||
} from '@opentelemetry/sdk-trace-node';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
import type { InstanceSettings } from 'n8n-core';
|
||||
|
||||
import type { OtelSettingsService } from '../otel-settings.service';
|
||||
import type { OtelConfig } from '../otel.config';
|
||||
import { ATTR } from '../otel.constants';
|
||||
import { OtelService } from '../otel.service';
|
||||
|
||||
import { N8N_VERSION } from '@/constants';
|
||||
|
||||
const { exportedSpans } = vi.hoisted(() => ({ exportedSpans: [] as ReadableSpan[] }));
|
||||
|
||||
vi.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
|
||||
OTLPTraceExporter: vi.fn().mockImplementation(function () {
|
||||
return {
|
||||
export: (spans: ReadableSpan[], resultCallback: (result: { code: number }) => void) => {
|
||||
exportedSpans.push(...spans);
|
||||
resultCallback({ code: 0 });
|
||||
},
|
||||
shutdown: async () => {},
|
||||
forceFlush: async () => {},
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('@opentelemetry/sdk-trace-base', async (importActual) => {
|
||||
const actual = await importActual<typeof import('@opentelemetry/sdk-trace-base')>();
|
||||
return { ...actual, BatchSpanProcessor: actual.SimpleSpanProcessor };
|
||||
});
|
||||
|
||||
vi.mock('@opentelemetry/resources', async (importActual) => {
|
||||
const actual = await importActual<typeof import('@opentelemetry/resources')>();
|
||||
return { ...actual, detectResources: () => actual.emptyResource() };
|
||||
});
|
||||
|
||||
const enabledSettings: OtelConfig = {
|
||||
enabled: true,
|
||||
exporterProtocol: 'http/protobuf',
|
||||
exporterEndpoint: 'http://localhost:4318',
|
||||
exporterTracingPath: '/v1/traces',
|
||||
exporterHeaders: '',
|
||||
exporterServiceName: 'n8n',
|
||||
tracesSampleRate: 1,
|
||||
startupConnectivityTimeoutMs: 2_000,
|
||||
includeNodeSpans: true,
|
||||
injectOutbound: true,
|
||||
productionExecutionsOnly: true,
|
||||
};
|
||||
|
||||
const disabledSettings: OtelConfig = { ...enabledSettings, enabled: false };
|
||||
|
||||
const traceparentPattern = /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/;
|
||||
const markedContext = ROOT_CONTEXT.setValue(createContextKey('marker'), 'marked');
|
||||
|
||||
function activeContextInside(ctx: Context): Context {
|
||||
return context.with(ctx, () => context.active());
|
||||
}
|
||||
|
||||
function globalTracerProviderDelegate() {
|
||||
const globalProvider = trace.getTracerProvider();
|
||||
return globalProvider instanceof ProxyTracerProvider
|
||||
? globalProvider.getDelegate()
|
||||
: globalProvider;
|
||||
}
|
||||
|
||||
function registerForeignProvider() {
|
||||
const exporter = new InMemorySpanExporter();
|
||||
const provider = new NodeTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });
|
||||
provider.register();
|
||||
return { provider, exporter };
|
||||
}
|
||||
|
||||
function exportedSpanNames() {
|
||||
return exportedSpans.map((span) => span.name);
|
||||
}
|
||||
|
||||
describe('OtelService tracer provider', () => {
|
||||
const fetchMock = vi.fn();
|
||||
const outboundHttp = {
|
||||
transport: () => ({ asCustomFetch: () => fetchMock }),
|
||||
} as unknown as OutboundHttp;
|
||||
let otelSettingsService: ReturnType<typeof mock<OtelSettingsService>>;
|
||||
let logger: ReturnType<typeof mock<Logger>>;
|
||||
let service: OtelService;
|
||||
let foreign: ReturnType<typeof registerForeignProvider> | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
exportedSpans.length = 0;
|
||||
fetchMock.mockResolvedValue({ ok: true });
|
||||
otelSettingsService = mock<OtelSettingsService>();
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
logger = mock<Logger>();
|
||||
service = new OtelService(
|
||||
otelSettingsService,
|
||||
mock<InstanceSettings>({ instanceId: 'inst-1', instanceType: 'main' }),
|
||||
logger,
|
||||
outboundHttp,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await service.shutdown();
|
||||
await foreign?.provider.shutdown();
|
||||
foreign = undefined;
|
||||
trace.disable();
|
||||
context.disable();
|
||||
propagation.disable();
|
||||
});
|
||||
|
||||
describe('when another library owns the global API', () => {
|
||||
beforeEach(() => {
|
||||
foreign = registerForeignProvider();
|
||||
});
|
||||
|
||||
it('exports n8n spans through its own provider and leaves the foreign exporter empty', async () => {
|
||||
await service.init();
|
||||
|
||||
service.getTracer('n8n-workflow').startSpan('workflow.execute').end();
|
||||
|
||||
expect(exportedSpanNames()).toEqual(['workflow.execute']);
|
||||
expect(exportedSpans[0].resource.attributes).toMatchObject({
|
||||
[ATTR.OTEL_SERVICE_NAME]: 'n8n',
|
||||
[ATTR.OTEL_SERVICE_VERSION]: N8N_VERSION,
|
||||
[ATTR.INSTANCE_ID]: 'inst-1',
|
||||
[ATTR.INSTANCE_ROLE]: 'main',
|
||||
});
|
||||
expect(foreign?.exporter.getFinishedSpans()).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('keeps exporting after a restart and leaves the foreign globals in place', async () => {
|
||||
await service.init();
|
||||
await service.restart();
|
||||
|
||||
service.getTracer('n8n-workflow').startSpan('workflow.execute').end();
|
||||
trace.getTracer('foreign').startSpan('GET /webhook').end();
|
||||
|
||||
expect(exportedSpanNames()).toEqual(['workflow.execute']);
|
||||
expect(foreign?.exporter.getFinishedSpans().map((span) => span.name)).toEqual([
|
||||
'GET /webhook',
|
||||
]);
|
||||
expect(globalTracerProviderDelegate()).toBe(foreign?.provider);
|
||||
expect(activeContextInside(markedContext)).toBe(markedContext);
|
||||
});
|
||||
|
||||
it('logs the foreign owner once for the lifetime of the service', async () => {
|
||||
await service.init();
|
||||
await service.restart();
|
||||
|
||||
expect(logger.info).toHaveBeenCalledTimes(1);
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
expect.stringContaining('runs on its own tracer provider'),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('when the global API is free', () => {
|
||||
it('registers its provider, so third-party tracers export through the module', async () => {
|
||||
await service.init();
|
||||
|
||||
trace.getTracer('third-party').startSpan('http.request').end();
|
||||
const headers: Record<string, string> = {};
|
||||
const span = service.getTracer('n8n-workflow').startSpan('workflow.execute');
|
||||
propagation.inject(trace.setSpan(context.active(), span), headers);
|
||||
span.end();
|
||||
|
||||
expect(exportedSpanNames()).toEqual(['http.request', 'workflow.execute']);
|
||||
expect(headers.traceparent).toMatch(traceparentPattern);
|
||||
expect(activeContextInside(markedContext)).toBe(markedContext);
|
||||
expect(logger.info).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('swaps the registered provider on restart and keeps the context manager', async () => {
|
||||
await service.init();
|
||||
await service.restart();
|
||||
|
||||
trace.getTracer('third-party').startSpan('http.request').end();
|
||||
|
||||
expect(exportedSpanNames()).toEqual(['http.request']);
|
||||
expect(activeContextInside(markedContext)).toBe(markedContext);
|
||||
});
|
||||
|
||||
it('keeps the context manager after shutdown and hands out no-op tracers', async () => {
|
||||
await service.init();
|
||||
await service.shutdown();
|
||||
|
||||
expect(activeContextInside(markedContext)).toBe(markedContext);
|
||||
expect(service.getTracer('n8n-workflow').startSpan('workflow.execute').isRecording()).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('registers on the first enabled restart after a disabled boot', async () => {
|
||||
otelSettingsService.loadSettings
|
||||
.mockResolvedValueOnce(disabledSettings)
|
||||
.mockResolvedValueOnce(enabledSettings);
|
||||
await service.init();
|
||||
expect(service.getTracer('n8n-workflow').startSpan('workflow.execute').isRecording()).toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
await service.restart();
|
||||
service.getTracer('n8n-workflow').startSpan('workflow.execute').end();
|
||||
trace.getTracer('third-party').startSpan('http.request').end();
|
||||
|
||||
expect(exportedSpanNames()).toEqual(['workflow.execute', 'http.request']);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,9 +1,10 @@
|
|||
import type { Logger } from '@n8n/backend-common';
|
||||
import type { OutboundHttp } from '@n8n/backend-network';
|
||||
import { context, diag, metrics, propagation, trace } from '@opentelemetry/api';
|
||||
import { diag } from '@opentelemetry/api';
|
||||
import { OTLPTraceExporter as OTLPGrpcTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base';
|
||||
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
import type { InstanceSettings } from 'n8n-core';
|
||||
|
||||
|
|
@ -12,8 +13,9 @@ import type { OtelConfig } from '../otel.config';
|
|||
import { ATTR, OTEL_TEST_SPAN_NAME } from '../otel.constants';
|
||||
import { OtelService } from '../otel.service';
|
||||
|
||||
const start = vi.fn();
|
||||
const register = vi.fn();
|
||||
const shutdown = vi.fn();
|
||||
const providerGetTracer = vi.fn();
|
||||
|
||||
// The connectivity-check fetch, obtained via outboundHttp.transport().asCustomFetch().
|
||||
// checkEndpointReachability ignores the response and only catches network errors.
|
||||
|
|
@ -86,15 +88,6 @@ const mockSpanEnd = vi.fn();
|
|||
const mockStartSpan = vi.fn();
|
||||
const mockGetTracer = vi.fn();
|
||||
|
||||
vi.mock('@opentelemetry/sdk-node', () => ({
|
||||
NodeSDK: vi.fn().mockImplementation(function () {
|
||||
return {
|
||||
start,
|
||||
shutdown,
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('@opentelemetry/exporter-trace-otlp-proto', () => ({
|
||||
OTLPTraceExporter: vi.fn().mockImplementation(function () {
|
||||
return {
|
||||
|
|
@ -122,6 +115,7 @@ vi.mock('@grpc/grpc-js', () => ({
|
|||
}));
|
||||
|
||||
vi.mock('@opentelemetry/sdk-trace-base', () => ({
|
||||
BatchSpanProcessor: vi.fn(),
|
||||
BasicTracerProvider: vi.fn().mockImplementation(function (config: {
|
||||
spanProcessors?: unknown[];
|
||||
}) {
|
||||
|
|
@ -136,21 +130,24 @@ vi.mock('@opentelemetry/sdk-trace-base', () => ({
|
|||
}));
|
||||
|
||||
vi.mock('@opentelemetry/resources', () => ({
|
||||
resourceFromAttributes: vi.fn().mockReturnValue({}),
|
||||
resourceFromAttributes: vi.fn().mockReturnValue({ merge: () => ({}) }),
|
||||
detectResources: vi.fn().mockReturnValue({}),
|
||||
envDetector: {},
|
||||
processDetector: {},
|
||||
hostDetector: {},
|
||||
}));
|
||||
|
||||
vi.mock('@opentelemetry/sdk-trace-node', () => ({
|
||||
TraceIdRatioBasedSampler: vi.fn().mockImplementation(function () {
|
||||
return {};
|
||||
}),
|
||||
NodeTracerProvider: vi.fn().mockImplementation(function () {
|
||||
return { register, shutdown, getTracer: providerGetTracer };
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock('@opentelemetry/api', async () => ({
|
||||
...(await vi.importActual<typeof import('@opentelemetry/api')>('@opentelemetry/api')),
|
||||
trace: { disable: vi.fn() },
|
||||
context: { disable: vi.fn() },
|
||||
propagation: { disable: vi.fn() },
|
||||
metrics: { disable: vi.fn() },
|
||||
DiagLogLevel: { WARN: 'WARN' },
|
||||
diag: { setLogger: vi.fn() },
|
||||
}));
|
||||
|
|
@ -201,20 +198,20 @@ describe('OtelService', () => {
|
|||
});
|
||||
|
||||
describe('init', () => {
|
||||
it('does not start SDK when enabled is false', async () => {
|
||||
it('does not start a tracer provider when enabled is false', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValue(disabledSettings);
|
||||
|
||||
await service.init();
|
||||
|
||||
expect(start).not.toHaveBeenCalled();
|
||||
expect(NodeTracerProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('starts SDK when enabled is true', async () => {
|
||||
it('starts a tracer provider when enabled is true', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
|
||||
await service.init();
|
||||
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('builds the HTTP exporter with the traces path and no gRPC exporter', async () => {
|
||||
|
|
@ -241,7 +238,7 @@ describe('OtelService', () => {
|
|||
await service.init();
|
||||
await flushPromises();
|
||||
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to connect to OpenTelemetry OTLP endpoint during startup',
|
||||
expect.objectContaining({ endpoint: 'http://localhost:4318/v1/traces' }),
|
||||
|
|
@ -259,7 +256,7 @@ describe('OtelService', () => {
|
|||
expect.objectContaining({ url: 'http://collector.example.com:4317' }),
|
||||
);
|
||||
expect(OTLPTraceExporter).not.toHaveBeenCalled();
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('lowercases only the endpoint scheme before it reaches the exporter', async () => {
|
||||
|
|
@ -408,7 +405,7 @@ describe('OtelService', () => {
|
|||
await service.init();
|
||||
await flushPromises();
|
||||
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to connect to OpenTelemetry OTLP endpoint during startup',
|
||||
|
|
@ -427,7 +424,7 @@ describe('OtelService', () => {
|
|||
await flushPromises();
|
||||
|
||||
expect(newClient).not.toHaveBeenCalled();
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
'Failed to connect to OpenTelemetry OTLP endpoint during startup',
|
||||
expect.objectContaining({ endpoint: 'not a valid endpoint' }),
|
||||
|
|
@ -436,7 +433,7 @@ describe('OtelService', () => {
|
|||
});
|
||||
|
||||
describe('restart', () => {
|
||||
it('shuts down existing SDK then reloads settings and starts a new one', async () => {
|
||||
it('shuts down the existing tracer provider then reloads settings and starts a new one', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
|
||||
await service.init();
|
||||
|
|
@ -447,10 +444,10 @@ describe('OtelService', () => {
|
|||
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
expect(otelSettingsService.loadSettings).toHaveBeenCalledTimes(1);
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
expect(NodeTracerProvider).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not start SDK after restart when reloaded settings have enabled=false', async () => {
|
||||
it('does not start a tracer provider after restart when reloaded settings have enabled=false', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValueOnce(enabledSettings);
|
||||
otelSettingsService.loadSettings.mockResolvedValueOnce(disabledSettings);
|
||||
|
||||
|
|
@ -459,11 +456,11 @@ describe('OtelService', () => {
|
|||
|
||||
await service.restart();
|
||||
|
||||
expect(start).not.toHaveBeenCalled();
|
||||
expect(NodeTracerProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('SDK startup failure', () => {
|
||||
describe('tracer provider startup failure', () => {
|
||||
const exporterFailure = () => {
|
||||
vi.mocked(OTLPTraceExporter).mockImplementationOnce(function () {
|
||||
throw new Error('exporter unavailable');
|
||||
|
|
@ -476,7 +473,7 @@ describe('OtelService', () => {
|
|||
|
||||
await expect(service.init()).resolves.not.toThrow();
|
||||
|
||||
expect(start).not.toHaveBeenCalled();
|
||||
expect(NodeTracerProvider).not.toHaveBeenCalled();
|
||||
expect(fetchMock).not.toHaveBeenCalled();
|
||||
expect(logger.error).toHaveBeenCalledTimes(1);
|
||||
expect(logger.error).toHaveBeenCalledWith(
|
||||
|
|
@ -485,9 +482,9 @@ describe('OtelService', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('tears down partially installed providers when the SDK start call throws', async () => {
|
||||
it('shuts the provider down when registering it with the global API throws', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
start.mockImplementationOnce(() => {
|
||||
register.mockImplementationOnce(() => {
|
||||
throw new Error('provider registration failed');
|
||||
});
|
||||
|
||||
|
|
@ -498,7 +495,6 @@ describe('OtelService', () => {
|
|||
{ error: 'provider registration failed' },
|
||||
);
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
expect(trace.disable).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('leaves the service in a non-exporting state when a restart fails', async () => {
|
||||
|
|
@ -509,7 +505,7 @@ describe('OtelService', () => {
|
|||
|
||||
await expect(service.restart()).resolves.not.toThrow();
|
||||
|
||||
expect(start).not.toHaveBeenCalled();
|
||||
expect(NodeTracerProvider).not.toHaveBeenCalled();
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
|
||||
shutdown.mockClear();
|
||||
|
|
@ -520,23 +516,20 @@ describe('OtelService', () => {
|
|||
});
|
||||
|
||||
describe('shutdown', () => {
|
||||
it('disables all four OTel globals so the next SDK start can re-register providers', async () => {
|
||||
it('shuts the running provider down', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
await service.init();
|
||||
|
||||
await service.shutdown();
|
||||
|
||||
expect(trace.disable).toHaveBeenCalledTimes(1);
|
||||
expect(context.disable).toHaveBeenCalledTimes(1);
|
||||
expect(propagation.disable).toHaveBeenCalledTimes(1);
|
||||
expect(metrics.disable).toHaveBeenCalledTimes(1);
|
||||
expect(shutdown).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not throw when called before init', async () => {
|
||||
await expect(service.shutdown()).resolves.not.toThrow();
|
||||
});
|
||||
|
||||
it('does not throw when the SDK shutdown rejects (e.g. exporter flush failure)', async () => {
|
||||
it('does not throw when the provider shutdown rejects (e.g. exporter flush failure)', async () => {
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
await service.init();
|
||||
shutdown.mockRejectedValueOnce(new Error('connect ECONNREFUSED 127.0.0.1:9'));
|
||||
|
|
@ -545,6 +538,24 @@ describe('OtelService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('getTracer', () => {
|
||||
it('hands out a no-op tracer before a provider has started', () => {
|
||||
const span = service.getTracer('n8n-workflow').startSpan('workflow.execute');
|
||||
|
||||
expect(span.isRecording()).toBe(false);
|
||||
});
|
||||
|
||||
it("hands out the running provider's tracer", async () => {
|
||||
const tracer = { startSpan: vi.fn() };
|
||||
providerGetTracer.mockReturnValue(tracer);
|
||||
otelSettingsService.loadSettings.mockResolvedValue(enabledSettings);
|
||||
await service.init();
|
||||
|
||||
expect(service.getTracer('n8n-workflow')).toBe(tracer);
|
||||
expect(providerGetTracer).toHaveBeenCalledWith('n8n-workflow');
|
||||
});
|
||||
});
|
||||
|
||||
describe('diagnostics logger', () => {
|
||||
beforeEach(() => {
|
||||
(
|
||||
|
|
@ -806,7 +817,7 @@ describe('OtelService', () => {
|
|||
await service.sendTestTrace(connection);
|
||||
|
||||
expect(BasicTracerProvider).toHaveBeenCalledTimes(1);
|
||||
expect(start).not.toHaveBeenCalled();
|
||||
expect(NodeTracerProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
describe('over gRPC', () => {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,9 @@ import * as utils from '@test-integration/utils';
|
|||
|
||||
import { OtelTestProvider } from './otel-test-provider';
|
||||
import { TestNodeWithTracing } from './test-node-with-tracing';
|
||||
import { OtelSettingsService } from '../../otel-settings.service';
|
||||
import { OtelConfig } from '../../otel.config';
|
||||
import { OtelService } from '../../otel.service';
|
||||
|
||||
const BASE_DIR = path.resolve(__dirname, '../../../../../..');
|
||||
|
||||
|
|
@ -51,6 +53,8 @@ export async function initOtelTestEnvironment() {
|
|||
|
||||
await testModules.loadModules(['otel']);
|
||||
await testDb.init();
|
||||
Container.set(OtelService, otel.asOtelService());
|
||||
await Container.get(OtelSettingsService).loadSettings();
|
||||
await Container.get(ModuleRegistry).initModules('main');
|
||||
Container.get(LicenseState).setLicenseProvider({
|
||||
isLicensed: (feature) => feature === LICENSE_FEATURES.OTEL_CUSTOM_SPAN_ATTRIBUTES,
|
||||
|
|
|
|||
|
|
@ -1,17 +1,19 @@
|
|||
import type { Context, ContextManager } from '@opentelemetry/api';
|
||||
import { context, propagation, ROOT_CONTEXT, trace } from '@opentelemetry/api';
|
||||
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
||||
import { context, ROOT_CONTEXT, trace } from '@opentelemetry/api';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
InMemorySpanExporter,
|
||||
NodeTracerProvider,
|
||||
SimpleSpanProcessor,
|
||||
} from '@opentelemetry/sdk-trace-node';
|
||||
import { AsyncLocalStorage } from 'node:async_hooks';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import type { OtelService } from '../../otel.service';
|
||||
|
||||
/**
|
||||
* Minimal stand-in for `@opentelemetry/context-async-hooks`'
|
||||
* `AsyncLocalStorageContextManager` (the one `NodeSDK` registers by default in
|
||||
* production) — avoids adding that package as a test-only dependency. Without
|
||||
* `AsyncLocalStorageContextManager` (the one `NodeTracerProvider.register()`
|
||||
* installs in production) — avoids adding that package as a test-only dependency. Without
|
||||
* a real context manager registered, `context.with()` is a no-op (the default
|
||||
* `NoopContextManager` ignores the context it's given), so span nesting via
|
||||
* ambient context can't be exercised at all.
|
||||
|
|
@ -46,11 +48,14 @@ class AsyncLocalStorageTestContextManager implements ContextManager {
|
|||
}
|
||||
|
||||
/**
|
||||
* Disposable OTel test harness. Sets up an in-memory tracer
|
||||
* that captures spans for assertion, and tears down cleanly.
|
||||
* Disposable OTel test harness. Sets up an in-memory tracer provider that
|
||||
* captures spans for assertion, registers it as the global tracer provider
|
||||
* (so the module under test sees the slot taken, like it does next to
|
||||
* Sentry), and tears down cleanly.
|
||||
*
|
||||
* Usage:
|
||||
* const otel = OtelTestProvider.create();
|
||||
* const tracer = new ExecutionLevelTracer(otel.asOtelService(), ...);
|
||||
* // ... run code that creates spans ...
|
||||
* expect(otel.getFinishedSpans()).toHaveLength(1);
|
||||
* otel.reset(); // between tests
|
||||
|
|
@ -62,24 +67,28 @@ class AsyncLocalStorageTestContextManager implements ContextManager {
|
|||
*/
|
||||
export class OtelTestProvider {
|
||||
private constructor(
|
||||
private readonly provider: BasicTracerProvider,
|
||||
readonly provider: NodeTracerProvider,
|
||||
private readonly exporter: InMemorySpanExporter,
|
||||
private readonly contextManagerEnabled: boolean,
|
||||
) {}
|
||||
|
||||
static create(options: { withContextManager?: boolean } = {}): OtelTestProvider {
|
||||
const exporter = new InMemorySpanExporter();
|
||||
const provider = new BasicTracerProvider({
|
||||
const provider = new NodeTracerProvider({
|
||||
spanProcessors: [new SimpleSpanProcessor(exporter)],
|
||||
});
|
||||
trace.setGlobalTracerProvider(provider);
|
||||
propagation.setGlobalPropagator(new W3CTraceContextPropagator());
|
||||
if (options.withContextManager) {
|
||||
context.setGlobalContextManager(new AsyncLocalStorageTestContextManager());
|
||||
}
|
||||
return new OtelTestProvider(provider, exporter, options.withContextManager ?? false);
|
||||
}
|
||||
|
||||
/** An `OtelService` stand-in that hands out this provider's tracers. */
|
||||
asOtelService(): OtelService {
|
||||
return mock<OtelService>({ getTracer: (name: string) => this.provider.getTracer(name) });
|
||||
}
|
||||
|
||||
getFinishedSpans() {
|
||||
return this.exporter.getFinishedSpans();
|
||||
}
|
||||
|
|
@ -92,7 +101,6 @@ export class OtelTestProvider {
|
|||
this.exporter.reset();
|
||||
await this.provider.shutdown();
|
||||
trace.disable();
|
||||
propagation.disable();
|
||||
if (this.contextManagerEnabled) context.disable();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,15 @@
|
|||
import { Logger } from '@n8n/backend-common';
|
||||
import { Service } from '@n8n/di';
|
||||
import type { Context, Exception, Span } from '@opentelemetry/api';
|
||||
import { context, propagation, SpanStatusCode, trace } from '@opentelemetry/api';
|
||||
import {
|
||||
context,
|
||||
defaultTextMapGetter,
|
||||
defaultTextMapSetter,
|
||||
ROOT_CONTEXT,
|
||||
SpanStatusCode,
|
||||
trace,
|
||||
} from '@opentelemetry/api';
|
||||
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
||||
import type { ExecutionStatus } from 'n8n-workflow';
|
||||
|
||||
import {
|
||||
|
|
@ -13,9 +21,11 @@ import {
|
|||
} from './execution-level-tracer.types';
|
||||
import { OtelSettingsService } from './otel-settings.service';
|
||||
import { ATTR } from './otel.constants';
|
||||
import { OtelService } from './otel.service';
|
||||
import type { TracingContext } from './tracing-context';
|
||||
|
||||
const TRACER_NAME = 'n8n-workflow';
|
||||
const propagator = new W3CTraceContextPropagator();
|
||||
const UNKNOWN_ERROR_TYPE = 'UnknownError';
|
||||
const OBJECT_ERROR_TYPE = 'Object';
|
||||
|
||||
|
|
@ -24,31 +34,28 @@ function isError(status: ExecutionStatus): boolean {
|
|||
}
|
||||
|
||||
type TrackedSpan = { span: Span };
|
||||
type TrackedWorkflowSpan = TrackedSpan & { projectAttributes: Record<string, string> };
|
||||
|
||||
@Service()
|
||||
export class ExecutionLevelTracer {
|
||||
private readonly activeWorkflowSpans = new Map<string, TrackedSpan>();
|
||||
private readonly activeWorkflowSpans = new Map<string, TrackedWorkflowSpan>();
|
||||
private readonly activeNodeSpansByExecutionId = new Map<string, Map<string, TrackedSpan>>();
|
||||
private tracer = trace.getTracer(TRACER_NAME);
|
||||
|
||||
/**
|
||||
* Called by OtelService after a SDK restart so this instance picks up the
|
||||
* new NodeTracerProvider. Without this, the cached NodeTracer stays bound
|
||||
* to the old (shutdown) provider and all spans are silently dropped.
|
||||
*/
|
||||
refreshTracer(): void {
|
||||
this.tracer = trace.getTracer(TRACER_NAME);
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly otelService: OtelService,
|
||||
private readonly otelSettingsService: OtelSettingsService,
|
||||
private readonly logger: Logger,
|
||||
) {}
|
||||
|
||||
private get tracer() {
|
||||
return this.otelService.getTracer(TRACER_NAME);
|
||||
}
|
||||
|
||||
startWorkflow(params: StartWorkflowParams) {
|
||||
try {
|
||||
const parentCtx = this.parseTraceParentHeaders(params.tracingContext);
|
||||
const links = this.buildContinuationLinks(params.linkTo);
|
||||
const projectAttributes = buildProjectAttributes(params.project);
|
||||
|
||||
const span = this.tracer.startSpan(
|
||||
'workflow.execute',
|
||||
|
|
@ -59,21 +66,18 @@ export class ExecutionLevelTracer {
|
|||
[ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '',
|
||||
[ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount,
|
||||
[ATTR.EXECUTION_ID]: params.executionId,
|
||||
...(params.project?.id && { [ATTR.PROJECT_ID]: params.project.id }),
|
||||
...buildCustomAttributes(
|
||||
ATTR.WORKFLOW_CUSTOM_PREFIX,
|
||||
params.workflow?.customAttributes,
|
||||
),
|
||||
...buildCustomAttributes(ATTR.PROJECT_CUSTOM_PREFIX, params.project?.customAttributes),
|
||||
...projectAttributes,
|
||||
},
|
||||
links,
|
||||
},
|
||||
parentCtx,
|
||||
);
|
||||
|
||||
this.activeWorkflowSpans.set(params.executionId, {
|
||||
span,
|
||||
});
|
||||
this.activeWorkflowSpans.set(params.executionId, { span, projectAttributes });
|
||||
return toTracingParentContext(span);
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to start workflow span', {
|
||||
|
|
@ -122,10 +126,10 @@ export class ExecutionLevelTracer {
|
|||
|
||||
startNode(params: StartNodeParams): void {
|
||||
try {
|
||||
// We should always have the node running in a workflow so parentCtx should never be null
|
||||
const parentCtx = this.findWorkflowSpanContext(params.executionId);
|
||||
// We should always have the node running in a workflow so the tracked span should never be missing
|
||||
const tracked = this.activeWorkflowSpans.get(params.executionId);
|
||||
|
||||
if (!parentCtx) {
|
||||
if (!tracked) {
|
||||
this.logger.warn(
|
||||
'Trying to start a node without a pre-existing parent workflow trace - ignoring',
|
||||
);
|
||||
|
|
@ -140,9 +144,10 @@ export class ExecutionLevelTracer {
|
|||
[ATTR.NODE_NAME]: params.node.name,
|
||||
[ATTR.NODE_TYPE]: params.node.type,
|
||||
[ATTR.NODE_TYPE_VERSION]: params.node.typeVersion,
|
||||
...tracked.projectAttributes,
|
||||
},
|
||||
},
|
||||
parentCtx,
|
||||
trace.setSpan(context.active(), tracked.span),
|
||||
);
|
||||
|
||||
let executionNodes = this.activeNodeSpansByExecutionId.get(params.executionId);
|
||||
|
|
@ -220,7 +225,7 @@ export class ExecutionLevelTracer {
|
|||
const span = this.findMostSpecificSpan(executionId, nodeName);
|
||||
if (!span) return;
|
||||
|
||||
propagation.inject(trace.setSpan(context.active(), span), headers);
|
||||
propagator.inject(trace.setSpan(ROOT_CONTEXT, span), headers, defaultTextMapSetter);
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to inject trace headers', {
|
||||
executionId,
|
||||
|
|
@ -230,15 +235,14 @@ export class ExecutionLevelTracer {
|
|||
}
|
||||
}
|
||||
|
||||
private parseTraceParentHeaders(tracingContext?: TracingContext) {
|
||||
return tracingContext
|
||||
? propagation.extract(context.active(), tracingContext)
|
||||
: context.active();
|
||||
private parseTraceParentHeaders(tracingContext?: TracingContext): Context {
|
||||
if (!tracingContext) return ROOT_CONTEXT;
|
||||
return propagator.extract(ROOT_CONTEXT, tracingContext, defaultTextMapGetter);
|
||||
}
|
||||
|
||||
private buildContinuationLinks(linkTo?: TracingContext) {
|
||||
if (!linkTo) return undefined;
|
||||
const extracted = propagation.extract(context.active(), linkTo);
|
||||
const extracted = propagator.extract(ROOT_CONTEXT, linkTo, defaultTextMapGetter);
|
||||
const spanContext = trace.getSpanContext(extracted);
|
||||
if (!spanContext) return undefined;
|
||||
return [
|
||||
|
|
@ -249,11 +253,6 @@ export class ExecutionLevelTracer {
|
|||
];
|
||||
}
|
||||
|
||||
private findWorkflowSpanContext(executionId: string) {
|
||||
const tracked = this.activeWorkflowSpans.get(executionId);
|
||||
return tracked ? trace.setSpan(context.active(), tracked.span) : undefined;
|
||||
}
|
||||
|
||||
private findMostSpecificSpan(executionId: string, nodeName?: string): Span | undefined {
|
||||
return (
|
||||
(nodeName
|
||||
|
|
@ -286,6 +285,14 @@ function buildCustomAttributes(
|
|||
return result;
|
||||
}
|
||||
|
||||
function buildProjectAttributes(project: StartWorkflowParams['project']): Record<string, string> {
|
||||
if (!project) return {};
|
||||
return {
|
||||
[ATTR.PROJECT_ID]: project.id,
|
||||
...buildCustomAttributes(ATTR.PROJECT_CUSTOM_PREFIX, project.customAttributes),
|
||||
};
|
||||
}
|
||||
|
||||
function buildNodeEndAttributes(params: EndNodeParams): Record<string, string | number> {
|
||||
const attrs: Record<string, string | number> = {
|
||||
[ATTR.NODE_ITEMS_INPUT]: params.inputItemCount,
|
||||
|
|
@ -297,7 +304,7 @@ function buildNodeEndAttributes(params: EndNodeParams): Record<string, string |
|
|||
|
||||
function toTracingParentContext(span: Span): TracingContext {
|
||||
const carrier: Record<string, string> = {};
|
||||
propagation.inject(trace.setSpan(context.active(), span), carrier);
|
||||
propagator.inject(trace.setSpan(ROOT_CONTEXT, span), carrier, defaultTextMapSetter);
|
||||
return { traceparent: carrier.traceparent, tracestate: carrier.tracestate };
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,6 @@ export class OtelLifecycleHandler {
|
|||
@OnPubSubEvent('reload-otel-config')
|
||||
async onReloadOtelConfig(): Promise<void> {
|
||||
await this.otelService.restart();
|
||||
this.tracer.refreshTracer();
|
||||
}
|
||||
|
||||
private shouldTrace(ctx: { type: string; mode: WorkflowExecuteMode }): boolean {
|
||||
|
|
|
|||
|
|
@ -2,18 +2,24 @@ import type { Metadata } from '@grpc/grpc-js';
|
|||
import { Logger } from '@n8n/backend-common';
|
||||
import { OutboundHttp } from '@n8n/backend-network';
|
||||
import { Service } from '@n8n/di';
|
||||
import type { DiagLogger } from '@opentelemetry/api';
|
||||
import { DiagLogLevel, diag, context, metrics, propagation, trace } from '@opentelemetry/api';
|
||||
import type { DiagLogger, Tracer, TracerProvider } from '@opentelemetry/api';
|
||||
import { DiagLogLevel, ProxyTracerProvider, diag, trace } from '@opentelemetry/api';
|
||||
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
||||
import { resourceFromAttributes } from '@opentelemetry/resources';
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import {
|
||||
detectResources,
|
||||
envDetector,
|
||||
hostDetector,
|
||||
processDetector,
|
||||
resourceFromAttributes,
|
||||
} from '@opentelemetry/resources';
|
||||
import {
|
||||
BasicTracerProvider,
|
||||
BatchSpanProcessor,
|
||||
type ReadableSpan,
|
||||
type SpanExporter,
|
||||
type SpanProcessor,
|
||||
} from '@opentelemetry/sdk-trace-base';
|
||||
import { TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-node';
|
||||
import { NodeTracerProvider, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-node';
|
||||
import { InstanceSettings } from 'n8n-core';
|
||||
import { OperationalError } from 'n8n-workflow';
|
||||
|
||||
|
|
@ -29,10 +35,22 @@ export type OtelTestTraceResult = { success: true } | { success: false; error: s
|
|||
const stripEmptyResolutionNote = (message: string) =>
|
||||
message.replace(/\s*Resolution note:\s*$/, '');
|
||||
|
||||
// Deprecated in `@opentelemetry/api` 1.x, but the only public source of a no-op tracer provider.
|
||||
const noopTracerProvider = new ProxyTracerProvider();
|
||||
|
||||
function registeredGlobalTracerProvider(): TracerProvider | undefined {
|
||||
const globalProvider = trace.getTracerProvider();
|
||||
const delegate =
|
||||
globalProvider instanceof ProxyTracerProvider ? globalProvider.getDelegate() : globalProvider;
|
||||
return delegate === noopTracerProvider.getDelegate() ? undefined : delegate;
|
||||
}
|
||||
|
||||
@Service()
|
||||
export class OtelService {
|
||||
private static isDiagnosticsLoggerConfigured = false;
|
||||
private sdk?: NodeSDK;
|
||||
private provider?: NodeTracerProvider;
|
||||
private ownsGlobalApi = false;
|
||||
private hasLoggedForeignGlobalApiOwner = false;
|
||||
|
||||
constructor(
|
||||
private readonly otelSettingsService: OtelSettingsService,
|
||||
|
|
@ -52,9 +70,13 @@ export class OtelService {
|
|||
await this.start(settings);
|
||||
}
|
||||
|
||||
getTracer(name: string): Tracer {
|
||||
return (this.provider ?? noopTracerProvider).getTracer(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends a single `n8n.test_trace` span to the given OTLP endpoint and waits
|
||||
* for the exporter's result. Unlike the long-running SDK (which batches spans
|
||||
* for the exporter's result. Unlike the long-running provider (which batches spans
|
||||
* fire-and-forget), this uses a throwaway provider/exporter so the collector's
|
||||
* response — success, a rejection (HTTP status or gRPC status code), or a
|
||||
* network error — can be reported back to the caller. Runs independently of
|
||||
|
|
@ -111,9 +133,9 @@ export class OtelService {
|
|||
if (!settings.enabled) return;
|
||||
|
||||
this.configureDiagnosticsLogger();
|
||||
let sdk: NodeSDK;
|
||||
let provider: NodeTracerProvider;
|
||||
try {
|
||||
sdk = await this.startSdk(settings);
|
||||
provider = await this.startProvider(settings);
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to start OpenTelemetry tracing, so tracing stays off', {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
|
|
@ -122,47 +144,67 @@ export class OtelService {
|
|||
return;
|
||||
}
|
||||
|
||||
void this.checkEndpointReachability(settings, sdk);
|
||||
void this.checkEndpointReachability(settings, provider);
|
||||
}
|
||||
|
||||
async shutdown(): Promise<void> {
|
||||
// Cleared before the flush, so a probe that fails meanwhile sees that its SDK is gone.
|
||||
const sdk = this.sdk;
|
||||
this.sdk = undefined;
|
||||
// Cleared before the flush, so a probe that fails meanwhile sees that its provider is gone.
|
||||
const provider = this.provider;
|
||||
this.provider = undefined;
|
||||
try {
|
||||
await sdk?.shutdown();
|
||||
await provider?.shutdown();
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
'Failed to cleanly shut down OpenTelemetry SDK (exporter flush may have failed)',
|
||||
'Failed to cleanly shut down OpenTelemetry tracer provider (exporter flush may have failed)',
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
);
|
||||
} finally {
|
||||
// Unregister the global providers so the next NodeSDK.start() can register
|
||||
// new ones. Without this, OTel's allowOverride=false guard blocks
|
||||
// re-registration and the restart silently fails.
|
||||
trace?.disable();
|
||||
context?.disable();
|
||||
propagation?.disable();
|
||||
metrics?.disable();
|
||||
}
|
||||
}
|
||||
|
||||
private async startSdk(settings: OtelConfig): Promise<NodeSDK> {
|
||||
private async startProvider(settings: OtelConfig): Promise<NodeTracerProvider> {
|
||||
const traceExporter = await this.createTraceExporter(settings);
|
||||
|
||||
this.sdk = new NodeSDK({
|
||||
resource: resourceFromAttributes({
|
||||
[ATTR.OTEL_SERVICE_NAME]: settings.exporterServiceName,
|
||||
[ATTR.OTEL_SERVICE_VERSION]: N8N_VERSION,
|
||||
[ATTR.INSTANCE_ID]: this.instanceSettings.instanceId,
|
||||
[ATTR.INSTANCE_ROLE]: this.instanceSettings.instanceType,
|
||||
}),
|
||||
traceExporter,
|
||||
this.provider = new NodeTracerProvider({
|
||||
resource: this.buildResource(settings.exporterServiceName),
|
||||
sampler: new TraceIdRatioBasedSampler(settings.tracesSampleRate),
|
||||
spanProcessors: [new BatchSpanProcessor(traceExporter)],
|
||||
});
|
||||
this.registerGlobalApi(this.provider);
|
||||
return this.provider;
|
||||
}
|
||||
|
||||
this.sdk.start();
|
||||
return this.sdk;
|
||||
private buildResource(serviceName: string) {
|
||||
return resourceFromAttributes({
|
||||
[ATTR.OTEL_SERVICE_NAME]: serviceName,
|
||||
[ATTR.OTEL_SERVICE_VERSION]: N8N_VERSION,
|
||||
[ATTR.INSTANCE_ID]: this.instanceSettings.instanceId,
|
||||
[ATTR.INSTANCE_ROLE]: this.instanceSettings.instanceType,
|
||||
}).merge(detectResources({ detectors: [envDetector, processDetector, hostDetector] }));
|
||||
}
|
||||
|
||||
private registerGlobalApi(provider: NodeTracerProvider): void {
|
||||
if (this.ownsGlobalApi) {
|
||||
trace.disable();
|
||||
this.ownsGlobalApi = trace.setGlobalTracerProvider(provider);
|
||||
return;
|
||||
}
|
||||
|
||||
if (registeredGlobalTracerProvider()) {
|
||||
this.logForeignGlobalApiOwner();
|
||||
return;
|
||||
}
|
||||
|
||||
provider.register();
|
||||
this.ownsGlobalApi = registeredGlobalTracerProvider() === provider;
|
||||
}
|
||||
|
||||
private logForeignGlobalApiOwner(): void {
|
||||
if (this.hasLoggedForeignGlobalApiOwner) return;
|
||||
|
||||
this.hasLoggedForeignGlobalApiOwner = true;
|
||||
this.logger.info(
|
||||
'Another library owns the global OpenTelemetry API, so n8n workflow tracing runs on its own tracer provider',
|
||||
);
|
||||
}
|
||||
|
||||
private async createTraceExporter(
|
||||
|
|
@ -284,7 +326,7 @@ export class OtelService {
|
|||
|
||||
private async checkEndpointReachability(
|
||||
settings: OtelConfig,
|
||||
startedSdk: NodeSDK,
|
||||
startedProvider: NodeTracerProvider,
|
||||
): Promise<void> {
|
||||
const url = this.resolveExporterUrl(settings);
|
||||
const timeoutMs = settings.startupConnectivityTimeoutMs;
|
||||
|
|
@ -296,8 +338,8 @@ export class OtelService {
|
|||
await this.probeHttpEndpoint(url, timeoutMs);
|
||||
}
|
||||
} catch (error) {
|
||||
// A restart or shutdown replaced the SDK this probe belongs to.
|
||||
if (this.sdk !== startedSdk) return;
|
||||
// A restart or shutdown replaced the provider this probe belongs to.
|
||||
if (this.provider !== startedProvider) return;
|
||||
|
||||
this.logger.error('Failed to connect to OpenTelemetry OTLP endpoint during startup', {
|
||||
endpoint: url,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { Time } from '@n8n/constants';
|
||||
|
||||
import type { PubSub } from './pubsub/pubsub.types';
|
||||
|
||||
export const JOB_TYPE_NAME = 'job';
|
||||
|
|
@ -11,6 +13,12 @@ export const WORKER_RESPONSE_PUBSUB_CHANNEL = 'n8n.worker-response';
|
|||
/** Pubsub channel for MCP relay messages between main instances in multi-main queue mode. */
|
||||
export const MCP_RELAY_PUBSUB_CHANNEL = 'n8n.mcp-relay';
|
||||
|
||||
/** How often to re-issue SUBSCRIBE on the pubsub subscriber connection. Short enough to keep idle NAT/IPVS flows alive. */
|
||||
export const SUBSCRIBER_LIVENESS_INTERVAL_MS = 30 * Time.seconds.toMilliseconds;
|
||||
|
||||
/** How long to wait for the SUBSCRIBE reply before dropping the subscriber connection so ioredis reconnects. */
|
||||
export const SUBSCRIBER_LIVENESS_TIMEOUT_MS = 10 * Time.seconds.toMilliseconds;
|
||||
|
||||
/**
|
||||
* Max allowed size in bytes of a message relayed over the pubsub channel. Events
|
||||
* exceeding this are skipped (or trimmed) rather than bloating the channel.
|
||||
|
|
|
|||
|
|
@ -20,6 +20,12 @@ describe('Subscriber', () => {
|
|||
const executionsConfig = mockInstance(ExecutionsConfig, { mode: 'queue' });
|
||||
const globalConfig = mockInstance(GlobalConfig, { redis: { prefix: 'n8n' } });
|
||||
|
||||
function getHandler(event: string) {
|
||||
const call = client.on.mock.calls.find(([e]) => e === event);
|
||||
expect(call).toBeDefined();
|
||||
return call![1] as () => void;
|
||||
}
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should init Redis client in scaling mode', () => {
|
||||
const subscriber = new Subscriber(
|
||||
|
|
@ -82,6 +88,149 @@ describe('Subscriber', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('reconnect', () => {
|
||||
beforeEach(() => {
|
||||
client.on.mockClear();
|
||||
client.subscribe.mockClear();
|
||||
});
|
||||
|
||||
it('should resubscribe to all channels when the client recovers from a lost connection', async () => {
|
||||
const subscriber = new Subscriber(
|
||||
mockLogger(),
|
||||
mock(),
|
||||
mock(),
|
||||
redisClientService,
|
||||
executionsConfig,
|
||||
globalConfig,
|
||||
);
|
||||
await subscriber.subscribe(subscriber.getCommandChannel());
|
||||
await subscriber.subscribe(subscriber.getWorkerResponseChannel());
|
||||
client.subscribe.mockClear();
|
||||
|
||||
getHandler('close')();
|
||||
getHandler('ready')();
|
||||
await vi.waitFor(() => expect(client.subscribe).toHaveBeenCalledTimes(2));
|
||||
|
||||
expect(client.subscribe).toHaveBeenCalledWith('n8n:n8n.commands', expect.any(Function));
|
||||
expect(client.subscribe).toHaveBeenCalledWith(
|
||||
'n8n:n8n.worker-response',
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it('should retry on the next ready when resubscribing fails', async () => {
|
||||
const scopedLogger = mock<Logger>();
|
||||
const logger = mock<Logger>({ scoped: vi.fn().mockReturnValue(scopedLogger) });
|
||||
const subscriber = new Subscriber(
|
||||
logger,
|
||||
mock(),
|
||||
mock(),
|
||||
redisClientService,
|
||||
executionsConfig,
|
||||
globalConfig,
|
||||
);
|
||||
await subscriber.subscribe(subscriber.getCommandChannel());
|
||||
client.subscribe.mockClear();
|
||||
client.subscribe.mockRejectedValueOnce(new Error('write ECONNRESET'));
|
||||
|
||||
getHandler('close')();
|
||||
getHandler('ready')();
|
||||
await vi.waitFor(() =>
|
||||
expect(scopedLogger.error).toHaveBeenCalledWith(
|
||||
'Failed to resubscribe to pubsub channels after Redis reconnect',
|
||||
expect.any(Object),
|
||||
),
|
||||
);
|
||||
|
||||
getHandler('ready')();
|
||||
await vi.waitFor(() => expect(client.subscribe).toHaveBeenCalledTimes(2));
|
||||
});
|
||||
|
||||
it('should not resubscribe on the initial ready', async () => {
|
||||
const subscriber = new Subscriber(
|
||||
mockLogger(),
|
||||
mock(),
|
||||
mock(),
|
||||
redisClientService,
|
||||
executionsConfig,
|
||||
globalConfig,
|
||||
);
|
||||
await subscriber.subscribe(subscriber.getCommandChannel());
|
||||
client.subscribe.mockClear();
|
||||
|
||||
getHandler('ready')();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(client.subscribe).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('liveness', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
client.subscribe.mockReset();
|
||||
client.disconnect.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
async function subscribedSubscriber() {
|
||||
const subscriber = new Subscriber(
|
||||
mockLogger(),
|
||||
mock(),
|
||||
mock(),
|
||||
redisClientService,
|
||||
executionsConfig,
|
||||
globalConfig,
|
||||
);
|
||||
await subscriber.subscribe(subscriber.getCommandChannel());
|
||||
await subscriber.subscribe(subscriber.getWorkerResponseChannel());
|
||||
client.subscribe.mockReset();
|
||||
return subscriber;
|
||||
}
|
||||
|
||||
it('should re-issue SUBSCRIBE for all channels periodically', async () => {
|
||||
const subscriber = await subscribedSubscriber();
|
||||
client.subscribe.mockResolvedValue(2);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000 + 10_000);
|
||||
|
||||
expect(client.subscribe).toHaveBeenCalledTimes(1);
|
||||
expect(client.subscribe).toHaveBeenCalledWith('n8n:n8n.commands', 'n8n:n8n.worker-response');
|
||||
expect(client.disconnect).not.toHaveBeenCalled();
|
||||
subscriber.shutdown();
|
||||
});
|
||||
|
||||
it('should drop the connection when SUBSCRIBE gets no reply in time', async () => {
|
||||
const subscriber = await subscribedSubscriber();
|
||||
client.subscribe.mockReturnValue(new Promise(() => {}));
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000 + 10_000);
|
||||
|
||||
expect(client.subscribe).toHaveBeenCalledTimes(1);
|
||||
expect(client.disconnect).toHaveBeenCalledWith(true);
|
||||
subscriber.shutdown();
|
||||
});
|
||||
|
||||
it('should not check before any channel was requested', async () => {
|
||||
const subscriber = new Subscriber(
|
||||
mockLogger(),
|
||||
mock(),
|
||||
mock(),
|
||||
redisClientService,
|
||||
executionsConfig,
|
||||
globalConfig,
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
|
||||
expect(client.subscribe).not.toHaveBeenCalled();
|
||||
subscriber.shutdown();
|
||||
});
|
||||
});
|
||||
|
||||
describe('prefix isolation', () => {
|
||||
it('should apply configured prefix when subscribing to channels', async () => {
|
||||
const customConfig = mockInstance(GlobalConfig, { redis: { prefix: 'n8n-instance-1' } });
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import {
|
|||
COMMAND_PUBSUB_CHANNEL,
|
||||
WORKER_RESPONSE_PUBSUB_CHANNEL,
|
||||
MCP_RELAY_PUBSUB_CHANNEL,
|
||||
SUBSCRIBER_LIVENESS_INTERVAL_MS,
|
||||
SUBSCRIBER_LIVENESS_TIMEOUT_MS,
|
||||
} from '../constants';
|
||||
|
||||
/**
|
||||
|
|
@ -45,6 +47,12 @@ export class Subscriber {
|
|||
|
||||
private readonly debouncedHandlers = new Map<string, ReturnType<typeof debounce>>();
|
||||
|
||||
private readonly channels = new Set<string>();
|
||||
|
||||
private lostConnection = false;
|
||||
|
||||
private livenessTimer?: NodeJS.Timeout;
|
||||
|
||||
constructor(
|
||||
private readonly logger: Logger,
|
||||
private readonly instanceSettings: InstanceSettings,
|
||||
|
|
@ -66,6 +74,21 @@ export class Subscriber {
|
|||
|
||||
this.client = this.redisClientService.createClient({ type: 'subscriber(n8n)' });
|
||||
|
||||
// ioredis replays SUBSCRIBE on reconnect without awaiting it, so a failed replay leaves
|
||||
// the connection ready with zero subscriptions. Re-issue our own on every recovery.
|
||||
this.client.on('close', () => {
|
||||
this.lostConnection = true;
|
||||
});
|
||||
this.client.on('ready', () => {
|
||||
if (!this.lostConnection) return;
|
||||
this.lostConnection = false;
|
||||
void this.resubscribe();
|
||||
});
|
||||
|
||||
this.livenessTimer = setInterval(async () => {
|
||||
await this.checkLiveness();
|
||||
}, SUBSCRIBER_LIVENESS_INTERVAL_MS).unref();
|
||||
|
||||
const handlerFn = (msg: PubSub.Command | PubSub.WorkerResponse) => {
|
||||
this.pubsubEventBus.emit(this.eventNameFrom(msg), msg.payload);
|
||||
};
|
||||
|
|
@ -145,11 +168,13 @@ export class Subscriber {
|
|||
|
||||
// @TODO: Use `@OnShutdown()` decorator
|
||||
shutdown() {
|
||||
clearInterval(this.livenessTimer);
|
||||
for (const handler of this.debouncedHandlers.values()) handler.cancel();
|
||||
this.client.disconnect();
|
||||
}
|
||||
|
||||
async subscribe(channel: string) {
|
||||
this.channels.add(channel);
|
||||
await this.client.subscribe(channel, (error) => {
|
||||
if (error) {
|
||||
this.logger.error(`Failed to subscribe to channel ${channel}`, { error });
|
||||
|
|
@ -160,6 +185,43 @@ export class Subscriber {
|
|||
});
|
||||
}
|
||||
|
||||
private async resubscribe() {
|
||||
if (this.channels.size === 0) return;
|
||||
try {
|
||||
for (const channel of this.channels) await this.subscribe(channel);
|
||||
this.logger.info('Resubscribed to pubsub channels after Redis reconnect', {
|
||||
channels: [...this.channels],
|
||||
});
|
||||
} catch (error) {
|
||||
this.lostConnection = true;
|
||||
this.logger.error('Failed to resubscribe to pubsub channels after Redis reconnect', {
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A subscriber connection is idle by nature, so a half-open socket (peer gone, no RST
|
||||
* received) is never detected by ioredis. Re-issuing SUBSCRIBE is idempotent, travels over
|
||||
* the subscriber connection in both single-node and cluster mode (PING would not), and
|
||||
* confirms the subscriptions still exist. No reply in time drops the socket so ioredis
|
||||
* reconnects and `resubscribe` runs.
|
||||
*/
|
||||
private async checkLiveness() {
|
||||
if (this.channels.size === 0) return;
|
||||
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
setTimeout(() => reject(new Error('timeout')), SUBSCRIBER_LIVENESS_TIMEOUT_MS).unref();
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([this.client.subscribe(...this.channels), timeout]);
|
||||
} catch (error) {
|
||||
this.logger.warn('Pubsub subscriber connection is unresponsive, reconnecting', { error });
|
||||
this.client.disconnect(true);
|
||||
}
|
||||
}
|
||||
|
||||
private eventNameFrom(msg: PubSub.Command | PubSub.WorkerResponse) {
|
||||
return 'command' in msg ? msg.command : msg.response;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "n8n-core",
|
||||
"version": "2.41.0",
|
||||
"version": "2.41.2",
|
||||
"description": "Core functionality of n8n",
|
||||
"main": "dist/index",
|
||||
"types": "dist/index.d.ts",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@n8n/chat",
|
||||
"version": "1.40.0",
|
||||
"version": "1.40.1",
|
||||
"scripts": {
|
||||
"clean": "rimraf dist .turbo",
|
||||
"dev": "pnpm run --dir=../storybook dev --initial-path=/docs/chat-chat--docs",
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user