mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 21:49:36 +02:00
feat(core): Add a guardrail runner to the agent loop (#39621)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Anne Aguirre
GitHub
parent
5b499d512f
commit
3745c835b2
|
|
@ -0,0 +1,190 @@
|
|||
[
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a calculator. When asked to add numbers, use the add_numbers tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is 2 + 3? Use the tool once. If the tool fails, say so in one sentence."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "add_numbers",
|
||||
"description": "Add two numbers together and return the result",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "number",
|
||||
"description": "First number"
|
||||
},
|
||||
"b": {
|
||||
"type": "number",
|
||||
"description": "Second number"
|
||||
}
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMypYKp2r2Wk9BhvPE6\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01AvHg8o8w8rvyMtUJqa9Zxy\",\"name\":\"add_numbers\",\"input\":{\"a\":2,\"b\":3},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":661,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":71,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:43:01Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:43:01Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:43:01Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:43:01Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a4023737eaa88168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:43:01 GMT",
|
||||
"request-id": "req_011CfNMypGwuiyYNFo85wMx3",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-1d7d2a59dc02bae4c3a2d2d7f6dd7f77-609995466a131c70-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
},
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a calculator. When asked to add numbers, use the add_numbers tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is 2 + 3? Use the tool once. If the tool fails, say so in one sentence."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01AvHg8o8w8rvyMtUJqa9Zxy",
|
||||
"name": "add_numbers",
|
||||
"input": {
|
||||
"a": 2,
|
||||
"b": 3
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01AvHg8o8w8rvyMtUJqa9Zxy",
|
||||
"content": "Error: Tool call stopped by guardrail: tool.blocked",
|
||||
"is_error": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "add_numbers",
|
||||
"description": "Add two numbers together and return the result",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "number",
|
||||
"description": "First number"
|
||||
},
|
||||
"b": {
|
||||
"type": "number",
|
||||
"description": "Second number"
|
||||
}
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMystyrRQQwxMfaZbR9\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The tool failed to execute due to a guardrail block.\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":765,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":16,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:43:02Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:43:02Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:43:02Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:43:02Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a402373cec268168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:43:02 GMT",
|
||||
"request-id": "req_011CfNMysebz3hvMi5bsjqLr",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-4d97d181f5846c7c30e698bbe47e6ea5-5e40d87c0e2e651d-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
[
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a file manager. When asked to delete a file, use the delete_file tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Delete the file /tmp/guardrail.txt"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "delete_file",
|
||||
"description": "Delete a file at the given path",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "File path to delete"
|
||||
}
|
||||
},
|
||||
"required": ["path"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMyveSbeictGQzrWfvq\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_019Wed5RVopf1ThLeEUf7Ndj\",\"name\":\"delete_file\",\"input\":{\"path\":\"/tmp/guardrail.txt\"},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":630,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":61,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:43:03Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:43:03Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:43:02Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:43:03Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a4023740c9728168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:43:03 GMT",
|
||||
"request-id": "req_011CfNMyvMKL29koHpHie9Qx",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-4533af4453ae3fd61a218dec0dcb5edb-77739d4e760ba069-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
},
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a file manager. When asked to delete a file, use the delete_file tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Delete the file /tmp/guardrail.txt"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_019Wed5RVopf1ThLeEUf7Ndj",
|
||||
"name": "delete_file",
|
||||
"input": {
|
||||
"path": "/tmp/guardrail.txt"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_019Wed5RVopf1ThLeEUf7Ndj",
|
||||
"content": "{\"deleted\":true,\"path\":\"/tmp/guardrail.txt\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "delete_file",
|
||||
"description": "Delete a file at the given path",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "File path to delete"
|
||||
}
|
||||
},
|
||||
"required": ["path"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMyz92vUy6yf98GFnqw\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"The file `/tmp/guardrail.txt` has been successfully deleted.\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":719,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":19,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:43:03Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:43:03Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:43:03Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:43:03Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a40237460f158168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:43:03 GMT",
|
||||
"request-id": "req_011CfNMyyttob2JoJoZYSS2P",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-4ea94ac832c02f45d872fda225fdc80d-b5a3a403193144cc-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1 @@
|
|||
[]
|
||||
|
|
@ -0,0 +1 @@
|
|||
[]
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
[
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a calculator. When asked to add numbers, use the add_numbers tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is 2 + 3? Use the tool."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "add_numbers",
|
||||
"description": "Add two numbers together and return the result",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "number",
|
||||
"description": "First number"
|
||||
},
|
||||
"b": {
|
||||
"type": "number",
|
||||
"description": "Second number"
|
||||
}
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMyfHD9pXKUCYZXWbu6\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01Dj6HQiSmYctMpqzcE1Lw9n\",\"name\":\"add_numbers\",\"input\":{\"a\":2,\"b\":3},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":649,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":71,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:42:59Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:42:59Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:42:59Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:42:59Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a4023729bc6d8168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:42:59 GMT",
|
||||
"request-id": "req_011CfNMyebnKy4nzubssNvTo",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-e3ba3820e5aa2d3227d8033792438e24-b469f989fcd37da0-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
},
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a calculator. When asked to add numbers, use the add_numbers tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is 2 + 3? Use the tool."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01Dj6HQiSmYctMpqzcE1Lw9n",
|
||||
"name": "add_numbers",
|
||||
"input": {
|
||||
"a": 2,
|
||||
"b": 3
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_result",
|
||||
"tool_use_id": "toolu_01Dj6HQiSmYctMpqzcE1Lw9n",
|
||||
"content": "{\"result\":5}"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "add_numbers",
|
||||
"description": "Add two numbers together and return the result",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "number",
|
||||
"description": "First number"
|
||||
},
|
||||
"b": {
|
||||
"type": "number",
|
||||
"description": "Second number"
|
||||
}
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMyiZQk7PxBFXNx2mZf\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"text\",\"text\":\"2 + 3 = **5**\"}],\"container\":null,\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":736,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":13,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "9999000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:43:00Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:43:00Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:42:59Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "11999000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:43:00Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a402372f28488168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:43:00 GMT",
|
||||
"request-id": "req_011CfNMyiKGd6qhCaXEdb1Pm",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-6bebf8ce8daaf6361a000d6410ff062c-9c1afba04f672cdf-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
[
|
||||
{
|
||||
"scope": "https://api.anthropic.com:443",
|
||||
"method": "POST",
|
||||
"path": "/v1/messages",
|
||||
"body": {
|
||||
"model": "claude-haiku-4-5",
|
||||
"max_tokens": 64000,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are a calculator. When asked to add numbers, use the add_numbers tool. Be concise."
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is 2 + 3? Use the tool."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "add_numbers",
|
||||
"description": "Add two numbers together and return the result",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"a": {
|
||||
"type": "number",
|
||||
"description": "First number"
|
||||
},
|
||||
"b": {
|
||||
"type": "number",
|
||||
"description": "Second number"
|
||||
}
|
||||
},
|
||||
"required": ["a", "b"],
|
||||
"additionalProperties": false,
|
||||
"$schema": "http://json-schema.org/draft-07/schema#"
|
||||
}
|
||||
}
|
||||
],
|
||||
"tool_choice": {
|
||||
"type": "auto"
|
||||
}
|
||||
},
|
||||
"status": 200,
|
||||
"response": "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CfNMykyXDFPEgh9h3Be7z\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_01YaASNbMtirUpkLmjJFMDp9\",\"name\":\"add_numbers\",\"input\":{\"a\":2,\"b\":3},\"caller\":{\"type\":\"direct\"}}],\"container\":null,\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":649,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":71,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"},\"diagnostics\":null}",
|
||||
"rawHeaders": {
|
||||
"anthropic-ratelimit-input-tokens-limit": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-remaining": "10000000",
|
||||
"anthropic-ratelimit-input-tokens-reset": "2026-09-24T13:43:00Z",
|
||||
"anthropic-ratelimit-output-tokens-limit": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-remaining": "2000000",
|
||||
"anthropic-ratelimit-output-tokens-reset": "2026-09-24T13:43:00Z",
|
||||
"anthropic-ratelimit-requests-limit": "20000",
|
||||
"anthropic-ratelimit-requests-remaining": "19999",
|
||||
"anthropic-ratelimit-requests-reset": "2026-09-24T13:43:00Z",
|
||||
"anthropic-ratelimit-tokens-limit": "12000000",
|
||||
"anthropic-ratelimit-tokens-remaining": "12000000",
|
||||
"anthropic-ratelimit-tokens-reset": "2026-09-24T13:43:00Z",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"cf-ray": "a4023732be728168-LHR",
|
||||
"connection": "keep-alive",
|
||||
"content-security-policy": "default-src 'none'; frame-ancestors 'none'",
|
||||
"content-type": "application/json",
|
||||
"date": "Thu, 24 Sep 2026 13:43:01 GMT",
|
||||
"request-id": "req_011CfNMykit5QN5zvHPdwZLh",
|
||||
"server": "cloudflare",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
|
||||
"traceresponse": "00-d5ba23ebec625ed0fc5f447f57b53c49-fc09062d845260f3-01",
|
||||
"vary": "Accept-Encoding",
|
||||
"x-robots-tag": "none"
|
||||
},
|
||||
"responseIsBinary": false
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
import { expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
chunksOfType,
|
||||
collectStreamChunks,
|
||||
createAgentWithAddTool,
|
||||
createAgentWithInterruptibleTool,
|
||||
describeIf,
|
||||
findAllToolResults,
|
||||
findLastTextContent,
|
||||
getModel,
|
||||
} from './helpers';
|
||||
import { Agent } from '../../index';
|
||||
import type {
|
||||
GuardrailDecision,
|
||||
GuardrailModelCallContext,
|
||||
GuardrailToolCallContext,
|
||||
GuardrailsOptions,
|
||||
ModelGuardrail,
|
||||
TokenUsage,
|
||||
} from '../../types';
|
||||
|
||||
const describe = describeIf('anthropic');
|
||||
|
||||
const IDS = { agentId: 'agent-1', threadId: 'thread-1' };
|
||||
|
||||
type Decide = {
|
||||
/** Called with the 1-based index of the model call. */
|
||||
before?: (callNumber: number) => GuardrailDecision | undefined;
|
||||
beforeTool?: (ctx: GuardrailToolCallContext) => GuardrailDecision | undefined;
|
||||
};
|
||||
|
||||
/** A hook that records every call and decides through the given functions. */
|
||||
function recordingGuardrail(decide: Decide = {}) {
|
||||
const calls = {
|
||||
before: [] as GuardrailModelCallContext[],
|
||||
after: [] as Array<{ ctx: GuardrailModelCallContext; usage: TokenUsage | undefined }>,
|
||||
beforeTool: [] as GuardrailToolCallContext[],
|
||||
afterTool: [] as Array<{ ctx: GuardrailToolCallContext; result: unknown }>,
|
||||
};
|
||||
const hook: ModelGuardrail = {
|
||||
async before(ctx) {
|
||||
calls.before.push(ctx);
|
||||
return await Promise.resolve(decide.before?.(calls.before.length));
|
||||
},
|
||||
async after(ctx, usage) {
|
||||
calls.after.push({ ctx, usage });
|
||||
await Promise.resolve();
|
||||
},
|
||||
async beforeTool(ctx) {
|
||||
calls.beforeTool.push(ctx);
|
||||
return await Promise.resolve(decide.beforeTool?.(ctx));
|
||||
},
|
||||
async afterTool(ctx, result) {
|
||||
calls.afterTool.push({ ctx, result });
|
||||
await Promise.resolve();
|
||||
},
|
||||
};
|
||||
const guardrails: GuardrailsOptions = { hooks: [hook], ...IDS };
|
||||
return { guardrails, calls };
|
||||
}
|
||||
|
||||
const stop = (code: string): GuardrailDecision => ({ action: 'stop', code });
|
||||
|
||||
describe('guardrails integration', () => {
|
||||
it('refuses the first model call in generate', async () => {
|
||||
const { guardrails, calls } = recordingGuardrail({ before: () => stop('test.first') });
|
||||
const agent = new Agent('guardrail-first-call')
|
||||
.model(getModel('anthropic'))
|
||||
.instructions('You are a concise assistant. Reply with one short sentence.');
|
||||
|
||||
const result = await agent.generate('Say hello.', { guardrails });
|
||||
|
||||
expect(result.finishReason).toBe('guardrail');
|
||||
expect(result.guardrail).toEqual({ code: 'test.first' });
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(findLastTextContent(result.messages)).toBeUndefined();
|
||||
expect(calls.before).toHaveLength(1);
|
||||
expect(calls.after).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('refuses the first model call in stream', async () => {
|
||||
const { guardrails } = recordingGuardrail({ before: () => stop('test.first') });
|
||||
const agent = new Agent('guardrail-first-call-stream')
|
||||
.model(getModel('anthropic'))
|
||||
.instructions('You are a concise assistant. Reply with one short sentence.');
|
||||
|
||||
const { stream } = await agent.stream('Say hello.', { guardrails });
|
||||
const chunks = await collectStreamChunks(stream);
|
||||
|
||||
const finishChunks = chunksOfType(chunks, 'finish');
|
||||
expect(finishChunks).toHaveLength(1);
|
||||
expect(finishChunks[0]).toMatchObject({
|
||||
finishReason: 'guardrail',
|
||||
guardrail: { code: 'test.first' },
|
||||
});
|
||||
expect(chunksOfType(chunks, 'text-delta')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('sees every model call and its usage', async () => {
|
||||
const { guardrails, calls } = recordingGuardrail();
|
||||
const agent = createAgentWithAddTool('anthropic');
|
||||
|
||||
const result = await agent.generate('What is 2 + 3? Use the tool.', { guardrails });
|
||||
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(calls.before.length).toBeGreaterThanOrEqual(2);
|
||||
expect(calls.after).toHaveLength(calls.before.length);
|
||||
|
||||
const callIds = new Set(calls.before.map((ctx) => ctx.callId));
|
||||
expect(callIds.size).toBe(calls.before.length);
|
||||
for (const ctx of calls.before) {
|
||||
expect(ctx).toMatchObject({ source: 'turn', model: getModel('anthropic'), ...IDS });
|
||||
}
|
||||
|
||||
let totalTokens = 0;
|
||||
for (const { usage } of calls.after) {
|
||||
expect(usage?.totalTokens).toBeGreaterThan(0);
|
||||
totalTokens += usage?.totalTokens ?? 0;
|
||||
}
|
||||
expect(totalTokens).toBe(result.usage?.totalTokens);
|
||||
});
|
||||
|
||||
it('stops the run after the crossing call', async () => {
|
||||
const { guardrails, calls } = recordingGuardrail({
|
||||
before: (callNumber) => (callNumber === 1 ? { action: 'allow' } : stop('test.second')),
|
||||
});
|
||||
const agent = createAgentWithAddTool('anthropic');
|
||||
|
||||
const result = await agent.generate('What is 2 + 3? Use the tool.', { guardrails });
|
||||
|
||||
const toolResults = findAllToolResults(result.messages);
|
||||
expect(toolResults).toHaveLength(1);
|
||||
expect(toolResults[0]).toMatchObject({
|
||||
toolName: 'add_numbers',
|
||||
state: 'resolved',
|
||||
output: { result: 5 },
|
||||
});
|
||||
expect(calls.after).toHaveLength(1);
|
||||
expect(result.finishReason).toBe('guardrail');
|
||||
expect(result.guardrail).toEqual({ code: 'test.second' });
|
||||
});
|
||||
|
||||
it('blocks one tool call and lets the model continue', async () => {
|
||||
const { guardrails, calls } = recordingGuardrail({
|
||||
beforeTool: (ctx) => (ctx.toolName === 'add_numbers' ? stop('tool.blocked') : undefined),
|
||||
});
|
||||
const agent = createAgentWithAddTool('anthropic');
|
||||
|
||||
const result = await agent.generate(
|
||||
'What is 2 + 3? Use the tool once. If the tool fails, say so in one sentence.',
|
||||
{ guardrails },
|
||||
);
|
||||
|
||||
expect(calls.beforeTool.length).toBeGreaterThanOrEqual(1);
|
||||
expect(calls.afterTool).toHaveLength(0);
|
||||
const toolResults = findAllToolResults(result.messages);
|
||||
expect(toolResults.length).toBeGreaterThanOrEqual(1);
|
||||
for (const toolResult of toolResults) {
|
||||
expect(toolResult.state).toBe('rejected');
|
||||
expect(JSON.stringify(toolResult.state === 'rejected' && toolResult.error)).toContain(
|
||||
'tool.blocked',
|
||||
);
|
||||
}
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(findLastTextContent(result.messages)).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
it('checks a suspended tool once', async () => {
|
||||
const { guardrails, calls } = recordingGuardrail();
|
||||
const agent = createAgentWithInterruptibleTool('anthropic');
|
||||
|
||||
const first = await agent.generate('Delete the file /tmp/guardrail.txt', { guardrails });
|
||||
|
||||
expect(first.pendingSuspend).toHaveLength(1);
|
||||
expect(calls.beforeTool).toHaveLength(1);
|
||||
expect(calls.beforeTool[0]).toMatchObject({ toolName: 'delete_file', ...IDS });
|
||||
expect(calls.afterTool).toHaveLength(0);
|
||||
|
||||
const { runId, toolCallId } = first.pendingSuspend![0];
|
||||
const resumed = await agent.resume(
|
||||
'generate',
|
||||
{ approved: true },
|
||||
{ runId, toolCallId, guardrails },
|
||||
);
|
||||
|
||||
expect(resumed.finishReason).toBe('stop');
|
||||
expect(calls.beforeTool).toHaveLength(1);
|
||||
expect(calls.afterTool).toHaveLength(1);
|
||||
expect(calls.afterTool[0].result).toMatchObject({ deleted: true });
|
||||
});
|
||||
});
|
||||
|
|
@ -6,6 +6,13 @@ export type {
|
|||
BuiltEpisodicMemoryCaptureStore,
|
||||
BuiltEpisodicMemoryStore,
|
||||
BuiltGuardrail,
|
||||
GuardrailDecision,
|
||||
GuardrailModelCallContext,
|
||||
GuardrailModelCallSource,
|
||||
GuardrailStop,
|
||||
GuardrailToolCallContext,
|
||||
GuardrailsOptions,
|
||||
ModelGuardrail,
|
||||
PiiDetectionType,
|
||||
BuiltEval,
|
||||
RunOptions,
|
||||
|
|
|
|||
|
|
@ -16,7 +16,12 @@ import type { RuntimeSkillSource } from '../../skills/types';
|
|||
import type { CheckpointStore, ModelConfig, SerializableAgentState } from '../../types';
|
||||
import { AgentEvent } from '../../types/runtime/event';
|
||||
import type { AgentEventData } from '../../types/runtime/event';
|
||||
import type { ExecutionOptions, StreamChunk } from '../../types/sdk/agent';
|
||||
import type { ExecutionOptions, GenerateResult, StreamChunk } from '../../types/sdk/agent';
|
||||
import type {
|
||||
GuardrailDecision,
|
||||
GuardrailsOptions,
|
||||
ModelGuardrail,
|
||||
} from '../../types/sdk/guardrail';
|
||||
import type { AgentDbMessage, ContentToolCall, Message } from '../../types/sdk/message';
|
||||
import type {
|
||||
BuiltTool,
|
||||
|
|
@ -1426,6 +1431,374 @@ describe('AgentRuntime — empty stop turn retry', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('AgentRuntime — guardrails', () => {
|
||||
beforeEach(() => {
|
||||
generateText.mockReset();
|
||||
streamText.mockReset();
|
||||
});
|
||||
|
||||
const stopDecision = (code: string): GuardrailDecision => ({ action: 'stop', code });
|
||||
|
||||
type GuardrailSpies = {
|
||||
[K in keyof Required<ModelGuardrail>]: MockedFunction<NonNullable<ModelGuardrail[K]>>;
|
||||
};
|
||||
|
||||
/** A hook whose every method is a spy. `before`/`beforeTool` allow unless overridden. */
|
||||
function makeGuardrail(overrides: Partial<GuardrailSpies> = {}): GuardrailSpies {
|
||||
return {
|
||||
before: vi.fn<NonNullable<ModelGuardrail['before']>>().mockResolvedValue({ action: 'allow' }),
|
||||
after: vi.fn<NonNullable<ModelGuardrail['after']>>().mockResolvedValue(undefined),
|
||||
beforeTool: vi
|
||||
.fn<NonNullable<ModelGuardrail['beforeTool']>>()
|
||||
.mockResolvedValue({ action: 'allow' }),
|
||||
afterTool: vi.fn<NonNullable<ModelGuardrail['afterTool']>>().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function guardrailsOption(hook: ModelGuardrail): GuardrailsOptions {
|
||||
return { hooks: [hook], agentId: 'agent-1', threadId: 'thread-1' };
|
||||
}
|
||||
|
||||
/** A `stop` turn with no output; the loop retries it. */
|
||||
function makeGenerateEmpty() {
|
||||
return {
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 10, outputTokens: 0, totalTokens: 10 },
|
||||
response: { messages: [] },
|
||||
toolCalls: [],
|
||||
};
|
||||
}
|
||||
|
||||
function createRuntimeWithEchoTool(handler: (input: { v: string }) => Promise<unknown>) {
|
||||
const echoTool = new ToolBuilder('echo')
|
||||
.description('Echo the input')
|
||||
.input(z.object({ v: z.string() }))
|
||||
.handler(handler)
|
||||
.build();
|
||||
return new AgentRuntime({
|
||||
name: 'test',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
instructions: 'test',
|
||||
tools: [echoTool],
|
||||
checkpointStorage: 'memory',
|
||||
});
|
||||
}
|
||||
|
||||
function findToolCallBlock(messages: GenerateResult['messages']): ContentToolCall {
|
||||
const assistantMsg = messages.find(
|
||||
(m) =>
|
||||
isLlmMessage(m) && m.role === 'assistant' && m.content.some((c) => c.type === 'tool-call'),
|
||||
) as Message;
|
||||
return assistantMsg.content.find((c) => c.type === 'tool-call') as ContentToolCall;
|
||||
}
|
||||
|
||||
it('ends a generate run with finishReason guardrail when before() stops', async () => {
|
||||
const hook = makeGuardrail({ before: vi.fn().mockResolvedValue(stopDecision('test.stop')) });
|
||||
const { runtime } = createRuntime();
|
||||
|
||||
const result = await runtime.generate('hi', { guardrails: guardrailsOption(hook) });
|
||||
|
||||
expect(generateText).not.toHaveBeenCalled();
|
||||
expect(result.finishReason).toBe('guardrail');
|
||||
expect(result.guardrail).toEqual({ code: 'test.stop' });
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(hook.after).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ends a stream run with a finish chunk that carries the guardrail stop', async () => {
|
||||
const hook = makeGuardrail({ before: vi.fn().mockResolvedValue(stopDecision('test.stop')) });
|
||||
const { runtime } = createRuntime();
|
||||
|
||||
const { stream } = await runtime.stream('hi', { guardrails: guardrailsOption(hook) });
|
||||
const chunks = await collectChunks(stream);
|
||||
|
||||
expect(streamText).not.toHaveBeenCalled();
|
||||
const finish = chunks[chunks.length - 1];
|
||||
expect(finish).toMatchObject({
|
||||
type: 'finish',
|
||||
finishReason: 'guardrail',
|
||||
guardrail: { code: 'test.stop' },
|
||||
});
|
||||
expect(chunks.some((c) => c.type === 'text-delta')).toBe(false);
|
||||
});
|
||||
|
||||
it('reports canContinue false at the completing boundary after a guardrail stop', async () => {
|
||||
const hook = makeGuardrail({ before: vi.fn().mockResolvedValue(stopDecision('test.stop')) });
|
||||
const { runtime } = createRuntime();
|
||||
const boundaries: Array<{ completing: boolean; canContinue: boolean }> = [];
|
||||
|
||||
const result = await runtime.generate('hi', {
|
||||
guardrails: guardrailsOption(hook),
|
||||
onInputBoundary: async (boundary) => {
|
||||
boundaries.push({ completing: boundary.completing, canContinue: boundary.canContinue });
|
||||
return [];
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finishReason).toBe('guardrail');
|
||||
expect(generateText).not.toHaveBeenCalled();
|
||||
// The run's first boundary, then one completing boundary after the stop.
|
||||
// The loop ends there instead of draining queued input per iteration.
|
||||
expect(boundaries).toEqual([
|
||||
{ completing: false, canContinue: true },
|
||||
{ completing: true, canContinue: false },
|
||||
]);
|
||||
});
|
||||
|
||||
it('ends the run even when a boundary returns input after a guardrail stop', async () => {
|
||||
const hook = makeGuardrail({ before: vi.fn().mockResolvedValue(stopDecision('test.stop')) });
|
||||
const { runtime } = createRuntime();
|
||||
const steered: AgentDbMessage = {
|
||||
id: 'steered-input',
|
||||
createdAt: new Date(),
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: 'queued while stopping' }],
|
||||
};
|
||||
const onInputBoundary = vi.fn(async () => [structuredClone(steered)]);
|
||||
|
||||
const result = await runtime.generate('hi', {
|
||||
guardrails: guardrailsOption(hook),
|
||||
onInputBoundary,
|
||||
});
|
||||
|
||||
expect(result.finishReason).toBe('guardrail');
|
||||
expect(result.guardrail).toEqual({ code: 'test.stop' });
|
||||
expect(generateText).not.toHaveBeenCalled();
|
||||
expect(onInputBoundary).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('calls before() and after() once with the same context when the call is allowed', async () => {
|
||||
generateText.mockResolvedValue(makeGenerateSuccess('Done'));
|
||||
const hook = makeGuardrail();
|
||||
const { runtime } = createRuntime();
|
||||
|
||||
const result = await runtime.generate('hi', { guardrails: guardrailsOption(hook) });
|
||||
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(result.guardrail).toBeUndefined();
|
||||
expect(hook.before).toHaveBeenCalledTimes(1);
|
||||
expect(hook.after).toHaveBeenCalledTimes(1);
|
||||
|
||||
const [beforeCtx] = hook.before.mock.calls[0];
|
||||
const [afterCtx, usage] = hook.after.mock.calls[0];
|
||||
expect(afterCtx).toBe(beforeCtx);
|
||||
expect(beforeCtx).toMatchObject({
|
||||
callId: expect.any(String),
|
||||
source: 'turn',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
agentId: 'agent-1',
|
||||
threadId: 'thread-1',
|
||||
});
|
||||
expect(usage).toMatchObject({ promptTokens: 10, completionTokens: 5, totalTokens: 15 });
|
||||
});
|
||||
|
||||
it('checks each empty-turn retry as a new call', async () => {
|
||||
generateText
|
||||
.mockResolvedValueOnce(makeGenerateEmpty())
|
||||
.mockResolvedValueOnce(makeGenerateSuccess('Recovered'));
|
||||
const hook = makeGuardrail();
|
||||
const { runtime } = createRuntime();
|
||||
|
||||
const result = await runtime.generate('hi', { guardrails: guardrailsOption(hook) });
|
||||
|
||||
expect(generateText).toHaveBeenCalledTimes(2);
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(hook.before).toHaveBeenCalledTimes(2);
|
||||
expect(hook.after).toHaveBeenCalledTimes(2);
|
||||
const [first] = hook.before.mock.calls[0];
|
||||
const [second] = hook.before.mock.calls[1];
|
||||
expect(first.callId).not.toBe(second.callId);
|
||||
});
|
||||
|
||||
it('ends the run when a guardrail stops the empty-turn retry', async () => {
|
||||
generateText.mockResolvedValue(makeGenerateEmpty());
|
||||
const before = vi
|
||||
.fn<NonNullable<ModelGuardrail['before']>>()
|
||||
.mockResolvedValueOnce({ action: 'allow' })
|
||||
.mockResolvedValueOnce(stopDecision('test.retry'));
|
||||
const hook = makeGuardrail({ before });
|
||||
const { runtime } = createRuntime();
|
||||
|
||||
const result = await runtime.generate('hi', { guardrails: guardrailsOption(hook) });
|
||||
|
||||
expect(generateText).toHaveBeenCalledTimes(1);
|
||||
expect(result.finishReason).toBe('guardrail');
|
||||
expect(result.guardrail).toEqual({ code: 'test.retry' });
|
||||
expect(hook.after).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('refuses a tool call when beforeTool() stops and lets the model continue', async () => {
|
||||
generateText
|
||||
.mockResolvedValueOnce(makeGenerateWithToolCall('tc-1', 'echo', { v: 'x' }))
|
||||
.mockResolvedValueOnce(makeGenerateSuccess('Understood'));
|
||||
const handler = vi.fn(async ({ v }: { v: string }) => await Promise.resolve({ echoed: v }));
|
||||
const hook = makeGuardrail({
|
||||
beforeTool: vi.fn().mockResolvedValue(stopDecision('tool.stop')),
|
||||
});
|
||||
const runtime = createRuntimeWithEchoTool(handler);
|
||||
|
||||
const result = await runtime.generate('go', { guardrails: guardrailsOption(hook) });
|
||||
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
expect(hook.afterTool).not.toHaveBeenCalled();
|
||||
expect(generateText).toHaveBeenCalledTimes(2);
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(result.guardrail).toBeUndefined();
|
||||
|
||||
const call = findToolCallBlock(result.messages);
|
||||
expect(call.state).toBe('rejected');
|
||||
expect(call.state === 'rejected' && call.error).toContain('tool.stop');
|
||||
});
|
||||
|
||||
it('passes the handler result to afterTool() when the tool call is allowed', async () => {
|
||||
generateText
|
||||
.mockResolvedValueOnce(makeGenerateWithToolCall('tc-1', 'echo', { v: 'x' }))
|
||||
.mockResolvedValueOnce(makeGenerateSuccess('Done'));
|
||||
const hook = makeGuardrail();
|
||||
const runtime = createRuntimeWithEchoTool(
|
||||
async ({ v }) => await Promise.resolve({ echoed: v }),
|
||||
);
|
||||
|
||||
const result = await runtime.generate('go', { guardrails: guardrailsOption(hook) });
|
||||
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(1);
|
||||
expect(hook.afterTool).toHaveBeenCalledTimes(1);
|
||||
|
||||
const [beforeCtx] = hook.beforeTool.mock.calls[0];
|
||||
const [afterCtx, toolResult] = hook.afterTool.mock.calls[0];
|
||||
expect(beforeCtx).toMatchObject({
|
||||
toolCallId: 'tc-1',
|
||||
toolName: 'echo',
|
||||
input: { v: 'x' },
|
||||
runId: expect.any(String),
|
||||
agentId: 'agent-1',
|
||||
threadId: 'thread-1',
|
||||
});
|
||||
expect(afterCtx).toBe(beforeCtx);
|
||||
expect(toolResult).toEqual({ echoed: 'x' });
|
||||
});
|
||||
|
||||
it('does not run beforeTool() again when a suspended tool resumes', async () => {
|
||||
const approvalTool = new ToolBuilder('delete')
|
||||
.description('Delete a record')
|
||||
.input(z.object({ id: z.string() }))
|
||||
.requireApproval()
|
||||
.handler(async ({ id }: { id: string }) => await Promise.resolve({ deleted: id }))
|
||||
.build();
|
||||
generateText
|
||||
.mockResolvedValueOnce(makeGenerateWithToolCall('tc-1', 'delete', { id: 'rec-1' }))
|
||||
.mockResolvedValueOnce(makeGenerateSuccess('Done'));
|
||||
const hook = makeGuardrail();
|
||||
const guardrails = guardrailsOption(hook);
|
||||
const runtime = new AgentRuntime({
|
||||
name: 'test',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
instructions: 'test',
|
||||
tools: [approvalTool],
|
||||
checkpointStorage: 'memory',
|
||||
});
|
||||
|
||||
const firstResult = await runtime.generate('Delete record rec-1', { guardrails });
|
||||
expect(firstResult.finishReason).toBe('tool-calls');
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(1);
|
||||
expect(hook.afterTool).not.toHaveBeenCalled();
|
||||
|
||||
const { runId, toolCallId } = firstResult.pendingSuspend![0];
|
||||
const resumeResult = await runtime.resume(
|
||||
'generate',
|
||||
{ approved: true },
|
||||
{ runId, toolCallId, guardrails },
|
||||
);
|
||||
|
||||
expect(resumeResult.finishReason).toBe('stop');
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(1);
|
||||
expect(hook.afterTool).toHaveBeenCalledTimes(1);
|
||||
const [, toolResult] = hook.afterTool.mock.calls[0];
|
||||
expect(toolResult).toEqual({ deleted: 'rec-1' });
|
||||
});
|
||||
|
||||
it('checks an unexecuted sibling when the suspended call resumes', async () => {
|
||||
const suspendTool = makeSuspendingTool('suspend_tool', async (_input, ctx) => {
|
||||
if (ctx.resumeData) return { approved: true };
|
||||
return await ctx.suspend({ reason: 'needs approval' });
|
||||
});
|
||||
const siblingHandler = vi.fn(async () => await Promise.resolve({ done: true }));
|
||||
const siblingTool = makeMockTool('normal_tool', siblingHandler);
|
||||
const { runtime } = createRuntimeWithTools([suspendTool, siblingTool], 1);
|
||||
const hook = makeGuardrail();
|
||||
const guardrails = guardrailsOption(hook);
|
||||
|
||||
generateText
|
||||
.mockResolvedValueOnce(
|
||||
makeGenerateWithToolCalls([
|
||||
{ toolCallId: 'tc-1', toolName: 'suspend_tool', args: { value: 'a' } },
|
||||
{ toolCallId: 'tc-2', toolName: 'normal_tool', args: { value: 'b' } },
|
||||
]),
|
||||
)
|
||||
.mockResolvedValueOnce(makeGenerateSuccess('Done'));
|
||||
|
||||
const first = await runtime.generate('run tools', { guardrails });
|
||||
|
||||
expect(first.finishReason).toBe('tool-calls');
|
||||
expect(siblingHandler).not.toHaveBeenCalled();
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(1);
|
||||
expect(hook.beforeTool.mock.calls[0][0]).toMatchObject({ toolName: 'suspend_tool' });
|
||||
|
||||
const { runId, toolCallId } = first.pendingSuspend![0];
|
||||
const resumed = await runtime.resume(
|
||||
'generate',
|
||||
{ approved: true },
|
||||
{ runId, toolCallId, guardrails },
|
||||
);
|
||||
|
||||
expect(resumed.finishReason).toBe('stop');
|
||||
expect(siblingHandler).toHaveBeenCalledTimes(1);
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(2);
|
||||
expect(hook.beforeTool.mock.calls[1][0]).toMatchObject({
|
||||
toolCallId: 'tc-2',
|
||||
toolName: 'normal_tool',
|
||||
});
|
||||
expect(hook.afterTool).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('checks an unexecuted pending tool when resume targets it', async () => {
|
||||
const handler = vi.fn(async (_input: unknown, ctx: InterruptibleToolContext) => {
|
||||
if (ctx.resumeData) return { approved: true };
|
||||
return await ctx.suspend({ reason: 'needs approval' });
|
||||
});
|
||||
const suspendTool = makeSuspendingTool('suspend_tool', handler);
|
||||
const { runtime } = createRuntimeWithTools([suspendTool], 1);
|
||||
const hook = makeGuardrail();
|
||||
const guardrails = guardrailsOption(hook);
|
||||
|
||||
generateText.mockResolvedValueOnce(
|
||||
makeGenerateWithToolCalls([
|
||||
{ toolCallId: 'tc-1', toolName: 'suspend_tool', args: { value: 'a' } },
|
||||
{ toolCallId: 'tc-2', toolName: 'suspend_tool', args: { value: 'b' } },
|
||||
]),
|
||||
);
|
||||
|
||||
const first = await runtime.generate('run tools', { guardrails });
|
||||
|
||||
expect(first.finishReason).toBe('tool-calls');
|
||||
expect(runtime.getState().pendingToolCalls['tc-2']?.suspended).toBe(false);
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(1);
|
||||
|
||||
const resumed = await runtime.resume(
|
||||
'generate',
|
||||
{ approved: true },
|
||||
{ runId: first.runId, toolCallId: 'tc-2', guardrails },
|
||||
);
|
||||
|
||||
expect(resumed.finishReason).toBe('tool-calls');
|
||||
expect(hook.beforeTool).toHaveBeenCalledTimes(2);
|
||||
expect(hook.beforeTool.mock.calls[1][0]).toMatchObject({ toolCallId: 'tc-2' });
|
||||
expect(handler).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('AgentRuntime — volatile instruction provider', () => {
|
||||
beforeEach(() => {
|
||||
generateText.mockReset();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,170 @@
|
|||
import type {
|
||||
GuardrailDecision,
|
||||
GuardrailModelCallContext,
|
||||
GuardrailToolCallContext,
|
||||
ModelGuardrail,
|
||||
TokenUsage,
|
||||
} from '../../../types';
|
||||
import { GuardrailRunner } from '../guardrail-runner';
|
||||
|
||||
const allow: GuardrailDecision = { action: 'allow' };
|
||||
const stop = (code: string): GuardrailDecision => ({ action: 'stop', code });
|
||||
|
||||
function makeHook(overrides: Partial<ModelGuardrail> = {}) {
|
||||
return {
|
||||
before: vi.fn<NonNullable<ModelGuardrail['before']>>().mockResolvedValue(allow),
|
||||
after: vi.fn<NonNullable<ModelGuardrail['after']>>().mockResolvedValue(undefined),
|
||||
beforeTool: vi.fn<NonNullable<ModelGuardrail['beforeTool']>>().mockResolvedValue(allow),
|
||||
afterTool: vi.fn<NonNullable<ModelGuardrail['afterTool']>>().mockResolvedValue(undefined),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function runnerFor(hooks: ModelGuardrail[]): GuardrailRunner {
|
||||
const runner = GuardrailRunner.from({ hooks, agentId: 'agent-1', threadId: 'thread-1' });
|
||||
if (!runner) throw new Error('expected a runner');
|
||||
return runner;
|
||||
}
|
||||
|
||||
const modelCtx: GuardrailModelCallContext = {
|
||||
callId: 'call-1',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
source: 'turn',
|
||||
};
|
||||
|
||||
const toolCtx: GuardrailToolCallContext = {
|
||||
toolCallId: 'tc-1',
|
||||
toolName: 'echo',
|
||||
input: { v: 'x' },
|
||||
runId: 'run-1',
|
||||
};
|
||||
|
||||
describe('GuardrailRunner', () => {
|
||||
describe('from()', () => {
|
||||
it('returns undefined without options', () => {
|
||||
expect(GuardrailRunner.from(undefined)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for an empty hook list', () => {
|
||||
expect(GuardrailRunner.from({ hooks: [] })).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns a runner when at least one hook is present', () => {
|
||||
expect(GuardrailRunner.from({ hooks: [makeHook()] })).toBeInstanceOf(GuardrailRunner);
|
||||
});
|
||||
});
|
||||
|
||||
describe('modelCallContext()', () => {
|
||||
it('mints a distinct callId for every call', () => {
|
||||
const runner = runnerFor([makeHook()]);
|
||||
|
||||
const first = runner.modelCallContext('turn', 'openai/gpt-4o-mini');
|
||||
const second = runner.modelCallContext('turn', 'openai/gpt-4o-mini');
|
||||
|
||||
expect(first.callId).toEqual(expect.any(String));
|
||||
expect(first.callId).not.toBe(second.callId);
|
||||
});
|
||||
|
||||
it('copies ids, model and source', () => {
|
||||
const runner = runnerFor([makeHook()]);
|
||||
|
||||
const ctx = runner.modelCallContext('turn', 'openai/gpt-4o-mini');
|
||||
|
||||
expect(ctx).toMatchObject({
|
||||
agentId: 'agent-1',
|
||||
threadId: 'thread-1',
|
||||
model: 'openai/gpt-4o-mini',
|
||||
source: 'turn',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('toolCallContext()', () => {
|
||||
it('copies the call fields and the ids', () => {
|
||||
const runner = runnerFor([makeHook()]);
|
||||
|
||||
const ctx = runner.toolCallContext(toolCtx);
|
||||
|
||||
expect(ctx).toEqual({ ...toolCtx, agentId: 'agent-1', threadId: 'thread-1' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('before()', () => {
|
||||
it('returns the first stop and does not call later hooks', async () => {
|
||||
const first = makeHook();
|
||||
const second = makeHook({ before: vi.fn().mockResolvedValue(stop('a')) });
|
||||
const third = makeHook({ before: vi.fn().mockResolvedValue(stop('b')) });
|
||||
|
||||
const result = await runnerFor([first, second, third]).before(modelCtx);
|
||||
|
||||
expect(result).toEqual({ code: 'a' });
|
||||
expect(first.before).toHaveBeenCalledWith(modelCtx);
|
||||
expect(second.before).toHaveBeenCalledWith(modelCtx);
|
||||
expect(third.before).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips hooks without before and returns undefined when all allow', async () => {
|
||||
const noBefore: ModelGuardrail = { after: vi.fn() };
|
||||
const allowing = makeHook();
|
||||
const silent = makeHook({ before: vi.fn().mockResolvedValue(undefined) });
|
||||
|
||||
const result = await runnerFor([noBefore, allowing, silent]).before(modelCtx);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(allowing.before).toHaveBeenCalledTimes(1);
|
||||
expect(silent.before).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('after()', () => {
|
||||
it('calls every hook with the same ctx and usage', async () => {
|
||||
const usage: TokenUsage = { promptTokens: 10, completionTokens: 5, totalTokens: 15 };
|
||||
const first = makeHook();
|
||||
const noAfter: ModelGuardrail = { before: vi.fn() };
|
||||
const second = makeHook();
|
||||
|
||||
await runnerFor([first, noAfter, second]).after(modelCtx, usage);
|
||||
|
||||
expect(first.after).toHaveBeenCalledWith(modelCtx, usage);
|
||||
expect(second.after).toHaveBeenCalledWith(modelCtx, usage);
|
||||
});
|
||||
});
|
||||
|
||||
describe('beforeTool()', () => {
|
||||
it('returns the first stop and does not call later hooks', async () => {
|
||||
const first = makeHook();
|
||||
const second = makeHook({ beforeTool: vi.fn().mockResolvedValue(stop('tool.a')) });
|
||||
const third = makeHook({ beforeTool: vi.fn().mockResolvedValue(stop('tool.b')) });
|
||||
|
||||
const result = await runnerFor([first, second, third]).beforeTool(toolCtx);
|
||||
|
||||
expect(result).toEqual({ code: 'tool.a' });
|
||||
expect(first.beforeTool).toHaveBeenCalledWith(toolCtx);
|
||||
expect(third.beforeTool).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('skips hooks without beforeTool and returns undefined when all allow', async () => {
|
||||
const noBeforeTool: ModelGuardrail = { afterTool: vi.fn() };
|
||||
const allowing = makeHook();
|
||||
|
||||
const result = await runnerFor([noBeforeTool, allowing]).beforeTool(toolCtx);
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
expect(allowing.beforeTool).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('afterTool()', () => {
|
||||
it('calls every hook with the same ctx and result', async () => {
|
||||
const first = makeHook();
|
||||
const noAfterTool: ModelGuardrail = { beforeTool: vi.fn() };
|
||||
const second = makeHook();
|
||||
const result = { echoed: 'x' };
|
||||
|
||||
await runnerFor([first, noAfterTool, second]).afterTool(toolCtx, result);
|
||||
|
||||
expect(first.afterTool).toHaveBeenCalledWith(toolCtx, result);
|
||||
expect(second.afterTool).toHaveBeenCalledWith(toolCtx, result);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
import type {
|
||||
GuardrailModelCallContext,
|
||||
GuardrailModelCallSource,
|
||||
GuardrailStop,
|
||||
GuardrailToolCallContext,
|
||||
GuardrailsOptions,
|
||||
ModelGuardrail,
|
||||
TokenUsage,
|
||||
} from '../../types';
|
||||
|
||||
type GuardrailIds = Pick<GuardrailsOptions, 'agentId' | 'threadId'>;
|
||||
|
||||
/**
|
||||
* Calls the host-supplied guardrail hooks around model and tool calls. Hooks
|
||||
* run in list order, one at a time. The first `stop` wins. The runner holds
|
||||
* no per-run state; each call site asks it fresh.
|
||||
*/
|
||||
export class GuardrailRunner {
|
||||
private constructor(
|
||||
private readonly hooks: ModelGuardrail[],
|
||||
private readonly ids: GuardrailIds,
|
||||
) {}
|
||||
|
||||
/** `undefined` when there is nothing to run, so call sites can skip all work. */
|
||||
static from(options: GuardrailsOptions | undefined): GuardrailRunner | undefined {
|
||||
if (!options || options.hooks.length === 0) return undefined;
|
||||
return new GuardrailRunner(options.hooks, {
|
||||
agentId: options.agentId,
|
||||
threadId: options.threadId,
|
||||
});
|
||||
}
|
||||
|
||||
modelCallContext(source: GuardrailModelCallSource, model: string): GuardrailModelCallContext {
|
||||
return { callId: crypto.randomUUID(), ...this.ids, model, source };
|
||||
}
|
||||
|
||||
toolCallContext(call: {
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
input: unknown;
|
||||
runId: string;
|
||||
}): GuardrailToolCallContext {
|
||||
return { ...call, ...this.ids };
|
||||
}
|
||||
|
||||
async before(ctx: GuardrailModelCallContext): Promise<GuardrailStop | undefined> {
|
||||
for (const hook of this.hooks) {
|
||||
const decision = await hook.before?.(ctx);
|
||||
if (decision?.action === 'stop') return { code: decision.code };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async after(ctx: GuardrailModelCallContext, usage: TokenUsage | undefined): Promise<void> {
|
||||
for (const hook of this.hooks) {
|
||||
await hook.after?.(ctx, usage);
|
||||
}
|
||||
}
|
||||
|
||||
async beforeTool(ctx: GuardrailToolCallContext): Promise<GuardrailStop | undefined> {
|
||||
for (const hook of this.hooks) {
|
||||
const decision = await hook.beforeTool?.(ctx);
|
||||
if (decision?.action === 'stop') return { code: decision.code };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async afterTool(ctx: GuardrailToolCallContext, result: unknown): Promise<void> {
|
||||
for (const hook of this.hooks) {
|
||||
await hook.afterTool?.(ctx, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -48,9 +48,11 @@ import type {
|
|||
ExecutionOptions,
|
||||
ResumeOptions,
|
||||
} from '../../types/sdk/agent';
|
||||
import type { GuardrailStop } from '../../types/sdk/guardrail';
|
||||
import type { AgentMessage, ContentToolCall } from '../../types/sdk/message';
|
||||
import { getModelIdString } from '../../utils/model';
|
||||
import { removeToolResultRun } from '../../workspace';
|
||||
import { GuardrailRunner } from '../guardrails/guardrail-runner';
|
||||
import { createFilteredLogger } from '../logger';
|
||||
import { MemoryOrchestrator } from '../memory/memory-orchestrator';
|
||||
import { sanitizeOffloadedToolResultsForMemory } from '../memory/tool-result-memory';
|
||||
|
|
@ -118,6 +120,7 @@ interface LoopState {
|
|||
maxIterations: number;
|
||||
iterationCount: number;
|
||||
reachedStopCondition: boolean;
|
||||
guardrailStop?: GuardrailStop;
|
||||
}
|
||||
|
||||
type ToolBatchSettlement<T> = { suspended: false } | { suspended: true; result: T };
|
||||
|
|
@ -724,6 +727,10 @@ export class AgentRuntime {
|
|||
|
||||
for (; state.iterationCount < state.maxIterations; state.iterationCount++) {
|
||||
const hasInput = await this.consumeAdditionalInput(prepared, sink, state);
|
||||
// A guardrail stop is terminal. The boundary above fired with
|
||||
// canContinue: false, so the consumer released queued input instead of
|
||||
// consuming it. End the run even if a consumer still returned messages.
|
||||
if (state.guardrailStop) break;
|
||||
if (state.reachedStopCondition && !hasInput) break;
|
||||
state.reachedStopCondition = false;
|
||||
const settlement = await this.runLoopIteration(prepared, sink, state);
|
||||
|
|
@ -743,6 +750,7 @@ export class AgentRuntime {
|
|||
finishReason: state.lastFinishReason,
|
||||
usage: state.totalUsage,
|
||||
structuredOutput: state.structuredOutput,
|
||||
guardrail: state.guardrailStop,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -760,7 +768,9 @@ export class AgentRuntime {
|
|||
messages: sanitizeOffloadedToolResultsForMemory(ctx.list.turnDelta()),
|
||||
lastCreatedAt: ctx.list.messages().at(-1)?.createdAt.getTime() ?? 0,
|
||||
completing: state.reachedStopCondition,
|
||||
canContinue: state.iterationCount < state.maxIterations,
|
||||
// A guardrail stop is terminal: queued input must be released for the
|
||||
// next run, not consumed into a run that makes no more model calls.
|
||||
canContinue: state.iterationCount < state.maxIterations && !state.guardrailStop,
|
||||
});
|
||||
// Only committed input enters the live list. Its IDs and timestamps stay stable.
|
||||
ctx.list.addInput(messages);
|
||||
|
|
@ -844,6 +854,7 @@ export class AgentRuntime {
|
|||
persistence: ctx.options?.persistence,
|
||||
telemetry: ctx.runTelemetry,
|
||||
executionCounter: ctx.options?.executionCounter,
|
||||
guardrails: ctx.options?.guardrails,
|
||||
approvalContext: ctx.options?.approvalContext,
|
||||
abortSignal: ctx.abortScope.signal,
|
||||
isAborted: () => ctx.abortScope.isAborted,
|
||||
|
|
@ -889,6 +900,13 @@ export class AgentRuntime {
|
|||
this.eventBus.emit({ type: AgentEvent.TurnStart });
|
||||
const { toolMap, modelCallContext } = await this.prepareModelCall(ctx);
|
||||
const turn = await this.callModelWithRetries(ctx, sink, state, modelCallContext);
|
||||
// A guardrail refused the call. Usage from any discarded empty attempt
|
||||
// is already folded in; end the run without a new model turn.
|
||||
if (!turn) {
|
||||
state.lastFinishReason = 'guardrail';
|
||||
state.reachedStopCondition = true;
|
||||
return { suspended: false };
|
||||
}
|
||||
this.assertNotAborted(ctx.abortScope);
|
||||
for (const message of ctx.list.inputDelta()) ctx.acceptedInputIds.add(message.id);
|
||||
|
||||
|
|
@ -975,14 +993,30 @@ export class AgentRuntime {
|
|||
sink: RunOutputSink<T>,
|
||||
state: LoopState,
|
||||
modelCallContext: ModelCallContext,
|
||||
): Promise<ModelTurnResult> {
|
||||
let turn = await sink.callModel(modelCallContext);
|
||||
): Promise<ModelTurnResult | undefined> {
|
||||
const guardrails = GuardrailRunner.from(ctx.options?.guardrails);
|
||||
const callModel = async (): Promise<ModelTurnResult | undefined> => {
|
||||
const guardCtx = guardrails?.modelCallContext('turn', this.modelIdString);
|
||||
if (guardrails && guardCtx) {
|
||||
const stop = await guardrails.before(guardCtx);
|
||||
if (stop) {
|
||||
state.guardrailStop = stop;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
const turn = await sink.callModel(modelCallContext);
|
||||
if (guardrails && guardCtx) await guardrails.after(guardCtx, turn.usage);
|
||||
return turn;
|
||||
};
|
||||
|
||||
let turn = await callModel();
|
||||
// Retry empty stop turns. Charge each attempt before checking for cancellation.
|
||||
for (let retry = 0; retry < MAX_EMPTY_TURN_RETRIES && isEmptyModelTurn(turn); retry++) {
|
||||
for (let retry = 0; turn && retry < MAX_EMPTY_TURN_RETRIES && isEmptyModelTurn(turn); retry++) {
|
||||
this.recordTurnUsage(ctx, sink, state, turn);
|
||||
this.assertNotAborted(ctx.abortScope);
|
||||
turn = await sink.callModel(modelCallContext);
|
||||
turn = await callModel();
|
||||
}
|
||||
if (!turn) return undefined;
|
||||
this.recordTurnUsage(ctx, sink, state, turn);
|
||||
return turn;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ export class GenerateSink implements RunOutputSink<GenerateResult> {
|
|||
...(structuredOutput !== undefined && { structuredOutput }),
|
||||
...(this.toolCallSummary.length > 0 && { toolCalls: this.toolCallSummary }),
|
||||
getState: () => this.services.getState(),
|
||||
...(emission.guardrail && { guardrail: emission.guardrail }),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -471,6 +471,7 @@ export class StreamSink implements RunOutputSink<void> {
|
|||
...(costUsage && { usage: costUsage }),
|
||||
model: this.services.modelId,
|
||||
...(structuredOutput !== undefined && { structuredOutput }),
|
||||
...(emission.guardrail && { guardrail: emission.guardrail }),
|
||||
});
|
||||
this.services.updateState({ status: 'success', messageList: list.serialize() });
|
||||
this.services.emitAgentEnd(list.responseDelta());
|
||||
|
|
|
|||
|
|
@ -200,6 +200,7 @@ export class ToolCallExecutor {
|
|||
persistence: ctx.persistence,
|
||||
resolvedTelemetry: ctx.telemetry,
|
||||
executionCounter: ctx.executionCounter,
|
||||
guardrails: ctx.guardrails,
|
||||
approvalContext: ctx.approvalContext,
|
||||
abortSignal: ctx.abortSignal,
|
||||
countToolCall: true,
|
||||
|
|
@ -467,6 +468,7 @@ export class ToolCallExecutor {
|
|||
persistence: ctx.persistence,
|
||||
telemetry: ctx.telemetry,
|
||||
executionCounter: ctx.executionCounter,
|
||||
guardrails: ctx.guardrails,
|
||||
approvalContext: ctx.approvalContext,
|
||||
abortSignal: ctx.abortSignal,
|
||||
isAborted: ctx.isAborted,
|
||||
|
|
@ -523,9 +525,11 @@ export class ToolCallExecutor {
|
|||
resumeData,
|
||||
resolvedTelemetry: ctx.telemetry,
|
||||
executionCounter: ctx.executionCounter,
|
||||
guardrails: ctx.guardrails,
|
||||
approvalContext: ctx.approvalContext,
|
||||
abortSignal: ctx.abortSignal,
|
||||
countToolCall: false,
|
||||
previouslySuspended: entry.suspended,
|
||||
...(entry.suspended
|
||||
? {
|
||||
suspendPayload: entry.suspendPayload,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import type { AgentMessage, ContentToolCall, Message } from '../../types/sdk/mes
|
|||
import type { JSONValue } from '../../types/utils/json';
|
||||
import { parseWithSchema } from '../../utils/parse';
|
||||
import { isZodSchema } from '../../utils/zod';
|
||||
import { GuardrailRunner } from '../guardrails/guardrail-runner';
|
||||
import { incrementToolCallCount } from '../loop/execution-counter';
|
||||
import { stringifyError } from '../loop/runtime-helpers';
|
||||
import type { AgentMessageList } from '../model/message-list';
|
||||
|
|
@ -124,6 +125,25 @@ export class ToolCallRunner {
|
|||
if (!validation.ok) return validation.outcome;
|
||||
const input = validation.input;
|
||||
|
||||
const guardrails = GuardrailRunner.from(params.guardrails);
|
||||
const guardCtx = guardrails?.toolCallContext({
|
||||
toolCallId,
|
||||
toolName,
|
||||
input,
|
||||
runId: params.runId,
|
||||
});
|
||||
// Skip only a call that already suspended. An unexecuted pending call
|
||||
// is still a first execution, even when resume data is present.
|
||||
if (guardrails && guardCtx && !params.previouslySuspended) {
|
||||
const stop = await guardrails.beforeTool(guardCtx);
|
||||
if (stop) {
|
||||
return await this.toolError(
|
||||
params,
|
||||
new Error(`Tool call stopped by guardrail: ${stop.code}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldEmitToolExecutionStart(builtTool, resumeData)) {
|
||||
this.eventBus.emit({
|
||||
type: AgentEvent.ToolExecutionStart,
|
||||
|
|
@ -133,13 +153,15 @@ export class ToolCallRunner {
|
|||
});
|
||||
}
|
||||
|
||||
return await this.executeValidatedToolCall(params, builtTool, input);
|
||||
return await this.executeValidatedToolCall(params, builtTool, input, guardrails, guardCtx);
|
||||
}
|
||||
|
||||
private async executeValidatedToolCall(
|
||||
params: ProcessToolCallParams,
|
||||
builtTool: BuiltTool,
|
||||
input: JSONValue,
|
||||
guardrails: GuardrailRunner | undefined,
|
||||
guardCtx: ReturnType<GuardrailRunner['toolCallContext']> | undefined,
|
||||
): Promise<ToolCallOutcome> {
|
||||
let toolResult: unknown;
|
||||
const suspension: InterruptedToolSuspension = {
|
||||
|
|
@ -169,6 +191,9 @@ export class ToolCallRunner {
|
|||
if (isSuspendedToolResult(toolResult)) {
|
||||
return await this.buildSuspendedOutcome(params, builtTool, toolResult);
|
||||
}
|
||||
|
||||
// Final result only: a call that suspended reports once, after resume.
|
||||
if (guardrails && guardCtx) await guardrails.afterTool(guardCtx, toolResult);
|
||||
return await this.buildSuccessOutcome(params, builtTool, input, toolResult);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -210,3 +210,13 @@ export type {
|
|||
ResolvedCredential,
|
||||
CredentialListItem,
|
||||
} from './sdk/credential-provider';
|
||||
|
||||
export type {
|
||||
GuardrailsOptions,
|
||||
ModelGuardrail,
|
||||
GuardrailModelCallSource,
|
||||
GuardrailModelCallContext,
|
||||
GuardrailToolCallContext,
|
||||
GuardrailDecision,
|
||||
GuardrailStop,
|
||||
} from './sdk/guardrail';
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import type {
|
|||
} from 'ai';
|
||||
|
||||
import type { FinishReason, SerializableAgentState, TokenUsage } from '../index';
|
||||
import type { GuardrailStop } from '../sdk/guardrail';
|
||||
import type { ExecutionOptions, RunOptions } from '../sdk/agent';
|
||||
import type { AgentDbMessage, AgentMessage } from '../sdk/message';
|
||||
import type { JSONObject } from '../utils/json';
|
||||
|
|
@ -87,6 +88,7 @@ export interface CompleteEmission {
|
|||
finishReason: FinishReason;
|
||||
usage: TokenUsage | undefined;
|
||||
structuredOutput: unknown;
|
||||
guardrail?: GuardrailStop;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import type {
|
|||
AgentExecutionCounter,
|
||||
BuiltTelemetry,
|
||||
BuiltTool,
|
||||
GuardrailsOptions,
|
||||
PendingToolCall,
|
||||
ToolSuspendOptions,
|
||||
ToolApprovalContext,
|
||||
|
|
@ -104,6 +105,7 @@ export interface ToolBatchContext {
|
|||
persistence?: AgentPersistenceOptions;
|
||||
telemetry?: BuiltTelemetry;
|
||||
executionCounter?: AgentExecutionCounter;
|
||||
guardrails?: GuardrailsOptions;
|
||||
approvalContext?: ToolApprovalContext;
|
||||
abortSignal: AbortSignal;
|
||||
isAborted: () => boolean;
|
||||
|
|
@ -122,10 +124,13 @@ export interface ProcessToolCallParams extends ToolCallIdentity {
|
|||
resumeData?: unknown;
|
||||
resolvedTelemetry?: BuiltTelemetry;
|
||||
executionCounter?: AgentExecutionCounter;
|
||||
guardrails?: GuardrailsOptions;
|
||||
approvalContext?: ToolApprovalContext;
|
||||
abortSignal?: AbortSignal;
|
||||
/** Whether this counts as a new tool-call invocation. Default `true`; `false` on resume. */
|
||||
countToolCall?: boolean;
|
||||
/** The call already suspended, so `beforeTool` already ran. */
|
||||
previouslySuspended?: boolean;
|
||||
/** Checkpointed suspend payload of the tool call being resumed. */
|
||||
suspendPayload?: unknown;
|
||||
/** Checkpointed private continuation of the tool call being resumed. */
|
||||
|
|
|
|||
|
|
@ -20,12 +20,13 @@ import type {
|
|||
import type { SerializedMessageList } from '../runtime/message-list';
|
||||
import type { BuiltTelemetry } from '../telemetry';
|
||||
import type { JSONObject, JSONValue } from '../utils/json';
|
||||
|
||||
import type { GuardrailsOptions, GuardrailStop } from './guardrail';
|
||||
export type SmoothStreamOptions = NonNullable<Parameters<typeof smoothStream>[0]>;
|
||||
|
||||
export const FINISH_REASONS = [
|
||||
'stop',
|
||||
'max-iterations',
|
||||
'guardrail',
|
||||
'length',
|
||||
'content-filter',
|
||||
'tool-calls',
|
||||
|
|
@ -162,6 +163,7 @@ export type StreamChunk = ContentMetadata &
|
|||
usage?: TokenUsage;
|
||||
model?: string;
|
||||
structuredOutput?: unknown;
|
||||
guardrail?: GuardrailStop;
|
||||
}
|
||||
| { type: 'error'; error: unknown }
|
||||
| {
|
||||
|
|
@ -215,6 +217,7 @@ export interface AgentInputBoundary {
|
|||
messages: AgentDbMessage[];
|
||||
lastCreatedAt: number;
|
||||
completing: boolean;
|
||||
/** False when the run cannot accept more input: max iterations reached, or a terminal stop such as a guardrail refusal. */
|
||||
canContinue: boolean;
|
||||
}
|
||||
|
||||
|
|
@ -281,6 +284,7 @@ export interface ExecutionOptions {
|
|||
* persistence-backed CheckpointStore; recover via `crashResume()`.
|
||||
*/
|
||||
stepCheckpoints?: boolean;
|
||||
guardrails?: GuardrailsOptions;
|
||||
}
|
||||
|
||||
export interface PersistedExecutionOptions {
|
||||
|
|
@ -341,6 +345,7 @@ export interface GenerateResult {
|
|||
/** The model ID used for this generation (e.g. 'anthropic/claude-haiku-4-5'). */
|
||||
model?: string;
|
||||
finishReason?: FinishReason;
|
||||
guardrail?: GuardrailStop;
|
||||
providerMetadata?: Record<string, unknown>;
|
||||
/** Tool calls made during the run (with merged results when available). */
|
||||
toolCalls?: ToolResultEntry[];
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import type { TokenUsage } from './agent';
|
||||
|
||||
export type GuardrailType = 'pii' | 'prompt-injection' | 'moderation' | 'custom';
|
||||
export type GuardrailStrategy = 'block' | 'redact' | 'warn';
|
||||
/** Re-exported so the detection vocabulary has one definition (see `@n8n/utils`). */
|
||||
|
|
@ -9,3 +11,44 @@ export interface BuiltGuardrail {
|
|||
readonly strategy: 'block' | 'redact' | 'warn';
|
||||
/** @internal */ readonly _config: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type GuardrailDecision = { action: 'allow' } | { action: 'stop'; code: string };
|
||||
|
||||
export type GuardrailStop = { code: string };
|
||||
export type GuardrailModelCallSource = 'turn' | 'title' | 'memory';
|
||||
|
||||
export interface GuardrailModelCallContext {
|
||||
/** New id for every attempt, including an empty-turn retry. */
|
||||
callId: string;
|
||||
agentId?: string;
|
||||
threadId?: string;
|
||||
model: string;
|
||||
source: GuardrailModelCallSource;
|
||||
}
|
||||
|
||||
export interface GuardrailToolCallContext {
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
input: unknown;
|
||||
runId: string;
|
||||
agentId?: string;
|
||||
threadId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hooks around model and tool calls. A `before*` hook returns `undefined` or
|
||||
* `allow` to continue, or `stop` to refuse the call.
|
||||
*/
|
||||
export interface ModelGuardrail {
|
||||
before?(ctx: GuardrailModelCallContext): Promise<GuardrailDecision | undefined>;
|
||||
after?(ctx: GuardrailModelCallContext, usage: TokenUsage | undefined): Promise<void>;
|
||||
beforeTool?(ctx: GuardrailToolCallContext): Promise<GuardrailDecision | undefined>;
|
||||
afterTool?(ctx: GuardrailToolCallContext, result: unknown): Promise<void>;
|
||||
}
|
||||
|
||||
/** Host-supplied on `ExecutionOptions`. The SDK never looks up settings. */
|
||||
export interface GuardrailsOptions {
|
||||
hooks: ModelGuardrail[];
|
||||
agentId?: string;
|
||||
threadId?: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ function statusForFinishReason(finishReason: FinishReason): DiscoveryStreamStatu
|
|||
case 'length':
|
||||
case 'content-filter':
|
||||
case 'error':
|
||||
case 'guardrail':
|
||||
case 'other':
|
||||
return 'errored';
|
||||
default: {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user