From b3cfccf4276e3987a522464ab80ca4d011048dc3 Mon Sep 17 00:00:00 2001 From: Oleg Ivaniv Date: Wed, 6 May 2026 15:51:50 +0200 Subject: [PATCH] fix(instance-ai): normalize langsmith tool messages --- .../__tests__/langsmith-tracing.test.ts | 92 ++++++++++ .../src/tracing/langsmith-tracing.ts | 157 +++++++++++++++++- 2 files changed, 246 insertions(+), 3 deletions(-) diff --git a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts index 613542e7d51..dcb2e84f46a 100644 --- a/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts +++ b/packages/@n8n/instance-ai/src/tracing/__tests__/langsmith-tracing.test.ts @@ -737,6 +737,98 @@ describe('createInstanceAiTraceContext', () => { }); }); + it('normalizes AI SDK tool messages for LangSmith chat rendering', () => { + const span = { + attributes: { + 'ai.prompt.messages': JSON.stringify([ + { + role: 'user', + content: [{ type: 'text', text: 'find my Slack credential' }], + }, + { + role: 'assistant', + content: [ + { type: 'text', text: 'Checking credentials.' }, + { + type: 'tool-call', + toolCallId: 'toolu-1', + toolName: 'credentials', + input: { action: 'list', name: 'Slack account' }, + providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } } }, + }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: 'toolu-1', + toolName: 'credentials', + output: { + ok: true, + items: [{ name: 'Slack account', apiKey: 'sk-secret' }], + }, + }, + ], + }, + ]), + 'ai.prompt.tools': [ + JSON.stringify({ + type: 'function', + name: 'credentials', + description: 'List credentials', + input_schema: { + type: 'object', + properties: { + action: { type: 'string' }, + name: { type: 'string' }, + }, + }, + }), + ], + }, + }; + + const redacted = redactLangSmithTelemetrySpan(span) as { + attributes: Record; + }; + const prompt = jsonParse<{ input: Array> }>( + redacted.attributes['gen_ai.prompt'] as string, + ); + + expect(prompt.input[1]).toEqual({ + role: 'assistant', + content: 'Checking credentials.', + tool_calls: [ + { + id: 'toolu-1', + type: 'function', + function: { + name: 'credentials', + arguments: JSON.stringify({ action: 'list', name: 'Slack account' }), + }, + }, + ], + }); + expect(prompt.input[2]).toEqual({ + role: 'tool', + tool_call_id: 'toolu-1', + name: 'credentials', + content: JSON.stringify({ + ok: true, + items: [{ name: 'Slack account', apiKey: '[redacted]' }], + }), + }); + + const originalMessages = jsonParse>>( + redacted.attributes['ai.prompt.messages'] as string, + ); + expect(JSON.stringify(originalMessages)).toContain('Slack account'); + expect(JSON.stringify(originalMessages)).not.toContain('[redacted-depth-limit]'); + expect(JSON.stringify(prompt)).not.toContain('sk-secret'); + }); + it('finishes OTel child spans with their parent linkage', async () => { const tracing = await createInstanceAiTraceContext({ threadId: 'thread-1', diff --git a/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts b/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts index 1d45a4cde70..b5786d61422 100644 --- a/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts +++ b/packages/@n8n/instance-ai/src/tracing/langsmith-tracing.ts @@ -623,8 +623,13 @@ function redactTelemetryAttribute(key: string, value: unknown): unknown { return '[redacted]'; } + const maxDepth = + key === 'ai.prompt.messages' || key === GEN_AI_PROMPT + ? MAX_PROMPT_SCHEMA_TRACE_DEPTH + : MAX_TRACE_DEPTH; + if (typeof value !== 'string') { - return redactTelemetryJsonValue(value, key); + return redactTelemetryJsonValue(value, key, 0, maxDepth); } const trimmed = value.trim(); @@ -634,7 +639,7 @@ function redactTelemetryAttribute(key: string, value: unknown): unknown { ) { try { const parsed: unknown = JSON.parse(trimmed); - return JSON.stringify(redactTelemetryJsonValue(parsed, key)); + return JSON.stringify(redactTelemetryJsonValue(parsed, key, 0, maxDepth)); } catch { return redactSecretString(value); } @@ -671,6 +676,152 @@ function parseTelemetryTools(value: unknown): unknown[] | undefined { return tools.length > 0 ? tools : undefined; } +function readToolCallPayload(part: Record): unknown { + if ('input' in part) return part.input; + if ('args' in part) return part.args; + if ('arguments' in part) return part.arguments; + return {}; +} + +function readToolResultPayload(part: Record): unknown { + if ('output' in part) return part.output; + if ('result' in part) return part.result; + if ('content' in part) return part.content; + return ''; +} + +function stringifyToolPayload(value: unknown): string { + if (value === undefined || value === null) { + return ''; + } + + if (typeof value === 'string') { + return value; + } + + try { + return JSON.stringify(value) ?? ''; + } catch { + return '[unserializable]'; + } +} + +function normalizeAssistantMessageForLangSmith(message: Record): unknown { + const content = message.content; + if (!Array.isArray(content)) { + return message; + } + + const textParts: string[] = []; + const toolCalls: Array> = []; + + for (const part of content) { + if (!isRecord(part)) continue; + + if (part.type === 'text' && typeof part.text === 'string') { + textParts.push(part.text); + continue; + } + + if (part.type !== 'tool-call') continue; + + const toolCallId = + typeof part.toolCallId === 'string' + ? part.toolCallId + : typeof part.id === 'string' + ? part.id + : undefined; + const toolName = + typeof part.toolName === 'string' + ? part.toolName + : typeof part.name === 'string' + ? part.name + : undefined; + if (!toolCallId || !toolName) continue; + + toolCalls.push({ + id: toolCallId, + type: 'function', + function: { + name: toolName, + arguments: stringifyToolPayload(readToolCallPayload(part)), + }, + }); + } + + if (toolCalls.length === 0) { + return message; + } + + return { + role: 'assistant', + content: textParts.join('\n'), + tool_calls: toolCalls, + }; +} + +function normalizeToolMessageForLangSmith(message: Record): unknown[] { + const content = message.content; + if (!Array.isArray(content)) { + return [message]; + } + + const normalizedMessages: unknown[] = []; + for (const part of content) { + if (!isRecord(part) || part.type !== 'tool-result') { + continue; + } + + const toolCallId = + typeof part.toolCallId === 'string' + ? part.toolCallId + : typeof part.id === 'string' + ? part.id + : undefined; + const toolName = + typeof part.toolName === 'string' + ? part.toolName + : typeof part.name === 'string' + ? part.name + : undefined; + if (!toolCallId) continue; + + normalizedMessages.push({ + role: 'tool', + tool_call_id: toolCallId, + ...(toolName ? { name: toolName } : {}), + content: stringifyToolPayload(readToolResultPayload(part)), + }); + } + + return normalizedMessages.length > 0 ? normalizedMessages : [message]; +} + +function normalizeMessagesForLangSmith(messages: unknown[]): unknown[] { + const normalizedMessages: unknown[] = []; + + for (const message of messages) { + if (!isRecord(message) || typeof message.role !== 'string') { + normalizedMessages.push(message); + continue; + } + + if (message.role === 'assistant') { + normalizedMessages.push(normalizeAssistantMessageForLangSmith(message)); + continue; + } + + if (message.role === 'tool') { + normalizedMessages.push(...normalizeToolMessageForLangSmith(message)); + continue; + } + + normalizedMessages.push(message); + } + + return normalizedMessages; +} + function stableStringify(value: unknown): string { if (Array.isArray(value)) { return `[${value.map((entry) => stableStringify(entry)).join(',')}]`; @@ -759,7 +910,7 @@ function enrichLangSmithPromptAttribute(attributes: Record): vo return; } - const prompt: Record = { input: messages }; + const prompt: Record = { input: normalizeMessagesForLangSmith(messages) }; if (tools) { prompt.tools = tools; }