fix(instance-ai): normalize langsmith tool messages

This commit is contained in:
Oleg Ivaniv 2026-05-06 15:51:50 +02:00
parent bccb6d0717
commit b3cfccf427
No known key found for this signature in database
2 changed files with 246 additions and 3 deletions

View File

@ -737,6 +737,98 @@ describe('createInstanceAiTraceContext', () => {
});
});
it('normalizes AI SDK tool messages for LangSmith chat rendering', () => {
const span = {
attributes: {
'ai.prompt.messages': JSON.stringify([
{
role: 'user',
content: [{ type: 'text', text: 'find my Slack credential' }],
},
{
role: 'assistant',
content: [
{ type: 'text', text: 'Checking credentials.' },
{
type: 'tool-call',
toolCallId: 'toolu-1',
toolName: 'credentials',
input: { action: 'list', name: 'Slack account' },
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral' } } },
},
],
},
{
role: 'tool',
content: [
{
type: 'tool-result',
toolCallId: 'toolu-1',
toolName: 'credentials',
output: {
ok: true,
items: [{ name: 'Slack account', apiKey: 'sk-secret' }],
},
},
],
},
]),
'ai.prompt.tools': [
JSON.stringify({
type: 'function',
name: 'credentials',
description: 'List credentials',
input_schema: {
type: 'object',
properties: {
action: { type: 'string' },
name: { type: 'string' },
},
},
}),
],
},
};
const redacted = redactLangSmithTelemetrySpan(span) as {
attributes: Record<string, unknown>;
};
const prompt = jsonParse<{ input: Array<Record<string, unknown>> }>(
redacted.attributes['gen_ai.prompt'] as string,
);
expect(prompt.input[1]).toEqual({
role: 'assistant',
content: 'Checking credentials.',
tool_calls: [
{
id: 'toolu-1',
type: 'function',
function: {
name: 'credentials',
arguments: JSON.stringify({ action: 'list', name: 'Slack account' }),
},
},
],
});
expect(prompt.input[2]).toEqual({
role: 'tool',
tool_call_id: 'toolu-1',
name: 'credentials',
content: JSON.stringify({
ok: true,
items: [{ name: 'Slack account', apiKey: '[redacted]' }],
}),
});
const originalMessages = jsonParse<Array<Record<string, unknown>>>(
redacted.attributes['ai.prompt.messages'] as string,
);
expect(JSON.stringify(originalMessages)).toContain('Slack account');
expect(JSON.stringify(originalMessages)).not.toContain('[redacted-depth-limit]');
expect(JSON.stringify(prompt)).not.toContain('sk-secret');
});
it('finishes OTel child spans with their parent linkage', async () => {
const tracing = await createInstanceAiTraceContext({
threadId: 'thread-1',

View File

@ -623,8 +623,13 @@ function redactTelemetryAttribute(key: string, value: unknown): unknown {
return '[redacted]';
}
const maxDepth =
key === 'ai.prompt.messages' || key === GEN_AI_PROMPT
? MAX_PROMPT_SCHEMA_TRACE_DEPTH
: MAX_TRACE_DEPTH;
if (typeof value !== 'string') {
return redactTelemetryJsonValue(value, key);
return redactTelemetryJsonValue(value, key, 0, maxDepth);
}
const trimmed = value.trim();
@ -634,7 +639,7 @@ function redactTelemetryAttribute(key: string, value: unknown): unknown {
) {
try {
const parsed: unknown = JSON.parse(trimmed);
return JSON.stringify(redactTelemetryJsonValue(parsed, key));
return JSON.stringify(redactTelemetryJsonValue(parsed, key, 0, maxDepth));
} catch {
return redactSecretString(value);
}
@ -671,6 +676,152 @@ function parseTelemetryTools(value: unknown): unknown[] | undefined {
return tools.length > 0 ? tools : undefined;
}
function readToolCallPayload(part: Record<string, unknown>): unknown {
if ('input' in part) return part.input;
if ('args' in part) return part.args;
if ('arguments' in part) return part.arguments;
return {};
}
function readToolResultPayload(part: Record<string, unknown>): unknown {
if ('output' in part) return part.output;
if ('result' in part) return part.result;
if ('content' in part) return part.content;
return '';
}
function stringifyToolPayload(value: unknown): string {
if (value === undefined || value === null) {
return '';
}
if (typeof value === 'string') {
return value;
}
try {
return JSON.stringify(value) ?? '';
} catch {
return '[unserializable]';
}
}
function normalizeAssistantMessageForLangSmith(message: Record<string, unknown>): unknown {
const content = message.content;
if (!Array.isArray(content)) {
return message;
}
const textParts: string[] = [];
const toolCalls: Array<Record<string, unknown>> = [];
for (const part of content) {
if (!isRecord(part)) continue;
if (part.type === 'text' && typeof part.text === 'string') {
textParts.push(part.text);
continue;
}
if (part.type !== 'tool-call') continue;
const toolCallId =
typeof part.toolCallId === 'string'
? part.toolCallId
: typeof part.id === 'string'
? part.id
: undefined;
const toolName =
typeof part.toolName === 'string'
? part.toolName
: typeof part.name === 'string'
? part.name
: undefined;
if (!toolCallId || !toolName) continue;
toolCalls.push({
id: toolCallId,
type: 'function',
function: {
name: toolName,
arguments: stringifyToolPayload(readToolCallPayload(part)),
},
});
}
if (toolCalls.length === 0) {
return message;
}
return {
role: 'assistant',
content: textParts.join('\n'),
tool_calls: toolCalls,
};
}
function normalizeToolMessageForLangSmith(message: Record<string, unknown>): unknown[] {
const content = message.content;
if (!Array.isArray(content)) {
return [message];
}
const normalizedMessages: unknown[] = [];
for (const part of content) {
if (!isRecord(part) || part.type !== 'tool-result') {
continue;
}
const toolCallId =
typeof part.toolCallId === 'string'
? part.toolCallId
: typeof part.id === 'string'
? part.id
: undefined;
const toolName =
typeof part.toolName === 'string'
? part.toolName
: typeof part.name === 'string'
? part.name
: undefined;
if (!toolCallId) continue;
normalizedMessages.push({
role: 'tool',
tool_call_id: toolCallId,
...(toolName ? { name: toolName } : {}),
content: stringifyToolPayload(readToolResultPayload(part)),
});
}
return normalizedMessages.length > 0 ? normalizedMessages : [message];
}
function normalizeMessagesForLangSmith(messages: unknown[]): unknown[] {
const normalizedMessages: unknown[] = [];
for (const message of messages) {
if (!isRecord(message) || typeof message.role !== 'string') {
normalizedMessages.push(message);
continue;
}
if (message.role === 'assistant') {
normalizedMessages.push(normalizeAssistantMessageForLangSmith(message));
continue;
}
if (message.role === 'tool') {
normalizedMessages.push(...normalizeToolMessageForLangSmith(message));
continue;
}
normalizedMessages.push(message);
}
return normalizedMessages;
}
function stableStringify(value: unknown): string {
if (Array.isArray(value)) {
return `[${value.map((entry) => stableStringify(entry)).join(',')}]`;
@ -759,7 +910,7 @@ function enrichLangSmithPromptAttribute(attributes: Record<string, unknown>): vo
return;
}
const prompt: Record<string, unknown> = { input: messages };
const prompt: Record<string, unknown> = { input: normalizeMessagesForLangSmith(messages) };
if (tools) {
prompt.tools = tools;
}