feat(core): Rework LangSmith tracing (no-changelog) (#30017)

This commit is contained in:
oleg 2026-05-20 12:06:11 +02:00 committed by GitHub
parent 3ee618b35b
commit c8ac2fb1f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 4426 additions and 1351 deletions

View File

@ -1,4 +1,12 @@
const mockExporterConfigs: unknown[] = [];
type MockExportResult = { code: number; error?: Error };
type MockExportCallback = (result: MockExportResult) => void;
type MockExporter = {
type: 'exporter';
export: jest.Mock<void, [unknown[], MockExportCallback]>;
shutdown: jest.Mock<Promise<void>, []>;
};
const mockExporterInstances: MockExporter[] = [];
const mockBatchProcessorInputs: unknown[] = [];
const mockBatchProcessorInstances: Array<{
forceFlush: jest.Mock<Promise<void>, []>;
@ -19,7 +27,15 @@ const mockProvider = {
jest.mock('langsmith/experimental/otel/exporter', () => ({
LangSmithOTLPTraceExporter: jest.fn((config: unknown) => {
mockExporterConfigs.push(config);
return { type: 'exporter' };
const exporter: MockExporter = {
type: 'exporter',
export: jest.fn((_: unknown[], resultHandler: MockExportCallback) => {
resultHandler({ code: 0 });
}),
shutdown: jest.fn(async () => await Promise.resolve()),
};
mockExporterInstances.push(exporter);
return exporter;
}),
}));
@ -59,6 +75,7 @@ describe('LangSmithTelemetry', () => {
beforeEach(() => {
mockExporterConfigs.length = 0;
mockExporterInstances.length = 0;
mockBatchProcessorInputs.length = 0;
mockBatchProcessorInstances.length = 0;
mockProviderConfigs.length = 0;
@ -78,21 +95,16 @@ describe('LangSmithTelemetry', () => {
}
});
it('passes proxy headers and derived OTLP URL to the LangSmith exporter', async () => {
it('passes static proxy headers and derived OTLP URL to the LangSmith exporter', async () => {
const transformExportedSpan = (span: unknown) => span;
const getHeaders = jest.fn(async () => {
await Promise.resolve();
return { Authorization: 'Bearer proxy-token' } satisfies Record<string, string>;
});
const built = await new LangSmithTelemetry({
apiKey: '-',
project: 'instance-ai',
endpoint: 'https://ai-proxy.test/langsmith',
headers: getHeaders,
headers: { Authorization: 'Bearer proxy-token' },
transformExportedSpan,
}).build();
expect(getHeaders).toHaveBeenCalledTimes(1);
expect(mockExporterConfigs).toEqual([
{
apiKey: '-',
@ -102,7 +114,7 @@ describe('LangSmithTelemetry', () => {
url: 'https://ai-proxy.test/langsmith/otel/v1/traces',
},
]);
expect(mockBatchProcessorInputs).toEqual([{ type: 'exporter' }]);
expect(mockBatchProcessorInputs).toEqual([mockExporterInstances[0]]);
expect(mockProviderConfigs).toHaveLength(1);
const providerConfig = mockProviderConfigs[0] as { spanProcessors: unknown[] };
expect(providerConfig.spanProcessors).toHaveLength(1);
@ -118,6 +130,84 @@ describe('LangSmithTelemetry', () => {
expect(process.env.LANGCHAIN_TRACING_V2).toBe('true');
});
it('resolves function headers for every export request', async () => {
const getHeaders = jest
.fn<Promise<Record<string, string>>, []>()
.mockResolvedValueOnce({ Authorization: 'Bearer proxy-token-1' })
.mockResolvedValueOnce({ Authorization: 'Bearer proxy-token-2' });
await new LangSmithTelemetry({
apiKey: '-',
project: 'instance-ai',
endpoint: 'https://ai-proxy.test/langsmith',
headers: getHeaders,
}).build();
expect(getHeaders).not.toHaveBeenCalled();
expect(mockExporterConfigs).toEqual([]);
expect(mockBatchProcessorInputs).toHaveLength(1);
const exporter = mockBatchProcessorInputs[0] as {
export(spans: unknown[], resultCallback: MockExportCallback): void;
};
const firstSpan = { name: 'first' };
const secondSpan = { name: 'second' };
const firstResult = await new Promise<MockExportResult>((resolve) => {
exporter.export([firstSpan], resolve);
});
const secondResult = await new Promise<MockExportResult>((resolve) => {
exporter.export([secondSpan], resolve);
});
expect(firstResult).toEqual({ code: 0 });
expect(secondResult).toEqual({ code: 0 });
expect(getHeaders).toHaveBeenCalledTimes(2);
expect(mockExporterConfigs).toEqual([
{
apiKey: '-',
projectName: 'instance-ai',
headers: { Authorization: 'Bearer proxy-token-1' },
url: 'https://ai-proxy.test/langsmith/otel/v1/traces',
},
{
apiKey: '-',
projectName: 'instance-ai',
headers: { Authorization: 'Bearer proxy-token-2' },
url: 'https://ai-proxy.test/langsmith/otel/v1/traces',
},
]);
expect(mockExporterInstances[0].export).toHaveBeenCalledWith([firstSpan], expect.any(Function));
expect(mockExporterInstances[1].export).toHaveBeenCalledWith(
[secondSpan],
expect.any(Function),
);
});
it('reports export failure when function headers reject', async () => {
const refreshError = new Error('could not refresh headers');
const getHeaders = jest
.fn<Promise<Record<string, string>>, []>()
.mockRejectedValueOnce(refreshError);
await new LangSmithTelemetry({
apiKey: '-',
project: 'instance-ai',
headers: getHeaders,
}).build();
const exporter = mockBatchProcessorInputs[0] as {
export(spans: unknown[], resultCallback: MockExportCallback): void;
};
const result = await new Promise<MockExportResult>((resolve) => {
exporter.export([{ name: 'span' }], resolve);
});
expect(result).toEqual({ code: 1, error: refreshError });
expect(getHeaders).toHaveBeenCalledTimes(1);
expect(mockExporterConfigs).toEqual([]);
});
it('does not allow endpoint overrides when using an engine-resolved key', async () => {
const telemetry = new LangSmithTelemetry({
project: 'instance-ai',

View File

@ -39,12 +39,66 @@ interface BatchSpanProcessorConstructor {
new (exporter: unknown): SpanProcessorLike;
}
interface ExportResultLike {
code: number;
error?: Error;
}
type ExportResultCallback = (result: ExportResultLike) => void;
interface SpanExporterLike {
export(spans: unknown[], resultCallback: ExportResultCallback): void;
shutdown(): Promise<void>;
}
interface LangSmithOTLPTraceExporterConfig {
apiKey?: string;
projectName?: string;
url?: string;
headers?: Record<string, string>;
transformExportedSpan?: (span: unknown) => unknown;
}
interface LangSmithOTLPTraceExporterConstructor {
new (cfg?: LangSmithOTLPTraceExporterConfig): SpanExporterLike;
}
interface LangSmithRunTree {
getSharedClient(): {
awaitPendingTraceBatches(): Promise<void>;
};
}
const OTEL_EXPORT_RESULT_FAILED = 1;
function toExportError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
function createHeaderRefreshingLangSmithExporter(
LangSmithOTLPTraceExporter: LangSmithOTLPTraceExporterConstructor,
getExporterConfig: (headers: Record<string, string>) => LangSmithOTLPTraceExporterConfig,
getHeaders: () => Promise<Record<string, string>>,
): SpanExporterLike {
return {
export(spans, resultCallback) {
void (async () => {
try {
const headers = await getHeaders();
const exporter = new LangSmithOTLPTraceExporter(getExporterConfig(headers));
exporter.export(spans, resultCallback);
} catch (error) {
resultCallback({ code: OTEL_EXPORT_RESULT_FAILED, error: toExportError(error) });
}
})();
},
async shutdown() {
await Promise.resolve();
},
};
}
function isOtelSpanLike(value: unknown): value is OtelSpanLike {
return (
value !== null &&
@ -174,7 +228,10 @@ export interface LangSmithTelemetryConfig {
* as `${endpoint}/otel/v1/traces`. Use this for custom collectors or testing.
*/
url?: string;
/** Default headers to send with LangSmith OTLP export requests. */
/**
* Default headers to send with LangSmith OTLP export requests.
* Callback headers are resolved per export request.
*/
headers?: Record<string, string> | (() => Promise<Record<string, string>>);
/** Optional hook for redacting or annotating spans before LangSmith export. */
transformExportedSpan?: (span: unknown) => unknown;
@ -199,13 +256,7 @@ async function createLangSmithTracer(
};
const { LangSmithOTLPTraceExporter } = (await import('langsmith/experimental/otel/exporter')) as {
LangSmithOTLPTraceExporter: new (cfg?: {
apiKey?: string;
projectName?: string;
url?: string;
headers?: Record<string, string>;
transformExportedSpan?: (span: unknown) => unknown;
}) => unknown;
LangSmithOTLPTraceExporter: LangSmithOTLPTraceExporterConstructor;
};
const { BatchSpanProcessor } = (await import('@opentelemetry/sdk-trace-base')) as {
BatchSpanProcessor: BatchSpanProcessorConstructor;
@ -223,9 +274,10 @@ async function createLangSmithTracer(
? undefined
: (config?.url ??
(config?.endpoint ? `${config.endpoint.replace(/\/$/, '')}/otel/v1/traces` : undefined));
const headers = typeof config?.headers === 'function' ? await config.headers() : config?.headers;
const exporter = new LangSmithOTLPTraceExporter({
const buildExporterConfig = (
headers?: Record<string, string>,
): LangSmithOTLPTraceExporterConfig => ({
apiKey,
projectName: config?.project,
...(headers ? { headers } : {}),
@ -235,6 +287,16 @@ async function createLangSmithTracer(
...(url ? { url } : {}),
});
const headers = config?.headers;
const exporter =
typeof headers === 'function'
? createHeaderRefreshingLangSmithExporter(
LangSmithOTLPTraceExporter,
buildExporterConfig,
headers,
)
: new LangSmithOTLPTraceExporter(buildExporterConfig(headers));
const processor = createLangSmithSpanProcessor({
exporter,
BatchSpanProcessor,

View File

@ -2807,6 +2807,32 @@ describe('AgentRuntime — telemetry propagation', () => {
expect(expTelemetry.recordOutputs).toBe(false);
});
it('uses updated telemetry config for later runs', async () => {
generateText.mockResolvedValue(makeGenerateSuccess());
const updatedTelemetry: BuiltTelemetry = {
...baseTelemetry,
functionId: 'updated-agent',
metadata: { env: 'updated' },
};
const runtime = new AgentRuntime({
name: 'telemetry-test',
model: 'openai/gpt-4o-mini',
instructions: 'test',
eventBus: new AgentEventBus(),
telemetry: baseTelemetry,
});
runtime.setTelemetry(updatedTelemetry);
await runtime.generate('hello');
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const callArgs = generateText.mock.calls[0][0] as Record<string, unknown>;
const expTelemetry = callArgs.experimental_telemetry as Record<string, unknown>;
expect(expTelemetry.functionId).toBe('updated-agent');
expect(expTelemetry.metadata).toEqual({ env: 'updated' });
});
it('wraps generate calls in a telemetry root span when the tracer supports active spans', async () => {
generateText.mockResolvedValue(makeGenerateSuccess());
const span = {

View File

@ -69,6 +69,7 @@ import {
toAiSdkProviderTools,
toAiSdkTools,
} from './tool-adapter';
import { Telemetry } from '../sdk/telemetry';
import { AgentEvent } from '../types/runtime/event';
import type { AgentEventData } from '../types/runtime/event';
import type {
@ -362,6 +363,10 @@ export class AgentRuntime {
};
}
setTelemetry(telemetry: BuiltTelemetry | undefined): void {
this.config.telemetry = telemetry;
}
/**
* Wait for in-flight background tasks (title generation, future
* observer cycles) to settle. Safe to call multiple times.
@ -748,14 +753,7 @@ export class AgentRuntime {
/** Best-effort flush of telemetry provider. Never throws. */
private async flushTelemetry(options?: ExecutionOptions): Promise<void> {
try {
const resolved = this.resolveTelemetry(options);
if (resolved?.provider) {
await resolved.provider.forceFlush();
}
} catch {
// Telemetry flush is best-effort — never block the response or mask the real error.
}
await Telemetry.forceFlush(this.resolveTelemetry(options));
}
/** Map resolved telemetry to AI SDK's experimental_telemetry shape. */

View File

@ -0,0 +1,80 @@
import type { BuiltTelemetry } from '../../types';
import { Agent } from '../agent';
// Mock provider packages so createModel() doesn't fail when no API key is set.
jest.mock('@ai-sdk/openai', () => ({
createOpenAI: () => () => ({ provider: 'openai', modelId: 'mock', specificationVersion: 'v3' }),
}));
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
type AiImport = typeof import('ai');
jest.mock('ai', () => {
const actual = jest.requireActual<AiImport>('ai');
return {
...actual,
generateText: jest.fn(),
};
});
// eslint-disable-next-line @typescript-eslint/no-require-imports
const { generateText } = require('ai') as {
generateText: jest.Mock;
};
function makeGenerateSuccess(text = 'OK') {
return {
finishReason: 'stop',
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
response: {
messages: [
{
role: 'assistant',
content: [{ type: 'text', text }],
},
],
},
toolCalls: [],
};
}
function makeTelemetry(functionId: string): BuiltTelemetry {
return {
enabled: true,
functionId,
metadata: { functionId },
recordInputs: true,
recordOutputs: true,
integrations: [],
tracer: { startSpan: jest.fn() },
};
}
describe('Agent telemetry', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('updates telemetry on an already-built runtime', async () => {
generateText.mockResolvedValue(makeGenerateSuccess());
const agent = new Agent('agent')
.model('openai/gpt-4o-mini')
.instructions('test')
.telemetry(makeTelemetry('initial-agent'));
await agent.generate('first');
agent.telemetry(makeTelemetry('updated-agent'));
await agent.generate('second');
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const firstCall = generateText.mock.calls[0][0] as Record<string, unknown>;
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const secondCall = generateText.mock.calls[1][0] as Record<string, unknown>;
const firstTelemetry = firstCall.experimental_telemetry as Record<string, unknown>;
const secondTelemetry = secondCall.experimental_telemetry as Record<string, unknown>;
expect(firstTelemetry.functionId).toBe('initial-agent');
expect(secondTelemetry.functionId).toBe('updated-agent');
expect(secondTelemetry.metadata).toEqual({ functionId: 'updated-agent' });
});
});

View File

@ -171,3 +171,37 @@ describe('Telemetry.shutdown()', () => {
await Telemetry.shutdown(built);
});
});
describe('Telemetry.forceFlush()', () => {
it('calls provider.forceFlush() when provider exists', async () => {
const forceFlushMock = jest.fn().mockResolvedValue(undefined);
const built = await new Telemetry().build();
const withProvider = {
...built,
provider: { forceFlush: forceFlushMock, shutdown: jest.fn() },
};
await Telemetry.forceFlush(withProvider);
expect(forceFlushMock).toHaveBeenCalled();
});
it('swallows provider.forceFlush() errors', async () => {
const built = await new Telemetry().build();
const withProvider = {
...built,
provider: {
forceFlush: jest.fn().mockRejectedValue(new Error('flush failed')),
shutdown: jest.fn(),
},
};
await expect(Telemetry.forceFlush(withProvider)).resolves.toBeUndefined();
});
it('does nothing when no provider exists', async () => {
const built = await new Telemetry().build();
await expect(Telemetry.forceFlush(built)).resolves.toBeUndefined();
});
});

View File

@ -344,6 +344,7 @@ export class Agent implements BuiltAgent, AgentBuilder {
} else {
this.telemetryBuilder = undefined;
this.telemetryConfig = t;
this.runtime?.setTelemetry(t);
}
return this;
}

View File

@ -309,4 +309,13 @@ export class Telemetry {
await telemetry.provider.shutdown();
}
}
/** Best-effort provider flush. Telemetry export must not affect agent execution. */
static async forceFlush(telemetry: BuiltTelemetry | undefined): Promise<void> {
try {
await telemetry?.provider?.forceFlush();
} catch {
// Telemetry flush is best-effort — never block the response or mask the real error.
}
}
}

View File

@ -17,6 +17,15 @@ All Instance AI configuration is done via environment variables.
| `N8N_INSTANCE_AI_BROWSER_MCP` | boolean | `false` | Enable Chrome DevTools MCP for browser-assisted credential setup |
| `N8N_INSTANCE_AI_LOCAL_GATEWAY_DISABLED` | boolean | `false` | Disable the local gateway (filesystem, shell, browser) for all users |
### Tracing
| Variable | Type | Default | Description |
|----------|------|---------|-------------|
| `N8N_DIAGNOSTICS_ENABLED` | boolean | `true` | When set to `false`, Instance AI tracing is disabled. |
| `LANGSMITH_API_KEY` / `LANGCHAIN_API_KEY` | string | unset | Enables direct LangSmith export for local and self-hosted setups. |
| `LANGSMITH_ENDPOINT` / `LANGCHAIN_ENDPOINT` | string | unset | Optional direct LangSmith endpoint override. |
| `LANGSMITH_TRACING` / `LANGCHAIN_TRACING_V2` | boolean | unset | LangSmith SDK tracing flags. `false` disables tracing; `true` enables direct tracing when direct LangSmith credentials or endpoints are configured. |
### Memory
| Variable | Type | Default | Description |

View File

@ -14,28 +14,12 @@ import {
} from '../tool-registry';
import { createAllTools, createOrchestratorDomainTools, createOrchestrationTools } from '../tools';
import { createToolsFromLocalMcpServer } from '../tools/filesystem/create-tools-from-mcp-server';
import { ALWAYS_LOADED_TOOL_NAMES, CHECKPOINT_FOLLOW_UP_TOOL_NAMES } from '../tools/tool-ids';
import { buildAgentTraceInputs, mergeTraceRunInputs } from '../tracing/langsmith-tracing';
import type { CreateInstanceAgentOptions, InstanceAiToolRegistry } from '../types';
// ── Agent factory ───────────────────────────────────────────────────────────
const ALWAYS_LOADED_TOOLS = new Set([
'plan',
'create-tasks',
'delegate',
'ask-user',
'credentials',
'workflows',
'build-workflow-with-agent',
'verify-built-workflow',
'research',
'evals',
'web-search',
'fetch-url',
]);
const CHECKPOINT_FOLLOW_UP_TOOLS = new Set(['complete-checkpoint', 'executions']);
function splitDeferredTools(
tools: InstanceAiToolRegistry,
options: { isCheckpointFollowUp?: boolean } = {},
@ -45,8 +29,8 @@ function splitDeferredTools(
for (const [name, tool] of tools) {
if (
ALWAYS_LOADED_TOOLS.has(name) ||
(options.isCheckpointFollowUp && CHECKPOINT_FOLLOW_UP_TOOLS.has(name))
ALWAYS_LOADED_TOOL_NAMES.has(name) ||
(options.isCheckpointFollowUp && CHECKPOINT_FOLLOW_UP_TOOL_NAMES.has(name))
) {
coreTools.set(name, tool);
} else {

View File

@ -8,10 +8,11 @@ import { z } from 'zod';
import { sanitizeInputSchema } from '../agent/sanitize-mcp-schemas';
import type { InstanceAiContext } from '../types';
import { CREDENTIALS_TOOL_ID } from './tool-ids';
// ── Constants ──────────────────────────────────────────────────────────────
export const CREDENTIALS_TOOL_ID = 'credentials';
export { CREDENTIALS_TOOL_ID };
const DEFAULT_LIMIT = 50;

View File

@ -9,10 +9,11 @@ import { z } from 'zod';
import { sanitizeInputSchema } from '../agent/sanitize-mcp-schemas';
import type { InstanceAiContext } from '../types';
import { DATA_TABLES_TOOL_ID } from './tool-ids';
// ── Shared schemas ─────────────────────────────────────────────────────────
export const DATA_TABLES_TOOL_ID = 'data-tables';
export { DATA_TABLES_TOOL_ID };
const columnTypeSchema = z.enum(['string', 'number', 'boolean', 'date']);

View File

@ -4,8 +4,8 @@ import { isParseableAttachment } from '../parsers/structured-file-parser';
import { createToolRegistry } from '../tool-registry';
import type { InstanceAiContext, InstanceAiToolRegistry, OrchestrationContext } from '../types';
import { createParseFileTool } from './attachments/parse-file.tool';
import { createCredentialsTool, CREDENTIALS_TOOL_ID } from './credentials.tool';
import { createDataTablesTool, DATA_TABLES_TOOL_ID } from './data-tables.tool';
import { createCredentialsTool } from './credentials.tool';
import { createDataTablesTool } from './data-tables.tool';
import { createEvalsTool } from './evals/evals.tool';
import { createExecutionsTool } from './executions.tool';
import { createNodesTool } from './nodes.tool';
@ -18,8 +18,9 @@ import { createPlanTool } from './orchestration/plan.tool';
import { createReportVerificationVerdictTool } from './orchestration/report-verification-verdict.tool';
import { createVerifyBuiltWorkflowTool } from './orchestration/verify-built-workflow.tool';
import { createResearchTool } from './research.tool';
import { ASK_USER_TOOL_ID, createAskUserTool } from './shared/ask-user.tool';
import { createAskUserTool } from './shared/ask-user.tool';
import { createTaskControlTool } from './task-control.tool';
import { DOMAIN_TOOL_IDS, ORCHESTRATION_TOOL_IDS } from './tool-ids';
import { createApplyWorkflowCredentialsTool } from './workflows/apply-workflow-credentials.tool';
import { createBuildWorkflowTool } from './workflows/build-workflow.tool';
import { createWorkflowsTool } from './workflows.tool';
@ -31,20 +32,20 @@ import { createWorkspaceTool } from './workspace.tool';
*/
export function createAllTools(context: InstanceAiContext): InstanceAiToolRegistry {
const tools: Array<[string, BuiltTool]> = [
['workflows', createWorkflowsTool(context)],
['evals', createEvalsTool(context)],
['executions', createExecutionsTool(context)],
[CREDENTIALS_TOOL_ID, createCredentialsTool(context)],
[DATA_TABLES_TOOL_ID, createDataTablesTool(context)],
['workspace', createWorkspaceTool(context)],
['research', createResearchTool(context)],
['nodes', createNodesTool(context)],
[ASK_USER_TOOL_ID, createAskUserTool()],
['build-workflow', createBuildWorkflowTool(context)],
[DOMAIN_TOOL_IDS.WORKFLOWS, createWorkflowsTool(context)],
[DOMAIN_TOOL_IDS.EVALS, createEvalsTool(context)],
[DOMAIN_TOOL_IDS.EXECUTIONS, createExecutionsTool(context)],
[DOMAIN_TOOL_IDS.CREDENTIALS, createCredentialsTool(context)],
[DOMAIN_TOOL_IDS.DATA_TABLES, createDataTablesTool(context)],
[DOMAIN_TOOL_IDS.WORKSPACE, createWorkspaceTool(context)],
[DOMAIN_TOOL_IDS.RESEARCH, createResearchTool(context)],
[DOMAIN_TOOL_IDS.NODES, createNodesTool(context)],
[DOMAIN_TOOL_IDS.ASK_USER, createAskUserTool()],
[DOMAIN_TOOL_IDS.BUILD_WORKFLOW, createBuildWorkflowTool(context)],
];
if (context.currentUserAttachments?.some(isParseableAttachment)) {
tools.push(['parse-file', createParseFileTool(context)]);
tools.push([DOMAIN_TOOL_IDS.PARSE_FILE, createParseFileTool(context)]);
}
return createToolRegistry(tools);
@ -56,19 +57,19 @@ export function createAllTools(context: InstanceAiContext): InstanceAiToolRegist
*/
export function createOrchestratorDomainTools(context: InstanceAiContext): InstanceAiToolRegistry {
const tools: Array<[string, BuiltTool]> = [
['workflows', createWorkflowsTool(context, 'orchestrator')],
['evals', createEvalsTool(context)],
['executions', createExecutionsTool(context)],
[CREDENTIALS_TOOL_ID, createCredentialsTool(context)],
[DATA_TABLES_TOOL_ID, createDataTablesTool(context, 'orchestrator')],
['workspace', createWorkspaceTool(context)],
['research', createResearchTool(context)],
['nodes', createNodesTool(context, 'orchestrator')],
[ASK_USER_TOOL_ID, createAskUserTool()],
[DOMAIN_TOOL_IDS.WORKFLOWS, createWorkflowsTool(context, 'orchestrator')],
[DOMAIN_TOOL_IDS.EVALS, createEvalsTool(context)],
[DOMAIN_TOOL_IDS.EXECUTIONS, createExecutionsTool(context)],
[DOMAIN_TOOL_IDS.CREDENTIALS, createCredentialsTool(context)],
[DOMAIN_TOOL_IDS.DATA_TABLES, createDataTablesTool(context, 'orchestrator')],
[DOMAIN_TOOL_IDS.WORKSPACE, createWorkspaceTool(context)],
[DOMAIN_TOOL_IDS.RESEARCH, createResearchTool(context)],
[DOMAIN_TOOL_IDS.NODES, createNodesTool(context, 'orchestrator')],
[DOMAIN_TOOL_IDS.ASK_USER, createAskUserTool()],
];
if (context.currentUserAttachments?.some(isParseableAttachment)) {
tools.push(['parse-file', createParseFileTool(context)]);
tools.push([DOMAIN_TOOL_IDS.PARSE_FILE, createParseFileTool(context)]);
}
return createToolRegistry(tools);
@ -80,25 +81,37 @@ export function createOrchestratorDomainTools(context: InstanceAiContext): Insta
*/
export function createOrchestrationTools(context: OrchestrationContext): InstanceAiToolRegistry {
const tools: Array<[string, BuiltTool]> = [
['plan', createPlanWithAgentTool(context)],
['create-tasks', createPlanTool(context)],
['task-control', createTaskControlTool(context)],
['delegate', createDelegateTool(context)],
['build-workflow-with-agent', createBuildWorkflowAgentTool(context)],
['complete-checkpoint', createCompleteCheckpointTool(context)],
[ORCHESTRATION_TOOL_IDS.PLAN, createPlanWithAgentTool(context)],
[ORCHESTRATION_TOOL_IDS.CREATE_TASKS, createPlanTool(context)],
[ORCHESTRATION_TOOL_IDS.TASK_CONTROL, createTaskControlTool(context)],
[ORCHESTRATION_TOOL_IDS.DELEGATE, createDelegateTool(context)],
[ORCHESTRATION_TOOL_IDS.BUILD_WORKFLOW_WITH_AGENT, createBuildWorkflowAgentTool(context)],
[ORCHESTRATION_TOOL_IDS.COMPLETE_CHECKPOINT, createCompleteCheckpointTool(context)],
];
if (context.browserMcpConfig || hasGatewayBrowserTools(context)) {
tools.push(['browser-credential-setup', createBrowserCredentialSetupTool(context)]);
tools.push([
ORCHESTRATION_TOOL_IDS.BROWSER_CREDENTIAL_SETUP,
createBrowserCredentialSetupTool(context),
]);
}
if (context.workflowTaskService) {
tools.push(['report-verification-verdict', createReportVerificationVerdictTool(context)]);
tools.push([
ORCHESTRATION_TOOL_IDS.REPORT_VERIFICATION_VERDICT,
createReportVerificationVerdictTool(context),
]);
}
if (context.workflowTaskService && context.domainContext) {
tools.push(['verify-built-workflow', createVerifyBuiltWorkflowTool(context)]);
tools.push(['apply-workflow-credentials', createApplyWorkflowCredentialsTool(context)]);
tools.push([
ORCHESTRATION_TOOL_IDS.VERIFY_BUILT_WORKFLOW,
createVerifyBuiltWorkflowTool(context),
]);
tools.push([
ORCHESTRATION_TOOL_IDS.APPLY_WORKFLOW_CREDENTIALS,
createApplyWorkflowCredentialsTool(context),
]);
}
return createToolRegistry(tools);

View File

@ -56,7 +56,7 @@ const PLANNER_DOMAIN_TOOL_NAMES = [
/** Research tools added when available. */
const PLANNER_RESEARCH_TOOL_NAMES = ['research'];
const RELEVANT_PRIOR_TOOL_NAMES = new Set([
const RELEVANT_PRIOR_TOOL_NAMES = new Set<string>([
ASK_USER_TOOL_ID,
CREDENTIALS_TOOL_ID,
DATA_TABLES_TOOL_ID,

View File

@ -2,7 +2,9 @@ import { Tool } from '@n8n/agents';
import { nanoid } from 'nanoid';
import { z } from 'zod';
export const ASK_USER_TOOL_ID = 'ask-user';
import { ASK_USER_TOOL_ID } from '../tool-ids';
export { ASK_USER_TOOL_ID };
const questionSchema = z.object({
id: z.string().describe('Unique question identifier'),

View File

@ -0,0 +1,62 @@
export const DOMAIN_TOOL_IDS = {
WORKFLOWS: 'workflows',
EVALS: 'evals',
EXECUTIONS: 'executions',
CREDENTIALS: 'credentials',
DATA_TABLES: 'data-tables',
WORKSPACE: 'workspace',
RESEARCH: 'research',
NODES: 'nodes',
ASK_USER: 'ask-user',
BUILD_WORKFLOW: 'build-workflow',
PARSE_FILE: 'parse-file',
} as const;
export const ORCHESTRATION_TOOL_IDS = {
PLAN: 'plan',
SUBMIT_PLAN: 'submit-plan',
ADD_PLAN_ITEM: 'add-plan-item',
REMOVE_PLAN_ITEM: 'remove-plan-item',
CREATE_TASKS: 'create-tasks',
TASK_CONTROL: 'task-control',
DELEGATE: 'delegate',
BUILD_WORKFLOW_WITH_AGENT: 'build-workflow-with-agent',
MANAGE_DATA_TABLES_WITH_AGENT: 'manage-data-tables-with-agent',
RESEARCH_WITH_AGENT: 'research-with-agent',
BROWSER_CREDENTIAL_SETUP: 'browser-credential-setup',
COMPLETE_CHECKPOINT: 'complete-checkpoint',
VERIFY_BUILT_WORKFLOW: 'verify-built-workflow',
REPORT_VERIFICATION_VERDICT: 'report-verification-verdict',
APPLY_WORKFLOW_CREDENTIALS: 'apply-workflow-credentials',
} as const;
export const WORKSPACE_TOOL_IDS = {
WRITE_FILE: 'write-file',
SUBMIT_WORKFLOW: 'submit-workflow',
} as const;
export const CREDENTIALS_TOOL_ID = DOMAIN_TOOL_IDS.CREDENTIALS;
export const DATA_TABLES_TOOL_ID = DOMAIN_TOOL_IDS.DATA_TABLES;
export const ASK_USER_TOOL_ID = DOMAIN_TOOL_IDS.ASK_USER;
export const ORCHESTRATION_TOOL_NAMES = new Set<string>(Object.values(ORCHESTRATION_TOOL_IDS));
export const ALWAYS_LOADED_TOOL_NAMES = new Set<string>([
ORCHESTRATION_TOOL_IDS.PLAN,
ORCHESTRATION_TOOL_IDS.CREATE_TASKS,
ORCHESTRATION_TOOL_IDS.DELEGATE,
DOMAIN_TOOL_IDS.ASK_USER,
DOMAIN_TOOL_IDS.CREDENTIALS,
DOMAIN_TOOL_IDS.WORKFLOWS,
ORCHESTRATION_TOOL_IDS.BUILD_WORKFLOW_WITH_AGENT,
ORCHESTRATION_TOOL_IDS.VERIFY_BUILT_WORKFLOW,
DOMAIN_TOOL_IDS.RESEARCH,
DOMAIN_TOOL_IDS.EVALS,
'web-search',
'fetch-url',
]);
export const CHECKPOINT_FOLLOW_UP_TOOL_NAMES = new Set<string>([
ORCHESTRATION_TOOL_IDS.COMPLETE_CHECKPOINT,
DOMAIN_TOOL_IDS.EXECUTIONS,
]);

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,34 @@
export function formatTraceLabel(value: string): string {
return value
.trim()
.replace(/[._\s]+/g, '-')
.replace(/-+/g, '-')
.replace(/^-+|-+$/g, '');
}
export function formatAgentRoleLabel(role: string): string {
return formatTraceLabel(role.replace(/^instance-ai[._-]?/, ''));
}
export function formatResumeReasonLabel(reason: unknown): string {
if (typeof reason !== 'string' || reason.trim().length === 0) {
return 'checkpoint';
}
return reason
.trim()
.replace(/[._-]+/g, ' ')
.replace(/\s+/g, ' ');
}
export function formatInternalOperationLabel(operationName: string): string {
return formatAgentRoleLabel(operationName);
}
export function formatTelemetryFunctionId(agentRole: string): string {
if (agentRole.startsWith('instance-ai.')) {
return agentRole;
}
return `instance-ai.${agentRole.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '')}`;
}

File diff suppressed because it is too large Load Diff

View File

@ -22,7 +22,7 @@ const SECRET_VALUE_PATTERNS: readonly RegExp[] = [
// AWS access key id
/\bAKIA[0-9A-Z]{16}\b/g,
// Generic `password=...` / `api_key=...` / `secret=...` style assignments
/\b(?:password|passwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token)\s*[:=]\s*\S+/gi,
/\b(?:password|passwd|secret|api[_-]?key|authorization|access[_-]?token|refresh[_-]?token|id[_-]?token|session[_-]?token|auth[_-]?token)\s*[:=]\s*\S+/gi,
];
export function scrubSecretsInText(input: string): string {

View File

@ -116,6 +116,7 @@ import type { InstanceAiAgentNode, InstanceAiEvent } from '@n8n/api-types';
import {
resumeAgentRun,
type ManagedBackgroundTask,
type InstanceAiTraceContext,
type SpawnBackgroundTaskOptions,
type SpawnBackgroundTaskResult,
type SpawnManagedBackgroundTaskOptions,
@ -575,6 +576,7 @@ type TerminalGuardOrderServiceInternals = {
signal: AbortSignal;
abortController: AbortController;
snapshotStorage: unknown;
tracing?: InstanceAiTraceContext;
},
) => Promise<void>;
};
@ -1555,6 +1557,49 @@ describe('InstanceAiService — terminal response guard wiring', () => {
thread_id: 'thread-a',
});
});
it('rebinds resumed agents to resume trace telemetry', async () => {
const service = createTerminalGuardOrderService();
const abortController = new AbortController();
const telemetry = { enabled: true };
const agent = { telemetry: jest.fn() };
const tracing = {
traceKind: 'orchestrator_resume',
actorRun: { id: 'actor-run' },
getTelemetry: jest.fn(() => telemetry),
withActiveSpan: jest.fn(async (_run: unknown, fn: () => Promise<unknown>) => await fn()),
} as unknown as InstanceAiTraceContext;
jest.mocked(resumeAgentRun).mockResolvedValueOnce({
status: 'completed',
agentRunId: 'agent-run-1',
text: Promise.resolve('done'),
workSummary: { toolCalls: [], totalToolCalls: 0, totalToolErrors: 0 },
});
await service.processResumedStream(
agent,
{},
{
runId: 'run-1',
agentRunId: 'agent-run-1',
threadId: 'thread-a',
user: fakeUser,
toolCallId: 'tool-call-1',
signal: abortController.signal,
abortController,
snapshotStorage: {},
tracing,
},
);
expect(tracing.getTelemetry).toHaveBeenCalledWith({
agentRole: 'orchestrator',
functionId: 'instance-ai.orchestrator',
executionMode: 'resume',
});
expect(agent.telemetry).toHaveBeenCalledWith(telemetry);
expect(tracing.withActiveSpan).toHaveBeenCalledWith(tracing.actorRun, expect.any(Function));
});
});
describe('InstanceAiService — AI temporary workflow cleanup', () => {

View File

@ -122,6 +122,16 @@ function getErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function isTelemetryConfigurableAgent(
agent: unknown,
): agent is { telemetry: (telemetry: unknown) => void } {
return (
typeof agent === 'object' &&
agent !== null &&
typeof Reflect.get(agent, 'telemetry') === 'function'
);
}
const INSTANCE_AI_CHECKPOINT_PRUNE_RETRY_MS = 30 * 1000;
function isTextMessagePart(part: unknown): part is { type: 'text'; text: string } {
@ -3809,6 +3819,25 @@ export class InstanceAiService {
let messageTraceFinalization: MessageTraceFinalization | undefined;
try {
if (opts.tracing?.getTelemetry && isTelemetryConfigurableAgent(agent)) {
try {
agent.telemetry(
opts.tracing.getTelemetry({
agentRole: 'orchestrator',
functionId: 'instance-ai.orchestrator',
executionMode:
opts.tracing.traceKind === 'orchestrator_resume' ? 'resume' : 'foreground',
}),
);
} catch (error) {
this.logger.warn('Failed to configure Instance AI resume tracing', {
error: getErrorMessage(error),
threadId: opts.threadId,
runId: opts.runId,
});
}
}
const result = opts.tracing
? await opts.tracing.withActiveSpan(opts.tracing.actorRun, async () => {
return await resumeAgentRun(