feat(core): Emit opt-in start marker spans for workflow and node executions (#40762)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ocdejong 2026-10-09 14:09:39 +00:00 committed by GitHub
parent 88a2531558
commit c7135451eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
31 changed files with 546 additions and 42 deletions

View File

@ -40,6 +40,26 @@ describe('UpdateOtelSettingsDto', () => {
expect(result.data.exporterProtocol).toBe('http/protobuf');
});
it('defaults the start span flags to false when omitted (body predates the fields)', () => {
const result = UpdateOtelSettingsDto.safeParse(validSettings);
assert(result.success, 'Expected a body without the start span flags to stay valid');
expect(result.data.emitWorkflowStartSpan).toBe(false);
expect(result.data.emitNodeStartSpan).toBe(false);
});
it('accepts the start span flags', () => {
const result = UpdateOtelSettingsDto.safeParse({
...validSettings,
emitWorkflowStartSpan: true,
emitNodeStartSpan: false,
});
assert(result.success, 'Expected the start span flags to be valid');
expect(result.data.emitWorkflowStartSpan).toBe(true);
expect(result.data.emitNodeStartSpan).toBe(false);
});
it('accepts a full body', () => {
const result = UpdateOtelSettingsDto.safeParse(validSettings);
expect(result.success).toBe(true);

View File

@ -20,6 +20,8 @@ export class OtelSettingsPublicDto extends Z.class(
tracesSampleRate: z.number().openapi(docs.tracesSampleRate),
startupConnectivityTimeoutMs: z.number().openapi(docs.startupConnectivityTimeoutMs),
includeNodeSpans: z.boolean().openapi(docs.includeNodeSpans),
emitWorkflowStartSpan: z.boolean().openapi(docs.emitWorkflowStartSpan),
emitNodeStartSpan: z.boolean().openapi(docs.emitNodeStartSpan),
injectOutbound: z.boolean().openapi(docs.injectOutbound),
productionExecutionsOnly: z.boolean().openapi(docs.productionExecutionsOnly),
},
@ -43,6 +45,8 @@ export class UpdateOtelSettingsPublicDto extends Z.class(
docs.startupConnectivityTimeoutMs,
),
includeNodeSpans: fields.includeNodeSpans.openapi(docs.includeNodeSpans),
emitWorkflowStartSpan: fields.emitWorkflowStartSpan.openapi(docs.emitWorkflowStartSpan),
emitNodeStartSpan: fields.emitNodeStartSpan.openapi(docs.emitNodeStartSpan),
injectOutbound: fields.injectOutbound.openapi(docs.injectOutbound),
productionExecutionsOnly: fields.productionExecutionsOnly.openapi(
docs.productionExecutionsOnly,

View File

@ -3,7 +3,8 @@ import type { ZodOpenAPIMetadata } from '@asteasolutions/zod-to-openapi';
export const otelSettingsPublicDescription =
'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a ' +
'full replacement: every field must be provided, except `exporterProtocol`, which defaults to ' +
'`http/protobuf` when omitted. Fields managed declaratively via environment variables are ' +
'`http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which ' +
'default to `false` when omitted. Fields managed declaratively via environment variables are ' +
'returned with their effective value. A write can repeat these values, but changing them returns 409.';
export const otelSettingsPublicFieldDocs = {
@ -63,6 +64,21 @@ export const otelSettingsPublicFieldDocs = {
description: 'Whether to emit a span for each node execution in addition to the workflow span.',
example: true,
},
emitWorkflowStartSpan: {
description:
'Whether to emit a `workflow.execute.started` span with a duration of 0 ms when each ' +
'execution or resumed segment starts. Optional: a write that omits it selects `false`, but an ' +
'instance that sets this field to `true` with an environment variable rejects that write with 409.',
example: false,
},
emitNodeStartSpan: {
description:
'Whether to emit a `node.execute.started` span with a duration of 0 ms when each node span ' +
'starts. This doubles the number of node spans. Has no effect when `includeNodeSpans` is ' +
'false. Optional: a write that omits it selects `false`, but an instance that sets this field ' +
'to `true` with an environment variable rejects that write with 409.',
example: false,
},
injectOutbound: {
description:
'Whether to inject trace context headers into outbound HTTP requests made by nodes.',

View File

@ -25,6 +25,9 @@ export class UpdateOtelSettingsDto extends Z.class({
tracesSampleRate: z.number().min(0).max(1),
startupConnectivityTimeoutMs: z.number().int().nonnegative(),
includeNodeSpans: z.boolean(),
// Defaulted so a body from before these fields existed still parses.
emitWorkflowStartSpan: z.boolean().default(false),
emitNodeStartSpan: z.boolean().default(false),
injectOutbound: z.boolean(),
productionExecutionsOnly: z.boolean(),
}) {}

View File

@ -62,6 +62,31 @@ project name holds the user's name and email.
This key is n8n's single source of truth for the execution identity. Inbound and outbound
baggage (GOV-143) decides what it reads from this key and what it adds to baggage.
### Start marker spans
A span leaves n8n only when it ends. A marker span tells a trace backend that an execution
or a node started, before the real span ends.
| Environment variable | Default | Marker |
| ------------------------------------------ | ------- | -------------------------------------------- |
| `N8N_OTEL_TRACES_EMIT_WORKFLOW_START_SPAN` | `false` | `workflow.execute.started` under each workflow span |
| `N8N_OTEL_TRACES_EMIT_NODE_START_SPAN` | `false` | `node.execute.started` under each node span |
- A marker is a child of the span that it announces. Its duration is 0 ms.
- A marker has the start attributes of its parent and the execution identity. The node
marker does not have node custom tags, because n8n sets them when the node ends.
- Each `workflow.execute` span gets one workflow marker. A resume after a Wait node starts a
new segment with its own `workflow.execute` span.
- In queue mode, main and the worker each send a `workflow.execute` span for each segment,
so each segment gets two workflow markers. The marker from main shows when main queued the
execution. The marker from the worker shows when the worker started it. The resource
attribute `n8n.instance.role` (`main` or `worker`) tells them apart.
- The node marker follows the rules of `node.execute`. With
`N8N_OTEL_TRACES_INCLUDE_NODE_SPANS=false`, no node marker exists. The node flag doubles
the number of node spans.
- Both flags follow `N8N_OTEL_TRACES_PRODUCTION_ONLY`. Set them on workers too.
- A marker that fails logs a warning. It does not stop the real span or the execution.
### Module architecture
```mermaid
graph TD

View File

@ -1,5 +1,5 @@
import type { Logger } from '@n8n/backend-common';
import type { TextMapPropagator } from '@opentelemetry/api';
import type { TextMapPropagator, Tracer } from '@opentelemetry/api';
import { context, propagation, SpanStatusCode, trace } from '@opentelemetry/api';
import { hrTimeToMilliseconds } from '@opentelemetry/core';
import type { ExecutionStatus } from 'n8n-workflow';
@ -10,6 +10,7 @@ import { ExecutionLevelTracer } from '../execution-level-tracer';
import type { StartWorkflowParams } from '../execution-level-tracer.types';
import type { OtelSettingsService } from '../otel-settings.service';
import type { OtelConfig } from '../otel.config';
import type { OtelService } from '../otel.service';
import { OtelTestProvider } from './support/otel-test-provider';
describe('ExecutionLevelTracer', () => {
@ -1323,4 +1324,145 @@ describe('ExecutionLevelTracer', () => {
expect(headers).not.toHaveProperty('baggage');
});
});
describe('start marker spans', () => {
const node = { id: 'n1', name: 'MyNode', type: 'test', typeVersion: 1 };
const project = { id: 'proj-1', name: 'Finance', customAttributes: { env: 'prod' } };
const workflow = { ...defaultWorkflow, customAttributes: { team: 'ops' } };
const identity = {
'n8n.execution.id': 'exec-marker',
'n8n.workflow.id': 'wf-1',
'n8n.workflow.name': 'Test',
'n8n.project.id': 'proj-1',
'n8n.project.name': 'Finance',
};
const runSegment = (emitStartSpan: boolean, params: Partial<StartWorkflowParams> = {}) => {
tracer.startWorkflow({
executionId: 'exec-marker',
workflow,
project,
emitStartSpan,
...params,
});
tracer.startNode({ executionId: 'exec-marker', node, emitStartSpan });
tracer.endNode({ executionId: 'exec-marker', node, inputItemCount: 1, outputItemCount: 1 });
tracer.endWorkflow({
executionId: 'exec-marker',
status: 'success',
mode: 'trigger',
isRetry: false,
});
};
const spansNamed = (name: string) => otel.getFinishedSpans().filter((s) => s.name === name);
it('should emit no marker when the flags are off', () => {
runSegment(false);
expect(otel.getFinishedSpans().map((s) => s.name)).toEqual([
'node.execute',
'workflow.execute',
]);
});
it('should emit one zero-duration workflow marker as a child of the workflow span', () => {
runSegment(true);
const [workflowSpan] = spansNamed('workflow.execute');
const markers = spansNamed('workflow.execute.started');
expect(markers).toHaveLength(1);
const [marker] = markers;
expect(marker.parentSpanContext?.spanId).toBe(workflowSpan.spanContext().spanId);
expect(marker.spanContext().traceId).toBe(workflowSpan.spanContext().traceId);
expect(marker.duration).toEqual([0, 0]);
expect(marker.startTime).toEqual(workflowSpan.startTime);
expect(marker.endTime).toEqual(workflowSpan.startTime);
});
it('should give the workflow marker the start attributes and identity of the workflow span', () => {
runSegment(true);
const [marker] = spansNamed('workflow.execute.started');
expect(marker.attributes).toEqual({
'n8n.workflow.version_id': 'v1',
'n8n.workflow.node_count': 2,
'n8n.workflow.custom.team': 'ops',
'n8n.project.custom.env': 'prod',
...identity,
});
});
it('should emit one zero-duration node marker as a child of each node span', () => {
runSegment(true);
const [nodeSpan] = spansNamed('node.execute');
const markers = spansNamed('node.execute.started');
expect(markers).toHaveLength(1);
const [marker] = markers;
expect(marker.parentSpanContext?.spanId).toBe(nodeSpan.spanContext().spanId);
expect(marker.duration).toEqual([0, 0]);
expect(marker.startTime).toEqual(nodeSpan.startTime);
expect(marker.endTime).toEqual(nodeSpan.startTime);
});
it('should give the node marker the node attributes and identity, without node custom tags', () => {
runSegment(true);
const [marker] = spansNamed('node.execute.started');
expect(marker.attributes).toEqual({
'n8n.node.id': 'n1',
'n8n.node.name': 'MyNode',
'n8n.node.type': 'test',
'n8n.node.type_version': 1,
'n8n.project.custom.env': 'prod',
...identity,
});
});
it('should emit one workflow marker for each segment', () => {
runSegment(true);
runSegment(true, { workflow: { ...workflow, name: 'Renamed' } });
const workflowSpans = spansNamed('workflow.execute');
const markers = spansNamed('workflow.execute.started');
expect(markers).toHaveLength(2);
expect(markers.map((m) => m.parentSpanContext?.spanId)).toEqual(
workflowSpans.map((s) => s.spanContext().spanId),
);
expect(markers.map((m) => m.attributes['n8n.workflow.name'])).toEqual(['Test', 'Renamed']);
});
it('should keep the real spans and only warn when a marker fails', () => {
const failingMarkers = mock<OtelService>({
getTracer: (name: string) => {
const real = otel.provider.getTracer(name);
return {
startSpan: (spanName, options, ctx) => {
if (spanName.endsWith('.started')) throw new Error('marker failed');
return real.startSpan(spanName, options, ctx);
},
} as Tracer;
},
});
tracer = new ExecutionLevelTracer(failingMarkers, makeOtelSettingsService(), logger);
expect(() => runSegment(true)).not.toThrow();
expect(otel.getFinishedSpans().map((s) => s.name)).toEqual([
'node.execute',
'workflow.execute',
]);
expect(logger.warn).toHaveBeenCalledWith(
'Failed to emit start marker span',
expect.objectContaining({ spanName: 'workflow.execute.started', error: 'marker failed' }),
);
expect(logger.warn).toHaveBeenCalledWith(
'Failed to emit start marker span',
expect.objectContaining({ spanName: 'node.execute.started', error: 'marker failed' }),
);
});
});
});

View File

@ -47,6 +47,8 @@ function makeOtelSettingsService(
enabled: true,
productionExecutionsOnly: false,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
exporterProtocol: 'http/protobuf',
exporterEndpoint: 'http://localhost:4318',
exporterTracingPath: '/v1/traces',
@ -374,6 +376,38 @@ describe('OtelLifecycleHandler', () => {
expect(traceContextService.get).toHaveBeenCalledWith('exec-sub');
});
it('should not ask for a workflow start span when the flag is off', async () => {
traceContextService.get.mockResolvedValueOnce(undefined);
await handler.onWorkflowStart(baseCtx);
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: false }),
);
});
it('should ask for a workflow start span when the flag is on', async () => {
otelSettingsService._settings.emitWorkflowStartSpan = true;
traceContextService.get.mockResolvedValueOnce(undefined);
await handler.onWorkflowStart(baseCtx);
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should ask for a workflow start span without a workflow instance (queue-mode main)', async () => {
otelSettingsService._settings.emitWorkflowStartSpan = true;
traceContextService.get.mockResolvedValueOnce(undefined);
await handler.onWorkflowStart({ ...baseCtx, workflowInstance: undefined });
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should always persist generated spanContext', async () => {
traceContextService.get.mockResolvedValueOnce(parentTracingContext);
@ -706,6 +740,23 @@ describe('OtelLifecycleHandler', () => {
);
});
it('should ask for a workflow start span on resume when the flag is on', async () => {
otelSettingsService._settings.emitWorkflowStartSpan = true;
traceContextService.get.mockResolvedValueOnce(prePauseContext);
await handler.onWorkflowResume({
type: 'workflowExecuteResume',
workflow: { id: 'wf-1', name: 'Test', versionId: 'v1', nodes: [], connections: {} },
workflowInstance: createWorkflowInstance(),
executionData: undefined as never,
executionId: 'exec-resume',
} as never);
expect(tracer.startWorkflow).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should start a root span when no pre-wait context is persisted', async () => {
traceContextService.get.mockResolvedValueOnce(undefined);
@ -1030,7 +1081,21 @@ describe('OtelLifecycleHandler', () => {
it('should call tracer.startNode with the resolved node', () => {
handler.onNodeStart(makeStartCtx());
expect(tracer.startNode).toHaveBeenCalledWith({ executionId: 'exec-1', node });
expect(tracer.startNode).toHaveBeenCalledWith({
executionId: 'exec-1',
node,
emitStartSpan: false,
});
});
it('should ask for a node start span when the flag is on', () => {
otelSettingsService._settings.emitNodeStartSpan = true;
handler.onNodeStart(makeStartCtx());
expect(tracer.startNode).toHaveBeenCalledWith(
expect.objectContaining({ emitStartSpan: true }),
);
});
it('should not call tracer.startNode when node is not found in workflow', () => {

View File

@ -27,6 +27,8 @@ const baseSettings: OtelConfig = {
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: false,
};

View File

@ -88,6 +88,8 @@ describe('OtelSettingsService', () => {
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],
@ -308,6 +310,8 @@ describe('OtelSettingsService', () => {
tracesSampleRate: 0.7,
startupConnectivityTimeoutMs: 5_000,
includeNodeSpans: false,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: false,
};

View File

@ -25,6 +25,8 @@ import {
createFailingWorkflowFixture,
createWaitWorkflowFixture,
} from './support/otel-workflow-fixtures';
import { OtelSettingsService } from '../otel-settings.service';
import { OtelConfig } from '../otel.config';
let otel: OtelTestProvider;
let workflowRunner: WorkflowRunner;
@ -219,6 +221,67 @@ describe('OTEL Workflow Tracing Integration', () => {
});
});
describe('Start marker spans', () => {
const setStartSpanFlags = async (enabled: boolean) => {
const config = Container.get(OtelConfig);
config.emitWorkflowStartSpan = enabled;
config.emitNodeStartSpan = enabled;
await Container.get(OtelSettingsService).loadSettings();
};
beforeAll(async () => await setStartSpanFlags(true));
afterAll(async () => await setStartSpanFlags(false));
it('should emit one workflow marker and one marker for each node span', async () => {
const project = await createTeamProject();
const workflow = await createWorkflow(createMultiNodeWorkflowFixture(), project);
const executionId = await executeWorkflow(workflowRunner, workflow, project.id);
await waitForExecution(executionRepository, executionId);
const spans = otel.getFinishedSpans();
const workflowSpan = spans.find((s) => s.name === 'workflow.execute')!;
const nodeSpanIds = spans
.filter((s) => s.name === 'node.execute')
.map((s) => s.spanContext().spanId);
const workflowMarkers = spans.filter((s) => s.name === 'workflow.execute.started');
const nodeMarkers = spans.filter((s) => s.name === 'node.execute.started');
expect(workflowMarkers).toHaveLength(1);
expect(workflowMarkers[0].parentSpanContext?.spanId).toBe(workflowSpan.spanContext().spanId);
expect(workflowMarkers[0].attributes['n8n.execution.id']).toBe(executionId);
expect(nodeMarkers.map((m) => m.parentSpanContext?.spanId).sort()).toEqual(nodeSpanIds.sort());
for (const marker of [...workflowMarkers, ...nodeMarkers]) {
expect(marker.duration).toEqual([0, 0]);
}
});
it('should emit a workflow marker for each segment of a resumed execution', async () => {
const project = await createTeamProject();
const workflow = await createWorkflow(createWaitWorkflowFixture(), project);
const executionId = await executeWorkflow(workflowRunner, workflow, project.id);
await waitForExecutionStatus(executionRepository, executionId, 'waiting');
await Container.get(WorkflowRepository).update(workflow.id, { name: 'Renamed during wait' });
await Container.get(WaitTracker).startExecution(executionId);
await waitForExecutionStatus(executionRepository, executionId, 'success');
const spans = otel.getFinishedSpans();
const parked = spans.find((s) => s.attributes['n8n.execution.status'] === 'waiting')!;
const resumed = spans.find((s) => s.attributes['n8n.execution.status'] === 'success')!;
const markerOf = (segment: typeof parked) =>
spans.filter(
(s) =>
s.name === 'workflow.execute.started' &&
s.parentSpanContext?.spanId === segment.spanContext().spanId,
);
expect(spans.filter((s) => s.name === 'workflow.execute.started')).toHaveLength(2);
expect(markerOf(parked)).toHaveLength(1);
expect(markerOf(resumed)).toHaveLength(1);
expect(markerOf(resumed)[0].attributes['n8n.workflow.name']).toBe('Renamed during wait');
});
});
describe('Custom Telemetry Tags', () => {
const createWorkflowWithCustomTagsFixture = () => ({
nodes: [

View File

@ -54,6 +54,8 @@ const enabledSettings: OtelConfig = {
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
};

View File

@ -162,6 +162,8 @@ const enabledSettings: OtelConfig = {
tracesSampleRate: 1,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
};

View File

@ -1,6 +1,6 @@
import { Logger } from '@n8n/backend-common';
import { Service } from '@n8n/di';
import type { Context, Exception, Span } from '@opentelemetry/api';
import type { Attributes, Context, Exception, Span, TimeInput } from '@opentelemetry/api';
import {
context,
defaultTextMapGetter,
@ -9,7 +9,7 @@ import {
SpanStatusCode,
trace,
} from '@opentelemetry/api';
import { W3CTraceContextPropagator } from '@opentelemetry/core';
import { isTimeInputHrTime, W3CTraceContextPropagator } from '@opentelemetry/core';
import type { ExecutionStatus } from 'n8n-workflow';
import { WorkflowCrashedError } from '@/errors/workflow-crashed.error';
@ -76,31 +76,34 @@ export class ExecutionLevelTracer {
params.project?.customAttributes,
);
const span = this.tracer.startSpan(
'workflow.execute',
{
attributes: {
[ATTR.WORKFLOW_NAME]: params.workflow.name,
[ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '',
[ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount,
...buildCustomAttributes(
ATTR.WORKFLOW_CUSTOM_PREFIX,
params.workflow?.customAttributes,
),
...(params.project && { [ATTR.PROJECT_ID]: params.project.id }),
...projectCustomAttributes,
...identity,
},
links,
},
parentCtx,
);
const attributes = {
[ATTR.WORKFLOW_NAME]: params.workflow.name,
[ATTR.WORKFLOW_VERSION_ID]: params.workflow.versionId ?? '',
[ATTR.WORKFLOW_NODE_COUNT]: params.workflow.nodeCount,
...buildCustomAttributes(ATTR.WORKFLOW_CUSTOM_PREFIX, params.workflow?.customAttributes),
...(params.project && { [ATTR.PROJECT_ID]: params.project.id }),
...projectCustomAttributes,
...identity,
};
const span = this.tracer.startSpan('workflow.execute', { attributes, links }, parentCtx);
const workflowContext = trace.setSpan(parentCtx, span);
this.activeWorkflowSpans.set(params.executionId, {
span,
context: trace.setSpan(parentCtx, span),
context: workflowContext,
projectCustomAttributes,
});
if (params.emitStartSpan) {
this.emitStartMarker(
'workflow.execute.started',
attributes,
workflowContext,
params.executionId,
);
}
return toTracingParentContext(span, identity);
} catch (error) {
this.logger.warn('Failed to start workflow span', {
@ -207,19 +210,24 @@ export class ExecutionLevelTracer {
return;
}
const span = this.tracer.startSpan(
'node.execute',
{
attributes: {
[ATTR.NODE_ID]: params.node.id,
[ATTR.NODE_NAME]: params.node.name,
[ATTR.NODE_TYPE]: params.node.type,
[ATTR.NODE_TYPE_VERSION]: params.node.typeVersion,
...tracked.projectCustomAttributes,
},
},
tracked.context,
);
const attributes = {
[ATTR.NODE_ID]: params.node.id,
[ATTR.NODE_NAME]: params.node.name,
[ATTR.NODE_TYPE]: params.node.type,
[ATTR.NODE_TYPE_VERSION]: params.node.typeVersion,
...tracked.projectCustomAttributes,
};
const span = this.tracer.startSpan('node.execute', { attributes }, tracked.context);
if (params.emitStartSpan) {
this.emitStartMarker(
'node.execute.started',
attributes,
trace.setSpan(tracked.context, span),
params.executionId,
);
}
let executionNodes = this.activeNodeSpansByExecutionId.get(params.executionId);
@ -306,6 +314,24 @@ export class ExecutionLevelTracer {
}
}
private emitStartMarker(
name: string,
attributes: Attributes,
parentCtx: Context,
executionId: string,
): void {
try {
const startTime = getStartTime(trace.getSpan(parentCtx));
this.tracer.startSpan(name, { startTime, attributes }, parentCtx).end(startTime);
} catch (error) {
this.logger.warn('Failed to emit start marker span', {
executionId,
spanName: name,
error: error instanceof Error ? error.message : String(error),
});
}
}
private parseTraceParentHeaders(tracingContext?: TracingContext): Context {
if (!tracingContext) return ROOT_CONTEXT;
return propagator.extract(ROOT_CONTEXT, tracingContext, defaultTextMapGetter);
@ -394,6 +420,12 @@ function toTracingParentContext(span: Span, identity: ExecutionIdentity): Tracin
return { traceparent: carrier.traceparent, tracestate: carrier.tracestate, identity };
}
// Only a recorded SDK span has a start time. The API span type does not expose it.
function getStartTime(span: Span | undefined): TimeInput {
const startTime = span && 'startTime' in span ? span.startTime : undefined;
return isTimeInputHrTime(startTime) ? startTime : new Date();
}
function terminateSpan(span: Span, reason: string): void {
span.setAttribute(ATTR.NODE_TERMINATION_REASON, reason);
span.setStatus({ code: SpanStatusCode.ERROR });

View File

@ -28,6 +28,7 @@ export type StartWorkflowParams = {
savedIdentity?: ExecutionIdentity;
workflow: WorkflowContext;
project?: ProjectContext;
emitStartSpan?: boolean;
};
export type EndWorkflowParams = {
@ -59,6 +60,7 @@ type NodeTracingParams = Pick<INode, 'id' | 'name' | 'type' | 'typeVersion'>;
export type StartNodeParams = {
executionId: string;
node: NodeTracingParams;
emitStartSpan?: boolean;
};
type EndNodeError = {

View File

@ -108,6 +108,7 @@ export class OtelLifecycleHandler {
executionId: ctx.executionId,
tracingContext,
savedIdentity,
emitStartSpan: this.otelSettingsService.getSettings().emitWorkflowStartSpan,
project: this.buildProjectContext(project),
workflow: {
id: ctx.workflow.id,
@ -151,6 +152,7 @@ export class OtelLifecycleHandler {
tracingContext: previousWorkflowExecution,
linkTo: previousWorkflowExecution,
savedIdentity: toExecutionIdentity(previousWorkflowExecution?.identity),
emitStartSpan: this.otelSettingsService.getSettings().emitWorkflowStartSpan,
project: this.buildProjectContext(project),
workflow: {
id: ctx.workflow.id,
@ -214,6 +216,7 @@ export class OtelLifecycleHandler {
this.tracer.startNode({
executionId: ctx.executionId,
node,
emitStartSpan: this.otelSettingsService.getSettings().emitNodeStartSpan,
});
}

View File

@ -207,6 +207,8 @@ export class OtelSettingsService {
tracesSampleRate: pick('tracesSampleRate'),
startupConnectivityTimeoutMs: pick('startupConnectivityTimeoutMs'),
includeNodeSpans: pick('includeNodeSpans'),
emitWorkflowStartSpan: pick('emitWorkflowStartSpan'),
emitNodeStartSpan: pick('emitNodeStartSpan'),
injectOutbound: pick('injectOutbound'),
productionExecutionsOnly: pick('productionExecutionsOnly'),
};

View File

@ -33,6 +33,12 @@ export class OtelConfig {
@Env(OTEL_ENV_VARS.includeNodeSpans)
includeNodeSpans: boolean = true;
@Env(OTEL_ENV_VARS.emitWorkflowStartSpan)
emitWorkflowStartSpan: boolean = false;
@Env(OTEL_ENV_VARS.emitNodeStartSpan)
emitNodeStartSpan: boolean = false;
@Env(OTEL_ENV_VARS.injectOutbound)
injectOutbound: boolean = true;

View File

@ -13,6 +13,8 @@ export const OTEL_ENV_VARS = {
tracesSampleRate: 'N8N_OTEL_TRACES_SAMPLE_RATE',
startupConnectivityTimeoutMs: 'N8N_OTEL_STARTUP_CONNECTIVITY_TIMEOUT_MS',
includeNodeSpans: 'N8N_OTEL_TRACES_INCLUDE_NODE_SPANS',
emitWorkflowStartSpan: 'N8N_OTEL_TRACES_EMIT_WORKFLOW_START_SPAN',
emitNodeStartSpan: 'N8N_OTEL_TRACES_EMIT_NODE_START_SPAN',
injectOutbound: 'N8N_OTEL_TRACES_INJECT_OUTBOUND',
productionExecutionsOnly: 'N8N_OTEL_TRACES_PRODUCTION_ONLY',
} as const;

View File

@ -63,7 +63,7 @@ export class OtelPublicController {
@ApiKeyScope('otel:manage')
@ApiSummary('Set the OpenTelemetry configuration')
@ApiDescription(
'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The one exception is `exporterProtocol`, which defaults to `http/protobuf` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.',
'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The exceptions are `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.',
)
@ApiTags(tags)
@ApiResponse(200, OtelSettingsPublicDto)

View File

@ -2,7 +2,7 @@ operationId: updateOtelSettings
tags:
- SettingsOtel
summary: Set the OpenTelemetry configuration
description: 'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The one exception is `exporterProtocol`, which defaults to `http/protobuf` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.'
description: 'Set the OpenTelemetry configuration. This is a full replacement: every field must be provided, and a partial body is rejected. The exceptions are `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. The update takes effect exactly as it would from the UI, using the same validation, and is applied to the running instance immediately. Fields managed declaratively via environment variables are read-only: attempting to change one is rejected with 409, while re-submitting its current value (as returned by GET) is accepted. Requires the `otel:manage` scope.'
x-required-scope: otel:manage
x-eov-operation-id: unreachable
x-eov-operation-handler: v1/handlers/decorator-routed.handler
@ -60,6 +60,16 @@ requestBody:
type: boolean
description: Whether to emit a span for each node execution in addition to the workflow span.
example: true
emitWorkflowStartSpan:
type: boolean
default: false
description: 'Whether to emit a `workflow.execute.started` span with a duration of 0 ms when each execution or resumed segment starts. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
emitNodeStartSpan:
type: boolean
default: false
description: 'Whether to emit a `node.execute.started` span with a duration of 0 ms when each node span starts. This doubles the number of node spans. Has no effect when `includeNodeSpans` is false. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
injectOutbound:
type: boolean
description: Whether to inject trace context headers into outbound HTTP requests made by nodes.
@ -80,7 +90,7 @@ requestBody:
- injectOutbound
- productionExecutionsOnly
additionalProperties: false
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
responses:
'200':
description: Operation successful.

View File

@ -13,6 +13,8 @@ export function toOtelSettingsResponse(config: OtelConfig): OtelSettingsResponse
tracesSampleRate: config.tracesSampleRate,
startupConnectivityTimeoutMs: config.startupConnectivityTimeoutMs,
includeNodeSpans: config.includeNodeSpans,
emitWorkflowStartSpan: config.emitWorkflowStartSpan,
emitNodeStartSpan: config.emitNodeStartSpan,
injectOutbound: config.injectOutbound,
productionExecutionsOnly: config.productionExecutionsOnly,
};

View File

@ -45,6 +45,14 @@ properties:
type: boolean
description: Whether to emit a span for each node execution in addition to the workflow span.
example: true
emitWorkflowStartSpan:
type: boolean
description: 'Whether to emit a `workflow.execute.started` span with a duration of 0 ms when each execution or resumed segment starts. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
emitNodeStartSpan:
type: boolean
description: 'Whether to emit a `node.execute.started` span with a duration of 0 ms when each node span starts. This doubles the number of node spans. Has no effect when `includeNodeSpans` is false. Optional: a write that omits it selects `false`, but an instance that sets this field to `true` with an environment variable rejects that write with 409.'
example: false
injectOutbound:
type: boolean
description: Whether to inject trace context headers into outbound HTTP requests made by nodes.
@ -63,7 +71,9 @@ required:
- tracesSampleRate
- startupConnectivityTimeoutMs
- includeNodeSpans
- emitWorkflowStartSpan
- emitNodeStartSpan
- injectOutbound
- productionExecutionsOnly
additionalProperties: false
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'
description: 'The OpenTelemetry configuration, matching the fields exposed in the UI. On a write this is a full replacement: every field must be provided, except `exporterProtocol`, which defaults to `http/protobuf` when omitted, and `emitWorkflowStartSpan` and `emitNodeStartSpan`, which default to `false` when omitted. Fields managed declaratively via environment variables are returned with their effective value. A write can repeat these values, but changing them returns 409.'

View File

@ -21,6 +21,8 @@ const validSettings = {
tracesSampleRate: 0.5,
startupConnectivityTimeoutMs: 3_000,
includeNodeSpans: false,
emitWorkflowStartSpan: true,
emitNodeStartSpan: true,
injectOutbound: false,
productionExecutionsOnly: false,
};
@ -101,6 +103,8 @@ describe('OpenTelemetry settings in Public API', () => {
'tracesSampleRate',
'startupConnectivityTimeoutMs',
'includeNodeSpans',
'emitWorkflowStartSpan',
'emitNodeStartSpan',
'injectOutbound',
'productionExecutionsOnly',
].sort(),
@ -248,6 +252,24 @@ describe('OpenTelemetry settings in Public API', () => {
expect(replaced.body.exporterProtocol).toBe('http/protobuf');
});
it('resets omitted start span flags to false (PUT is a full replacement)', async () => {
await testServer.publicApiAgentFor(owner).put('/settings/otel').send(validSettings);
const {
emitWorkflowStartSpan: _workflow,
emitNodeStartSpan: _node,
...bodyWithoutFlags
} = validSettings;
const response = await testServer
.publicApiAgentFor(owner)
.put('/settings/otel')
.send(bodyWithoutFlags);
expect(response.status).toBe(200);
expect(response.body.emitWorkflowStartSpan).toBe(false);
expect(response.body.emitNodeStartSpan).toBe(false);
});
it('stores the exporter endpoint without surrounding whitespace', async () => {
const response = await testServer
.publicApiAgentFor(owner)

View File

@ -3557,6 +3557,10 @@
"settings.opentelemetry.tracesSampleRate.slug": "of {max}",
"settings.opentelemetry.includeNodeSpans.label": "Include node spans",
"settings.opentelemetry.includeNodeSpans.description": "One span per node, or workflow-level spans only.",
"settings.opentelemetry.emitWorkflowStartSpan.label": "Send workflow start spans",
"settings.opentelemetry.emitWorkflowStartSpan.description": "Send a 0 ms span when a run starts, so it shows before the run ends.",
"settings.opentelemetry.emitNodeStartSpan.label": "Send node start spans",
"settings.opentelemetry.emitNodeStartSpan.description": "Send a 0 ms span when each node starts.",
"settings.opentelemetry.injectOutbound.label": "Inject outbound traceparent",
"settings.opentelemetry.injectOutbound.description": "Add a W3C traceparent header to outbound HTTP so downstream services join the same trace.",
"settings.opentelemetry.productionExecutionsOnly.label": "Track published workflows only",

View File

@ -59,6 +59,8 @@ const makeSettings = (overrides: Partial<OtelSettingsResponse> = {}): OtelSettin
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],
@ -238,6 +240,19 @@ describe('SettingsOpenTelemetryView', () => {
await waitFor(() => expect(getByTestId('settings-save-bar-save')).toBeInTheDocument());
});
it('disables the node start span checkbox while node spans are off', async () => {
getOtelSettingsMock.mockResolvedValue(makeSettings({ includeNodeSpans: false }));
const { getByTestId } = render();
await waitFor(() => expect(getByTestId('otel-emit-node-start-span')).toBeInTheDocument());
expect(getByTestId('otel-emit-node-start-span')).toBeDisabled();
await userEvent.click(getByTestId('otel-include-node-spans'));
await waitFor(() => expect(getByTestId('otel-emit-node-start-span')).toBeEnabled());
});
// ── save ──────────────────────────────────────────────────────────────────
it('calls updateOtelSettings and shows success toast when saving', async () => {
@ -440,6 +455,8 @@ describe('SettingsOpenTelemetryView', () => {
// Checkboxes
await userEvent.click(getByTestId('otel-include-node-spans'));
await userEvent.click(getByTestId('otel-emit-workflow-start-span'));
await userEvent.click(getByTestId('otel-emit-node-start-span'));
await userEvent.click(getByTestId('otel-inject-outbound'));
await userEvent.click(getByTestId('otel-production-only'));
@ -458,6 +475,8 @@ describe('SettingsOpenTelemetryView', () => {
startupConnectivityTimeoutMs: 5000,
tracesSampleRate: 0.5,
includeNodeSpans: true,
emitWorkflowStartSpan: true,
emitNodeStartSpan: true,
injectOutbound: true,
productionExecutionsOnly: true,
}),

View File

@ -703,6 +703,38 @@ watch(
</template>
</OtelSettingsRow>
<OtelSettingsRow
:title="i18n.baseText('settings.opentelemetry.emitWorkflowStartSpan.label')"
:description="i18n.baseText('settings.opentelemetry.emitWorkflowStartSpan.description')"
:env-tooltip="envTooltip('emitWorkflowStartSpan')"
>
<template #action>
<N8nCheckbox
:model-value="otelStore.settings.emitWorkflowStartSpan"
:disabled="isEnvManaged('emitWorkflowStartSpan')"
data-test-id="otel-emit-workflow-start-span"
@update:model-value="otelStore.settings.emitWorkflowStartSpan = Boolean($event)"
/>
</template>
</OtelSettingsRow>
<OtelSettingsRow
:title="i18n.baseText('settings.opentelemetry.emitNodeStartSpan.label')"
:description="i18n.baseText('settings.opentelemetry.emitNodeStartSpan.description')"
:env-tooltip="envTooltip('emitNodeStartSpan')"
>
<template #action>
<N8nCheckbox
:model-value="otelStore.settings.emitNodeStartSpan"
:disabled="
isEnvManaged('emitNodeStartSpan') || !otelStore.settings.includeNodeSpans
"
data-test-id="otel-emit-node-start-span"
@update:model-value="otelStore.settings.emitNodeStartSpan = Boolean($event)"
/>
</template>
</OtelSettingsRow>
<OtelSettingsRow
:title="i18n.baseText('settings.opentelemetry.injectOutbound.label')"
:description="i18n.baseText('settings.opentelemetry.injectOutbound.description')"

View File

@ -22,6 +22,8 @@ const makeSettings = (overrides: Partial<OtelSettingsResponse> = {}): OtelSettin
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],

View File

@ -13,6 +13,8 @@ export type OtelSettings = {
tracesSampleRate: number;
startupConnectivityTimeoutMs: number;
includeNodeSpans: boolean;
emitWorkflowStartSpan: boolean;
emitNodeStartSpan: boolean;
injectOutbound: boolean;
productionExecutionsOnly: boolean;
};

View File

@ -33,6 +33,8 @@ export const OTEL_FIELD_ENV_VARS = {
tracesSampleRate: 'N8N_OTEL_TRACES_SAMPLE_RATE',
startupConnectivityTimeoutMs: 'N8N_OTEL_STARTUP_CONNECTIVITY_TIMEOUT_MS',
includeNodeSpans: 'N8N_OTEL_TRACES_INCLUDE_NODE_SPANS',
emitWorkflowStartSpan: 'N8N_OTEL_TRACES_EMIT_WORKFLOW_START_SPAN',
emitNodeStartSpan: 'N8N_OTEL_TRACES_EMIT_NODE_START_SPAN',
injectOutbound: 'N8N_OTEL_TRACES_INJECT_OUTBOUND',
productionExecutionsOnly: 'N8N_OTEL_TRACES_PRODUCTION_ONLY',
} as const;

View File

@ -30,6 +30,8 @@ const makeSettings = (overrides: Partial<OtelSettingsResponse> = {}): OtelSettin
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
envManagedFields: [],

View File

@ -37,6 +37,8 @@ const defaultSettings: OtelSettings = {
tracesSampleRate: 1.0,
startupConnectivityTimeoutMs: 2_000,
includeNodeSpans: true,
emitWorkflowStartSpan: false,
emitNodeStartSpan: false,
injectOutbound: true,
productionExecutionsOnly: true,
};