refactor(core): Share LLM output JSON extraction and tracing header stripping (no-changelog) (#38050)

Co-authored-by: Cursor <cursoragent@cursor.com>

packages/@n8n/agents/src/evals/parse-judge-response.ts

@@ -1,3 +1,5 @@
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
+
 import type { EvalScore } from '../types';
 
 /**
@@ -5,11 +7,7 @@ import type { EvalScore } from '../types';
  * Handles JSON wrapped in markdown fences, plain JSON, or raw text.
  */
 export function parseJudgeResponse(text: string): EvalScore {
-	// Strip markdown code fences if present: ```json ... ``` or ``` ... ```
-	const stripped = text
-		.replace(/^```(?:json)?\s*\n?/i, '')
-		.replace(/\n?```\s*$/i, '')
-		.trim();
+	const stripped = extractJsonCandidate(text);
 
 	try {
 		const parsed = JSON.parse(stripped) as { pass?: boolean; score?: number; reasoning?: string };

packages/@n8n/agents/src/runtime/memory/observation-log-reflector.ts

@@ -1,3 +1,4 @@
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import { isRecord } from '@n8n/utils/is-record';
 
 import { uniqueStrings } from './memory-lifecycle';
@@ -64,7 +65,7 @@ export type RunObservationLogReflectorResult =
 export function parseObservationLogReflectionJson(output: string): ObservationLogReflection {
 	let parsed: unknown;
 	try {
-		parsed = JSON.parse(extractJsonObject(output));
+		parsed = JSON.parse(extractJsonCandidate(output));
 	} catch {
 		throw new Error('Reflector output must be valid JSON');
 	}
@@ -275,15 +276,6 @@ function normalizeReplacementParentId(
 	return activeById.has(parentId) && !removedIds.has(parentId) ? parentId : null;
 }
 
-function extractJsonObject(output: string): string {
-	const start = output.indexOf('{');
-	const end = output.lastIndexOf('}');
-	if (start === -1 || end === -1 || end < start) {
-		throw new Error('Reflector output did not contain a JSON object');
-	}
-	return output.slice(start, end + 1);
-}
-
 function readStringArray(value: unknown, fieldName: string): string[] {
 	if (!Array.isArray(value)) throw new Error(`Reflector field "${fieldName}" must be an array`);
 	const strings: string[] = [];

packages/@n8n/ai-utilities/AGENTS.md

@@ -0,0 +1,25 @@
+# AGENTS.md
+
+`@n8n/ai-utilities` owns shared helpers that are specific to AI nodes, LLMs,
+model configuration, and model output.
+
+## Public subpaths
+
+- `agent-config`
+- `fromai-helpers`
+- `generic-text-editor`
+- `http-proxy-agent`
+- `json-schema`
+- `llm-output`
+- `model-discovery`
+- `node-catalog`
+- `text-editor`
+- `tokenizer`
+- `web-search`
+
+## Package boundary
+
+- Put shared AI- or LLM-specific helpers in `@n8n/ai-utilities`.
+- Put generic helpers and generic secret or PII redaction in `@n8n/utils`.
+- Put workflow graph and traversal utilities in `n8n-workflow`.
+- Keep domain logic in the package that owns that domain.

packages/@n8n/ai-utilities/package.json

@@ -31,6 +31,9 @@
       "json-schema": [
         "dist/esm/json-schema/index.d.ts"
       ],
+      "llm-output": [
+        "dist/esm/llm-output/index.d.ts"
+      ],
       "web-search": [
         "dist/esm/web-search/index.d.ts"
       ],
@@ -85,6 +88,11 @@
       "import": "./dist/esm/json-schema/index.js",
       "require": "./dist/cjs/json-schema/index.js"
     },
+    "./llm-output": {
+      "types": "./dist/esm/llm-output/index.d.ts",
+      "import": "./dist/esm/llm-output/index.js",
+      "require": "./dist/cjs/llm-output/index.js"
+    },
     "./web-search": {
       "types": "./dist/esm/web-search/index.d.ts",
       "import": "./dist/esm/web-search/index.js",
@@ -126,6 +134,7 @@
     "@types/mime-types": "catalog:",
     "@types/lodash": "catalog:",
     "@types/proxy-from-env": "catalog:",
+    "fast-check": "catalog:",
     "fast-glob": "catalog:",
     "tsx": "catalog:",
     "typescript": "catalog:typescript",

packages/@n8n/ai-utilities/src/index.ts

@@ -91,6 +91,7 @@ export {
 	processDocument,
 } from './utils/vector-store/processDocuments';
 export type { ServerSentEventMessage } from './utils/sse';
+export { stripNonXHeaders } from './utils/strip-non-x-headers';
 
 // Converters
 export { getParametersJsonSchema } from './converters/tool';

packages/@n8n/ai-utilities/src/llm-output/extract-json.property.test.ts

@@ -0,0 +1,56 @@
+import fc from 'fast-check';
+
+import { extractJsonCandidate } from './extract-json';
+
+// Prose an LLM wraps around a payload: bracketed links and notes included.
+const proseArb = fc
+	.array(
+		fc.constantFrom(
+			'See',
+			'[the docs]',
+			'[link](https://x.test)',
+			'then:',
+			'result',
+			'(note)',
+			'Example',
+			'-',
+			'done.',
+		),
+		{ maxLength: 6 },
+	)
+	.map((words) => words.join(' '));
+const containerArb = fc.oneof(fc.dictionary(fc.string(), fc.jsonValue()), fc.array(fc.jsonValue()));
+const renderArb = fc.constantFrom<(json: string) => string>(
+	(json) => json,
+	(json) => '```json\n' + json + '\n```',
+	(json) => '```\n' + json + '\n```',
+);
+const separatorArb = fc.constantFrom(' ', '\n', ': ', '\n\n');
+
+describe('extractJsonCandidate properties', () => {
+	it('recovers any JSON container from surrounding prose, fenced or bare', () => {
+		fc.assert(
+			fc.property(
+				containerArb,
+				fc.boolean(),
+				renderArb,
+				proseArb,
+				proseArb,
+				separatorArb,
+				(value, pretty, render, before, after, separator) => {
+					const json = pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value);
+					// A fence inside a string value closes the outer fence early; known limit.
+					fc.pre(!json.includes('```'));
+					const input = `${before}${separator}${render(json)}${separator}${after}`;
+					let parsed: unknown;
+					try {
+						parsed = JSON.parse(extractJsonCandidate(input));
+					} catch {
+						return false;
+					}
+					return JSON.stringify(parsed) === JSON.stringify(value);
+				},
+			),
+		);
+	});
+});

packages/@n8n/ai-utilities/src/llm-output/extract-json.test.ts

@@ -0,0 +1,73 @@
+import { extractFencedJson, extractJsonCandidate } from './extract-json';
+
+describe('extractJsonCandidate', () => {
+	it.each([
+		['```json\n{"ok":true}\n```', '{"ok":true}'],
+		['```\n{"ok":true}\n```', '{"ok":true}'],
+		['Result:\n```json\n{"ok":true}\n```\nDone', '{"ok":true}'],
+	])('extracts JSON from fenced output', (input, expected) => {
+		expect(extractJsonCandidate(input)).toBe(expected);
+	});
+
+	it('extracts a bare JSON object from surrounding prose', () => {
+		expect(extractJsonCandidate('Result: {"ok":true} Done')).toBe('{"ok":true}');
+	});
+
+	it.each([
+		['[{"a":1},{"a":2}]', '[{"a":1},{"a":2}]'],
+		['Rows:\n[{"a":1},{"a":2}]', '[{"a":1},{"a":2}]'],
+		['```json\n[1,2]\n```', '[1,2]'],
+		['See [the docs] then {"ok":true}', '{"ok":true}'],
+		['See [the docs] first: [1, 2, 3]', '[1, 2, 3]'],
+		['[link](https://x.test) then [{"a":1}] done', '[{"a":1}]'],
+		[
+			'Example {"x":1} applies. Answer: {"answer":true,"reasoning":"ok"}',
+			'{"answer":true,"reasoning":"ok"}',
+		],
+		['{"items":[1,2]}', '{"items":[1,2]}'],
+		['{"a":"]"} tail', '{"a":"]"}'],
+	])(
+		'keeps array payloads intact without letting prose brackets shadow or join them',
+		(input, expected) => {
+			expect(extractJsonCandidate(input)).toBe(expected);
+		},
+	);
+
+	it.each([
+		['Ran ```pnpm test``` then {"ok":true}', '{"ok":true}'],
+		['{"note": "run ```pnpm test``` twice"}', '{"note": "run ```pnpm test``` twice"}'],
+		[
+			'{"pass":true,"reasoning":"see ```json\\n{\\"x\\":1}\\n``` ok"}',
+			'{"pass":true,"reasoning":"see ```json\\n{\\"x\\":1}\\n``` ok"}',
+		],
+		[
+			'```json\n{"hint":"run ```pnpm test``` first","ok":true}\n```',
+			'{"hint":"run ```pnpm test``` first","ok":true}',
+		],
+		['```json\n{"broken": \n```\nFixed: {"ok":true}', '{"ok":true}'],
+	])('ignores fenced blocks that are not the payload', (input, expected) => {
+		expect(extractJsonCandidate(input)).toBe(expected);
+	});
+
+	it('returns the fenced value when it is a JSON scalar', () => {
+		expect(extractJsonCandidate('```json\n42\n```')).toBe('42');
+	});
+
+	it('returns trimmed text when no JSON candidate exists', () => {
+		expect(extractJsonCandidate('  not JSON  ')).toBe('not JSON');
+	});
+
+	it('gives up on bracket noise in linear time', () => {
+		// Every unmatched opener used to rescan the whole suffix; 40k chars took over a second.
+		const input = '{ '.repeat(20_000).trim();
+		const start = performance.now();
+		expect(extractJsonCandidate(input)).toBe(input);
+		expect(performance.now() - start).toBeLessThan(100);
+	});
+});
+
+describe('extractFencedJson', () => {
+	it('returns undefined when the output has no fenced block', () => {
+		expect(extractFencedJson('{"ok":true}')).toBeUndefined();
+	});
+});

packages/@n8n/ai-utilities/src/llm-output/extract-json.ts

@@ -0,0 +1,75 @@
+const FENCED_BLOCK = /```(?:json)?\s*\n?([\s\S]*?)```/i;
+const JSON_CONTAINER_START = /^[[{]/;
+
+export function extractFencedJson(text: string): string | undefined {
+	return FENCED_BLOCK.exec(text)?.[1].trim();
+}
+
+/** Index of the bracket closing the one at `start`, or -1 when the text ends
+ *  first. String literals are skipped so brackets inside values don't count. */
+function findBalancedEnd(text: string, start: number): number {
+	let depth = 0;
+	let inString = false;
+	for (let i = start; i < text.length; i++) {
+		const ch = text[i];
+		if (inString) {
+			if (ch === '\\') i++;
+			else if (ch === '"') inString = false;
+		} else if (ch === '"') {
+			inString = true;
+		} else if (ch === '{' || ch === '[') {
+			depth++;
+		} else if (ch === '}' || ch === ']') {
+			depth--;
+			if (depth === 0) return i;
+		}
+	}
+	return -1;
+}
+
+function parsesAsJson(text: string): boolean {
+	try {
+		JSON.parse(text);
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/** Longest balanced `{...}` or `[...]` span that parses as JSON. Bracketed
+ *  prose (`[docs]`, `[link](url)`) and example snippets before the payload are
+ *  skipped instead of being glued onto it or shadowing it. */
+function extractJsonContainer(text: string): string | undefined {
+	let best: string | undefined;
+	// Every opener starts its own scan, so a response full of unmatched brackets
+	// would rescan the same suffix from each one (quadratic). Cap the total
+	// scanned characters: a payload is found within a few scans, one per level
+	// of prose or truncated wrapper around it.
+	let budget = text.length * 32;
+	for (let start = 0; start < text.length && budget > 0; start++) {
+		if (best !== undefined && text.length - start <= best.length) break;
+		if (text[start] !== '{' && text[start] !== '[') continue;
+		const end = findBalancedEnd(text, start);
+		budget -= (end === -1 ? text.length : end + 1) - start;
+		if (end === -1) continue;
+		const candidate = text.slice(start, end + 1);
+		if ((best === undefined || candidate.length > best.length) && parsesAsJson(candidate)) {
+			best = candidate;
+			start = end;
+		}
+	}
+	return best;
+}
+
+export function extractJsonCandidate(text: string): string {
+	const trimmed = text.trim();
+	// A response that already starts with a JSON container is the payload itself;
+	// backtick pairs inside one of its string values must not shadow it. Elsewhere,
+	// trust a fenced block only when it is JSON-shaped and parses as JSON.
+	// An inner ``` can cut the fence short, but that fence must not shadow the payload.
+	const fenced = JSON_CONTAINER_START.test(trimmed) ? undefined : extractFencedJson(trimmed);
+	if (fenced !== undefined && JSON_CONTAINER_START.test(fenced) && parsesAsJson(fenced))
+		return fenced;
+
+	return extractJsonContainer(trimmed) ?? fenced ?? trimmed;
+}

packages/@n8n/ai-utilities/src/llm-output/index.ts

@@ -0,0 +1 @@
+export { extractFencedJson, extractJsonCandidate } from './extract-json';

packages/@n8n/ai-utilities/src/utils/__tests__/strip-non-x-headers.test.ts

@@ -0,0 +1,28 @@
+import { stripNonXHeaders } from '../strip-non-x-headers';
+
+describe('stripNonXHeaders', () => {
+	it('removes non-x headers and preserves x-prefixed headers', () => {
+		const error = {
+			headers: {
+				authorization: 'secret',
+				'x-request-id': 'request-id',
+				'X-Request-Id': 'mixed-case',
+			},
+		};
+
+		stripNonXHeaders(error);
+
+		expect(error.headers).toEqual({
+			'x-request-id': 'request-id',
+			'X-Request-Id': 'mixed-case',
+		});
+	});
+
+	it('does nothing when headers are absent', () => {
+		const error = new Error('failed');
+
+		stripNonXHeaders(error);
+
+		expect(error).toEqual(new Error('failed'));
+	});
+});

packages/@n8n/ai-utilities/src/utils/n8n-llm-tracing.ts

@@ -14,6 +14,7 @@ import { NodeConnectionTypes, NodeError, NodeOperationError } from 'n8n-workflow
 
 import { logAiEvent } from './log-ai-event';
 import { redactHeaderValues } from './redact-headers';
+import { stripNonXHeaders } from './strip-non-x-headers';
 import { estimateTokensFromStringList } from './tokenizer/token-estimator';
 
 /** Normalized token usage returned by TokensUsageParser. */
@@ -251,17 +252,7 @@ export class N8nLlmTracing extends BaseCallbackHandler {
 	async handleLLMError(error: IDataObject | Error, runId: string, parentRunId?: string) {
 		const runDetails = this.runsMap[runId] ?? { index: Object.keys(this.runsMap).length };
 
-		// Filter out non-x- headers to avoid leaking sensitive information in logs
-		// eslint-disable-next-line no-prototype-builtins
-		if (typeof error === 'object' && error?.hasOwnProperty('headers')) {
-			const errorWithHeaders = error as { headers: Record<string, unknown> };
-
-			Object.keys(errorWithHeaders.headers).forEach((key) => {
-				if (!key.startsWith('x-')) {
-					delete errorWithHeaders.headers[key];
-				}
-			});
-		}
+		stripNonXHeaders(error);
 
 		if (error instanceof NodeError) {
 			if (this.options.errorDescriptionMapper) {

packages/@n8n/ai-utilities/src/utils/strip-non-x-headers.ts

@@ -0,0 +1,13 @@
+import { isRecord } from '@n8n/utils/is-record';
+import type { IDataObject } from 'n8n-workflow';
+
+export function stripNonXHeaders(error: IDataObject | Error): void {
+	if (!Object.prototype.hasOwnProperty.call(error, 'headers')) return;
+
+	const headers: unknown = Reflect.get(error, 'headers');
+	if (!isRecord(headers)) return;
+
+	for (const key of Object.keys(headers)) {
+		if (!key.toLowerCase().startsWith('x-')) delete headers[key];
+	}
+}

packages/@n8n/instance-ai/evaluations/utils/llm-judge.ts

@@ -6,7 +6,8 @@
  * the parsing of fenced/bare JSON, and the verdict shape.
  */
 
-const FENCED_JSON = /```(?:json)?\s*\n?([\s\S]*?)```/;
+import { extractFencedJson } from '@n8n/ai-utilities/llm-output';
+
 const BARE_JSON_OBJECT = /\{[\s\S]*\}/;
 
 /**
@@ -81,9 +82,9 @@ function isJudgeVerdict(value: unknown): value is JudgeVerdict {
  * Returns `undefined` when no valid verdict is found.
  */
 export function parseJudgeVerdict(text: string): JudgeVerdict | undefined {
-	const fenceMatch = text.match(FENCED_JSON);
-	const fenced = fenceMatch ? tryParse(fenceMatch[1].trim()) : tryParse(text.trim());
-	if (fenced) return fenced;
+	const fenced = extractFencedJson(text);
+	const first = tryParse(fenced ?? text.trim());
+	if (first) return first;
 
 	const objectMatch = text.match(BARE_JSON_OBJECT);
 	const bare = objectMatch ? tryParse(objectMatch[0]) : undefined;

packages/@n8n/instance-ai/src/utils/generate-validated-json.ts

@@ -9,6 +9,7 @@
  */
 
 import type { ModelConfig } from '@n8n/agents';
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import type { z } from 'zod';
 
 import { createEvalAgent, extractText } from './eval-agents';
@@ -30,12 +31,6 @@ export interface GenerateValidatedJsonOptions<T> {
 	fallbackModelConfig?: ModelConfig;
 }
 
-function stripMarkdownFences(text: string): string {
-	const trimmed = text.trim();
-	const fencedMatch = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
-	return fencedMatch ? fencedMatch[1].trim() : trimmed;
-}
-
 export async function generateValidatedJson<T>(
 	agentName: string,
 	options: GenerateValidatedJsonOptions<T>,
@@ -57,7 +52,7 @@ export async function generateValidatedJson<T>(
 
 	let parsed: unknown;
 	try {
-		parsed = JSON.parse(stripMarkdownFences(text));
+		parsed = JSON.parse(extractJsonCandidate(text));
 	} catch {
 		return { ok: false, reason: 'invalid_json' };
 	}

packages/@n8n/nodes-langchain/nodes/llms/N8nNonEstimatingTracing.ts

@@ -7,7 +7,7 @@ import type {
 } from '@langchain/core/load/serializable';
 import type { BaseMessage } from '@langchain/core/messages';
 import type { LLMResult } from '@langchain/core/outputs';
-import { logAiEvent, redactHeaderValues } from '@n8n/ai-utilities';
+import { logAiEvent, redactHeaderValues, stripNonXHeaders } from '@n8n/ai-utilities';
 import pick from 'lodash/pick';
 import type { IDataObject, ISupplyDataFunctions, JsonObject } from 'n8n-workflow';
 import { NodeConnectionTypes, NodeError, NodeOperationError } from 'n8n-workflow';
@@ -145,16 +145,7 @@ export class N8nNonEstimatingTracing extends BaseCallbackHandler {
 	async handleLLMError(error: IDataObject | Error, runId: string, parentRunId?: string) {
 		const runDetails = this.runsMap[runId] ?? { index: Object.keys(this.runsMap).length };
 
-		// Filter out non-x- headers to avoid leaking sensitive information in logs
-		if (typeof error === 'object' && error?.hasOwnProperty('headers')) {
-			const errorWithHeaders = error as { headers: Record<string, unknown> };
-
-			Object.keys(errorWithHeaders.headers).forEach((key) => {
-				if (!key.startsWith('x-')) {
-					delete errorWithHeaders.headers[key];
-				}
-			});
-		}
+		stripNonXHeaders(error);
 
 		if (error instanceof NodeError) {
 			if (this.options.errorDescriptionMapper) {

packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts

@@ -1,3 +1,4 @@
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import type { InstanceAiEvalAgentScenarioSeed } from '@n8n/api-types';
 import { createEvalAgent, extractText } from '@n8n/instance-ai';
 import { jsonParse } from 'n8n-workflow';
@@ -90,8 +91,9 @@ const MAX_SEED_ATTEMPTS = 2;
 const SEED_LLM_TIMEOUT_MS = 300_000;
 
 function parseSeed(raw: string): Omit<InstanceAiEvalAgentScenarioSeed, 'warnings'> | undefined {
-	const text = raw.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?\s*```\s*$/i, '');
-	const parsed = jsonParse<Record<string, unknown>>(text, { fallbackValue: {} });
+	const parsed = jsonParse<Record<string, unknown>>(extractJsonCandidate(raw), {
+		fallbackValue: {},
+	});
 
 	const openingMessage = typeof parsed.openingMessage === 'string' ? parsed.openingMessage : '';
 	if (openingMessage.trim().length === 0) return undefined;

packages/cli/src/modules/instance-ai/eval/mock-utils.ts

@@ -1,4 +1,5 @@
 import type { FetchFn } from '@n8n/agents';
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import type { Logger } from '@n8n/backend-common';
 import { createEvalAgent, extractText } from '@n8n/instance-ai';
 import { jsonParse } from 'n8n-workflow';
@@ -11,10 +12,6 @@ import { jsonParse } from 'n8n-workflow';
 const GENERATOR_TIMEOUT_MS = 120_000;
 const MAX_GENERATOR_ATTEMPTS = 2;
 
-function fenceStrip(raw: string): string {
-	return raw.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?\s*```\s*$/i, '');
-}
-
 /**
  * Generate JSON with the eval agent and validate its shape, retrying once on
  * unusable output. Returns undefined when both attempts fail — callers supply
@@ -34,7 +31,7 @@ export async function generateJson<T>(
 				providerOptions: { anthropic: { maxTokens: 4096 } },
 				abortSignal: AbortSignal.timeout(GENERATOR_TIMEOUT_MS),
 			});
-			const parsed = jsonParse<unknown>(fenceStrip(extractText(result)), {
+			const parsed = jsonParse<unknown>(extractJsonCandidate(extractText(result)), {
 				fallbackValue: undefined,
 			});
 			const validated = validate(parsed);

packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts

@@ -1,3 +1,4 @@
+import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import { Logger } from '@n8n/backend-common';
 import { Container } from '@n8n/di';
 import { createEvalAgent, extractText } from '@n8n/instance-ai';
@@ -699,11 +700,7 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 				abortSignal: AbortSignal.timeout(HINT_LLM_TIMEOUT_MS),
 			});
 
-			const text = extractText(result)
-				.replace(/^```(?:json)?\s*\n?/i, '')
-				.replace(/\n?\s*```\s*$/i, '')
-				.trim();
-
+			const text = extractJsonCandidate(extractText(result));
 			const parsed: Record<string, unknown> = jsonParse(text);
 
 			// globalContext may come back as a string or object — normalize to string

pnpm-lock.yaml

@@ -1354,6 +1354,9 @@ importers:
       axios:
         specifier: 1.18.0
         version: 1.18.0(patch_hash=149e256a2a7b632497650b32816716ced972ab02a5ab00fbd8a5158a51722c49)(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1)
+      fast-check:
+        specifier: 'catalog:'
+        version: 3.23.2
       fast-glob:
         specifier: 'catalog:'
         version: 3.3.3