From fcf21f5efc633fa59034893cf10a73ae6c2177cb Mon Sep 17 00:00:00 2001 From: Robin Braumann <50590409+bjorger@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:53:35 +0000 Subject: [PATCH] refactor(core): Share LLM output JSON extraction and tracing header stripping (no-changelog) (#38050) Co-authored-by: Cursor --- .../agents/src/evals/parse-judge-response.ts | 8 +- .../memory/observation-log-reflector.ts | 12 +-- packages/@n8n/ai-utilities/AGENTS.md | 25 +++++++ packages/@n8n/ai-utilities/package.json | 9 +++ packages/@n8n/ai-utilities/src/index.ts | 1 + .../llm-output/extract-json.property.test.ts | 56 ++++++++++++++ .../src/llm-output/extract-json.test.ts | 73 ++++++++++++++++++ .../src/llm-output/extract-json.ts | 75 +++++++++++++++++++ .../@n8n/ai-utilities/src/llm-output/index.ts | 1 + .../__tests__/strip-non-x-headers.test.ts | 28 +++++++ .../ai-utilities/src/utils/n8n-llm-tracing.ts | 13 +--- .../src/utils/strip-non-x-headers.ts | 13 ++++ .../evaluations/utils/llm-judge.ts | 9 ++- .../src/utils/generate-validated-json.ts | 9 +-- .../nodes/llms/N8nNonEstimatingTracing.ts | 13 +--- .../instance-ai/eval/agent-scenario-seed.ts | 6 +- .../modules/instance-ai/eval/mock-utils.ts | 7 +- .../instance-ai/eval/workflow-analysis.ts | 7 +- pnpm-lock.yaml | 3 + 19 files changed, 308 insertions(+), 60 deletions(-) create mode 100644 packages/@n8n/ai-utilities/AGENTS.md create mode 100644 packages/@n8n/ai-utilities/src/llm-output/extract-json.property.test.ts create mode 100644 packages/@n8n/ai-utilities/src/llm-output/extract-json.test.ts create mode 100644 packages/@n8n/ai-utilities/src/llm-output/extract-json.ts create mode 100644 packages/@n8n/ai-utilities/src/llm-output/index.ts create mode 100644 packages/@n8n/ai-utilities/src/utils/__tests__/strip-non-x-headers.test.ts create mode 100644 packages/@n8n/ai-utilities/src/utils/strip-non-x-headers.ts diff --git a/packages/@n8n/agents/src/evals/parse-judge-response.ts b/packages/@n8n/agents/src/evals/parse-judge-response.ts index e8d7385a943..f435e0782d8 100644 --- a/packages/@n8n/agents/src/evals/parse-judge-response.ts +++ b/packages/@n8n/agents/src/evals/parse-judge-response.ts @@ -1,3 +1,5 @@ +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; + import type { EvalScore } from '../types'; /** @@ -5,11 +7,7 @@ import type { EvalScore } from '../types'; * Handles JSON wrapped in markdown fences, plain JSON, or raw text. */ export function parseJudgeResponse(text: string): EvalScore { - // Strip markdown code fences if present: ```json ... ``` or ``` ... ``` - const stripped = text - .replace(/^```(?:json)?\s*\n?/i, '') - .replace(/\n?```\s*$/i, '') - .trim(); + const stripped = extractJsonCandidate(text); try { const parsed = JSON.parse(stripped) as { pass?: boolean; score?: number; reasoning?: string }; diff --git a/packages/@n8n/agents/src/runtime/memory/observation-log-reflector.ts b/packages/@n8n/agents/src/runtime/memory/observation-log-reflector.ts index 8dcff412a55..f12bb92e82f 100644 --- a/packages/@n8n/agents/src/runtime/memory/observation-log-reflector.ts +++ b/packages/@n8n/agents/src/runtime/memory/observation-log-reflector.ts @@ -1,3 +1,4 @@ +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import { isRecord } from '@n8n/utils/is-record'; import { uniqueStrings } from './memory-lifecycle'; @@ -64,7 +65,7 @@ export type RunObservationLogReflectorResult = export function parseObservationLogReflectionJson(output: string): ObservationLogReflection { let parsed: unknown; try { - parsed = JSON.parse(extractJsonObject(output)); + parsed = JSON.parse(extractJsonCandidate(output)); } catch { throw new Error('Reflector output must be valid JSON'); } @@ -275,15 +276,6 @@ function normalizeReplacementParentId( return activeById.has(parentId) && !removedIds.has(parentId) ? parentId : null; } -function extractJsonObject(output: string): string { - const start = output.indexOf('{'); - const end = output.lastIndexOf('}'); - if (start === -1 || end === -1 || end < start) { - throw new Error('Reflector output did not contain a JSON object'); - } - return output.slice(start, end + 1); -} - function readStringArray(value: unknown, fieldName: string): string[] { if (!Array.isArray(value)) throw new Error(`Reflector field "${fieldName}" must be an array`); const strings: string[] = []; diff --git a/packages/@n8n/ai-utilities/AGENTS.md b/packages/@n8n/ai-utilities/AGENTS.md new file mode 100644 index 00000000000..31b2ca844d2 --- /dev/null +++ b/packages/@n8n/ai-utilities/AGENTS.md @@ -0,0 +1,25 @@ +# AGENTS.md + +`@n8n/ai-utilities` owns shared helpers that are specific to AI nodes, LLMs, +model configuration, and model output. + +## Public subpaths + +- `agent-config` +- `fromai-helpers` +- `generic-text-editor` +- `http-proxy-agent` +- `json-schema` +- `llm-output` +- `model-discovery` +- `node-catalog` +- `text-editor` +- `tokenizer` +- `web-search` + +## Package boundary + +- Put shared AI- or LLM-specific helpers in `@n8n/ai-utilities`. +- Put generic helpers and generic secret or PII redaction in `@n8n/utils`. +- Put workflow graph and traversal utilities in `n8n-workflow`. +- Keep domain logic in the package that owns that domain. diff --git a/packages/@n8n/ai-utilities/package.json b/packages/@n8n/ai-utilities/package.json index 0c39431590d..27fa83f816f 100644 --- a/packages/@n8n/ai-utilities/package.json +++ b/packages/@n8n/ai-utilities/package.json @@ -31,6 +31,9 @@ "json-schema": [ "dist/esm/json-schema/index.d.ts" ], + "llm-output": [ + "dist/esm/llm-output/index.d.ts" + ], "web-search": [ "dist/esm/web-search/index.d.ts" ], @@ -85,6 +88,11 @@ "import": "./dist/esm/json-schema/index.js", "require": "./dist/cjs/json-schema/index.js" }, + "./llm-output": { + "types": "./dist/esm/llm-output/index.d.ts", + "import": "./dist/esm/llm-output/index.js", + "require": "./dist/cjs/llm-output/index.js" + }, "./web-search": { "types": "./dist/esm/web-search/index.d.ts", "import": "./dist/esm/web-search/index.js", @@ -126,6 +134,7 @@ "@types/mime-types": "catalog:", "@types/lodash": "catalog:", "@types/proxy-from-env": "catalog:", + "fast-check": "catalog:", "fast-glob": "catalog:", "tsx": "catalog:", "typescript": "catalog:typescript", diff --git a/packages/@n8n/ai-utilities/src/index.ts b/packages/@n8n/ai-utilities/src/index.ts index cf8dac6ecb0..e4c7abc3cfa 100644 --- a/packages/@n8n/ai-utilities/src/index.ts +++ b/packages/@n8n/ai-utilities/src/index.ts @@ -91,6 +91,7 @@ export { processDocument, } from './utils/vector-store/processDocuments'; export type { ServerSentEventMessage } from './utils/sse'; +export { stripNonXHeaders } from './utils/strip-non-x-headers'; // Converters export { getParametersJsonSchema } from './converters/tool'; diff --git a/packages/@n8n/ai-utilities/src/llm-output/extract-json.property.test.ts b/packages/@n8n/ai-utilities/src/llm-output/extract-json.property.test.ts new file mode 100644 index 00000000000..8b06bd6742a --- /dev/null +++ b/packages/@n8n/ai-utilities/src/llm-output/extract-json.property.test.ts @@ -0,0 +1,56 @@ +import fc from 'fast-check'; + +import { extractJsonCandidate } from './extract-json'; + +// Prose an LLM wraps around a payload: bracketed links and notes included. +const proseArb = fc + .array( + fc.constantFrom( + 'See', + '[the docs]', + '[link](https://x.test)', + 'then:', + 'result', + '(note)', + 'Example', + '-', + 'done.', + ), + { maxLength: 6 }, + ) + .map((words) => words.join(' ')); +const containerArb = fc.oneof(fc.dictionary(fc.string(), fc.jsonValue()), fc.array(fc.jsonValue())); +const renderArb = fc.constantFrom<(json: string) => string>( + (json) => json, + (json) => '```json\n' + json + '\n```', + (json) => '```\n' + json + '\n```', +); +const separatorArb = fc.constantFrom(' ', '\n', ': ', '\n\n'); + +describe('extractJsonCandidate properties', () => { + it('recovers any JSON container from surrounding prose, fenced or bare', () => { + fc.assert( + fc.property( + containerArb, + fc.boolean(), + renderArb, + proseArb, + proseArb, + separatorArb, + (value, pretty, render, before, after, separator) => { + const json = pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value); + // A fence inside a string value closes the outer fence early; known limit. + fc.pre(!json.includes('```')); + const input = `${before}${separator}${render(json)}${separator}${after}`; + let parsed: unknown; + try { + parsed = JSON.parse(extractJsonCandidate(input)); + } catch { + return false; + } + return JSON.stringify(parsed) === JSON.stringify(value); + }, + ), + ); + }); +}); diff --git a/packages/@n8n/ai-utilities/src/llm-output/extract-json.test.ts b/packages/@n8n/ai-utilities/src/llm-output/extract-json.test.ts new file mode 100644 index 00000000000..f1b24a742d9 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/llm-output/extract-json.test.ts @@ -0,0 +1,73 @@ +import { extractFencedJson, extractJsonCandidate } from './extract-json'; + +describe('extractJsonCandidate', () => { + it.each([ + ['```json\n{"ok":true}\n```', '{"ok":true}'], + ['```\n{"ok":true}\n```', '{"ok":true}'], + ['Result:\n```json\n{"ok":true}\n```\nDone', '{"ok":true}'], + ])('extracts JSON from fenced output', (input, expected) => { + expect(extractJsonCandidate(input)).toBe(expected); + }); + + it('extracts a bare JSON object from surrounding prose', () => { + expect(extractJsonCandidate('Result: {"ok":true} Done')).toBe('{"ok":true}'); + }); + + it.each([ + ['[{"a":1},{"a":2}]', '[{"a":1},{"a":2}]'], + ['Rows:\n[{"a":1},{"a":2}]', '[{"a":1},{"a":2}]'], + ['```json\n[1,2]\n```', '[1,2]'], + ['See [the docs] then {"ok":true}', '{"ok":true}'], + ['See [the docs] first: [1, 2, 3]', '[1, 2, 3]'], + ['[link](https://x.test) then [{"a":1}] done', '[{"a":1}]'], + [ + 'Example {"x":1} applies. Answer: {"answer":true,"reasoning":"ok"}', + '{"answer":true,"reasoning":"ok"}', + ], + ['{"items":[1,2]}', '{"items":[1,2]}'], + ['{"a":"]"} tail', '{"a":"]"}'], + ])( + 'keeps array payloads intact without letting prose brackets shadow or join them', + (input, expected) => { + expect(extractJsonCandidate(input)).toBe(expected); + }, + ); + + it.each([ + ['Ran ```pnpm test``` then {"ok":true}', '{"ok":true}'], + ['{"note": "run ```pnpm test``` twice"}', '{"note": "run ```pnpm test``` twice"}'], + [ + '{"pass":true,"reasoning":"see ```json\\n{\\"x\\":1}\\n``` ok"}', + '{"pass":true,"reasoning":"see ```json\\n{\\"x\\":1}\\n``` ok"}', + ], + [ + '```json\n{"hint":"run ```pnpm test``` first","ok":true}\n```', + '{"hint":"run ```pnpm test``` first","ok":true}', + ], + ['```json\n{"broken": \n```\nFixed: {"ok":true}', '{"ok":true}'], + ])('ignores fenced blocks that are not the payload', (input, expected) => { + expect(extractJsonCandidate(input)).toBe(expected); + }); + + it('returns the fenced value when it is a JSON scalar', () => { + expect(extractJsonCandidate('```json\n42\n```')).toBe('42'); + }); + + it('returns trimmed text when no JSON candidate exists', () => { + expect(extractJsonCandidate(' not JSON ')).toBe('not JSON'); + }); + + it('gives up on bracket noise in linear time', () => { + // Every unmatched opener used to rescan the whole suffix; 40k chars took over a second. + const input = '{ '.repeat(20_000).trim(); + const start = performance.now(); + expect(extractJsonCandidate(input)).toBe(input); + expect(performance.now() - start).toBeLessThan(100); + }); +}); + +describe('extractFencedJson', () => { + it('returns undefined when the output has no fenced block', () => { + expect(extractFencedJson('{"ok":true}')).toBeUndefined(); + }); +}); diff --git a/packages/@n8n/ai-utilities/src/llm-output/extract-json.ts b/packages/@n8n/ai-utilities/src/llm-output/extract-json.ts new file mode 100644 index 00000000000..450824bf317 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/llm-output/extract-json.ts @@ -0,0 +1,75 @@ +const FENCED_BLOCK = /```(?:json)?\s*\n?([\s\S]*?)```/i; +const JSON_CONTAINER_START = /^[[{]/; + +export function extractFencedJson(text: string): string | undefined { + return FENCED_BLOCK.exec(text)?.[1].trim(); +} + +/** Index of the bracket closing the one at `start`, or -1 when the text ends + * first. String literals are skipped so brackets inside values don't count. */ +function findBalancedEnd(text: string, start: number): number { + let depth = 0; + let inString = false; + for (let i = start; i < text.length; i++) { + const ch = text[i]; + if (inString) { + if (ch === '\\') i++; + else if (ch === '"') inString = false; + } else if (ch === '"') { + inString = true; + } else if (ch === '{' || ch === '[') { + depth++; + } else if (ch === '}' || ch === ']') { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + +function parsesAsJson(text: string): boolean { + try { + JSON.parse(text); + return true; + } catch { + return false; + } +} + +/** Longest balanced `{...}` or `[...]` span that parses as JSON. Bracketed + * prose (`[docs]`, `[link](url)`) and example snippets before the payload are + * skipped instead of being glued onto it or shadowing it. */ +function extractJsonContainer(text: string): string | undefined { + let best: string | undefined; + // Every opener starts its own scan, so a response full of unmatched brackets + // would rescan the same suffix from each one (quadratic). Cap the total + // scanned characters: a payload is found within a few scans, one per level + // of prose or truncated wrapper around it. + let budget = text.length * 32; + for (let start = 0; start < text.length && budget > 0; start++) { + if (best !== undefined && text.length - start <= best.length) break; + if (text[start] !== '{' && text[start] !== '[') continue; + const end = findBalancedEnd(text, start); + budget -= (end === -1 ? text.length : end + 1) - start; + if (end === -1) continue; + const candidate = text.slice(start, end + 1); + if ((best === undefined || candidate.length > best.length) && parsesAsJson(candidate)) { + best = candidate; + start = end; + } + } + return best; +} + +export function extractJsonCandidate(text: string): string { + const trimmed = text.trim(); + // A response that already starts with a JSON container is the payload itself; + // backtick pairs inside one of its string values must not shadow it. Elsewhere, + // trust a fenced block only when it is JSON-shaped and parses as JSON. + // An inner ``` can cut the fence short, but that fence must not shadow the payload. + const fenced = JSON_CONTAINER_START.test(trimmed) ? undefined : extractFencedJson(trimmed); + if (fenced !== undefined && JSON_CONTAINER_START.test(fenced) && parsesAsJson(fenced)) + return fenced; + + return extractJsonContainer(trimmed) ?? fenced ?? trimmed; +} diff --git a/packages/@n8n/ai-utilities/src/llm-output/index.ts b/packages/@n8n/ai-utilities/src/llm-output/index.ts new file mode 100644 index 00000000000..649eb7c9336 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/llm-output/index.ts @@ -0,0 +1 @@ +export { extractFencedJson, extractJsonCandidate } from './extract-json'; diff --git a/packages/@n8n/ai-utilities/src/utils/__tests__/strip-non-x-headers.test.ts b/packages/@n8n/ai-utilities/src/utils/__tests__/strip-non-x-headers.test.ts new file mode 100644 index 00000000000..5673280124b --- /dev/null +++ b/packages/@n8n/ai-utilities/src/utils/__tests__/strip-non-x-headers.test.ts @@ -0,0 +1,28 @@ +import { stripNonXHeaders } from '../strip-non-x-headers'; + +describe('stripNonXHeaders', () => { + it('removes non-x headers and preserves x-prefixed headers', () => { + const error = { + headers: { + authorization: 'secret', + 'x-request-id': 'request-id', + 'X-Request-Id': 'mixed-case', + }, + }; + + stripNonXHeaders(error); + + expect(error.headers).toEqual({ + 'x-request-id': 'request-id', + 'X-Request-Id': 'mixed-case', + }); + }); + + it('does nothing when headers are absent', () => { + const error = new Error('failed'); + + stripNonXHeaders(error); + + expect(error).toEqual(new Error('failed')); + }); +}); diff --git a/packages/@n8n/ai-utilities/src/utils/n8n-llm-tracing.ts b/packages/@n8n/ai-utilities/src/utils/n8n-llm-tracing.ts index f9fb8009f10..09d93e40378 100644 --- a/packages/@n8n/ai-utilities/src/utils/n8n-llm-tracing.ts +++ b/packages/@n8n/ai-utilities/src/utils/n8n-llm-tracing.ts @@ -14,6 +14,7 @@ import { NodeConnectionTypes, NodeError, NodeOperationError } from 'n8n-workflow import { logAiEvent } from './log-ai-event'; import { redactHeaderValues } from './redact-headers'; +import { stripNonXHeaders } from './strip-non-x-headers'; import { estimateTokensFromStringList } from './tokenizer/token-estimator'; /** Normalized token usage returned by TokensUsageParser. */ @@ -251,17 +252,7 @@ export class N8nLlmTracing extends BaseCallbackHandler { async handleLLMError(error: IDataObject | Error, runId: string, parentRunId?: string) { const runDetails = this.runsMap[runId] ?? { index: Object.keys(this.runsMap).length }; - // Filter out non-x- headers to avoid leaking sensitive information in logs - // eslint-disable-next-line no-prototype-builtins - if (typeof error === 'object' && error?.hasOwnProperty('headers')) { - const errorWithHeaders = error as { headers: Record }; - - Object.keys(errorWithHeaders.headers).forEach((key) => { - if (!key.startsWith('x-')) { - delete errorWithHeaders.headers[key]; - } - }); - } + stripNonXHeaders(error); if (error instanceof NodeError) { if (this.options.errorDescriptionMapper) { diff --git a/packages/@n8n/ai-utilities/src/utils/strip-non-x-headers.ts b/packages/@n8n/ai-utilities/src/utils/strip-non-x-headers.ts new file mode 100644 index 00000000000..ddb50667e31 --- /dev/null +++ b/packages/@n8n/ai-utilities/src/utils/strip-non-x-headers.ts @@ -0,0 +1,13 @@ +import { isRecord } from '@n8n/utils/is-record'; +import type { IDataObject } from 'n8n-workflow'; + +export function stripNonXHeaders(error: IDataObject | Error): void { + if (!Object.prototype.hasOwnProperty.call(error, 'headers')) return; + + const headers: unknown = Reflect.get(error, 'headers'); + if (!isRecord(headers)) return; + + for (const key of Object.keys(headers)) { + if (!key.toLowerCase().startsWith('x-')) delete headers[key]; + } +} diff --git a/packages/@n8n/instance-ai/evaluations/utils/llm-judge.ts b/packages/@n8n/instance-ai/evaluations/utils/llm-judge.ts index d14b485571d..9b6504aa3f6 100644 --- a/packages/@n8n/instance-ai/evaluations/utils/llm-judge.ts +++ b/packages/@n8n/instance-ai/evaluations/utils/llm-judge.ts @@ -6,7 +6,8 @@ * the parsing of fenced/bare JSON, and the verdict shape. */ -const FENCED_JSON = /```(?:json)?\s*\n?([\s\S]*?)```/; +import { extractFencedJson } from '@n8n/ai-utilities/llm-output'; + const BARE_JSON_OBJECT = /\{[\s\S]*\}/; /** @@ -81,9 +82,9 @@ function isJudgeVerdict(value: unknown): value is JudgeVerdict { * Returns `undefined` when no valid verdict is found. */ export function parseJudgeVerdict(text: string): JudgeVerdict | undefined { - const fenceMatch = text.match(FENCED_JSON); - const fenced = fenceMatch ? tryParse(fenceMatch[1].trim()) : tryParse(text.trim()); - if (fenced) return fenced; + const fenced = extractFencedJson(text); + const first = tryParse(fenced ?? text.trim()); + if (first) return first; const objectMatch = text.match(BARE_JSON_OBJECT); const bare = objectMatch ? tryParse(objectMatch[0]) : undefined; diff --git a/packages/@n8n/instance-ai/src/utils/generate-validated-json.ts b/packages/@n8n/instance-ai/src/utils/generate-validated-json.ts index 1c00dab2232..0bff5efb1da 100644 --- a/packages/@n8n/instance-ai/src/utils/generate-validated-json.ts +++ b/packages/@n8n/instance-ai/src/utils/generate-validated-json.ts @@ -9,6 +9,7 @@ */ import type { ModelConfig } from '@n8n/agents'; +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import type { z } from 'zod'; import { createEvalAgent, extractText } from './eval-agents'; @@ -30,12 +31,6 @@ export interface GenerateValidatedJsonOptions { fallbackModelConfig?: ModelConfig; } -function stripMarkdownFences(text: string): string { - const trimmed = text.trim(); - const fencedMatch = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed); - return fencedMatch ? fencedMatch[1].trim() : trimmed; -} - export async function generateValidatedJson( agentName: string, options: GenerateValidatedJsonOptions, @@ -57,7 +52,7 @@ export async function generateValidatedJson( let parsed: unknown; try { - parsed = JSON.parse(stripMarkdownFences(text)); + parsed = JSON.parse(extractJsonCandidate(text)); } catch { return { ok: false, reason: 'invalid_json' }; } diff --git a/packages/@n8n/nodes-langchain/nodes/llms/N8nNonEstimatingTracing.ts b/packages/@n8n/nodes-langchain/nodes/llms/N8nNonEstimatingTracing.ts index 27fbee76028..388d5a75dc6 100644 --- a/packages/@n8n/nodes-langchain/nodes/llms/N8nNonEstimatingTracing.ts +++ b/packages/@n8n/nodes-langchain/nodes/llms/N8nNonEstimatingTracing.ts @@ -7,7 +7,7 @@ import type { } from '@langchain/core/load/serializable'; import type { BaseMessage } from '@langchain/core/messages'; import type { LLMResult } from '@langchain/core/outputs'; -import { logAiEvent, redactHeaderValues } from '@n8n/ai-utilities'; +import { logAiEvent, redactHeaderValues, stripNonXHeaders } from '@n8n/ai-utilities'; import pick from 'lodash/pick'; import type { IDataObject, ISupplyDataFunctions, JsonObject } from 'n8n-workflow'; import { NodeConnectionTypes, NodeError, NodeOperationError } from 'n8n-workflow'; @@ -145,16 +145,7 @@ export class N8nNonEstimatingTracing extends BaseCallbackHandler { async handleLLMError(error: IDataObject | Error, runId: string, parentRunId?: string) { const runDetails = this.runsMap[runId] ?? { index: Object.keys(this.runsMap).length }; - // Filter out non-x- headers to avoid leaking sensitive information in logs - if (typeof error === 'object' && error?.hasOwnProperty('headers')) { - const errorWithHeaders = error as { headers: Record }; - - Object.keys(errorWithHeaders.headers).forEach((key) => { - if (!key.startsWith('x-')) { - delete errorWithHeaders.headers[key]; - } - }); - } + stripNonXHeaders(error); if (error instanceof NodeError) { if (this.options.errorDescriptionMapper) { diff --git a/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts b/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts index 4a06cf41d9c..b1e516ac14f 100644 --- a/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts +++ b/packages/cli/src/modules/instance-ai/eval/agent-scenario-seed.ts @@ -1,3 +1,4 @@ +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import type { InstanceAiEvalAgentScenarioSeed } from '@n8n/api-types'; import { createEvalAgent, extractText } from '@n8n/instance-ai'; import { jsonParse } from 'n8n-workflow'; @@ -90,8 +91,9 @@ const MAX_SEED_ATTEMPTS = 2; const SEED_LLM_TIMEOUT_MS = 300_000; function parseSeed(raw: string): Omit | undefined { - const text = raw.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?\s*```\s*$/i, ''); - const parsed = jsonParse>(text, { fallbackValue: {} }); + const parsed = jsonParse>(extractJsonCandidate(raw), { + fallbackValue: {}, + }); const openingMessage = typeof parsed.openingMessage === 'string' ? parsed.openingMessage : ''; if (openingMessage.trim().length === 0) return undefined; diff --git a/packages/cli/src/modules/instance-ai/eval/mock-utils.ts b/packages/cli/src/modules/instance-ai/eval/mock-utils.ts index a7e5859206a..cd964786c29 100644 --- a/packages/cli/src/modules/instance-ai/eval/mock-utils.ts +++ b/packages/cli/src/modules/instance-ai/eval/mock-utils.ts @@ -1,4 +1,5 @@ import type { FetchFn } from '@n8n/agents'; +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import type { Logger } from '@n8n/backend-common'; import { createEvalAgent, extractText } from '@n8n/instance-ai'; import { jsonParse } from 'n8n-workflow'; @@ -11,10 +12,6 @@ import { jsonParse } from 'n8n-workflow'; const GENERATOR_TIMEOUT_MS = 120_000; const MAX_GENERATOR_ATTEMPTS = 2; -function fenceStrip(raw: string): string { - return raw.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?\s*```\s*$/i, ''); -} - /** * Generate JSON with the eval agent and validate its shape, retrying once on * unusable output. Returns undefined when both attempts fail — callers supply @@ -34,7 +31,7 @@ export async function generateJson( providerOptions: { anthropic: { maxTokens: 4096 } }, abortSignal: AbortSignal.timeout(GENERATOR_TIMEOUT_MS), }); - const parsed = jsonParse(fenceStrip(extractText(result)), { + const parsed = jsonParse(extractJsonCandidate(extractText(result)), { fallbackValue: undefined, }); const validated = validate(parsed); diff --git a/packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts b/packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts index 7025df41a14..eef83fe77f0 100644 --- a/packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts +++ b/packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts @@ -1,3 +1,4 @@ +import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output'; import { Logger } from '@n8n/backend-common'; import { Container } from '@n8n/di'; import { createEvalAgent, extractText } from '@n8n/instance-ai'; @@ -699,11 +700,7 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom abortSignal: AbortSignal.timeout(HINT_LLM_TIMEOUT_MS), }); - const text = extractText(result) - .replace(/^```(?:json)?\s*\n?/i, '') - .replace(/\n?\s*```\s*$/i, '') - .trim(); - + const text = extractJsonCandidate(extractText(result)); const parsed: Record = jsonParse(text); // globalContext may come back as a string or object — normalize to string diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1e9eefc6d2b..5d334e842fb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1354,6 +1354,9 @@ importers: axios: specifier: 1.18.0 version: 1.18.0(patch_hash=149e256a2a7b632497650b32816716ced972ab02a5ab00fbd8a5158a51722c49)(debug@4.4.3(supports-color@8.1.1))(supports-color@8.1.1) + fast-check: + specifier: 'catalog:' + version: 3.23.2 fast-glob: specifier: 'catalog:' version: 3.3.3