From a3f47117cf6484184b4b4604319cf3d3191b0e16 Mon Sep 17 00:00:00 2001 From: Johannes Nicolai Date: Wed, 14 Oct 2026 12:56:44 +0000 Subject: [PATCH] fix(OpenAI Node): Report token usage for image operations (#38463) Co-authored-by: Claude --- .../vendors/OpenAi/helpers/interfaces.ts | 8 ++ .../OpenAi/helpers/token-usage.test.ts | 127 ++++++++++++++++++ .../vendors/OpenAi/helpers/token-usage.ts | 48 +++++++ .../test/v2/actions/image/analyze.test.ts | 68 ++++++++++ .../v2/actions/image/edit.operation.test.ts | 100 ++++++++++++++ .../actions/image/generate.operation.test.ts | 65 +++++++++ .../v2/actions/image/analyze.operation.ts | 3 + .../OpenAi/v2/actions/image/edit.operation.ts | 6 +- .../v2/actions/image/generate.operation.ts | 12 +- 9 files changed, 434 insertions(+), 3 deletions(-) create mode 100644 packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.test.ts create mode 100644 packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.ts diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/interfaces.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/interfaces.ts index 3bf4878b463..7117fcf631e 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/interfaces.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/interfaces.ts @@ -84,6 +84,14 @@ export type ChatCompletion = { system_fingerprint: string; }; +/** + * Response shape of the `/images/generations` and `/images/edits` endpoints, including + * the `usage` block the images API returns. Note that it reports `input_tokens` / + * `output_tokens`, unlike the Chat Completions naming (`prompt_tokens` / + * `completion_tokens`) used by `ChatCompletion` above. + */ +export type ImageResponse = OpenAIClient.Images.ImagesResponse; + export type ThreadMessage = { id: string; object: string; diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.test.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.test.ts new file mode 100644 index 00000000000..afe1ad2f88f --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.test.ts @@ -0,0 +1,127 @@ +import type { IExecuteFunctions, ITaskMetadata } from 'n8n-workflow'; + +import { accumulateImageTokenUsage } from './token-usage'; + +type TokenUsage = NonNullable; + +/** + * Deliberately exercises the real `accumulateTokenUsage` rather than mocking it, so the + * assertions are about what actually reaches the execution metadata. + */ +const makeContext = (previous?: TokenUsage) => { + const setMetadata = vi.fn(); + const context = { + getExecuteData: vi.fn().mockReturnValue(previous ? { metadata: { tokenUsage: previous } } : {}), + setMetadata, + } as unknown as IExecuteFunctions; + + return { context, setMetadata }; +}; + +const storedUsage = (setMetadata: ReturnType): TokenUsage => + (setMetadata.mock.calls[0][0] as ITaskMetadata).tokenUsage as TokenUsage; + +describe('accumulateImageTokenUsage', () => { + it('should report both counts when the API returns them', () => { + const { context, setMetadata } = makeContext(); + + accumulateImageTokenUsage(context, { input_tokens: 350, output_tokens: 4160 }); + + expect(setMetadata).toHaveBeenCalledTimes(1); + expect(storedUsage(setMetadata)).toEqual({ inputTokens: 350, outputTokens: 4160 }); + }); + + it('should add to a total already accumulated earlier in the execution', () => { + const { context, setMetadata } = makeContext({ inputTokens: 100, outputTokens: 200 }); + + accumulateImageTokenUsage(context, { input_tokens: 350, output_tokens: 4160 }); + + expect(storedUsage(setMetadata)).toEqual({ inputTokens: 450, outputTokens: 4360 }); + }); + + it('should report nothing when the response has no usage block', () => { + const { context, setMetadata } = makeContext(); + + accumulateImageTokenUsage(context, undefined); + + expect(setMetadata).not.toHaveBeenCalled(); + }); + + it('should report nothing when a proxy returns an empty usage block', () => { + const { context, setMetadata } = makeContext(); + + accumulateImageTokenUsage(context, {}); + + expect(setMetadata).not.toHaveBeenCalled(); + }); + + it('should report nothing when usage carries only unrelated fields', () => { + const { context, setMetadata } = makeContext(); + + accumulateImageTokenUsage(context, { total_tokens: 1610 } as never); + + expect(setMetadata).not.toHaveBeenCalled(); + }); + + it('should still report the one count a partial usage block does provide', () => { + const { context, setMetadata } = makeContext(); + + accumulateImageTokenUsage(context, { input_tokens: 350 }); + + expect(storedUsage(setMetadata)).toEqual({ inputTokens: 350, outputTokens: 0 }); + }); + + it.each([ + ['string counts', { input_tokens: '350', output_tokens: '4160' }], + ['null counts', { input_tokens: null, output_tokens: null }], + ['NaN counts', { input_tokens: NaN, output_tokens: NaN }], + ['infinite counts', { input_tokens: Infinity, output_tokens: -Infinity }], + ['negative counts', { input_tokens: -350, output_tokens: -4160 }], + ['fractional counts', { input_tokens: 350.5, output_tokens: 4160.5 }], + ['counts beyond the safe integer range', { input_tokens: 2 ** 53, output_tokens: 2 ** 53 }], + ])('should ignore %s', (_label, usage) => { + const { context, setMetadata } = makeContext(); + + accumulateImageTokenUsage(context, usage as never); + + expect(setMetadata).not.toHaveBeenCalled(); + }); + + it('should never reduce a total already accumulated', () => { + const { context, setMetadata } = makeContext({ inputTokens: 100, outputTokens: 200 }); + + accumulateImageTokenUsage(context, { input_tokens: -350, output_tokens: -4160 } as never); + + expect(setMetadata).not.toHaveBeenCalled(); + }); + + it('should never store NaN, which would persist for the rest of the execution', () => { + const malformed = [ + undefined, + {}, + { total_tokens: 1610 }, + { input_tokens: undefined, output_tokens: undefined }, + { input_tokens: null, output_tokens: null }, + { input_tokens: NaN, output_tokens: NaN }, + { input_tokens: '350', output_tokens: '4160' }, + { input_tokens: -350, output_tokens: -4160 }, + { input_tokens: 350.5, output_tokens: 4160.5 }, + { input_tokens: 350 }, + { output_tokens: 4160 }, + ]; + + for (const usage of malformed) { + const { context, setMetadata } = makeContext({ inputTokens: 100, outputTokens: 200 }); + + accumulateImageTokenUsage(context, usage as never); + + if (setMetadata.mock.calls.length === 0) continue; + + const { inputTokens, outputTokens } = storedUsage(setMetadata); + expect(Number.isFinite(inputTokens)).toBe(true); + expect(Number.isFinite(outputTokens)).toBe(true); + expect(inputTokens).toBeGreaterThanOrEqual(100); + expect(outputTokens).toBeGreaterThanOrEqual(200); + } + }); +}); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.ts new file mode 100644 index 00000000000..d364428f5c1 --- /dev/null +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/helpers/token-usage.ts @@ -0,0 +1,48 @@ +import type { IExecuteFunctions } from 'n8n-workflow'; +import { accumulateTokenUsage } from 'n8n-workflow'; + +/** + * Token usage as the image and responses endpoints report it. Upstream both types + * declare these counts as required numbers, but an OpenAI-compatible proxy can answer + * with a partial or malformed `usage` block, so they are treated as untrusted here. + */ +type ImageTokenUsage = { + input_tokens?: number; + output_tokens?: number; +}; + +/** + * Accepts only what a token count can actually be: a non-negative integer. Nothing is + * coerced, so `null`, `undefined`, numeric strings, `NaN` and the infinities are all + * rejected — as are negative counts, which would otherwise *subtract* from the running + * total, and fractional ones, which are not a possible number of tokens. + */ +const asTokenCount = (value: unknown): number | undefined => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined; + +/** + * Reports token usage for an image operation, ignoring any count the API did not + * actually return as a number. + * + * `accumulateTokenUsage` adds to the running total with `??`, which only guards against + * `null`/`undefined`. Passing it a missing count would therefore store `NaN`, and since + * `NaN` is not nullish every later accumulation in the same execution would stay `NaN` + * too — leaving the whole run without usable token metrics (`NaN` also serializes to + * `null`). Guarding at the call site keeps one malformed response from poisoning the + * totals for every other node in the execution. + */ +export function accumulateImageTokenUsage( + context: IExecuteFunctions, + usage: ImageTokenUsage | undefined, +): void { + if (!usage) return; + + const inputTokens = asTokenCount(usage.input_tokens); + const outputTokens = asTokenCount(usage.output_tokens); + + // Nothing usable at all: report nothing rather than a confident zero, which is how + // a response without a `usage` block already behaves. + if (inputTokens === undefined && outputTokens === undefined) return; + + accumulateTokenUsage(context, inputTokens ?? 0, outputTokens ?? 0); +} diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/analyze.test.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/analyze.test.ts index 442c5c46883..328f0f33101 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/analyze.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/analyze.test.ts @@ -10,6 +10,15 @@ import { execute } from '../../../../v2/actions/image/analyze.operation'; vi.mock('../../../../helpers/binary-data'); vi.mock('../../../../transport'); +const { accumulateTokenUsageMock } = vi.hoisted(() => ({ + accumulateTokenUsageMock: vi.fn(), +})); + +vi.mock('n8n-workflow', async (importOriginal) => ({ + ...(await importOriginal()), + accumulateTokenUsage: accumulateTokenUsageMock, +})); + describe('Image Analyze Operation', () => { let mockExecuteFunctions: Mocked; let mockNode: INode; @@ -919,4 +928,63 @@ describe('Image Analyze Operation', () => { }); }); }); + + describe('token usage reporting', () => { + beforeEach(() => { + mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => { + const params = { + modelId: 'gpt-4o', + text: "What's in this image?", + inputType: 'url', + imageUrls: 'https://example.com/image1.jpg', + simplify: true, + options: {}, + }; + return params[paramName as keyof typeof params]; + }); + }); + + const baseResponse = { + id: 'response-123', + status: 'completed', + output: [ + { + type: 'message', + role: 'assistant', + content: [{ type: 'output_text', text: 'A mountain lake.' }], + }, + ], + }; + + it('should report token usage when the API returns a usage object', async () => { + apiRequestSpy.mockResolvedValue({ + ...baseResponse, + usage: { input_tokens: 1120, output_tokens: 96, total_tokens: 1216 }, + } as ChatResponse); + + await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).toHaveBeenCalledTimes(1); + expect(accumulateTokenUsageMock).toHaveBeenCalledWith(mockExecuteFunctions, 1120, 96); + }); + + it('should not report token usage when the API omits it', async () => { + apiRequestSpy.mockResolvedValue(baseResponse as ChatResponse); + + await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).not.toHaveBeenCalled(); + }); + + it('should not report token usage when a proxy returns usage without the counts', async () => { + apiRequestSpy.mockResolvedValue({ + ...baseResponse, + usage: { total_tokens: 1216 }, + } as unknown as ChatResponse); + + await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/edit.operation.test.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/edit.operation.test.ts index 94974328302..f364a6eef37 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/edit.operation.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/edit.operation.test.ts @@ -26,6 +26,15 @@ vi.mock('form-data', () => { return { default: MockFormData }; }); +const { accumulateTokenUsageMock } = vi.hoisted(() => ({ + accumulateTokenUsageMock: vi.fn(), +})); + +vi.mock('n8n-workflow', async (importOriginal) => ({ + ...(await importOriginal()), + accumulateTokenUsage: accumulateTokenUsageMock, +})); + describe('Image Edit Operation', () => { let mockExecuteFunctions: Mocked; let mockNode: INode; @@ -736,4 +745,95 @@ describe('Image Edit Operation', () => { expect(getBinaryDataFileSpy).toHaveBeenNthCalledWith(2, mockExecuteFunctions, 0, 'image2'); }); }); + + describe('token usage reporting', () => { + const mockBinaryFile = { + fileContent: Buffer.from('mock-image-data'), + contentType: 'image/png', + filename: 'data.png', + }; + + const mockEditedBinaryData = { + data: 'base64encodedimagedata', + mimeType: 'image/png', + fileName: 'data', + }; + + beforeEach(() => { + mockExecuteFunctions.getNodeParameter.mockImplementation((paramName: string) => { + const params = { + model: 'gpt-image-1', + prompt: 'Add a rainbow to this landscape', + images: { + values: [{ binaryPropertyName: 'data' }], + }, + n: 1, + size: '1024x1024', + quality: 'auto', + options: {}, + }; + return params[paramName as keyof typeof params]; + }); + + getBinaryDataFileSpy.mockResolvedValue(mockBinaryFile); + (mockExecuteFunctions.helpers.binaryToBuffer as Mock).mockResolvedValue( + mockBinaryFile.fileContent, + ); + (mockExecuteFunctions.helpers.prepareBinaryData as Mock).mockResolvedValue( + mockEditedBinaryData, + ); + }); + + it('should report token usage when the API returns a usage object', async () => { + apiRequestSpy.mockResolvedValue({ + data: [{ b64_json: 'base64encodedimagedata' }], + usage: { + input_tokens: 350, + output_tokens: 4160, + total_tokens: 4510, + input_tokens_details: { text_tokens: 27, image_tokens: 323 }, + }, + }); + + await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).toHaveBeenCalledTimes(1); + expect(accumulateTokenUsageMock).toHaveBeenCalledWith(mockExecuteFunctions, 350, 4160); + }); + + it('should not report token usage when the API omits it, and still return the image', async () => { + apiRequestSpy.mockResolvedValue({ + data: [{ b64_json: 'base64encodedimagedata' }], + }); + + const result = await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).not.toHaveBeenCalled(); + expect(result).toEqual([ + { + json: expect.objectContaining({ data: undefined }), + binary: { data: mockEditedBinaryData }, + pairedItem: { item: 0 }, + }, + ]); + }); + + it('should not report token usage when a proxy returns usage without the counts', async () => { + apiRequestSpy.mockResolvedValue({ + data: [{ b64_json: 'base64encodedimagedata' }], + usage: { total_tokens: 4510 }, + }); + + const result = await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).not.toHaveBeenCalled(); + expect(result).toEqual([ + { + json: expect.objectContaining({ data: undefined }), + binary: { data: mockEditedBinaryData }, + pairedItem: { item: 0 }, + }, + ]); + }); + }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/generate.operation.test.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/generate.operation.test.ts index 7ef610a7503..57ac357b8ba 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/generate.operation.test.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/test/v2/actions/image/generate.operation.test.ts @@ -7,6 +7,15 @@ import type { Mocked } from 'vitest'; vi.mock('../../../../transport'); +const { accumulateTokenUsageMock } = vi.hoisted(() => ({ + accumulateTokenUsageMock: vi.fn(), +})); + +vi.mock('n8n-workflow', async (importOriginal) => ({ + ...(await importOriginal()), + accumulateTokenUsage: accumulateTokenUsageMock, +})); + describe('Image Generate Operation', () => { let mockExecuteFunctions: Mocked; let mockNode: INode; @@ -264,4 +273,60 @@ describe('Image Generate Operation', () => { ); }); }); + + describe('token usage reporting', () => { + beforeEach(() => { + mockNode = makeNode(2.2); + mockExecuteFunctions.getNode.mockReturnValue(mockNode); + mockExecuteFunctions.getNodeParameter.mockImplementation( + (paramName: string, _i: unknown, _default: unknown, opts?: { extractValue?: boolean }) => { + if (paramName === 'modelId' && opts?.extractValue) return 'gpt-image-1'; + const params = { prompt: 'a cute cat', options: {} }; + return params[paramName as keyof typeof params]; + }, + ); + }); + + it('should report token usage when the API returns a usage object', async () => { + apiRequestSpy.mockResolvedValue({ + ...b64Response, + usage: { input_tokens: 42, output_tokens: 1568, total_tokens: 1610 }, + }); + + await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).toHaveBeenCalledTimes(1); + expect(accumulateTokenUsageMock).toHaveBeenCalledWith(mockExecuteFunctions, 42, 1568); + }); + + it('should not report token usage when the API omits it, and still return the image', async () => { + apiRequestSpy.mockResolvedValue(b64Response); + + const result = await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).not.toHaveBeenCalled(); + expect(result).toEqual([ + { + json: expect.objectContaining({ data: undefined }), + binary: { data: mockBinaryData }, + pairedItem: { item: 0 }, + }, + ]); + }); + + it('should not report token usage when a proxy returns usage without the counts', async () => { + apiRequestSpy.mockResolvedValue({ ...b64Response, usage: { total_tokens: 1610 } }); + + const result = await execute.call(mockExecuteFunctions, 0); + + expect(accumulateTokenUsageMock).not.toHaveBeenCalled(); + expect(result).toEqual([ + { + json: expect.objectContaining({ data: undefined }), + binary: { data: mockBinaryData }, + pairedItem: { item: 0 }, + }, + ]); + }); + }); }); diff --git a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/v2/actions/image/analyze.operation.ts b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/v2/actions/image/analyze.operation.ts index 270224bb981..950d2f99c81 100644 --- a/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/v2/actions/image/analyze.operation.ts +++ b/packages/@n8n/nodes-langchain/nodes/vendors/OpenAi/v2/actions/image/analyze.operation.ts @@ -8,6 +8,7 @@ import { NodeOperationError, updateDisplayOptions } from 'n8n-workflow'; import type { ResponseInputImage } from 'openai/resources/responses/responses'; import type { ChatContent, ChatResponse, ChatResponseRequest } from '../../../helpers/interfaces'; +import { accumulateImageTokenUsage } from '../../../helpers/token-usage'; import { apiRequest } from '../../../transport'; import { modelRLC } from '../descriptions'; import { getBinaryDataFile } from '../../../helpers/binary-data'; @@ -196,6 +197,8 @@ export async function execute(this: IExecuteFunctions, i: number): Promise ({ json: entry, @@ -122,7 +130,7 @@ export async function execute(this: IExecuteFunctions, i: number): Promise