mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-17 13:39:44 +02:00
feat(core): Offload large webhook responses to storage in scaling mode (#35036)
This commit is contained in:
parent
68e4c528b9
commit
2e6eb98e13
|
|
@ -166,13 +166,32 @@ export class ExecutionsConfig {
|
|||
maxDisplaySize: number = 100 * 1024 * 1024; // 100 MB
|
||||
|
||||
/**
|
||||
* Maximum size in MiB of a response body a worker relays back to main in scaling mode.
|
||||
* A larger response fails the node instead of being sent through the queue.
|
||||
* Maximum size in MiB of a webhook response a worker sends back to the main instance
|
||||
* inside a queue message, in queue mode.
|
||||
*
|
||||
* Relaying costs the queue a multiple of the body size,
|
||||
* so size this against the memory available to it,
|
||||
* and return large payloads as binary data to have them streamed from storage instead.
|
||||
* Redis holds several copies of a response while the message is in flight, so budget
|
||||
* about 1.5 times this value in Redis memory for each response in flight.
|
||||
*
|
||||
* With `N8N_WEBHOOK_RESPONSE_RELAY_OFFLOAD_ENABLED` turned on, a larger body goes to
|
||||
* the binary-data store and the main instance reads it back from there. Otherwise the
|
||||
* node fails. Only the body can go to the store, so a response whose headers alone
|
||||
* exceed this limit fails either way.
|
||||
*/
|
||||
@Env('N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX', positiveIntSchema)
|
||||
webhookResponseRelaySizeMaxMiB: number = 64;
|
||||
|
||||
/**
|
||||
* Whether a worker stores a response body over `N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX` in
|
||||
* the binary-data store, so the main instance can stream it to the client, instead of
|
||||
* failing the node.
|
||||
*
|
||||
* Needs `N8N_DEFAULT_BINARY_DATA_MODE` set to a mode with a store (`filesystem`,
|
||||
* `database`, `s3`, or `azure`), and a store every instance can read.
|
||||
*
|
||||
* Only the worker reads this setting. Turn it on once every main instance runs a version
|
||||
* that reads a stored body: an older main returns the storage reference instead of the
|
||||
* response body.
|
||||
*/
|
||||
@Env('N8N_WEBHOOK_RESPONSE_RELAY_OFFLOAD_ENABLED')
|
||||
webhookResponseRelayOffloadEnabled: boolean = false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -540,6 +540,7 @@ describe('GlobalConfig', () => {
|
|||
saveDataManualExecutions: true,
|
||||
maxDisplaySize: 100 * 1024 * 1024,
|
||||
webhookResponseRelaySizeMaxMiB: 64,
|
||||
webhookResponseRelayOffloadEnabled: false,
|
||||
},
|
||||
diagnostics: {
|
||||
enabled: true,
|
||||
|
|
|
|||
|
|
@ -307,6 +307,10 @@ export class McpServer {
|
|||
};
|
||||
}
|
||||
|
||||
hasSession(sessionId: string): boolean {
|
||||
return this.getTransport(sessionId) !== undefined;
|
||||
}
|
||||
|
||||
handleWorkerResponse(sessionId: string, messageId: string, result: unknown): void {
|
||||
const callId = messageId ? `${sessionId}_${messageId}` : sessionId;
|
||||
const pending = this.pendingResponses[callId];
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
import { UserError } from 'n8n-workflow';
|
||||
|
||||
/**
|
||||
* A response a worker cannot deliver back to the main instance because of its size.
|
||||
*
|
||||
* @param description How to make the response deliverable, which depends on
|
||||
* which limit it crossed: the size relayable inline through the queue, or the
|
||||
* size the binary-data store accepts for an offloaded body.
|
||||
*/
|
||||
export class WebhookResponseTooLargeError extends UserError {
|
||||
constructor(maxSizeInMiB: number) {
|
||||
super(
|
||||
`The response is too large to be sent back from the worker (limit is ${maxSizeInMiB} MiB)`,
|
||||
{
|
||||
description:
|
||||
'In scaling mode a response is relayed to the main instance through the queue, which limits how large it can be. Respond with binary data to have the payload streamed from storage instead, or raise N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX.',
|
||||
},
|
||||
);
|
||||
constructor(message: string, options: { description: string; cause?: unknown }) {
|
||||
super(message, options);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,16 @@
|
|||
import type { WorkflowEntity } from '@n8n/db';
|
||||
import { Container } from '@n8n/di';
|
||||
import type { INode, IWorkflowExecutionDataProcess } from 'n8n-workflow';
|
||||
import type { IDeferredPromise } from '@n8n/utils/promise/deferred-promise';
|
||||
import type {
|
||||
IExecuteResponsePromiseData,
|
||||
INode,
|
||||
IWorkflowExecutionDataProcess,
|
||||
} from 'n8n-workflow';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import type { ActiveExecutions } from '@/active-executions';
|
||||
import { ExecutionPersistence } from '@/executions/execution-persistence';
|
||||
import { WebhookResponseRelay } from '@/scaling/webhook-response-relay';
|
||||
import type { WorkflowRunner } from '@/workflow-runner';
|
||||
|
||||
import { executeWorkflow, type WorkflowToolContext } from '../workflow-tool-factory';
|
||||
|
|
@ -97,3 +104,104 @@ describe('executeWorkflow → eval instrumentation', () => {
|
|||
expect(runData.configureAdditionalData).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('executeWorkflow → webhook response', () => {
|
||||
const relay = mock<WebhookResponseRelay>();
|
||||
|
||||
const runnerResolving = (response: IExecuteResponsePromiseData) =>
|
||||
vi.fn(
|
||||
(
|
||||
_runData: IWorkflowExecutionDataProcess,
|
||||
_loadStaticData?: boolean,
|
||||
_realtime?: boolean,
|
||||
_restartExecutionId?: string,
|
||||
responsePromise?: IDeferredPromise<IExecuteResponsePromiseData>,
|
||||
) => {
|
||||
responsePromise?.resolve(response);
|
||||
return 'exec-1';
|
||||
},
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
Container.set(ExecutionPersistence, {
|
||||
findSingleExecution: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ status: 'success', data: { resultData: { runData: {} } } }),
|
||||
} as unknown as ExecutionPersistence);
|
||||
Container.set(WebhookResponseRelay, relay);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
Container.reset();
|
||||
});
|
||||
|
||||
it('embeds the restored body rather than the store reference it arrived as', async () => {
|
||||
const relayed = {
|
||||
body: { binaryData: { id: 'database:abc' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
} as IExecuteResponsePromiseData;
|
||||
const restored = { body: { hello: 'world' }, headers: {}, statusCode: 200 };
|
||||
relay.restoreOffloadedBody.mockResolvedValue(restored);
|
||||
|
||||
const result = await executeWorkflow(
|
||||
workflow,
|
||||
triggerNode,
|
||||
'manual',
|
||||
{},
|
||||
buildContext(runnerResolving(relayed)),
|
||||
false,
|
||||
);
|
||||
|
||||
expect(relay.restoreOffloadedBody).toHaveBeenCalledWith(relayed, {
|
||||
reclaim: true,
|
||||
context: { workflowId: 'wf-1', executionId: 'exec-1' },
|
||||
});
|
||||
expect(result.data?.response).toEqual(restored);
|
||||
});
|
||||
|
||||
const responseBodyOf = async (body: unknown) => {
|
||||
const relayed = { body: { binaryData: { id: 'database:abc' } } } as IExecuteResponsePromiseData;
|
||||
relay.restoreOffloadedBody.mockResolvedValue({ body, headers: {}, statusCode: 200 });
|
||||
|
||||
const result = await executeWorkflow(
|
||||
workflow,
|
||||
triggerNode,
|
||||
'manual',
|
||||
{},
|
||||
buildContext(runnerResolving(relayed)),
|
||||
false,
|
||||
);
|
||||
|
||||
return (result.data?.response as { body: unknown }).body;
|
||||
};
|
||||
|
||||
it('caps an oversized body, keeping a preview and its length', async () => {
|
||||
const body = { blob: 'x'.repeat(50_000) };
|
||||
|
||||
expect(await responseBodyOf(body)).toEqual({
|
||||
_truncated: true,
|
||||
_charLength: JSON.stringify(body).length,
|
||||
_preview: expect.stringMatching(/^\{"blob":"x{100}/),
|
||||
});
|
||||
});
|
||||
|
||||
// Whatever its size: measuring a Buffer costs 12x the body, so it is never serialized.
|
||||
it.each([
|
||||
['a large Buffer body', 4 * 1024 * 1024],
|
||||
['a small Buffer body', 5],
|
||||
])('describes %s by its size instead of serializing it', async (_label, byteLength) => {
|
||||
expect(await responseBodyOf(Buffer.alloc(byteLength))).toEqual({
|
||||
_truncated: true,
|
||||
_byteLength: byteLength,
|
||||
});
|
||||
});
|
||||
|
||||
// Reachable in regular mode only: a relayed body was serialized once already.
|
||||
it('describes a cyclic body it cannot serialize', async () => {
|
||||
const body: Record<string, unknown> = { name: 'cycle' };
|
||||
body.self = body;
|
||||
|
||||
expect(await responseBodyOf(body)).toEqual({ _truncated: true });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import { z } from 'zod';
|
|||
|
||||
import type { ActiveExecutions } from '@/active-executions';
|
||||
import { ExecutionPersistence } from '@/executions/execution-persistence';
|
||||
import { WebhookResponseRelay } from '@/scaling/webhook-response-relay';
|
||||
import type { WorkflowRunner } from '@/workflow-runner';
|
||||
|
||||
import type { InstrumentToolAdditionalData } from '../agent-runtime-instrumentation';
|
||||
|
|
@ -381,9 +382,13 @@ export async function executeWorkflow(
|
|||
|
||||
const result = await extractResult(executionId, allOutputs);
|
||||
if (isWorkflowToolResponse(webhookResponse)) {
|
||||
const response = await Container.get(WebhookResponseRelay).restoreOffloadedBody(
|
||||
webhookResponse,
|
||||
{ reclaim: true, context: { workflowId: workflow.id, executionId } },
|
||||
);
|
||||
result.data = {
|
||||
...(result.data ?? {}),
|
||||
response: truncateResultData({ response: webhookResponse }).response,
|
||||
response: truncateWebhookResponse(response),
|
||||
};
|
||||
}
|
||||
return result;
|
||||
|
|
@ -501,6 +506,47 @@ function truncateNodeOutput(items: unknown[]): unknown {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Caps a webhook response's body at {@link MAX_RESULT_CHARS}, describing it
|
||||
* instead of carrying it once over. Headers and status code pass through.
|
||||
*
|
||||
* @remarks A Buffer body is described without being serialized at all:
|
||||
* `JSON.stringify` turns it into one array element per byte, which costs about
|
||||
* twelve times the body and throws above V8's max string length. A body
|
||||
* `JSON.stringify` rejects (a cycle, a BigInt) is described bare, with neither
|
||||
* length nor preview.
|
||||
*/
|
||||
function truncateWebhookResponse(response: IExecuteResponsePromiseData): unknown {
|
||||
if (!isRecord(response)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const { body, ...rest } = response;
|
||||
|
||||
if (Buffer.isBuffer(body)) {
|
||||
return { ...rest, body: { _truncated: true, _byteLength: body.length } };
|
||||
}
|
||||
|
||||
let serialized: string;
|
||||
try {
|
||||
serialized = JSON.stringify(body) ?? '';
|
||||
} catch {
|
||||
return { ...rest, body: { _truncated: true } };
|
||||
}
|
||||
if (serialized.length <= MAX_RESULT_CHARS) {
|
||||
return response;
|
||||
}
|
||||
|
||||
return {
|
||||
...rest,
|
||||
body: {
|
||||
_truncated: true,
|
||||
_charLength: serialized.length,
|
||||
_preview: serialized.slice(0, MAX_RESULT_CHARS),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function truncateResultData(data: Record<string, unknown>): Record<string, unknown> {
|
||||
const serialized = JSON.stringify(data);
|
||||
if (serialized.length <= MAX_RESULT_CHARS) return data;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,12 @@ import type { Logger } from '@n8n/backend-common';
|
|||
import type { ExecutionsConfig } from '@n8n/config';
|
||||
import type { IExecutionResponse, ExecutionRepository, Project } from '@n8n/db';
|
||||
import { WorkflowPublishHistoryRepository } from '@n8n/db';
|
||||
import type { WorkflowExecute as ActualWorkflowExecute, InstanceSettings } from 'n8n-core';
|
||||
import type {
|
||||
WorkflowExecute as ActualWorkflowExecute,
|
||||
BinaryDataConfig,
|
||||
BinaryDataService,
|
||||
InstanceSettings,
|
||||
} from 'n8n-core';
|
||||
import { ExternalSecretsProxy } from 'n8n-core';
|
||||
import { mockInstance } from 'n8n-core/test/utils';
|
||||
import {
|
||||
|
|
@ -40,7 +45,7 @@ import { WorkflowStaticDataService } from '@/workflows/workflow-static-data.serv
|
|||
|
||||
import { JobProcessor } from '../job-processor';
|
||||
import type { Job } from '../scaling.types';
|
||||
import { ENCODED_BUFFER_KEY } from '../webhook-response-relay';
|
||||
import { ENCODED_BUFFER_KEY, WebhookResponseRelay } from '../webhook-response-relay';
|
||||
|
||||
mockInstance(WorkflowPublishHistoryRepository);
|
||||
mockInstance(VariablesService, {
|
||||
|
|
@ -74,14 +79,10 @@ const logger = mock<Logger>({
|
|||
scoped: vi.fn().mockImplementation(() => logger),
|
||||
});
|
||||
|
||||
const executionsConfigWith = (overrides: Partial<ExecutionsConfig> = {}) =>
|
||||
mock<ExecutionsConfig>({
|
||||
timeout: -1,
|
||||
maxTimeout: 3600,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const executionsConfig = executionsConfigWith();
|
||||
const executionsConfig = mock<ExecutionsConfig>({
|
||||
timeout: -1,
|
||||
maxTimeout: 3600,
|
||||
});
|
||||
|
||||
const successRun = (): IRun =>
|
||||
mock<IRun>({
|
||||
|
|
@ -131,6 +132,7 @@ describe('JobProcessor', () => {
|
|||
mock(),
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const result = await jobProcessor.processJob(mock<Job>());
|
||||
|
|
@ -161,6 +163,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>({ data: { executionId: 'execution-id', loadStaticData: false } });
|
||||
|
|
@ -195,6 +198,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -238,6 +242,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -286,6 +291,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const executionId = 'execution-id';
|
||||
|
|
@ -335,6 +341,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const executionId = 'execution-id';
|
||||
|
|
@ -373,6 +380,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const executionId = 'execution-id';
|
||||
|
|
@ -426,6 +434,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
await jobProcessor.processJob(mock<Job>());
|
||||
|
|
@ -473,6 +482,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(), // eventService
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -532,6 +542,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -582,6 +593,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(), // eventService
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -643,6 +655,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(), // eventService
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -730,6 +743,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(), // eventService
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -804,6 +818,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -906,6 +921,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -947,6 +963,91 @@ describe('JobProcessor', () => {
|
|||
expect(lastResponse.response).toEqual([{ result: 'tool response data' }]);
|
||||
});
|
||||
|
||||
it('should relay a tool result too large for the queue as an error', async () => {
|
||||
const executionRepository = mock<ExecutionRepository>();
|
||||
const executionPersistence = mock<ExecutionPersistence>();
|
||||
const toolNode = {
|
||||
name: 'HTTP Request',
|
||||
type: 'n8n-nodes-base.httpRequestTool',
|
||||
typeVersion: 4.4,
|
||||
parameters: {},
|
||||
position: [0, 0] as [number, number],
|
||||
};
|
||||
executionPersistence.findSingleExecution.mockResolvedValueOnce(
|
||||
mock<IExecutionResponse>({
|
||||
mode: 'trigger',
|
||||
workflowData: { id: 'wf-1', nodes: [toolNode], staticData: {} },
|
||||
data: mock<IRunExecutionData>({ executionData: undefined }),
|
||||
}),
|
||||
);
|
||||
executionPersistence.findSingleExecution.mockResolvedValueOnce(
|
||||
mock<IExecutionResponse>({
|
||||
status: 'success',
|
||||
workflowData: { id: 'wf-1', nodes: [toolNode], staticData: {} },
|
||||
data: mock<IRunExecutionData>({ resultData: { runData: {} } }),
|
||||
}),
|
||||
);
|
||||
|
||||
const nodeTypes = mock<NodeTypes>();
|
||||
nodeTypes.getByNameAndVersion.mockReturnValue({
|
||||
description: {
|
||||
name: 'httpRequestTool',
|
||||
outputs: [NodeConnectionTypes.AiTool],
|
||||
properties: [],
|
||||
},
|
||||
execute: vi.fn().mockResolvedValue([[{ json: { blob: 'x'.repeat(64) } }]]),
|
||||
} as never);
|
||||
|
||||
const relay = mock<WebhookResponseRelay>();
|
||||
relay.assertFitsInline.mockImplementation(() => {
|
||||
throw new WebhookResponseTooLargeError(
|
||||
'The response is too large to be sent back from the worker (limit is 1 MiB)',
|
||||
{ description: 'Raise N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX.' },
|
||||
);
|
||||
});
|
||||
|
||||
const jobProcessor = new JobProcessor(
|
||||
logger,
|
||||
executionRepository,
|
||||
executionPersistence,
|
||||
mock(),
|
||||
nodeTypes,
|
||||
{ hostId: 'worker-host-123' } as unknown as InstanceSettings,
|
||||
createManualExecutionServiceMock(),
|
||||
executionsConfig,
|
||||
mock(),
|
||||
relay,
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
job.data = {
|
||||
workflowId: 'wf-1',
|
||||
executionId: 'exec-mcp-tool-oversized',
|
||||
loadStaticData: false,
|
||||
isMcpExecution: true,
|
||||
mcpType: 'trigger',
|
||||
mcpSessionId: 'session-tool',
|
||||
mcpMessageId: 'msg-tool',
|
||||
mcpToolCall: {
|
||||
toolName: 'HTTP Request',
|
||||
arguments: { url: 'https://example.com' },
|
||||
sourceNodeName: 'HTTP Request',
|
||||
},
|
||||
};
|
||||
|
||||
await jobProcessor.processJob(job);
|
||||
|
||||
expect(relay.assertFitsInline).toHaveBeenCalledWith([{ blob: 'x'.repeat(64) }]);
|
||||
|
||||
const mcpResponseCalls = (job.progress as Mock).mock.calls.filter(
|
||||
(call: unknown[]) => (call[0] as { kind: string }).kind === 'mcp-response',
|
||||
);
|
||||
const lastResponse = mcpResponseCalls[mcpResponseCalls.length - 1][0] as {
|
||||
response: { error?: { message?: string } };
|
||||
};
|
||||
expect(lastResponse.response.error?.message).toContain('too large');
|
||||
});
|
||||
|
||||
describe('expression isolate for tool calls', () => {
|
||||
const toolNode = {
|
||||
name: 'HTTP Request',
|
||||
|
|
@ -993,6 +1094,7 @@ describe('JobProcessor', () => {
|
|||
createManualExecutionServiceMock(),
|
||||
executionsConfig,
|
||||
mock(), // eventService
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1128,6 +1230,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1235,6 +1338,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1329,6 +1433,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1418,6 +1523,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1517,6 +1623,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1602,6 +1709,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1693,6 +1801,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1769,6 +1878,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1807,6 +1917,7 @@ describe('JobProcessor', () => {
|
|||
mock(),
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
const run = mock<IRun>({
|
||||
status: 'waiting',
|
||||
|
|
@ -1836,6 +1947,7 @@ describe('JobProcessor', () => {
|
|||
mock(),
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
const run = mock<IRun>({
|
||||
status: 'success',
|
||||
|
|
@ -1885,6 +1997,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1951,6 +2064,7 @@ describe('JobProcessor', () => {
|
|||
manualExecutionService,
|
||||
executionsConfig,
|
||||
mock(),
|
||||
mock(),
|
||||
);
|
||||
|
||||
const job = mock<Job>();
|
||||
|
|
@ -1975,10 +2089,33 @@ describe('JobProcessor', () => {
|
|||
});
|
||||
|
||||
describe('webhook response relay', () => {
|
||||
const buildRelay = (webhookResponseRelaySizeMaxMiB: number, mode: BinaryDataConfig['mode']) => {
|
||||
const binaryDataService = mock<BinaryDataService>();
|
||||
binaryDataService.store.mockImplementation(async (_location, _body, binaryData) => ({
|
||||
...binaryData,
|
||||
id: `${mode}:stored-file-id`,
|
||||
}));
|
||||
|
||||
const relay = new WebhookResponseRelay(
|
||||
logger,
|
||||
binaryDataService,
|
||||
mock<BinaryDataConfig>({ mode }),
|
||||
mock<ExecutionsConfig>({
|
||||
webhookResponseRelaySizeMaxMiB,
|
||||
webhookResponseRelayOffloadEnabled: true,
|
||||
}),
|
||||
);
|
||||
|
||||
return { relay, binaryDataService };
|
||||
};
|
||||
|
||||
/** Runs a job and returns the hooks it registered, so `sendResponse` can be invoked. */
|
||||
const processJobAndCaptureHooks = async (
|
||||
webhookResponseRelaySizeMaxMiB: number,
|
||||
jobData: Partial<Job['data']> = {},
|
||||
mode: BinaryDataConfig['mode'] = 'default',
|
||||
) => {
|
||||
const { relay, binaryDataService } = buildRelay(webhookResponseRelaySizeMaxMiB, mode);
|
||||
const executionPersistence = mock<ExecutionPersistence>();
|
||||
executionPersistence.findSingleExecution.mockResolvedValue(
|
||||
mock<IExecutionResponse>({
|
||||
|
|
@ -1999,8 +2136,9 @@ describe('JobProcessor', () => {
|
|||
mock(),
|
||||
mock(),
|
||||
createManualExecutionServiceMock(),
|
||||
executionsConfigWith({ webhookResponseRelaySizeMaxMiB }),
|
||||
executionsConfig,
|
||||
mock(),
|
||||
relay,
|
||||
);
|
||||
|
||||
const job = mock<Job>({
|
||||
|
|
@ -2015,7 +2153,7 @@ describe('JobProcessor', () => {
|
|||
});
|
||||
await jobProcessor.processJob(job);
|
||||
|
||||
return { hooks: additionalData.hooks!, job };
|
||||
return { hooks: additionalData.hooks!, job, binaryDataService };
|
||||
};
|
||||
|
||||
it('should relay a response within the size limit, encoding a buffer body', async () => {
|
||||
|
|
@ -2034,7 +2172,27 @@ describe('JobProcessor', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should refuse to relay a response over the size limit', async () => {
|
||||
it('should offload a response over the size limit and relay a reference', async () => {
|
||||
const { hooks, job, binaryDataService } = await processJobAndCaptureHooks(1, {}, 'database');
|
||||
|
||||
await hooks.runHook('sendResponse', [
|
||||
{ body: { blob: 'x'.repeat(2 * 1024 * 1024) }, headers: {}, statusCode: 200 },
|
||||
]);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledTimes(1);
|
||||
expect(job.progress).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
kind: 'respond-to-webhook',
|
||||
response: expect.objectContaining({
|
||||
body: expect.objectContaining({
|
||||
binaryData: expect.objectContaining({ id: 'database:stored-file-id' }),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should refuse to relay a response over the size limit without a store', async () => {
|
||||
const { hooks, job } = await processJobAndCaptureHooks(1);
|
||||
const relayedBefore = (job.progress as Mock).mock.calls.length;
|
||||
|
||||
|
|
@ -2047,18 +2205,76 @@ describe('JobProcessor', () => {
|
|||
expect((job.progress as Mock).mock.calls).toHaveLength(relayedBefore);
|
||||
});
|
||||
|
||||
it('should refuse to relay an MCP response over the size limit', async () => {
|
||||
it('should offload an MCP response over the size limit and relay a reference', async () => {
|
||||
const { hooks, job, binaryDataService } = await processJobAndCaptureHooks(
|
||||
1,
|
||||
{ isMcpExecution: true, mcpSessionId: 'session-1', mcpType: 'trigger' },
|
||||
'database',
|
||||
);
|
||||
|
||||
await hooks.runHook('sendResponse', [
|
||||
{ body: { blob: 'x'.repeat(2 * 1024 * 1024) }, headers: {}, statusCode: 200 },
|
||||
]);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledTimes(1);
|
||||
expect(job.progress).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
kind: 'mcp-response',
|
||||
sessionId: 'session-1',
|
||||
response: expect.objectContaining({
|
||||
body: expect.objectContaining({
|
||||
binaryData: expect.objectContaining({ id: 'database:stored-file-id' }),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not relay an MCP Service response, which main reads from stored data', async () => {
|
||||
const { hooks, job } = await processJobAndCaptureHooks(1, {
|
||||
isMcpExecution: true,
|
||||
mcpSessionId: 'session-1',
|
||||
mcpType: 'service',
|
||||
});
|
||||
const relayedBefore = (job.progress as Mock).mock.calls.length;
|
||||
|
||||
await hooks.runHook('sendResponse', [
|
||||
{ body: Buffer.from('hello'), headers: {}, statusCode: 200 },
|
||||
]);
|
||||
|
||||
expect((job.progress as Mock).mock.calls).toHaveLength(relayedBefore);
|
||||
});
|
||||
|
||||
it('should not store an MCP Service response over the size limit', async () => {
|
||||
const { hooks, job, binaryDataService } = await processJobAndCaptureHooks(
|
||||
1,
|
||||
{ isMcpExecution: true, mcpSessionId: 'session-1', mcpType: 'service' },
|
||||
'database',
|
||||
);
|
||||
const relayedBefore = (job.progress as Mock).mock.calls.length;
|
||||
|
||||
await hooks.runHook('sendResponse', [
|
||||
{ body: { blob: 'x'.repeat(2 * 1024 * 1024) }, headers: {}, statusCode: 200 },
|
||||
]);
|
||||
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
expect((job.progress as Mock).mock.calls).toHaveLength(relayedBefore);
|
||||
});
|
||||
|
||||
it('should refuse to relay an oversized MCP payload with no body to offload', async () => {
|
||||
const { hooks, job, binaryDataService } = await processJobAndCaptureHooks(
|
||||
1,
|
||||
{ isMcpExecution: true, mcpSessionId: 'session-1', mcpType: 'trigger' },
|
||||
'database',
|
||||
);
|
||||
const relayedBefore = (job.progress as Mock).mock.calls.length;
|
||||
|
||||
await expect(
|
||||
hooks.runHook('sendResponse', [{ toolResult: 'x'.repeat(2 * 1024 * 1024) }]),
|
||||
).rejects.toThrow(WebhookResponseTooLargeError);
|
||||
|
||||
expect((job.progress as Mock).mock.calls).toHaveLength(relayedBefore);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { JOB_TYPE_NAME, QUEUE_NAME } from '../constants';
|
|||
import type { JobProcessor } from '../job-processor';
|
||||
import { ScalingService } from '../scaling.service';
|
||||
import type { Job, JobData, JobId, JobQueue } from '../scaling.types';
|
||||
import { ENCODED_BUFFER_KEY, type WebhookResponseRelay } from '../webhook-response-relay';
|
||||
|
||||
const queue = mock<JobQueue>({
|
||||
client: { ping: vi.fn() },
|
||||
|
|
@ -29,6 +30,27 @@ vi.mock('bull', () => ({
|
|||
}),
|
||||
}));
|
||||
|
||||
const { mcpServer } = vi.hoisted(() => ({
|
||||
mcpServer: {
|
||||
hasSession: vi.fn(),
|
||||
hasPendingResponse: vi.fn(),
|
||||
handleWorkerResponse: vi.fn(),
|
||||
setSessionStore: vi.fn(),
|
||||
setExecutionStrategy: vi.fn(),
|
||||
getPendingCallsManager: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('@n8n/n8n-nodes-langchain/mcp/core', () => ({
|
||||
McpServer: { instance: () => mcpServer },
|
||||
RedisSessionStore: vi.fn(function () {
|
||||
return {};
|
||||
}),
|
||||
QueuedExecutionStrategy: vi.fn(function () {
|
||||
return {};
|
||||
}),
|
||||
}));
|
||||
|
||||
describe('ScalingService', () => {
|
||||
const Bull = vi.mocked(BullModule.default);
|
||||
|
||||
|
|
@ -67,6 +89,7 @@ describe('ScalingService', () => {
|
|||
const jobProcessor = mock<JobProcessor>();
|
||||
const executionRepository = mock<ExecutionRepository>();
|
||||
const executionPersistence = mock<ExecutionPersistence>();
|
||||
const webhookResponseRelay = mock<WebhookResponseRelay>();
|
||||
|
||||
let scalingService: ScalingService;
|
||||
|
||||
|
|
@ -102,6 +125,7 @@ describe('ScalingService', () => {
|
|||
executionPersistence,
|
||||
instanceSettings,
|
||||
mock(),
|
||||
webhookResponseRelay,
|
||||
);
|
||||
|
||||
getRunningJobsCountSpy = vi.spyOn(scalingService, 'getRunningJobsCount');
|
||||
|
|
@ -370,6 +394,7 @@ describe('ScalingService', () => {
|
|||
mock(),
|
||||
instanceSettings,
|
||||
mock(),
|
||||
webhookResponseRelay,
|
||||
);
|
||||
|
||||
await scalingService.setupQueue();
|
||||
|
|
@ -407,6 +432,7 @@ describe('ScalingService', () => {
|
|||
mock(),
|
||||
instanceSettings,
|
||||
mock(),
|
||||
webhookResponseRelay,
|
||||
);
|
||||
|
||||
await scalingService.setupQueue();
|
||||
|
|
@ -439,6 +465,7 @@ describe('ScalingService', () => {
|
|||
mock(),
|
||||
instanceSettings,
|
||||
mock(),
|
||||
webhookResponseRelay,
|
||||
);
|
||||
|
||||
await scalingService.setupQueue();
|
||||
|
|
@ -474,6 +501,7 @@ describe('ScalingService', () => {
|
|||
mock(),
|
||||
instanceSettings,
|
||||
mock(),
|
||||
webhookResponseRelay,
|
||||
);
|
||||
|
||||
await scalingService.setupQueue();
|
||||
|
|
@ -585,5 +613,142 @@ describe('ScalingService', () => {
|
|||
// Should not throw for trigger type either
|
||||
expect(() => messageHandler('job-trigger', mcpTriggerResponseMessage)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should restore an offloaded body without reclaiming it on the session-owning main', async () => {
|
||||
await scalingService.setupQueue();
|
||||
mcpServer.hasSession.mockReturnValue(true);
|
||||
webhookResponseRelay.restoreOffloadedBody.mockImplementation(async (response) => response);
|
||||
|
||||
const messageHandler = queue.on.mock.calls.find(
|
||||
([event]) => (event as string) === 'global:progress',
|
||||
)?.[1] as (jobId: JobId, msg: unknown) => void;
|
||||
|
||||
const response = {
|
||||
body: { binaryData: { id: 'database:abc' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
};
|
||||
|
||||
messageHandler('job-trigger', {
|
||||
kind: 'mcp-response',
|
||||
executionId: 'exec-456',
|
||||
mcpType: 'trigger',
|
||||
sessionId: 'session-trigger',
|
||||
messageId: 'msg-trigger',
|
||||
response,
|
||||
workerId: 'worker-xyz',
|
||||
});
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(webhookResponseRelay.restoreOffloadedBody).toHaveBeenCalledWith(response, {
|
||||
reclaim: false,
|
||||
context: { executionId: 'exec-456' },
|
||||
}),
|
||||
);
|
||||
expect(mcpServer.handleWorkerResponse).toHaveBeenCalledWith(
|
||||
'session-trigger',
|
||||
'msg-trigger',
|
||||
response,
|
||||
);
|
||||
});
|
||||
|
||||
it('should decode a Buffer body the worker base64-encoded to relay it', async () => {
|
||||
await scalingService.setupQueue();
|
||||
mcpServer.hasSession.mockReturnValue(true);
|
||||
webhookResponseRelay.restoreOffloadedBody.mockImplementation(async (response) => response);
|
||||
|
||||
const messageHandler = queue.on.mock.calls.find(
|
||||
([event]) => (event as string) === 'global:progress',
|
||||
)?.[1] as (jobId: JobId, msg: unknown) => void;
|
||||
|
||||
messageHandler('job-trigger', {
|
||||
kind: 'mcp-response',
|
||||
executionId: 'exec-456',
|
||||
mcpType: 'trigger',
|
||||
sessionId: 'session-trigger',
|
||||
messageId: 'msg-trigger',
|
||||
response: {
|
||||
body: { [ENCODED_BUFFER_KEY]: Buffer.from('tool output').toString('base64') },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
},
|
||||
workerId: 'worker-xyz',
|
||||
});
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(webhookResponseRelay.restoreOffloadedBody).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ body: Buffer.from('tool output') }),
|
||||
{ reclaim: false, context: { executionId: 'exec-456' } },
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it('should not restore an offloaded body on a main that does not hold the session', async () => {
|
||||
await scalingService.setupQueue();
|
||||
mcpServer.hasSession.mockReturnValue(false);
|
||||
mcpServer.hasPendingResponse.mockReturnValue(false);
|
||||
|
||||
const messageHandler = queue.on.mock.calls.find(
|
||||
([event]) => (event as string) === 'global:progress',
|
||||
)?.[1] as (jobId: JobId, msg: unknown) => void;
|
||||
|
||||
messageHandler('job-trigger', {
|
||||
kind: 'mcp-response',
|
||||
executionId: 'exec-456',
|
||||
mcpType: 'trigger',
|
||||
sessionId: 'session-trigger',
|
||||
messageId: 'msg-trigger',
|
||||
response: {
|
||||
body: { binaryData: { id: 'database:abc' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
},
|
||||
workerId: 'worker-xyz',
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(mcpServer.hasSession).toHaveBeenCalledWith('session-trigger'));
|
||||
expect(mcpServer.hasPendingResponse).toHaveBeenCalledWith('session-trigger', 'msg-trigger');
|
||||
expect(webhookResponseRelay.restoreOffloadedBody).not.toHaveBeenCalled();
|
||||
expect(mcpServer.handleWorkerResponse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should deliver a response a pending call awaits when the transport is gone', async () => {
|
||||
await scalingService.setupQueue();
|
||||
mcpServer.hasSession.mockReturnValue(false);
|
||||
mcpServer.hasPendingResponse.mockReturnValue(true);
|
||||
webhookResponseRelay.restoreOffloadedBody.mockImplementation(async (response) => response);
|
||||
|
||||
const messageHandler = queue.on.mock.calls.find(
|
||||
([event]) => (event as string) === 'global:progress',
|
||||
)?.[1] as (jobId: JobId, msg: unknown) => void;
|
||||
|
||||
const response = {
|
||||
body: { binaryData: { id: 'database:abc' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
};
|
||||
|
||||
messageHandler('job-trigger', {
|
||||
kind: 'mcp-response',
|
||||
executionId: 'exec-456',
|
||||
mcpType: 'trigger',
|
||||
sessionId: 'session-trigger',
|
||||
messageId: 'msg-trigger',
|
||||
response,
|
||||
workerId: 'worker-xyz',
|
||||
});
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(mcpServer.handleWorkerResponse).toHaveBeenCalledWith(
|
||||
'session-trigger',
|
||||
'msg-trigger',
|
||||
response,
|
||||
),
|
||||
);
|
||||
expect(webhookResponseRelay.restoreOffloadedBody).toHaveBeenCalledWith(response, {
|
||||
reclaim: false,
|
||||
context: { executionId: 'exec-456' },
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,55 +1,753 @@
|
|||
import type { IExecuteResponsePromiseData } from 'n8n-workflow';
|
||||
import type { Logger } from '@n8n/backend-common';
|
||||
import type { ExecutionsConfig } from '@n8n/config';
|
||||
import { FileLocation, FileTooLargeError } from 'n8n-core';
|
||||
import type { BinaryDataConfig, BinaryDataService } from 'n8n-core';
|
||||
import { OperationalError } from 'n8n-workflow';
|
||||
import type { IBinaryData, IExecuteResponsePromiseData } from 'n8n-workflow';
|
||||
import { Readable } from 'node:stream';
|
||||
import { mock } from 'vitest-mock-extended';
|
||||
|
||||
import { WebhookResponseTooLargeError } from '@/errors/webhook-response-too-large.error';
|
||||
|
||||
import {
|
||||
assertRelayableSize,
|
||||
decodeRelayedWebhookResponse,
|
||||
ENCODED_BUFFER_KEY,
|
||||
encodeRelayedWebhookResponse,
|
||||
OFFLOADED_BODY_KIND_KEY,
|
||||
WebhookResponseRelay,
|
||||
} from '../webhook-response-relay';
|
||||
|
||||
const fullResponse = (body: unknown): IExecuteResponsePromiseData =>
|
||||
({ body, headers: {}, statusCode: 200 }) as IExecuteResponsePromiseData;
|
||||
|
||||
const ONE_MIB = 1024 * 1024;
|
||||
const SIZE_MAX_IN_MIB = 2;
|
||||
|
||||
describe('encodeRelayedWebhookResponse', () => {
|
||||
it('wraps a Buffer body in a base64 envelope', () => {
|
||||
const response = fullResponse(Buffer.from('hello'));
|
||||
const ctx = { workflowId: 'workflow-1', executionId: 'execution-1' };
|
||||
|
||||
const encoded = encodeRelayedWebhookResponse(response);
|
||||
const fullResponse = (body: unknown, headers: Record<string, string> = {}) =>
|
||||
({ body, headers, statusCode: 200 }) as IExecuteResponsePromiseData;
|
||||
|
||||
expect(encoded).toBe(response);
|
||||
expect(encoded).toEqual(
|
||||
expect.objectContaining({ body: { [ENCODED_BUFFER_KEY]: 'aGVsbG8=' } }),
|
||||
const bodyOf = (response: IExecuteResponsePromiseData) =>
|
||||
(response as { body: unknown }).body as Record<string, unknown>;
|
||||
|
||||
const headersOf = (response: IExecuteResponsePromiseData) =>
|
||||
(response as { headers: Record<string, string> }).headers;
|
||||
|
||||
type Harness = {
|
||||
relay: WebhookResponseRelay;
|
||||
binaryDataService: ReturnType<typeof mock<BinaryDataService>>;
|
||||
logger: ReturnType<typeof mock<Logger>>;
|
||||
};
|
||||
|
||||
const buildRelay = ({
|
||||
mode = 'database',
|
||||
offloadEnabled = true,
|
||||
}: { mode?: BinaryDataConfig['mode']; offloadEnabled?: boolean } = {}): Harness => {
|
||||
const logger = mock<Logger>();
|
||||
logger.scoped.mockReturnValue(logger);
|
||||
|
||||
const binaryDataService = mock<BinaryDataService>();
|
||||
binaryDataService.store.mockImplementation(async (_location, _body, binaryData) => ({
|
||||
...binaryData,
|
||||
id: `${mode}:stored-file-id`,
|
||||
}));
|
||||
|
||||
const relay = new WebhookResponseRelay(
|
||||
logger,
|
||||
binaryDataService,
|
||||
mock<BinaryDataConfig>({ mode }),
|
||||
mock<ExecutionsConfig>({
|
||||
webhookResponseRelaySizeMaxMiB: SIZE_MAX_IN_MIB,
|
||||
webhookResponseRelayOffloadEnabled: offloadEnabled,
|
||||
}),
|
||||
);
|
||||
|
||||
return { relay, binaryDataService, logger };
|
||||
};
|
||||
|
||||
/** A relayed response as it reaches main after the worker offloaded its body. */
|
||||
const offloadedResponse = (
|
||||
kind: 'buffer' | 'string' | 'json',
|
||||
binaryData: Partial<IBinaryData> = {},
|
||||
) =>
|
||||
({
|
||||
body: { binaryData: { id: 'database:abc', ...binaryData } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
[OFFLOADED_BODY_KIND_KEY]: kind,
|
||||
}) as IExecuteResponsePromiseData;
|
||||
|
||||
const offloadMarkerOf = (response: IExecuteResponsePromiseData) =>
|
||||
(response as Record<string, unknown>)[OFFLOADED_BODY_KIND_KEY];
|
||||
|
||||
describe('WebhookResponseRelay', () => {
|
||||
describe('prepare, within the limit', () => {
|
||||
it('wraps a Buffer body in a base64 envelope', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(Buffer.from('hello')), ctx);
|
||||
|
||||
expect(bodyOf(prepared)).toEqual({ [ENCODED_BUFFER_KEY]: 'aGVsbG8=' });
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a JSON body', { hello: 'world' }],
|
||||
['a nested JSON body', { outer: { inner: ['x'.repeat(ONE_MIB)] } }],
|
||||
['a string body', 'hello'],
|
||||
['a null body', null],
|
||||
['a binary-data reference', { binaryData: { id: 'database:abc' } }],
|
||||
])('leaves %s untouched', async (_label, body) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
expect(bodyOf(prepared)).toEqual(body);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves a cyclic body it cannot serialize untouched', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const body: Record<string, unknown> = { name: 'cycle' };
|
||||
body.self = body;
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
expect(bodyOf(prepared)).toBe(body);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps inline a body and headers that fit the limit together', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const response = fullResponse('x'.repeat(ONE_MIB / 2), { 'x-data': 'y'.repeat(ONE_MIB) });
|
||||
|
||||
await expect(relay.prepare(response, ctx)).resolves.toBe(response);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves a stream body untouched, whatever its size', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const stream = Readable.from('hello');
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(stream), ctx);
|
||||
|
||||
expect(bodyOf(prepared)).toBe(stream);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves a payload that is not a full response untouched', async () => {
|
||||
const { relay } = buildRelay();
|
||||
const payload = { toolResult: 'done' };
|
||||
|
||||
expect(await relay.prepare(payload, ctx)).toBe(payload);
|
||||
});
|
||||
});
|
||||
|
||||
describe('prepare, over the limit with a store', () => {
|
||||
it.each([
|
||||
['a Buffer body', Buffer.alloc(3 * ONE_MIB), 'buffer'],
|
||||
['a string body', 'x'.repeat(3 * ONE_MIB), 'string'],
|
||||
['a multi-byte string body', 'é'.repeat(2 * ONE_MIB), 'string'],
|
||||
['a JSON body', { blob: 'x'.repeat(3 * ONE_MIB) }, 'json'],
|
||||
['a JSON body nested in an array', { items: [{ blob: 'x'.repeat(3 * ONE_MIB) }] }, 'json'],
|
||||
])('offloads %s and records its form', async (_label, body, kind) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledTimes(1);
|
||||
expect(bodyOf(prepared)).toEqual({
|
||||
binaryData: expect.objectContaining({ id: 'database:stored-file-id' }),
|
||||
});
|
||||
expect(offloadMarkerOf(prepared)).toBe(kind);
|
||||
});
|
||||
|
||||
it('measures a Buffer body base64-encoded, the form it would travel in', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
// Under the limit raw, a third over it once encoded.
|
||||
const body = Buffer.alloc(1.75 * ONE_MIB);
|
||||
|
||||
await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('measures a string body in bytes, not in UTF-16 code units', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
// Half the limit in code units, one and a half times it in UTF-8 bytes.
|
||||
const body = 'é'.repeat(1.5 * ONE_MIB);
|
||||
|
||||
await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('offloads a body that fits the limit only without the rest of the response', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const response = fullResponse('x'.repeat(1.5 * ONE_MIB), { 'x-data': 'y'.repeat(ONE_MIB) });
|
||||
|
||||
await relay.prepare(response, ctx);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('stores the serialized body', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const body = { blob: 'x'.repeat(3 * ONE_MIB) };
|
||||
|
||||
await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
const [, stored] = binaryDataService.store.mock.calls[0];
|
||||
// `equals` over `toEqual`: deep equality walks a Buffer byte by byte.
|
||||
expect(Buffer.isBuffer(stored) && stored.equals(Buffer.from(JSON.stringify(body)))).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it.each(['filesystem', 'database', 's3', 'azure'] as const)(
|
||||
"stores under the execution's location in %s mode",
|
||||
async (mode) => {
|
||||
const { relay, binaryDataService } = buildRelay({ mode });
|
||||
|
||||
const prepared = await relay.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx);
|
||||
|
||||
expect(binaryDataService.store).toHaveBeenCalledWith(
|
||||
FileLocation.ofExecution(ctx.workflowId, ctx.executionId),
|
||||
expect.anything(),
|
||||
expect.anything(),
|
||||
);
|
||||
expect(bodyOf(prepared)).toEqual({
|
||||
binaryData: expect.objectContaining({ id: `${mode}:stored-file-id` }),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
['a string body', 'x'.repeat(3 * ONE_MIB), 'text/html; charset=utf-8'],
|
||||
['a JSON body', { blob: 'x'.repeat(3 * ONE_MIB) }, 'application/json; charset=utf-8'],
|
||||
])('keeps the content-type %s would have had inline', async (_label, body, contentType) => {
|
||||
const { relay } = buildRelay();
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(body), ctx);
|
||||
|
||||
expect(headersOf(prepared)['content-type']).toBe(contentType);
|
||||
});
|
||||
|
||||
it('leaves a content-type set by the node alone', async () => {
|
||||
const { relay } = buildRelay();
|
||||
const headers = { 'Content-Type': 'application/xml' };
|
||||
|
||||
const prepared = await relay.prepare(fullResponse('x'.repeat(3 * ONE_MIB), headers), ctx);
|
||||
|
||||
expect(headersOf(prepared)).toEqual(headers);
|
||||
});
|
||||
|
||||
it('sets no content-type for a Buffer body, which has none inline either', async () => {
|
||||
const { relay } = buildRelay();
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(Buffer.alloc(3 * ONE_MIB)), ctx);
|
||||
|
||||
expect(headersOf(prepared)['content-type']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('throws when the store reports no id, rather than relaying the body inline', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.store.mockImplementation(
|
||||
async (_location, _body, binaryData) => binaryData,
|
||||
);
|
||||
|
||||
await expect(relay.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)).rejects.toThrow(
|
||||
OperationalError,
|
||||
);
|
||||
});
|
||||
|
||||
it('propagates a store failure, rather than relaying the body inline', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.store.mockRejectedValue(new Error('store is down'));
|
||||
|
||||
await expect(relay.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)).rejects.toThrow(
|
||||
'store is down',
|
||||
);
|
||||
});
|
||||
|
||||
it('names the store limit when the store refuses the body for its size', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.store.mockRejectedValue(
|
||||
new FileTooLargeError({ fileSizeMb: 600, maxFileSizeMb: 512, fileId: 'file-1' }),
|
||||
);
|
||||
|
||||
const error = await relay
|
||||
.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)
|
||||
.catch((e: WebhookResponseTooLargeError) => e);
|
||||
|
||||
expect(error).toBeInstanceOf(WebhookResponseTooLargeError);
|
||||
expect((error as WebhookResponseTooLargeError).message).not.toContain('MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain('3 MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'N8N_BINARY_DATA_DATABASE_MAX_FILE_SIZE',
|
||||
);
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX',
|
||||
);
|
||||
expect((error as WebhookResponseTooLargeError).cause).toBeInstanceOf(FileTooLargeError);
|
||||
});
|
||||
|
||||
it('names every store mode without a size limit of its own', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.store.mockRejectedValue(
|
||||
new FileTooLargeError({ fileSizeMb: 600, maxFileSizeMb: 512, fileId: 'file-1' }),
|
||||
);
|
||||
|
||||
const error = await relay
|
||||
.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)
|
||||
.catch((e: WebhookResponseTooLargeError) => e);
|
||||
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'filesystem, s3 and azure',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a JSON body', { hello: 'world' }],
|
||||
['a string body', 'hello'],
|
||||
['a null body', null],
|
||||
['a binary-data reference', { binaryData: { id: 'database:abc' } }],
|
||||
])('leaves %s untouched', (_label, body) => {
|
||||
const encoded = encodeRelayedWebhookResponse(fullResponse(body));
|
||||
describe('prepare, over the limit with offload turned off', () => {
|
||||
it.each(['filesystem', 'database', 's3', 'azure'] as const)(
|
||||
'rejects a body in %s mode rather than storing it',
|
||||
async (mode) => {
|
||||
const { relay, binaryDataService } = buildRelay({ mode, offloadEnabled: false });
|
||||
|
||||
expect(encoded).toEqual(expect.objectContaining({ body }));
|
||||
await expect(relay.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)).rejects.toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('names the flag to turn on, not the binary-data mode to change', async () => {
|
||||
const { relay } = buildRelay({ offloadEnabled: false });
|
||||
|
||||
const error = await relay
|
||||
.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)
|
||||
.catch((e: WebhookResponseTooLargeError) => e);
|
||||
|
||||
expect((error as WebhookResponseTooLargeError).message).not.toContain('MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain('The limit is 2 MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'N8N_WEBHOOK_RESPONSE_RELAY_OFFLOAD_ENABLED',
|
||||
);
|
||||
expect((error as WebhookResponseTooLargeError).description).not.toContain(
|
||||
'N8N_DEFAULT_BINARY_DATA_MODE',
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps relaying a body within the limit inline', async () => {
|
||||
const { relay, binaryDataService } = buildRelay({ offloadEnabled: false });
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(Buffer.from('hello')), ctx);
|
||||
|
||||
expect(bodyOf(prepared)).toEqual({ [ENCODED_BUFFER_KEY]: 'aGVsbG8=' });
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves a stream body untouched', () => {
|
||||
const stream = Readable.from('hello');
|
||||
describe('prepare, over the limit without a store', () => {
|
||||
it.each([
|
||||
['a Buffer body', Buffer.alloc(3 * ONE_MIB)],
|
||||
['a string body', 'x'.repeat(3 * ONE_MIB)],
|
||||
['a multi-byte string body', 'é'.repeat(2 * ONE_MIB)],
|
||||
['a JSON body', { blob: 'x'.repeat(3 * ONE_MIB) }],
|
||||
['a JSON body nested in an array', { items: [{ blob: 'x'.repeat(3 * ONE_MIB) }] }],
|
||||
])('rejects %s where bytes are only kept in memory', async (_label, body) => {
|
||||
const { relay, binaryDataService } = buildRelay({ mode: 'default' });
|
||||
|
||||
const encoded = encodeRelayedWebhookResponse(fullResponse(stream));
|
||||
await expect(relay.prepare(fullResponse(body), ctx)).rejects.toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
expect(encoded).toEqual(expect.objectContaining({ body: stream }));
|
||||
it('names the limit and both ways out', async () => {
|
||||
const { relay } = buildRelay({ mode: 'default' });
|
||||
|
||||
const error = await relay
|
||||
.prepare(fullResponse('x'.repeat(3 * ONE_MIB)), ctx)
|
||||
.catch((e: WebhookResponseTooLargeError) => e);
|
||||
|
||||
expect((error as WebhookResponseTooLargeError).message).not.toContain('MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain('The limit is 2 MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'N8N_DEFAULT_BINARY_DATA_MODE',
|
||||
);
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves a payload that is not a full response untouched', () => {
|
||||
const payload = { toolResult: 'done' };
|
||||
describe('prepare, over the limit with nothing to offload', () => {
|
||||
it('rejects oversized headers, which cannot be stored, even with a store', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const response = fullResponse(null, { 'x-data': 'x'.repeat(3 * ONE_MIB) });
|
||||
|
||||
expect(encodeRelayedWebhookResponse(payload)).toBe(payload);
|
||||
await expect(relay.prepare(response, ctx)).rejects.toThrow(WebhookResponseTooLargeError);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an oversized payload that is not a full response', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
await expect(relay.prepare({ toolResult: 'x'.repeat(3 * ONE_MIB) }, ctx)).rejects.toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an oversized value beside a body that is within the limit', async () => {
|
||||
const { relay } = buildRelay();
|
||||
|
||||
await expect(
|
||||
relay.prepare({ body: 'small', extra: 'x'.repeat(3 * ONE_MIB) }, ctx),
|
||||
).rejects.toThrow(WebhookResponseTooLargeError);
|
||||
});
|
||||
|
||||
it('rejects an oversized body shaped like a binary-data reference', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const body = { binaryData: { id: 'database:abc' }, blob: 'x'.repeat(3 * ONE_MIB) };
|
||||
|
||||
await expect(relay.prepare(fullResponse(body), ctx)).rejects.toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects oversized headers beside a stream body, which is exempt from measuring', async () => {
|
||||
const { relay } = buildRelay();
|
||||
const response = fullResponse(Readable.from('hello'), {
|
||||
'x-data': 'x'.repeat(3 * ONE_MIB),
|
||||
});
|
||||
|
||||
await expect(relay.prepare(response, ctx)).rejects.toThrow(WebhookResponseTooLargeError);
|
||||
});
|
||||
|
||||
it('rejects a non-offloadable body and headers that only fit the limit apart', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const body = { binaryData: { id: 'database:abc' }, blob: 'x'.repeat(1.5 * ONE_MIB) };
|
||||
const response = fullResponse(body, { 'x-data': 'y'.repeat(1.5 * ONE_MIB) });
|
||||
|
||||
await expect(relay.prepare(response, ctx)).rejects.toThrow(WebhookResponseTooLargeError);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects an oversized payload shaped like a binary-data reference that is not a full response', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const payload = { binaryData: { id: 'database:abc' }, toolResult: 'x'.repeat(3 * ONE_MIB) };
|
||||
|
||||
await expect(relay.prepare(payload, ctx)).rejects.toThrow(WebhookResponseTooLargeError);
|
||||
expect(binaryDataService.store).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('names the limit and how to raise it', async () => {
|
||||
const { relay } = buildRelay();
|
||||
const response = fullResponse(null, { 'x-data': 'x'.repeat(3 * ONE_MIB) });
|
||||
|
||||
const error = await relay
|
||||
.prepare(response, ctx)
|
||||
.catch((e: WebhookResponseTooLargeError) => e);
|
||||
|
||||
expect((error as WebhookResponseTooLargeError).message).not.toContain('MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain('The limit is 2 MiB');
|
||||
expect((error as WebhookResponseTooLargeError).description).toContain(
|
||||
'N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('restoreOffloadedBody', () => {
|
||||
it.each([
|
||||
['buffer', Buffer.from('hello'), Buffer.from('hello')],
|
||||
['string', Buffer.from('hello'), 'hello'],
|
||||
['json', Buffer.from('{"hello":"world"}'), { hello: 'world' }],
|
||||
] as const)('restores an offloaded %s body', async (kind, stored, expected) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(stored);
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(offloadedResponse(kind), {
|
||||
reclaim: true,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(bodyOf(restored)).toEqual(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['an unknown form', 'stream'],
|
||||
['a non-string', 42],
|
||||
['null', null],
|
||||
] as const)('leaves a reference marked with %s untouched', async (_label, marker) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const response = {
|
||||
body: { binaryData: { id: 'database:abc' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
[OFFLOADED_BODY_KIND_KEY]: marker,
|
||||
} as IExecuteResponsePromiseData;
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(response, { reclaim: true, context: ctx });
|
||||
|
||||
expect(binaryDataService.getAsBuffer).not.toHaveBeenCalled();
|
||||
expect(bodyOf(restored)).toEqual({ binaryData: { id: 'database:abc' } });
|
||||
});
|
||||
|
||||
it('round-trips a body the worker offloaded', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const body = { blob: 'x'.repeat(3 * ONE_MIB) };
|
||||
|
||||
const prepared = await relay.prepare(fullResponse(body), ctx);
|
||||
const [, stored] = binaryDataService.store.mock.calls[0];
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(stored as Buffer);
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(prepared, { reclaim: true, context: ctx });
|
||||
|
||||
expect(bodyOf(restored)).toEqual(body);
|
||||
});
|
||||
|
||||
it('reclaims the storage once the body is read', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(Buffer.from('hello'));
|
||||
|
||||
await relay.restoreOffloadedBody(offloadedResponse('string'), {
|
||||
reclaim: true,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(binaryDataService.deleteManyByBinaryDataId).toHaveBeenCalledWith(['database:abc']);
|
||||
});
|
||||
|
||||
it('hands the body back without waiting for the storage to be reclaimed', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(Buffer.from('hello'));
|
||||
binaryDataService.deleteManyByBinaryDataId.mockReturnValue(new Promise<void>(() => {}));
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(offloadedResponse('string'), {
|
||||
reclaim: true,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(bodyOf(restored)).toEqual('hello');
|
||||
});
|
||||
|
||||
it('logs a reclaim that fails once the body has been handed back', async () => {
|
||||
const { relay, binaryDataService, logger } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(Buffer.from('hello'));
|
||||
binaryDataService.deleteManyByBinaryDataId.mockRejectedValue(new Error('store is down'));
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(offloadedResponse('string'), {
|
||||
reclaim: true,
|
||||
context: ctx,
|
||||
});
|
||||
await new Promise(setImmediate);
|
||||
|
||||
expect(bodyOf(restored)).toEqual('hello');
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'Failed to delete an offloaded webhook response body',
|
||||
expect.objectContaining({ binaryDataId: 'database:abc' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves the storage in place for other readers when not reclaiming', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(Buffer.from('hello'));
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(offloadedResponse('string'), {
|
||||
reclaim: false,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(bodyOf(restored)).toEqual('hello');
|
||||
expect(binaryDataService.deleteManyByBinaryDataId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('removes the offload marker once the body is restored', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockResolvedValue(Buffer.from('hello'));
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(offloadedResponse('string'), {
|
||||
reclaim: true,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(offloadMarkerOf(restored)).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['buffer', Buffer.alloc(0)],
|
||||
['string', ''],
|
||||
['json', {}],
|
||||
] as const)(
|
||||
'substitutes an empty %s body when the fetch fails for a shared reader',
|
||||
async (kind, expected) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockRejectedValue(new Error('gone'));
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(offloadedResponse(kind), {
|
||||
reclaim: false,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(bodyOf(restored)).toEqual(expected);
|
||||
expect(offloadMarkerOf(restored)).toBeUndefined();
|
||||
expect(binaryDataService.deleteManyByBinaryDataId).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('throws when the fetch fails for a sole reader, which owns the delivery', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockRejectedValue(new Error('gone'));
|
||||
|
||||
await expect(
|
||||
relay.restoreOffloadedBody(offloadedResponse('json'), { reclaim: true, context: ctx }),
|
||||
).rejects.toThrow(OperationalError);
|
||||
expect(binaryDataService.deleteManyByBinaryDataId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('names the store as the thing to check when the fetch fails', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockRejectedValue(new Error('gone'));
|
||||
|
||||
const error = await relay
|
||||
.restoreOffloadedBody(offloadedResponse('json'), { reclaim: true, context: ctx })
|
||||
.catch((e: OperationalError) => e);
|
||||
|
||||
expect((error as OperationalError).description).toContain('N8N_DEFAULT_BINARY_DATA_MODE');
|
||||
});
|
||||
|
||||
it('reports which execution the unreadable body belongs to', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockRejectedValue(new Error('gone'));
|
||||
|
||||
const error = await relay
|
||||
.restoreOffloadedBody(offloadedResponse('json'), { reclaim: true, context: ctx })
|
||||
.catch((e: OperationalError) => e);
|
||||
|
||||
expect((error as OperationalError).extra).toEqual({
|
||||
binaryDataId: 'database:abc',
|
||||
workflowId: 'workflow-1',
|
||||
executionId: 'execution-1',
|
||||
});
|
||||
});
|
||||
|
||||
it('reports which execution a body it could not restore belongs to', async () => {
|
||||
const { relay, binaryDataService, logger } = buildRelay();
|
||||
binaryDataService.getAsBuffer.mockRejectedValue(new Error('gone'));
|
||||
|
||||
await relay.restoreOffloadedBody(offloadedResponse('json'), {
|
||||
reclaim: false,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'Failed to restore an offloaded webhook response body',
|
||||
expect.objectContaining({
|
||||
binaryDataId: 'database:abc',
|
||||
workflowId: 'workflow-1',
|
||||
executionId: 'execution-1',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a genuine binary-data reference', { binaryData: { id: 'database:abc' } }],
|
||||
[
|
||||
'a reference whose marker sits inside the body, which only a workflow produces',
|
||||
{
|
||||
binaryData: { id: 'database:abc' },
|
||||
[OFFLOADED_BODY_KIND_KEY]: 'json',
|
||||
},
|
||||
],
|
||||
['a JSON body', { hello: 'world' }],
|
||||
['a null body', null],
|
||||
])('leaves %s untouched', async (_label, body) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(fullResponse(body), {
|
||||
reclaim: true,
|
||||
context: ctx,
|
||||
});
|
||||
|
||||
expect(bodyOf(restored)).toEqual(body);
|
||||
expect(binaryDataService.getAsBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves a reference with an unknown form marker untouched', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
const response = {
|
||||
body: { binaryData: { id: 'database:abc' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
[OFFLOADED_BODY_KIND_KEY]: 'zip',
|
||||
} as IExecuteResponsePromiseData;
|
||||
|
||||
const restored = await relay.restoreOffloadedBody(response, { reclaim: true, context: ctx });
|
||||
|
||||
expect(bodyOf(restored)).toEqual({ binaryData: { id: 'database:abc' } });
|
||||
expect(binaryDataService.getAsBuffer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('leaves a payload that is not a full response untouched', async () => {
|
||||
const { relay } = buildRelay();
|
||||
const payload = { toolResult: 'done' };
|
||||
|
||||
expect(await relay.restoreOffloadedBody(payload, { reclaim: true, context: ctx })).toBe(
|
||||
payload,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deleteOffloadedBody', () => {
|
||||
it('reclaims the storage of an offloaded body', async () => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
await relay.deleteOffloadedBody(offloadedResponse('json'), ctx);
|
||||
|
||||
expect(binaryDataService.deleteManyByBinaryDataId).toHaveBeenCalledWith(['database:abc']);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a genuine binary-data reference', { binaryData: { id: 'database:abc' } }],
|
||||
[
|
||||
'a reference whose marker sits inside the body, which only a workflow produces',
|
||||
{
|
||||
binaryData: { id: 'database:abc' },
|
||||
[OFFLOADED_BODY_KIND_KEY]: 'json',
|
||||
},
|
||||
],
|
||||
['a JSON body', { hello: 'world' }],
|
||||
['a null body', null],
|
||||
])('leaves %s alone', async (_label, body) => {
|
||||
const { relay, binaryDataService } = buildRelay();
|
||||
|
||||
await relay.deleteOffloadedBody(fullResponse(body), ctx);
|
||||
|
||||
expect(binaryDataService.deleteManyByBinaryDataId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('logs rather than throws when the store cannot delete', async () => {
|
||||
const { relay, binaryDataService, logger } = buildRelay();
|
||||
binaryDataService.deleteManyByBinaryDataId.mockRejectedValue(new Error('store is down'));
|
||||
|
||||
await expect(
|
||||
relay.deleteOffloadedBody(offloadedResponse('json'), ctx),
|
||||
).resolves.toBeUndefined();
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
'Failed to delete an offloaded webhook response body',
|
||||
expect.objectContaining({
|
||||
binaryDataId: 'database:abc',
|
||||
workflowId: 'workflow-1',
|
||||
executionId: 'execution-1',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('accepts a payload that is not a full response', async () => {
|
||||
const { relay } = buildRelay();
|
||||
|
||||
await expect(relay.deleteOffloadedBody({ toolResult: 'done' }, ctx)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -60,26 +758,31 @@ describe('decodeRelayedWebhookResponse', () => {
|
|||
const decoded = decodeRelayedWebhookResponse(response);
|
||||
|
||||
expect(decoded).toBe(response);
|
||||
expect((decoded as { body: Buffer }).body).toEqual(Buffer.from('hello'));
|
||||
expect(bodyOf(decoded)).toEqual(Buffer.from('hello'));
|
||||
});
|
||||
|
||||
it('round-trips a Buffer body', () => {
|
||||
it('round-trips a Buffer body the worker relayed inline', async () => {
|
||||
const { relay } = buildRelay();
|
||||
const body = Buffer.from([0x00, 0xff, 0x10]);
|
||||
|
||||
const decoded = decodeRelayedWebhookResponse(encodeRelayedWebhookResponse(fullResponse(body)));
|
||||
const decoded = decodeRelayedWebhookResponse(await relay.prepare(fullResponse(body), ctx));
|
||||
|
||||
expect((decoded as { body: Buffer }).body).toEqual(body);
|
||||
expect(bodyOf(decoded)).toEqual(body);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a JSON body', { hello: 'world' }],
|
||||
['a string body', 'hello'],
|
||||
['a null body', null],
|
||||
[
|
||||
'a binary-data reference, offloaded or genuine, which main streams instead',
|
||||
{ binaryData: { id: 'database:abc' } },
|
||||
],
|
||||
['an envelope whose payload is not a string', { [ENCODED_BUFFER_KEY]: 42 }],
|
||||
])('leaves %s untouched', (_label, body) => {
|
||||
const decoded = decodeRelayedWebhookResponse(fullResponse(body));
|
||||
|
||||
expect(decoded).toEqual(expect.objectContaining({ body }));
|
||||
expect(bodyOf(decoded)).toEqual(body);
|
||||
});
|
||||
|
||||
it('leaves a payload that is not a full response untouched', () => {
|
||||
|
|
@ -88,96 +791,3 @@ describe('decodeRelayedWebhookResponse', () => {
|
|||
expect(decodeRelayedWebhookResponse(payload)).toBe(payload);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertRelayableSize', () => {
|
||||
describe('within the limit', () => {
|
||||
it.each([
|
||||
['a Buffer body', Buffer.alloc(ONE_MIB)],
|
||||
['a string body', 'x'.repeat(ONE_MIB)],
|
||||
['a JSON body', { blob: 'x'.repeat(ONE_MIB) }],
|
||||
['a nested JSON body', { outer: { inner: ['x'.repeat(ONE_MIB)] } }],
|
||||
['a null body', null],
|
||||
['a stream body', Readable.from('hello')],
|
||||
['a binary-data reference', { binaryData: { id: 'database:abc' } }],
|
||||
])('accepts %s', (_label, body) => {
|
||||
expect(() => assertRelayableSize(fullResponse(body), 2)).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a payload that is not a full response', () => {
|
||||
expect(() => assertRelayableSize({ toolResult: 'done' }, 2)).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a cyclic body it cannot serialize', () => {
|
||||
const body: Record<string, unknown> = { name: 'cycle' };
|
||||
body.self = body;
|
||||
|
||||
expect(() => assertRelayableSize(fullResponse(body), 2)).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a body and headers each within the limit, measuring them separately', () => {
|
||||
const response: IExecuteResponsePromiseData = {
|
||||
body: 'x'.repeat(ONE_MIB),
|
||||
headers: { 'x-data': 'y'.repeat(ONE_MIB) },
|
||||
statusCode: 200,
|
||||
};
|
||||
|
||||
expect(() => assertRelayableSize(response, 2)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('over the limit', () => {
|
||||
it.each([
|
||||
['a Buffer body', Buffer.alloc(3 * ONE_MIB)],
|
||||
['a string body', 'x'.repeat(3 * ONE_MIB)],
|
||||
['a multi-byte string body', 'é'.repeat(2 * ONE_MIB)],
|
||||
['a JSON body', { blob: 'x'.repeat(3 * ONE_MIB) }],
|
||||
['a JSON body nested in an array', { items: [{ blob: 'x'.repeat(3 * ONE_MIB) }] }],
|
||||
])('rejects %s', (_label, body) => {
|
||||
expect(() => assertRelayableSize(fullResponse(body), 2)).toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a Buffer body only its base64 expansion pushes over the limit', () => {
|
||||
const body = Buffer.alloc(1.75 * ONE_MIB);
|
||||
|
||||
expect(() => assertRelayableSize(fullResponse(body), 2)).toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects a payload that is not a full response', () => {
|
||||
expect(() => assertRelayableSize({ toolResult: 'x'.repeat(3 * ONE_MIB) }, 2)).toThrow(
|
||||
WebhookResponseTooLargeError,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects oversized headers', () => {
|
||||
const response: IExecuteResponsePromiseData = {
|
||||
body: null,
|
||||
headers: { 'x-data': 'x'.repeat(3 * ONE_MIB) },
|
||||
statusCode: 200,
|
||||
};
|
||||
|
||||
expect(() => assertRelayableSize(response, 2)).toThrow(WebhookResponseTooLargeError);
|
||||
});
|
||||
|
||||
it('rejects an oversized value beside the body of a payload that is not a full response', () => {
|
||||
const payload = { body: 'small', extra: 'x'.repeat(3 * ONE_MIB) };
|
||||
|
||||
expect(() => assertRelayableSize(payload, 2)).toThrow(WebhookResponseTooLargeError);
|
||||
});
|
||||
|
||||
it('names the limit and how to raise it', () => {
|
||||
let error: WebhookResponseTooLargeError | undefined;
|
||||
try {
|
||||
assertRelayableSize(fullResponse('x'.repeat(3 * ONE_MIB)), 2);
|
||||
} catch (e) {
|
||||
error = e as WebhookResponseTooLargeError;
|
||||
}
|
||||
|
||||
expect(error?.message).toContain('limit is 2 MiB');
|
||||
expect(error?.description).toContain('N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ import type {
|
|||
RunningJob,
|
||||
SendChunkMessage,
|
||||
} from './scaling.types';
|
||||
import { assertRelayableSize, encodeRelayedWebhookResponse } from './webhook-response-relay';
|
||||
import { WebhookResponseRelay } from './webhook-response-relay';
|
||||
|
||||
/**
|
||||
* Responsible for processing jobs from the queue, i.e. running enqueued executions.
|
||||
|
|
@ -76,6 +76,7 @@ export class JobProcessor {
|
|||
private readonly manualExecutionService: ManualExecutionService,
|
||||
private readonly executionsConfig: ExecutionsConfig,
|
||||
private readonly eventService: EventService,
|
||||
private readonly webhookResponseRelay: WebhookResponseRelay,
|
||||
) {
|
||||
this.logger = this.logger.scoped('scaling');
|
||||
}
|
||||
|
|
@ -197,17 +198,27 @@ export class JobProcessor {
|
|||
}
|
||||
|
||||
lifecycleHooks.addHandler('sendResponse', async (response): Promise<void> => {
|
||||
assertRelayableSize(response, this.executionsConfig.webhookResponseRelaySizeMaxMiB);
|
||||
// An MCP Service call takes its result from the execution's stored data, so a
|
||||
// response relayed to main has no reader. Relaying one would also reach main
|
||||
// mid-execution, resolving the call on data the run has not finished writing.
|
||||
if (job.data.isMcpExecution && job.data.mcpSessionId && job.data.mcpType !== 'trigger') {
|
||||
return;
|
||||
}
|
||||
|
||||
const relayed = await this.webhookResponseRelay.prepare(response, {
|
||||
workflowId: job.data.workflowId,
|
||||
executionId,
|
||||
});
|
||||
|
||||
// Check if this is an MCP execution - broadcast response to all mains
|
||||
if (job.data.isMcpExecution && job.data.mcpSessionId) {
|
||||
const msg: McpResponseMessage = {
|
||||
kind: 'mcp-response',
|
||||
executionId,
|
||||
mcpType: job.data.mcpType ?? 'service',
|
||||
mcpType: 'trigger',
|
||||
sessionId: job.data.mcpSessionId,
|
||||
messageId: job.data.mcpMessageId ?? '',
|
||||
response,
|
||||
response: relayed,
|
||||
workerId: this.instanceSettings.hostId,
|
||||
};
|
||||
|
||||
|
|
@ -219,7 +230,7 @@ export class JobProcessor {
|
|||
const msg: RespondToWebhookMessage = {
|
||||
kind: 'respond-to-webhook',
|
||||
executionId,
|
||||
response: encodeRelayedWebhookResponse(response),
|
||||
response: relayed,
|
||||
workerId: this.instanceSettings.hostId,
|
||||
};
|
||||
|
||||
|
|
@ -377,8 +388,12 @@ export class JobProcessor {
|
|||
execution.data?.executionData?.runtimeData,
|
||||
),
|
||||
);
|
||||
|
||||
// A tool result is not a response, so it has no offload path: this limit
|
||||
// is all that keeps it from travelling through the queue unbounded.
|
||||
this.webhookResponseRelay.assertFitsInline(toolResult);
|
||||
} catch (error) {
|
||||
this.logger.error('Tool node execution failed for MCP Trigger', {
|
||||
this.logger.error('Tool call failed for MCP Trigger', {
|
||||
executionId,
|
||||
toolName,
|
||||
sourceNodeName,
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import type {
|
|||
JobMessage,
|
||||
JobFailedMessage,
|
||||
} from './scaling.types';
|
||||
import { decodeRelayedWebhookResponse } from './webhook-response-relay';
|
||||
import { decodeRelayedWebhookResponse, WebhookResponseRelay } from './webhook-response-relay';
|
||||
|
||||
@Service()
|
||||
export class ScalingService {
|
||||
|
|
@ -49,6 +49,7 @@ export class ScalingService {
|
|||
private readonly executionPersistence: ExecutionPersistence,
|
||||
private readonly instanceSettings: InstanceSettings,
|
||||
private readonly eventService: EventService,
|
||||
private readonly webhookResponseRelay: WebhookResponseRelay,
|
||||
) {
|
||||
this.logger = this.logger.scoped('scaling');
|
||||
}
|
||||
|
|
@ -484,7 +485,24 @@ export class ScalingService {
|
|||
} else {
|
||||
const { McpServer } = await import('@n8n/n8n-nodes-langchain/mcp/core');
|
||||
const mcpServer = McpServer.instance(this.logger);
|
||||
mcpServer.handleWorkerResponse(sessionId, messageId, response);
|
||||
|
||||
const holdsResponse =
|
||||
mcpServer.hasSession(sessionId) || mcpServer.hasPendingResponse(sessionId, messageId);
|
||||
|
||||
if (holdsResponse) {
|
||||
// Holding the session does not make this main the sole reader: a second
|
||||
// main recreates the transport when a request for the session lands on
|
||||
// it. So the stored body is left in place, and execution pruning
|
||||
// reclaims it. Restoring under this guard only spares the mains that
|
||||
// would discard the body a read of the whole thing.
|
||||
const decoded = decodeRelayedWebhookResponse(response);
|
||||
const toolResult = await this.webhookResponseRelay.restoreOffloadedBody(decoded, {
|
||||
reclaim: false,
|
||||
context: { executionId },
|
||||
});
|
||||
|
||||
mcpServer.handleWorkerResponse(sessionId, messageId, toolResult);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to handle MCP response', {
|
||||
|
|
|
|||
|
|
@ -1,114 +1,420 @@
|
|||
/**
|
||||
* Moves webhook response bodies from a worker back to main in scaling mode.
|
||||
* Moves response bodies from a worker back to main in scaling mode.
|
||||
*
|
||||
* A relayed body travels inline inside a single queue message, so its size
|
||||
* bounds what the queue must hold while that message is processed.
|
||||
* A small body travels inline inside the queue message.
|
||||
*
|
||||
* Where offload is enabled:
|
||||
* - a larger one is stored in the binary-data store
|
||||
* - and replaced with a reference
|
||||
* So the size of a response no longer bounds what the queue must hold.
|
||||
*/
|
||||
|
||||
import { Logger } from '@n8n/backend-common';
|
||||
import { ExecutionsConfig } from '@n8n/config';
|
||||
import { Service } from '@n8n/di';
|
||||
import { jsonSizeExceeds } from '@n8n/utils/json/json-size-exceeds';
|
||||
import { BINARY_ENCODING } from 'n8n-workflow';
|
||||
import type { IDataObject, IExecuteResponsePromiseData, IN8nHttpFullResponse } from 'n8n-workflow';
|
||||
import { BinaryDataConfig, BinaryDataService, FileLocation, FileTooLargeError } from 'n8n-core';
|
||||
import type { BinaryData } from 'n8n-core';
|
||||
import { BINARY_ENCODING, jsonParse, OperationalError } from 'n8n-workflow';
|
||||
import type {
|
||||
IBinaryData,
|
||||
IDataObject,
|
||||
IExecuteResponsePromiseData,
|
||||
IN8nHttpFullResponse,
|
||||
} from 'n8n-workflow';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { WebhookResponseTooLargeError } from '@/errors/webhook-response-too-large.error';
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
|
||||
/** Sentinel key marking a base64-encoded Buffer body relayed inline through the queue. */
|
||||
export const ENCODED_BUFFER_KEY = '__@N8nEncodedBuffer@__';
|
||||
|
||||
/**
|
||||
* Asserts that a payload is small enough to be relayed through the queue.
|
||||
*
|
||||
* @throws {WebhookResponseTooLargeError} When the payload exceeds `maxSizeInMiB`.
|
||||
*
|
||||
* @remarks The whole payload counts, headers included, the body and the rest
|
||||
* being measured separately, each against the limit. A Buffer body counts
|
||||
* base64-encoded, the form it travels in, and JSON content counts as an upper
|
||||
* bound of what it serializes to. A string body counts as its raw bytes, so the
|
||||
* escapes serialization adds to it are not counted, and neither is the queue
|
||||
* envelope around the payload.
|
||||
* Sentinel key recording an offloaded body's original form, so it can be restored.
|
||||
* It sits on the relayed response, beside `body`, never inside it:
|
||||
* - a body's content comes from the workflow
|
||||
* - the response envelope only from the relay
|
||||
* So only a reference the relay itself stored reads as offloaded.
|
||||
*/
|
||||
export function assertRelayableSize(
|
||||
payload: IExecuteResponsePromiseData,
|
||||
maxSizeInMiB: number,
|
||||
): void {
|
||||
if (exceedsRelayableSize(payload, maxSizeInMiB * 1024 * 1024)) {
|
||||
throw new WebhookResponseTooLargeError(maxSizeInMiB);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether any part of the payload would serialize to more than `maxBytes`. */
|
||||
function exceedsRelayableSize(payload: IExecuteResponsePromiseData, maxBytes: number): boolean {
|
||||
if (!isFullResponse(payload)) {
|
||||
return exceedsSize(payload, maxBytes);
|
||||
}
|
||||
|
||||
const { body, ...rest } = payload;
|
||||
return exceedsSize(body, maxBytes) || jsonSizeExceeds(rest, maxBytes);
|
||||
}
|
||||
|
||||
/** Whether the body would serialize to more than `maxBytes`. */
|
||||
function exceedsSize(body: IN8nHttpFullResponse['body'], maxBytes: number): boolean {
|
||||
if (Buffer.isBuffer(body)) {
|
||||
return base64Size(body.length) > maxBytes;
|
||||
}
|
||||
|
||||
if (typeof body === 'string') {
|
||||
return Buffer.byteLength(body) > maxBytes;
|
||||
}
|
||||
|
||||
if (isJsonObject(body)) {
|
||||
return jsonSizeExceeds(body, maxBytes);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function base64Size(byteLength: number): number {
|
||||
return Math.ceil(byteLength / 3) * 4;
|
||||
}
|
||||
|
||||
function isJsonObject(body: IN8nHttpFullResponse['body']): body is IDataObject {
|
||||
return (
|
||||
typeof body === 'object' &&
|
||||
body !== null &&
|
||||
!Buffer.isBuffer(body) &&
|
||||
!(body instanceof Readable)
|
||||
);
|
||||
}
|
||||
export const OFFLOADED_BODY_KIND_KEY = '__@N8nOffloadedBodyKind@__';
|
||||
|
||||
/**
|
||||
* Encodes a worker's webhook response into the form it travels in:
|
||||
* - a Buffer body is base64-encoded
|
||||
* - every other body passes through untouched.
|
||||
* The one mode with nowhere to offload a body to: it keeps bytes in memory, so a
|
||||
* body stored in it would travel inline after all.
|
||||
*
|
||||
* @param response Worker response. Mutated and returned.
|
||||
* @returns The same `response`, with a Buffer body wrapped in a base64 envelope.
|
||||
* Every other mode has a store, and whether main reaches it is the deployment's to
|
||||
* get right: a store only the instance that wrote the body can read surfaces when
|
||||
* main reads it back, rather than being refused up front.
|
||||
*/
|
||||
export function encodeRelayedWebhookResponse(
|
||||
response: IExecuteResponsePromiseData,
|
||||
): IExecuteResponsePromiseData {
|
||||
if (!isFullResponse(response)) {
|
||||
const IN_MEMORY_MODE: BinaryDataConfig['mode'] = 'default';
|
||||
|
||||
/** What Express sets on a string body sent inline through `res.send`. */
|
||||
const INLINE_STRING_CONTENT_TYPE = 'text/html; charset=utf-8';
|
||||
|
||||
/** What Express sets on a JSON body sent inline through `res.json`. */
|
||||
const INLINE_JSON_CONTENT_TYPE = 'application/json; charset=utf-8';
|
||||
|
||||
const TOO_LARGE_MESSAGE = 'The response is too large to be sent back from the worker';
|
||||
|
||||
const TOO_LARGE_FOR_STORE_MESSAGE = 'The response is too large for the binary-data store to hold';
|
||||
|
||||
const OFFLOAD_DISABLED_GUIDANCE =
|
||||
'In scaling mode a response over this size can be stored for the main instance to stream instead of failing. Set N8N_WEBHOOK_RESPONSE_RELAY_OFFLOAD_ENABLED to true on every worker, once every main instance runs a version that reads a stored body, or raise N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX.';
|
||||
|
||||
const BINARY_DATA_DOCS_LINK =
|
||||
"<a href='https://docs.n8n.io/deploy/host-n8n/configure-n8n/basic-configuration/use-environment-variables/binary-data' target='_blank'>in the docs</a>";
|
||||
|
||||
const NO_STORE_GUIDANCE = `In scaling mode a response over this size is stored for the main instance to stream, which the in-memory binary-data mode cannot do. Set N8N_DEFAULT_BINARY_DATA_MODE to a mode with a store, or raise N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX. The modes are described ${BINARY_DATA_DOCS_LINK}.`;
|
||||
|
||||
const UNREADABLE_BODY_GUIDANCE = `A response over N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX is stored by the instance that produced it, so N8N_DEFAULT_BINARY_DATA_MODE has to name a store every instance can read. The modes are described ${BINARY_DATA_DOCS_LINK}.`;
|
||||
|
||||
const NOT_OFFLOADABLE_GUIDANCE =
|
||||
'In scaling mode a response is relayed to the main instance through the queue, which limits how large it can be. Only a response body can be stored for the main instance to stream instead, so raise N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX to relay a payload this large.';
|
||||
|
||||
const STORE_LIMIT_GUIDANCE =
|
||||
'A response over N8N_WEBHOOK_RESPONSE_RELAY_SIZE_MAX is stored for the main instance to stream, so the store applies its own size limit above that one. In database mode raise N8N_BINARY_DATA_DATABASE_MAX_FILE_SIZE, up to the 1 GB a database column holds. The filesystem, s3 and azure modes have no such limit, so setting N8N_DEFAULT_BINARY_DATA_MODE to one of those lifts it, as long as every instance reads the same store.';
|
||||
|
||||
export type RelayContext = { workflowId: string; executionId: string };
|
||||
|
||||
export type RelayReadContext = Partial<RelayContext>;
|
||||
|
||||
const OFFLOADED_BODY_KINDS = ['buffer', 'string', 'json'] as const;
|
||||
|
||||
type OffloadedBodyKind = (typeof OFFLOADED_BODY_KINDS)[number];
|
||||
|
||||
type OffloadedBodyContent = Buffer | string | IDataObject;
|
||||
|
||||
type OffloadablePayload = {
|
||||
kind: OffloadedBodyKind;
|
||||
exceeds: (maxBytes: number) => boolean;
|
||||
serialize: () => Buffer;
|
||||
inlineContentType?: string;
|
||||
};
|
||||
|
||||
type OffloadedBody = {
|
||||
binaryData: IBinaryData & { id: string };
|
||||
kind: OffloadedBodyKind;
|
||||
};
|
||||
|
||||
/**
|
||||
* A full response that may carry the {@link OFFLOADED_BODY_KIND_KEY} marker.
|
||||
*
|
||||
* @remarks The marker is typed `unknown` because a relayed response is parsed
|
||||
* from the queue message, where nothing constrains what the key holds. Reading
|
||||
* it therefore goes through {@link isOffloadedBodyKind}.
|
||||
*/
|
||||
type OffloadMarkedResponse = IN8nHttpFullResponse &
|
||||
Partial<Record<typeof OFFLOADED_BODY_KIND_KEY, unknown>>;
|
||||
|
||||
/**
|
||||
* Carries a response body from a worker to main in scaling mode, inline inside
|
||||
* the queue message or through the binary-data store.
|
||||
*/
|
||||
@Service()
|
||||
export class WebhookResponseRelay {
|
||||
constructor(
|
||||
private readonly logger: Logger,
|
||||
private readonly binaryDataService: BinaryDataService,
|
||||
private readonly binaryDataConfig: BinaryDataConfig,
|
||||
private readonly executionsConfig: ExecutionsConfig,
|
||||
) {
|
||||
this.logger = this.logger.scoped('scaling');
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepares a worker's response to travel inside a queue message:
|
||||
* - the body of a response over the size limit is stored and replaced with a
|
||||
* reference, where offload is enabled
|
||||
* - a Buffer body staying inline is base64-encoded
|
||||
* - and any other body passes through.
|
||||
*
|
||||
* @param response Worker response. Mutated and returned.
|
||||
* @returns The same `response`.
|
||||
*
|
||||
* @throws {WebhookResponseTooLargeError} When:
|
||||
* - the response is over the limit and its body cannot be offloaded,
|
||||
* - or when what is left once the body is offloaded is over the limit on its own,
|
||||
* - or when the response has no offload path and is over the limit as a whole.
|
||||
*
|
||||
* A stream body is the one exception to measuring a whole response: it cannot be
|
||||
* measured without being consumed, so only the rest of the response is asserted on.
|
||||
*/
|
||||
async prepare(
|
||||
response: IExecuteResponsePromiseData,
|
||||
context: RelayContext,
|
||||
): Promise<IExecuteResponsePromiseData> {
|
||||
if (!hasResponseBody(response)) {
|
||||
this.assertFitsInline(response);
|
||||
return response;
|
||||
}
|
||||
|
||||
const { body, ...rest } = response;
|
||||
const offloadable = asOffloadablePayload(body);
|
||||
|
||||
if (!offloadable) {
|
||||
this.assertFitsInline(body instanceof Readable ? rest : response);
|
||||
return response;
|
||||
}
|
||||
|
||||
this.assertFitsInline(rest);
|
||||
|
||||
if (!this.exceedsInline(response, offloadable)) {
|
||||
return encodeBufferBody(response);
|
||||
}
|
||||
|
||||
this.assertOffloadAvailable();
|
||||
|
||||
return await this.offload(response, offloadable, context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces an offloaded body with its stored content.
|
||||
* Any other response passes through.
|
||||
*
|
||||
* @param response Relayed response. Mutated and returned.
|
||||
* @param reclaim Whether this caller is the sole reader of the response.
|
||||
* A sole reader deletes the stored body once read, without holding the returned
|
||||
* promise for it, and a body that cannot be fetched fails the delivery it owns.
|
||||
* Where the same relayed response reaches several readers, deleting would strand
|
||||
* the others, and a fetch failure degrades to an empty body of the original form
|
||||
* instead, never the reference itself.
|
||||
* @param context Where the response came from, reported on failure.
|
||||
* @returns The same `response`.
|
||||
* @throws {OperationalError} When `reclaim` is set and the stored content cannot be fetched.
|
||||
*/
|
||||
async restoreOffloadedBody<T>(
|
||||
response: T,
|
||||
{ reclaim, context }: { reclaim: boolean; context: RelayReadContext },
|
||||
): Promise<T> {
|
||||
if (!hasResponseBody(response)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const offloaded = asOffloadedBody(response);
|
||||
if (!offloaded) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const binaryDataId = offloaded.binaryData.id;
|
||||
|
||||
try {
|
||||
const buffer = await this.binaryDataService.getAsBuffer(offloaded.binaryData);
|
||||
response.body = deserializeBody(buffer, offloaded.kind);
|
||||
} catch (error) {
|
||||
if (reclaim) {
|
||||
throw new OperationalError('The stored webhook response body could not be read', {
|
||||
cause: error,
|
||||
description: UNREADABLE_BODY_GUIDANCE,
|
||||
extra: { binaryDataId, ...context },
|
||||
});
|
||||
}
|
||||
|
||||
return this.degradeToEmptyBody(response, offloaded, context, error);
|
||||
}
|
||||
|
||||
clearOffloadMarker(response);
|
||||
|
||||
if (reclaim) {
|
||||
void this.deleteStoredBody(binaryDataId, context);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
if (Buffer.isBuffer(response.body)) {
|
||||
response.body = { [ENCODED_BUFFER_KEY]: response.body.toString(BINARY_ENCODING) };
|
||||
/**
|
||||
* Reclaims the storage of an offloaded body.
|
||||
* A no-op for any other body.
|
||||
*
|
||||
* @param context Where the response came from, reported on failure.
|
||||
*
|
||||
* @remarks Failures are logged rather than thrown: the response has already
|
||||
* been delivered, and a body left behind is reclaimed by execution pruning
|
||||
* (`database`) or by store lifecycle rules (object stores).
|
||||
*/
|
||||
async deleteOffloadedBody(
|
||||
response: IExecuteResponsePromiseData,
|
||||
context: RelayReadContext,
|
||||
): Promise<void> {
|
||||
if (hasResponseBody(response)) {
|
||||
const offloaded = asOffloadedBody(response);
|
||||
if (offloaded) {
|
||||
await this.deleteStoredBody(offloaded.binaryData.id, context);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return response;
|
||||
/**
|
||||
* Asserts that a payload with no offload path is small enough to travel
|
||||
* inline inside a queue message.
|
||||
*
|
||||
* @throws {WebhookResponseTooLargeError} When the payload is over the limit.
|
||||
*/
|
||||
assertFitsInline(payload: unknown): void {
|
||||
if (exceedsInlineSize(payload, this.maxInlineBytes)) {
|
||||
throw new WebhookResponseTooLargeError(TOO_LARGE_MESSAGE, {
|
||||
description: withInlineLimit(NOT_OFFLOADABLE_GUIDANCE, this.maxInlineMib),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that a body over the limit has somewhere to be offloaded to.
|
||||
*
|
||||
* @throws {WebhookResponseTooLargeError} When offload is turned off, or when the
|
||||
* binary-data mode keeps bytes in memory.
|
||||
*/
|
||||
private assertOffloadAvailable(): void {
|
||||
if (!this.executionsConfig.webhookResponseRelayOffloadEnabled) {
|
||||
throw new WebhookResponseTooLargeError(TOO_LARGE_MESSAGE, {
|
||||
description: withInlineLimit(OFFLOAD_DISABLED_GUIDANCE, this.maxInlineMib),
|
||||
});
|
||||
}
|
||||
|
||||
if (this.binaryDataConfig.mode === IN_MEMORY_MODE) {
|
||||
throw new WebhookResponseTooLargeError(TOO_LARGE_MESSAGE, {
|
||||
description: withInlineLimit(NO_STORE_GUIDANCE, this.maxInlineMib),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private get maxInlineMib(): number {
|
||||
return this.executionsConfig.webhookResponseRelaySizeMaxMiB;
|
||||
}
|
||||
|
||||
private get maxInlineBytes(): number {
|
||||
return this.maxInlineMib * MIB;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `response` is too large to travel inline inside a queue message.
|
||||
*
|
||||
* @remarks Measured twice, because neither measure bounds the other: the body
|
||||
* alone, in the form it travels in, which counts a Buffer base64-encoded, and
|
||||
* the whole response as JSON, which adds the headers. The second is an upper
|
||||
* bound, and a loose one over a Buffer body, which serializes as an array of
|
||||
* numbers rather than as base64, so a Buffer may be offloaded below the limit.
|
||||
*/
|
||||
private exceedsInline(response: IN8nHttpFullResponse, body: OffloadablePayload): boolean {
|
||||
return body.exceeds(this.maxInlineBytes) || jsonSizeExceeds(response, this.maxInlineBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param response Mutated and returned.
|
||||
*
|
||||
* @throws Whatever storing the body fails with, leaving `response` untouched.
|
||||
*
|
||||
* @remarks A stored body is deliberately absent from run data, unlike a node's
|
||||
* binary data, because its lifetime is a single delivery: it is deleted once
|
||||
* read, so duplicating it to a parent execution would copy bytes already gone,
|
||||
* and counting it towards an execution's binary-data size would report bytes no
|
||||
* reader can fetch. It is still stored under the execution, so that pruning
|
||||
* reclaims a body no reader deleted: pruning deletes a location by prefix, so a
|
||||
* store implementing no prefix deletion leaves that to its lifecycle rules.
|
||||
*/
|
||||
private async offload(
|
||||
response: OffloadMarkedResponse,
|
||||
{ kind, serialize, inlineContentType }: OffloadablePayload,
|
||||
{ workflowId, executionId }: RelayContext,
|
||||
): Promise<OffloadMarkedResponse> {
|
||||
const existingContentType = contentTypeOf(response.headers);
|
||||
const contentType = existingContentType ?? inlineContentType;
|
||||
const location = FileLocation.ofExecution(workflowId, executionId);
|
||||
|
||||
const stored = await this.storeBody(location, serialize(), contentType);
|
||||
|
||||
// Main streams a stored body instead of sending it through `res.json` or
|
||||
// `res.send`, so the `content-type` Express would have set has to travel
|
||||
// with it, keeping a response's headers independent of its size.
|
||||
if (contentType !== undefined && existingContentType === undefined) {
|
||||
response.headers ??= {};
|
||||
response.headers['content-type'] = contentType;
|
||||
}
|
||||
|
||||
response.body = { binaryData: stored };
|
||||
response[OFFLOADED_BODY_KIND_KEY] = kind;
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns The stored body's binary-data reference, its `id` set.
|
||||
* @throws {WebhookResponseTooLargeError} When the store refuses the body for its own size limit.
|
||||
* @throws {OperationalError} When the store reports no id.
|
||||
*/
|
||||
private async storeBody(
|
||||
location: BinaryData.FileLocation,
|
||||
body: Buffer,
|
||||
contentType: string | undefined,
|
||||
): Promise<IBinaryData> {
|
||||
let stored: IBinaryData;
|
||||
|
||||
try {
|
||||
stored = await this.binaryDataService.store(location, body, {
|
||||
data: '',
|
||||
mimeType: contentType ?? 'application/octet-stream',
|
||||
fileName: 'webhook-response',
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof FileTooLargeError) {
|
||||
throw new WebhookResponseTooLargeError(TOO_LARGE_FOR_STORE_MESSAGE, {
|
||||
description: withResponseSize(STORE_LIMIT_GUIDANCE, body.length),
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (!stored.id) {
|
||||
throw new OperationalError('Binary-data store did not persist the webhook response body');
|
||||
}
|
||||
|
||||
return stored;
|
||||
}
|
||||
|
||||
private degradeToEmptyBody<T extends IN8nHttpFullResponse>(
|
||||
response: T,
|
||||
offloaded: OffloadedBody,
|
||||
context: RelayReadContext,
|
||||
error: unknown,
|
||||
): T {
|
||||
this.logger.warn('Failed to restore an offloaded webhook response body', {
|
||||
binaryDataId: offloaded.binaryData.id,
|
||||
...context,
|
||||
error,
|
||||
});
|
||||
|
||||
response.body = emptyBodyOf(offloaded.kind);
|
||||
clearOffloadMarker(response);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
private async deleteStoredBody(binaryDataId: string, context: RelayReadContext): Promise<void> {
|
||||
try {
|
||||
await this.binaryDataService.deleteManyByBinaryDataId([binaryDataId]);
|
||||
} catch (error) {
|
||||
this.logger.warn('Failed to delete an offloaded webhook response body', {
|
||||
binaryDataId,
|
||||
...context,
|
||||
error,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverses {@link encodeRelayedWebhookResponse} on main, decoding a base64
|
||||
* envelope back into a Buffer. Every other body passes through untouched.
|
||||
* Reverses the inline base64 envelope on main, restoring a Buffer body. Every
|
||||
* other body passes through, an offloaded one included: main streams that from
|
||||
* storage rather than materializing it.
|
||||
*
|
||||
* @param response Relayed response. Mutated and returned.
|
||||
* @returns The same `response`, with an encoded-buffer body restored to a Buffer.
|
||||
* @returns The same `response`.
|
||||
*/
|
||||
export function decodeRelayedWebhookResponse(
|
||||
response: IExecuteResponsePromiseData,
|
||||
): IExecuteResponsePromiseData {
|
||||
if (!isFullResponse(response)) {
|
||||
export function decodeRelayedWebhookResponse<T>(response: T): T {
|
||||
if (!hasResponseBody(response)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
|
|
@ -120,7 +426,125 @@ export function decodeRelayedWebhookResponse(
|
|||
return response;
|
||||
}
|
||||
|
||||
function isFullResponse(response: IExecuteResponsePromiseData): response is IN8nHttpFullResponse {
|
||||
function withInlineLimit(guidance: string, limitInMib: number): string {
|
||||
return `The limit is ${limitInMib} MiB. ${guidance}`;
|
||||
}
|
||||
|
||||
function withResponseSize(guidance: string, byteLength: number): string {
|
||||
const sizeInMib = Math.round((byteLength / MIB) * 100) / 100;
|
||||
return `The response is ${sizeInMib} MiB. ${guidance}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes `byteLength` bytes occupy once base64-encoded, the form a Buffer takes to
|
||||
* travel inline. Excludes the envelope around it, keeping this a lower bound.
|
||||
*/
|
||||
function base64Size(byteLength: number): number {
|
||||
return Math.ceil(byteLength / 3) * 4;
|
||||
}
|
||||
|
||||
function encodeBufferBody(response: IN8nHttpFullResponse): IN8nHttpFullResponse {
|
||||
if (Buffer.isBuffer(response.body)) {
|
||||
response.body = { [ENCODED_BUFFER_KEY]: response.body.toString(BINARY_ENCODING) };
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `payload` would serialize to more than `maxBytes` inside a queue
|
||||
* message. Everything is measured, offloadable or not — a binary-data-shaped
|
||||
* object included. Only a stream is exempt: it cannot be measured without
|
||||
* being consumed.
|
||||
*/
|
||||
function exceedsInlineSize(payload: unknown, maxBytes: number): boolean {
|
||||
const offloadable = asOffloadablePayload(payload);
|
||||
if (offloadable) {
|
||||
return offloadable.exceeds(maxBytes);
|
||||
}
|
||||
|
||||
return !(payload instanceof Readable) && jsonSizeExceeds(payload, maxBytes);
|
||||
}
|
||||
|
||||
function asOffloadablePayload(payload: unknown): OffloadablePayload | undefined {
|
||||
if (Buffer.isBuffer(payload)) {
|
||||
return {
|
||||
kind: 'buffer',
|
||||
exceeds: (maxBytes) => base64Size(payload.length) > maxBytes,
|
||||
serialize: () => payload,
|
||||
};
|
||||
}
|
||||
|
||||
if (typeof payload === 'string') {
|
||||
return {
|
||||
kind: 'string',
|
||||
exceeds: (maxBytes) => Buffer.byteLength(payload, 'utf8') > maxBytes,
|
||||
serialize: () => Buffer.from(payload, 'utf8'),
|
||||
inlineContentType: INLINE_STRING_CONTENT_TYPE,
|
||||
};
|
||||
}
|
||||
|
||||
if (isPlainJson(payload)) {
|
||||
return {
|
||||
kind: 'json',
|
||||
exceeds: (maxBytes) => jsonSizeExceeds(payload, maxBytes),
|
||||
serialize: () => Buffer.from(JSON.stringify(payload), 'utf8'),
|
||||
inlineContentType: INLINE_JSON_CONTENT_TYPE,
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function deserializeBody(buffer: Buffer, kind: OffloadedBodyKind): OffloadedBodyContent {
|
||||
switch (kind) {
|
||||
case 'buffer':
|
||||
return buffer;
|
||||
case 'string':
|
||||
return buffer.toString('utf8');
|
||||
case 'json':
|
||||
return jsonParse<IDataObject>(buffer.toString('utf8'));
|
||||
}
|
||||
}
|
||||
|
||||
function emptyBodyOf(kind: OffloadedBodyKind): OffloadedBodyContent {
|
||||
switch (kind) {
|
||||
case 'buffer':
|
||||
return Buffer.alloc(0);
|
||||
case 'string':
|
||||
return '';
|
||||
case 'json':
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function asOffloadedBody(response: IN8nHttpFullResponse): OffloadedBody | undefined {
|
||||
if (!isBinaryDataReference(response.body)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const marked: OffloadMarkedResponse = response;
|
||||
const kind = marked[OFFLOADED_BODY_KIND_KEY];
|
||||
if (isOffloadedBodyKind(kind)) {
|
||||
return {
|
||||
binaryData: response.body.binaryData,
|
||||
kind,
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isOffloadedBodyKind(value: unknown): value is OffloadedBodyKind {
|
||||
return typeof value === 'string' && OFFLOADED_BODY_KINDS.some((kind) => kind === value);
|
||||
}
|
||||
|
||||
function clearOffloadMarker(response: IN8nHttpFullResponse): void {
|
||||
const marked: OffloadMarkedResponse = response;
|
||||
delete marked[OFFLOADED_BODY_KIND_KEY];
|
||||
}
|
||||
|
||||
function hasResponseBody(response: unknown): response is IN8nHttpFullResponse {
|
||||
return typeof response === 'object' && response !== null && 'body' in response;
|
||||
}
|
||||
|
||||
|
|
@ -133,3 +557,37 @@ function encodedBufferIn(body: IN8nHttpFullResponse['body']): string | undefined
|
|||
const encoded = body[ENCODED_BUFFER_KEY];
|
||||
return typeof encoded === 'string' ? encoded : undefined;
|
||||
}
|
||||
|
||||
function isPlainJson(payload: unknown): payload is IDataObject {
|
||||
return (
|
||||
typeof payload === 'object' &&
|
||||
payload !== null &&
|
||||
!Buffer.isBuffer(payload) &&
|
||||
!(payload instanceof Readable) &&
|
||||
!isBinaryDataReference(payload)
|
||||
);
|
||||
}
|
||||
|
||||
function isBinaryDataReference(
|
||||
body: unknown,
|
||||
): body is IDataObject & { binaryData: IBinaryData & { id: string } } {
|
||||
if (typeof body !== 'object' || body === null || !('binaryData' in body)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const { binaryData } = body;
|
||||
return (
|
||||
typeof binaryData === 'object' &&
|
||||
binaryData !== null &&
|
||||
'id' in binaryData &&
|
||||
typeof binaryData.id === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
function contentTypeOf(headers: IN8nHttpFullResponse['headers']): string | undefined {
|
||||
const entry = Object.entries(headers ?? {}).find(
|
||||
([name]) => name.toLowerCase() === 'content-type',
|
||||
);
|
||||
|
||||
return typeof entry?.[1] === 'string' ? entry[1] : undefined;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ import type { Readable } from 'stream';
|
|||
import { finished } from 'stream/promises';
|
||||
import { mock, type MockProxy } from 'vitest-mock-extended';
|
||||
|
||||
import { WebhookResponseRelay } from '@/scaling/webhook-response-relay';
|
||||
|
||||
import {
|
||||
autoDetectResponseMode,
|
||||
handleFormRedirectionCase,
|
||||
|
|
@ -235,6 +237,7 @@ describe('setupResponseNodePromise', () => {
|
|||
const workflowStartNode = mock<INode>();
|
||||
const workflow = mock<Workflow>({ id: workflowId });
|
||||
const binaryDataService = mockInstance(BinaryDataService);
|
||||
const webhookResponseRelay = mockInstance(WebhookResponseRelay);
|
||||
const errorReporter = mockInstance(ErrorReporter);
|
||||
const logger = mockInstance(Logger);
|
||||
|
||||
|
|
@ -305,6 +308,86 @@ describe('setupResponseNodePromise', () => {
|
|||
expect(responseCallback).toHaveBeenCalledWith(null, { noWebhookResponse: true });
|
||||
});
|
||||
|
||||
test('should reclaim an offloaded body once it has been streamed', async () => {
|
||||
binaryDataService.getAsStream.mockResolvedValue(mock<Readable>());
|
||||
const response = {
|
||||
body: { binaryData: { id: 'binary-123' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
} as unknown as IN8nHttpFullResponse;
|
||||
|
||||
setupResponseNodePromise(
|
||||
responsePromise,
|
||||
res,
|
||||
responseCallback,
|
||||
workflowStartNode,
|
||||
executionId,
|
||||
workflow,
|
||||
);
|
||||
|
||||
responsePromise.resolve(response);
|
||||
await new Promise(process.nextTick);
|
||||
|
||||
expect(webhookResponseRelay.deleteOffloadedBody).toHaveBeenCalledWith(response, {
|
||||
workflowId,
|
||||
executionId,
|
||||
});
|
||||
});
|
||||
|
||||
test('should destroy the stream when the client goes away, so delivery settles', async () => {
|
||||
const stream = mock<Readable>();
|
||||
binaryDataService.getAsStream.mockResolvedValue(stream);
|
||||
|
||||
setupResponseNodePromise(
|
||||
responsePromise,
|
||||
res,
|
||||
responseCallback,
|
||||
workflowStartNode,
|
||||
executionId,
|
||||
workflow,
|
||||
);
|
||||
|
||||
responsePromise.resolve({
|
||||
body: { binaryData: { id: 'binary-123' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
} as unknown as IN8nHttpFullResponse);
|
||||
await new Promise(process.nextTick);
|
||||
|
||||
const closeHandler = res.once.mock.calls.find(([event]) => event === 'close')?.[1] as
|
||||
| (() => void)
|
||||
| undefined;
|
||||
expect(closeHandler).toBeDefined();
|
||||
expect(stream.destroy).not.toHaveBeenCalled();
|
||||
|
||||
closeHandler!();
|
||||
|
||||
expect(stream.destroy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('should reclaim an offloaded body even when streaming fails', async () => {
|
||||
binaryDataService.getAsStream.mockRejectedValue(new Error('store is down'));
|
||||
|
||||
setupResponseNodePromise(
|
||||
responsePromise,
|
||||
res,
|
||||
responseCallback,
|
||||
workflowStartNode,
|
||||
executionId,
|
||||
workflow,
|
||||
);
|
||||
|
||||
responsePromise.resolve({
|
||||
body: { binaryData: { id: 'binary-123' } },
|
||||
headers: {},
|
||||
statusCode: 200,
|
||||
} as unknown as IN8nHttpFullResponse);
|
||||
await new Promise(process.nextTick);
|
||||
|
||||
expect(webhookResponseRelay.deleteOffloadedBody).toHaveBeenCalled();
|
||||
expect(responseCallback).toHaveBeenCalledWith(expect.any(Error), {});
|
||||
});
|
||||
|
||||
test('should apply the status code to binary data responses', async () => {
|
||||
binaryDataService.getAsStream.mockResolvedValue(mock<Readable>());
|
||||
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ import { InternalServerError } from '@/errors/response-errors/internal-server.er
|
|||
import { NotFoundError } from '@/errors/response-errors/not-found.error';
|
||||
import { EventService } from '@/events/event.service';
|
||||
import { parseBody } from '@/middlewares';
|
||||
import { WebhookResponseRelay } from '@/scaling/webhook-response-relay';
|
||||
import {
|
||||
type AuthFailureReason,
|
||||
OAuthTokenVerifierProxy,
|
||||
|
|
@ -328,9 +329,17 @@ export function setupResponseNodePromise(
|
|||
}
|
||||
WebhookResponseHeaders.fromObject(response.headers).applyToResponse(res);
|
||||
applySandboxCSP(res);
|
||||
const stream = await Container.get(BinaryDataService).getAsStream(binaryData.id);
|
||||
stream.pipe(res, { end: false });
|
||||
await finished(stream);
|
||||
try {
|
||||
const stream = await Container.get(BinaryDataService).getAsStream(binaryData.id);
|
||||
res.once('close', () => stream.destroy());
|
||||
stream.pipe(res, { end: false });
|
||||
await finished(stream);
|
||||
} finally {
|
||||
void Container.get(WebhookResponseRelay).deleteOffloadedBody(response, {
|
||||
workflowId: workflow.id,
|
||||
executionId,
|
||||
});
|
||||
}
|
||||
responseCallback(null, { noWebhookResponse: true });
|
||||
} else if (Buffer.isBuffer(response.body)) {
|
||||
if (response.statusCode) {
|
||||
|
|
|
|||
|
|
@ -226,7 +226,7 @@ describe('deleteMany', () => {
|
|||
});
|
||||
|
||||
describe('deleteManyByFileId', () => {
|
||||
it('deletes the binary_data dirs of execution and custom file ids', async () => {
|
||||
it('deletes execution and custom file ids along with their metadata companions', async () => {
|
||||
const { fileId: executionFileId } = await manager.store(executionLocation, body, {});
|
||||
const customLocation = FileLocation.ofCustom({
|
||||
pathSegments: ['agents', 'a1', 'knowledge-files', 'f1'],
|
||||
|
|
@ -237,6 +237,18 @@ describe('deleteManyByFileId', () => {
|
|||
|
||||
await expect(manager.getAsBuffer(executionFileId)).rejects.toThrow(FileNotFoundError);
|
||||
await expect(manager.getAsBuffer(customFileId)).rejects.toThrow(FileNotFoundError);
|
||||
await expect(manager.getMetadata(executionFileId)).rejects.toThrow(FileNotFoundError);
|
||||
await expect(manager.getMetadata(customFileId)).rejects.toThrow(FileNotFoundError);
|
||||
});
|
||||
|
||||
it('leaves sibling files in the same binary_data dir intact', async () => {
|
||||
const { fileId: deletedFileId } = await manager.store(executionLocation, body, {});
|
||||
const { fileId: siblingFileId } = await manager.store(executionLocation, body, {});
|
||||
|
||||
await manager.deleteManyByFileId([deletedFileId]);
|
||||
|
||||
await expect(manager.getAsBuffer(deletedFileId)).rejects.toThrow(FileNotFoundError);
|
||||
expect((await manager.getAsBuffer(siblingFileId)).equals(body)).toBe(true);
|
||||
});
|
||||
|
||||
it('deletes a custom file nested under an execution-like prefix, not that execution', async () => {
|
||||
|
|
@ -273,9 +285,9 @@ describe('deleteManyByFileId', () => {
|
|||
|
||||
await manager.deleteManyByFileId([malformedId, fileId]);
|
||||
|
||||
expect(errorReporter.warn).toHaveBeenCalledWith(
|
||||
`Could not parse file ID ${malformedId}. Skip deletion`,
|
||||
);
|
||||
expect(errorReporter.warn).toHaveBeenCalledWith('Could not parse file ID. Skip deletion', {
|
||||
extra: { fileId: malformedId },
|
||||
});
|
||||
await expect(manager.getAsBuffer(fileId)).rejects.toThrow(FileNotFoundError);
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -116,10 +116,24 @@ describe('deletion', () => {
|
|||
expect(byteStore.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deleteManyByFileId is a no-op without prefix deletion, even for malformed ids', async () => {
|
||||
await manager.deleteManyByFileId([fileId, 'malformed-id']);
|
||||
it('deleteManyByFileId deletes objects by key, without metadata companions', async () => {
|
||||
await manager.deleteManyByFileId([fileId]);
|
||||
|
||||
expect(byteStore.delete).toHaveBeenCalledWith([fileId]);
|
||||
});
|
||||
|
||||
it('deleteManyByFileId warns on a malformed id without dropping the rest of the batch', async () => {
|
||||
await manager.deleteManyByFileId(['malformed-id', fileId]);
|
||||
|
||||
expect(byteStore.delete).toHaveBeenCalledWith([fileId]);
|
||||
expect(errorReporter.warn).toHaveBeenCalledWith('Could not parse file ID. Skip deletion', {
|
||||
extra: { fileId: 'malformed-id' },
|
||||
});
|
||||
});
|
||||
|
||||
it('deleteManyByFileId leaves the store untouched when every id is malformed', async () => {
|
||||
await manager.deleteManyByFileId(['malformed-id']);
|
||||
|
||||
expect(byteStore.delete).not.toHaveBeenCalled();
|
||||
expect(errorReporter.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,8 +18,9 @@ const EXECUTION_PATH_MATCHER = /^workflows\/([^/]+)\/executions\/([^/]+)\/binary
|
|||
* Backend differences are capability-driven:
|
||||
* - Backends with native object metadata (S3, Azure) store it on the object.
|
||||
* FS keeps it in companion `{fileId}.metadata` entries.
|
||||
* - Deletion happens only on backends that can delete by prefix (FS). Others
|
||||
* delegate deletion, e.g. to bucket lifecycle policies.
|
||||
* - Deletion by file id works on all backends. Location-wide deletion happens
|
||||
* only on backends that can delete by prefix (FS); others delegate it, e.g. to
|
||||
* bucket lifecycle policies.
|
||||
*/
|
||||
export class BinaryDataBlobManager implements BinaryData.Manager {
|
||||
constructor(
|
||||
|
|
@ -77,18 +78,22 @@ export class BinaryDataBlobManager implements BinaryData.Manager {
|
|||
}
|
||||
|
||||
async deleteManyByFileId(ids: string[]) {
|
||||
if (!this.byteStore.deletePrefix) return;
|
||||
|
||||
const locations = ids.flatMap((id) => {
|
||||
const keys = ids.flatMap((fileId) => {
|
||||
try {
|
||||
return [this.parseFileId(id)];
|
||||
this.parseFileId(fileId);
|
||||
} catch {
|
||||
this.errorReporter.warn(`Could not parse file ID ${id}. Skip deletion`);
|
||||
this.errorReporter.warn('Could not parse file ID. Skip deletion', {
|
||||
extra: { fileId },
|
||||
});
|
||||
return [];
|
||||
}
|
||||
|
||||
return this.byteStore.getMetadata ? [fileId] : [fileId, this.metadataKey(fileId)];
|
||||
});
|
||||
|
||||
await this.deleteBinaryDataDirs(locations);
|
||||
if (keys.length > 0) {
|
||||
await this.byteStore.delete(keys);
|
||||
}
|
||||
}
|
||||
|
||||
async copyByFileId(targetLocation: BinaryData.FileLocation, sourceFileId: string) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user