diff --git a/packages/@n8n/blob-storage/README.md b/packages/@n8n/blob-storage/README.md new file mode 100644 index 00000000000..856db0377e5 --- /dev/null +++ b/packages/@n8n/blob-storage/README.md @@ -0,0 +1,8 @@ +# @n8n/blob-storage + +Blob storage layer for n8n. Hosts the external-storage clients used to persist blobs (execution data, binary data) outside the database: + +- `ObjectStoreService` (`@n8n/blob-storage/object-store`): S3-compatible object storage +- `AzureBlobService` (`@n8n/blob-storage/azure-blob`): Azure Blob Storage + +The root entrypoint is free of cloud SDK imports: it exposes only types, configs, and stream utilities. The services live behind the subpath exports above and are meant to be loaded with `await import()` so the S3/Azure SDKs are pulled in only when external storage is configured. diff --git a/packages/@n8n/blob-storage/eslint.config.mjs b/packages/@n8n/blob-storage/eslint.config.mjs new file mode 100644 index 00000000000..dd494ad9eed --- /dev/null +++ b/packages/@n8n/blob-storage/eslint.config.mjs @@ -0,0 +1,49 @@ +import { defineConfig } from 'eslint/config'; +import { baseConfig } from '@n8n/eslint-config/base'; + +export default defineConfig( + baseConfig, + { + // Relax type-aware unsafe rules for untyped mock plumbing, mirroring n8n-core + files: ['**/__tests__/**/*.ts'], + rules: { + '@typescript-eslint/no-unsafe-assignment': 'warn', + '@typescript-eslint/no-unsafe-argument': 'warn', + '@typescript-eslint/no-unsafe-member-access': 'warn', + }, + }, + { + rules: { + 'unicorn/filename-case': ['error', { case: 'kebabCase' }], + + /** + * This package is full of AWS SDK params (`Bucket`, `Key`, `Delete`) and + * HTTP header maps (`content-type`, `x-amz-meta-filename`). + */ + '@typescript-eslint/naming-convention': [ + 'error', + { selector: 'default', format: ['camelCase'] }, + { selector: 'import', format: ['camelCase', 'PascalCase'] }, + { + selector: 'variable', + format: ['camelCase', 'snake_case', 'UPPER_CASE', 'PascalCase'], + leadingUnderscore: 'allowSingleOrDouble', + trailingUnderscore: 'allowSingleOrDouble', + }, + { + selector: 'property', + format: ['camelCase', 'snake_case', 'UPPER_CASE'], + leadingUnderscore: 'allowSingleOrDouble', + trailingUnderscore: 'allowSingleOrDouble', + }, + { selector: 'typeLike', format: ['PascalCase'] }, + { + selector: ['method', 'function', 'parameter'], + format: ['camelCase'], + leadingUnderscore: 'allowSingleOrDouble', + }, + { selector: ['objectLiteralProperty', 'typeProperty'], format: null }, + ], + }, + }, +); diff --git a/packages/@n8n/blob-storage/package.json b/packages/@n8n/blob-storage/package.json new file mode 100644 index 00000000000..c756263ac34 --- /dev/null +++ b/packages/@n8n/blob-storage/package.json @@ -0,0 +1,67 @@ +{ + "name": "@n8n/blob-storage", + "version": "1.0.0", + "description": "Blob storage layer for n8n, with filesystem, S3, and Azure Blob backends", + "scripts": { + "clean": "rimraf dist .turbo", + "dev": "pnpm watch", + "typecheck": "tsc --noEmit", + "build": "tsc -p tsconfig.build.json", + "build:unchecked": "tsc -p tsconfig.build.json --noCheck", + "format": "biome format --write .", + "format:check": "biome ci .", + "lint": "eslint . --quiet", + "lint:fix": "eslint . --fix", + "watch": "tsc -p tsconfig.build.json --watch", + "test": "vitest run", + "test:unit": "vitest run", + "test:dev": "vitest --silent=false" + }, + "main": "dist/index.js", + "module": "src/index.ts", + "types": "dist/index.d.ts", + "files": [ + "dist/**/*" + ], + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./object-store": { + "types": "./dist/object-store/index.d.ts", + "default": "./dist/object-store/index.js" + }, + "./azure-blob": { + "types": "./dist/azure-blob/index.d.ts", + "default": "./dist/azure-blob/index.js" + } + }, + "dependencies": { + "@aws-sdk/client-s3": "3.808.0", + "@azure/identity": "catalog:", + "@azure/storage-blob": "catalog:", + "@n8n/backend-common": "workspace:*", + "@n8n/backend-network": "workspace:*", + "@n8n/config": "workspace:*", + "@n8n/di": "workspace:*", + "@n8n/utils": "workspace:*", + "lodash": "catalog:", + "n8n-workflow": "workspace:*", + "reflect-metadata": "catalog:", + "zod": "catalog:" + }, + "devDependencies": { + "@n8n/eslint-config": "workspace:*", + "@n8n/typescript-config": "workspace:*", + "@n8n/vitest-config": "workspace:*", + "@types/lodash": "catalog:", + "@vitest/coverage-v8": "catalog:", + "eslint": "catalog:", + "typescript": "catalog:typescript", + "vite": "catalog:", + "vitest": "catalog:", + "vitest-mock-extended": "catalog:" + }, + "license": "LicenseRef-n8n-sustainable-use" +} diff --git a/packages/core/src/binary-data/__tests__/utils.test.ts b/packages/@n8n/blob-storage/src/__tests__/stream-utils.test.ts similarity index 95% rename from packages/core/src/binary-data/__tests__/utils.test.ts rename to packages/@n8n/blob-storage/src/__tests__/stream-utils.test.ts index d7329679ad6..f7fdbdb2d40 100644 --- a/packages/core/src/binary-data/__tests__/utils.test.ts +++ b/packages/@n8n/blob-storage/src/__tests__/stream-utils.test.ts @@ -1,9 +1,9 @@ import { UnexpectedError } from 'n8n-workflow'; import { Readable } from 'node:stream'; -import { createFixedSizeChunker } from '@/binary-data/utils'; +import { createFixedSizeChunker } from '../stream-utils'; -describe('BinaryData/utils', () => { +describe('stream-utils', () => { describe('createFixedSizeChunker', () => { const drain = async (source: Readable): Promise => { return await new Promise((resolve, reject) => { diff --git a/packages/core/src/binary-data/azure-blob/__tests__/azure-blob.service.ee.test.ts b/packages/@n8n/blob-storage/src/azure-blob/__tests__/azure-blob.service.ee.test.ts similarity index 100% rename from packages/core/src/binary-data/azure-blob/__tests__/azure-blob.service.ee.test.ts rename to packages/@n8n/blob-storage/src/azure-blob/__tests__/azure-blob.service.ee.test.ts diff --git a/packages/core/src/binary-data/azure-blob/azure-blob.config.ts b/packages/@n8n/blob-storage/src/azure-blob/azure-blob.config.ts similarity index 100% rename from packages/core/src/binary-data/azure-blob/azure-blob.config.ts rename to packages/@n8n/blob-storage/src/azure-blob/azure-blob.config.ts diff --git a/packages/core/src/binary-data/azure-blob/azure-blob.service.ee.ts b/packages/@n8n/blob-storage/src/azure-blob/azure-blob.service.ee.ts similarity index 93% rename from packages/core/src/binary-data/azure-blob/azure-blob.service.ee.ts rename to packages/@n8n/blob-storage/src/azure-blob/azure-blob.service.ee.ts index 6a4da3ff658..a39bfd8cf48 100644 --- a/packages/core/src/binary-data/azure-blob/azure-blob.service.ee.ts +++ b/packages/@n8n/blob-storage/src/azure-blob/azure-blob.service.ee.ts @@ -8,8 +8,8 @@ import { UnexpectedError, UserError } from 'n8n-workflow'; import { PassThrough, Readable, pipeline } from 'node:stream'; import { AzureBlobConfig } from './azure-blob.config'; -import type { BinaryData } from '../types'; -import { createFixedSizeChunker } from '../utils'; +import { createFixedSizeChunker } from '../stream-utils'; +import type { BlobMetadata, PreWriteBlobMetadata } from '../types'; @Service() export class AzureBlobService { @@ -71,7 +71,7 @@ export class AzureBlobService { } } - async put(blobName: string, body: Buffer, metadata: BinaryData.PreWriteMetadata = {}) { + async put(blobName: string, body: Buffer, metadata: PreWriteBlobMetadata = {}) { try { await this.containerClient.getBlockBlobClient(blobName).uploadData(body, { blobHTTPHeaders: { blobContentType: metadata.mimeType ?? 'application/octet-stream' }, @@ -139,11 +139,11 @@ export class AzureBlobService { } } - async getMetadata(blobName: string): Promise { + async getMetadata(blobName: string): Promise { try { const props = await this.containerClient.getBlockBlobClient(blobName).getProperties(); - const metadata: BinaryData.Metadata = { fileSize: props.contentLength ?? 0 }; + const metadata: BlobMetadata = { fileSize: props.contentLength ?? 0 }; if (props.contentType) metadata.mimeType = props.contentType; diff --git a/packages/@n8n/blob-storage/src/azure-blob/index.ts b/packages/@n8n/blob-storage/src/azure-blob/index.ts new file mode 100644 index 00000000000..ed93c1401eb --- /dev/null +++ b/packages/@n8n/blob-storage/src/azure-blob/index.ts @@ -0,0 +1,2 @@ +export { AzureBlobService } from './azure-blob.service.ee'; +export { AzureBlobConfig } from './azure-blob.config'; diff --git a/packages/@n8n/blob-storage/src/index.ts b/packages/@n8n/blob-storage/src/index.ts new file mode 100644 index 00000000000..dfa557998b1 --- /dev/null +++ b/packages/@n8n/blob-storage/src/index.ts @@ -0,0 +1,9 @@ +export type { BlobMetadata, PreWriteBlobMetadata } from './types'; +export { createFixedSizeChunker } from './stream-utils'; +export { ObjectStoreConfig } from './object-store/object-store.config'; +export type { MetadataResponseHeaders } from './object-store/types'; +export { AzureBlobConfig } from './azure-blob/azure-blob.config'; +// type-only: the service classes stay behind the `./object-store` and `./azure-blob` +// subpath exports so the root entrypoint never loads the S3/Azure SDKs +export type { ObjectStoreService } from './object-store/object-store.service.ee'; +export type { AzureBlobService } from './azure-blob/azure-blob.service.ee'; diff --git a/packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts b/packages/@n8n/blob-storage/src/object-store/__tests__/object-store.service.test.ts similarity index 100% rename from packages/core/src/binary-data/object-store/__tests__/object-store.service.test.ts rename to packages/@n8n/blob-storage/src/object-store/__tests__/object-store.service.test.ts diff --git a/packages/@n8n/blob-storage/src/object-store/index.ts b/packages/@n8n/blob-storage/src/object-store/index.ts new file mode 100644 index 00000000000..cdb9bb64099 --- /dev/null +++ b/packages/@n8n/blob-storage/src/object-store/index.ts @@ -0,0 +1,3 @@ +export { ObjectStoreService } from './object-store.service.ee'; +export { ObjectStoreConfig } from './object-store.config'; +export type { MetadataResponseHeaders } from './types'; diff --git a/packages/core/src/binary-data/object-store/object-store.config.ts b/packages/@n8n/blob-storage/src/object-store/object-store.config.ts similarity index 100% rename from packages/core/src/binary-data/object-store/object-store.config.ts rename to packages/@n8n/blob-storage/src/object-store/object-store.config.ts diff --git a/packages/core/src/binary-data/object-store/object-store.service.ee.ts b/packages/@n8n/blob-storage/src/object-store/object-store.service.ee.ts similarity index 98% rename from packages/core/src/binary-data/object-store/object-store.service.ee.ts rename to packages/@n8n/blob-storage/src/object-store/object-store.service.ee.ts index 03bbd2474dc..65f0c4f2335 100644 --- a/packages/core/src/binary-data/object-store/object-store.service.ee.ts +++ b/packages/@n8n/blob-storage/src/object-store/object-store.service.ee.ts @@ -25,8 +25,8 @@ import { PassThrough, Readable, pipeline } from 'node:stream'; import { ObjectStoreConfig } from './object-store.config'; import type { MetadataResponseHeaders } from './types'; -import type { BinaryData } from '../types'; -import { createFixedSizeChunker } from '../utils'; +import { createFixedSizeChunker } from '../stream-utils'; +import type { PreWriteBlobMetadata } from '../types'; /** How many per-key delete failures to name in the error before truncating, to keep the message bounded. */ const MAX_REPORTED_DELETE_ERRORS = 5; @@ -103,7 +103,7 @@ export class ObjectStoreService { /** * Upload an object to the configured bucket. */ - async put(filename: string, buffer: Buffer, metadata: BinaryData.PreWriteMetadata = {}) { + async put(filename: string, buffer: Buffer, metadata: PreWriteBlobMetadata = {}) { try { const params: PutObjectCommandInput = { Bucket: this.bucket, diff --git a/packages/core/src/binary-data/object-store/types.ts b/packages/@n8n/blob-storage/src/object-store/types.ts similarity index 71% rename from packages/core/src/binary-data/object-store/types.ts rename to packages/@n8n/blob-storage/src/object-store/types.ts index 54253f90676..c2a184346cc 100644 --- a/packages/core/src/binary-data/object-store/types.ts +++ b/packages/@n8n/blob-storage/src/object-store/types.ts @@ -1,4 +1,4 @@ -import type { BinaryData } from '../types'; +import type { PreWriteBlobMetadata } from '../types'; export type MetadataResponseHeaders = Record & { 'content-length'?: string; @@ -6,4 +6,4 @@ export type MetadataResponseHeaders = Record & { 'x-amz-meta-filename'?: string; etag?: string; 'last-modified'?: string; -} & BinaryData.PreWriteMetadata; +} & PreWriteBlobMetadata; diff --git a/packages/@n8n/blob-storage/src/stream-utils.ts b/packages/@n8n/blob-storage/src/stream-utils.ts new file mode 100644 index 00000000000..bb3008c447e --- /dev/null +++ b/packages/@n8n/blob-storage/src/stream-utils.ts @@ -0,0 +1,65 @@ +import { UnexpectedError } from 'n8n-workflow'; +import { Transform } from 'node:stream'; + +/** + * A `Transform` that re-emits its input as chunks of exactly `chunkSize` bytes, with a possibly smaller final chunk. + * `chunkSize` must be a positive integer: values `<= 0` throws an `UnexpectedError`. + * + * Between transforms the internal queue carries at most one partial chunk (< `chunkSize` bytes). + * + * Wire the upstream source into the chunker with `node:stream.pipeline()`, not plain `.pipe()`. + * `pipeline()` propagates errors from upstream to the chunker + * (so consumers see them) and propagates destroy from the chunker to upstream + * (so sockets don't dangle when the consumer aborts). + * `.pipe()` does neither. + */ +export function createFixedSizeChunker(chunkSize: number): Transform { + if (chunkSize <= 0) { + throw new UnexpectedError(`createFixedSizeChunker requires chunkSize > 0, got ${chunkSize}`); + } + + const queue: Buffer[] = []; + let queued = 0; + + const take = (size: number): Buffer => { + const out = Buffer.allocUnsafe(size); + let written = 0; + while (written < size) { + const head = queue[0]; + const need = size - written; + if (head.length <= need) { + head.copy(out, written); + written += head.length; + queue.shift(); + } else { + head.copy(out, written, 0, need); + queue[0] = head.subarray(need); + written += need; + } + } + queued -= size; + return out; + }; + + return new Transform({ + transform(chunk: Buffer, _encoding, done) { + queue.push(chunk); + queued += chunk.length; + while (queued >= chunkSize) { + this.push(take(chunkSize)); + } + done(); + }, + flush(done) { + if (queued > 0) { + this.push(take(queued)); + } + done(); + }, + destroy(error, done) { + queue.length = 0; + queued = 0; + done(error); + }, + }); +} diff --git a/packages/@n8n/blob-storage/src/types.ts b/packages/@n8n/blob-storage/src/types.ts new file mode 100644 index 00000000000..421473be3ba --- /dev/null +++ b/packages/@n8n/blob-storage/src/types.ts @@ -0,0 +1,9 @@ +/** Metadata stored alongside a blob. */ +export type BlobMetadata = { + fileName?: string; + mimeType?: string; + fileSize: number; +}; + +/** Caller-supplied metadata for a blob write, before its size is known. */ +export type PreWriteBlobMetadata = Omit; diff --git a/packages/@n8n/blob-storage/tsconfig.build.json b/packages/@n8n/blob-storage/tsconfig.build.json new file mode 100644 index 00000000000..e0f9e6d86b4 --- /dev/null +++ b/packages/@n8n/blob-storage/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": ["./tsconfig.json", "@n8n/typescript-config/tsconfig.build.go.json"], + "compilerOptions": { + "composite": true, + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/build.tsbuildinfo" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/__tests__/**"] +} diff --git a/packages/@n8n/blob-storage/tsconfig.json b/packages/@n8n/blob-storage/tsconfig.json new file mode 100644 index 00000000000..e8ea5d1264a --- /dev/null +++ b/packages/@n8n/blob-storage/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "@n8n/typescript-config/tsconfig.common.go.json", + "compilerOptions": { + "types": ["node", "vitest/globals"], + "tsBuildInfoFile": "dist/typecheck.tsbuildinfo", + "experimentalDecorators": true, + "emitDecoratorMetadata": true + }, + "include": ["src/**/*.ts"], + "references": [ + { "path": "../../workflow/tsconfig.build.cjs.json" }, + { "path": "../backend-common/tsconfig.build.json" }, + { "path": "../backend-network/tsconfig.build.json" }, + { "path": "../config/tsconfig.build.json" }, + { "path": "../di/tsconfig.build.json" } + ] +} diff --git a/packages/@n8n/blob-storage/vite.config.ts b/packages/@n8n/blob-storage/vite.config.ts new file mode 100644 index 00000000000..b746cf59f8a --- /dev/null +++ b/packages/@n8n/blob-storage/vite.config.ts @@ -0,0 +1,23 @@ +import { mergeConfig } from 'vite'; +import { createVitestConfigWithDecorators } from '@n8n/vitest-config/node-decorators'; + +export default mergeConfig( + createVitestConfigWithDecorators( + {}, + // Pin `zod` and `n8n-workflow` to their CJS build so cross-boundary `instanceof` + // (`ZodType`, `UnexpectedError`) holds against the externalized CJS dist. See + // `cjsPinAliases` in @n8n/vitest-config/node for the rationale. + { pinCjs: ['zod', 'n8n-workflow'] }, + ), + { + oxc: { + // OXC's TS transform ignores tsconfig's `emitDecoratorMetadata` — must be enabled + // explicitly here so `@n8n/config`'s `@Env(name, zodSchema) field: z.infer<...>` + // pattern works (the decorator reads `design:type` via `Reflect.getMetadata`). + decorator: { + legacy: true, + emitDecoratorMetadata: true, + }, + }, + }, +); diff --git a/packages/cli/package.json b/packages/cli/package.json index 418a21963c7..0bd7981b7d7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -149,6 +149,7 @@ "@n8n/api-types": "workspace:*", "@n8n/backend-common": "workspace:*", "@n8n/backend-network": "workspace:*", + "@n8n/blob-storage": "workspace:*", "@n8n/chat-hub": "workspace:*", "@n8n/client-oauth2": "workspace:*", "@n8n/config": "workspace:*", diff --git a/packages/cli/src/blob-storage/__tests__/azure-byte-store.test.ts b/packages/cli/src/blob-storage/__tests__/azure-byte-store.test.ts index 064ba9161cb..18f617780d8 100644 --- a/packages/cli/src/blob-storage/__tests__/azure-byte-store.test.ts +++ b/packages/cli/src/blob-storage/__tests__/azure-byte-store.test.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/unbound-method */ -import type { AzureBlobService } from 'n8n-core'; +import type { AzureBlobService } from '@n8n/blob-storage'; import type { Readable } from 'node:stream'; import { mock } from 'vitest-mock-extended'; diff --git a/packages/cli/src/blob-storage/__tests__/s3-byte-store.test.ts b/packages/cli/src/blob-storage/__tests__/s3-byte-store.test.ts index 87c63dd0ded..222ad4f4775 100644 --- a/packages/cli/src/blob-storage/__tests__/s3-byte-store.test.ts +++ b/packages/cli/src/blob-storage/__tests__/s3-byte-store.test.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/unbound-method */ -import type { ObjectStoreService } from 'n8n-core'; +import type { ObjectStoreService } from '@n8n/blob-storage'; import type { Readable } from 'node:stream'; import { mock } from 'vitest-mock-extended'; diff --git a/packages/cli/src/blob-storage/azure-byte-store.ee.ts b/packages/cli/src/blob-storage/azure-byte-store.ee.ts index b2904bb847a..87e574bea13 100644 --- a/packages/cli/src/blob-storage/azure-byte-store.ee.ts +++ b/packages/cli/src/blob-storage/azure-byte-store.ee.ts @@ -1,6 +1,6 @@ import { ensureError } from '@n8n/utils/errors/ensure-error'; import chunk from 'lodash/chunk'; -import type { AzureBlobService } from 'n8n-core'; +import type { AzureBlobService } from '@n8n/blob-storage'; import type { ByteStore, ByteStoreKey } from './types'; diff --git a/packages/cli/src/blob-storage/s3-byte-store.ee.ts b/packages/cli/src/blob-storage/s3-byte-store.ee.ts index bf700568a09..c0b6f887505 100644 --- a/packages/cli/src/blob-storage/s3-byte-store.ee.ts +++ b/packages/cli/src/blob-storage/s3-byte-store.ee.ts @@ -1,5 +1,5 @@ import { ensureError } from '@n8n/utils/errors/ensure-error'; -import type { ObjectStoreService } from 'n8n-core'; +import type { ObjectStoreService } from '@n8n/blob-storage'; import type { ByteStore, ByteStoreKey } from './types'; diff --git a/packages/cli/src/commands/base-command.ts b/packages/cli/src/commands/base-command.ts index 7eb25ff6555..2398ca7583a 100644 --- a/packages/cli/src/commands/base-command.ts +++ b/packages/cli/src/commands/base-command.ts @@ -21,8 +21,7 @@ import { ExecutionContextHookRegistry, StorageConfig, } from 'n8n-core'; -import { ObjectStoreConfig } from 'n8n-core/dist/binary-data/object-store/object-store.config'; -import { AzureBlobConfig } from 'n8n-core/dist/binary-data/azure-blob/azure-blob.config'; +import { AzureBlobConfig, ObjectStoreConfig } from '@n8n/blob-storage'; import { ensureError } from '@n8n/utils/errors/ensure-error'; import { Expression, sleep, UnexpectedError } from 'n8n-workflow'; @@ -374,9 +373,7 @@ export abstract class BaseCommand { protected async initObjectStoreIfConfigured() { if (Container.get(ObjectStoreConfig).bucket.name === '') return undefined; - const { ObjectStoreService } = await import( - 'n8n-core/dist/binary-data/object-store/object-store.service.ee.js' - ); + const { ObjectStoreService } = await import('@n8n/blob-storage/object-store'); const objectStoreService = Container.get(ObjectStoreService); await objectStoreService.init(); @@ -392,9 +389,7 @@ export abstract class BaseCommand { protected async initAzureStoreIfConfigured() { if (Container.get(AzureBlobConfig).containerName === '') return; - const { AzureBlobService } = await import( - 'n8n-core/dist/binary-data/azure-blob/azure-blob.service.ee.js' - ); + const { AzureBlobService } = await import('@n8n/blob-storage/azure-blob'); const azureBlobService = Container.get(AzureBlobService); await azureBlobService.init(); diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 38430dceebb..2ea18dcc807 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -36,6 +36,7 @@ { "path": "../@n8n/errors/tsconfig.build.json" }, { "path": "../@n8n/backend-common/tsconfig.build.json" }, { "path": "../@n8n/backend-test-utils/tsconfig.build.json" }, + { "path": "../@n8n/blob-storage/tsconfig.build.json" }, { "path": "../@n8n/di/tsconfig.build.json" }, { "path": "../@n8n/nodes-langchain/tsconfig.build.json" }, { "path": "../@n8n/permissions/tsconfig.build.json" }, diff --git a/packages/core/package.json b/packages/core/package.json index 37d5c8add71..70e7f73bab6 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -58,12 +58,10 @@ "zod": "catalog:" }, "dependencies": { - "@aws-sdk/client-s3": "3.808.0", - "@azure/identity": "catalog:", - "@azure/storage-blob": "catalog:", "@langchain/core": "catalog:", "@n8n/backend-common": "workspace:*", "@n8n/backend-network": "workspace:*", + "@n8n/blob-storage": "workspace:*", "@n8n/client-oauth2": "workspace:*", "@n8n/config": "workspace:*", "@n8n/constants": "workspace:*", diff --git a/packages/core/src/binary-data/__tests__/azure-blob.manager.test.ts b/packages/core/src/binary-data/__tests__/azure-blob.manager.test.ts index 9489a13bbf9..fbcf81b9422 100644 --- a/packages/core/src/binary-data/__tests__/azure-blob.manager.test.ts +++ b/packages/core/src/binary-data/__tests__/azure-blob.manager.test.ts @@ -1,7 +1,7 @@ +import { AzureBlobService } from '@n8n/blob-storage/azure-blob'; import fs from 'node:fs/promises'; import { Readable } from 'node:stream'; -import { AzureBlobService } from '@/binary-data/azure-blob/azure-blob.service.ee'; import { AzureBlobManager } from '@/binary-data/azure-blob.manager'; import type { BinaryData } from '@/binary-data/types'; import { mockInstance, toFileId, toStream } from '@test/utils'; diff --git a/packages/core/src/binary-data/__tests__/object-store.manager.test.ts b/packages/core/src/binary-data/__tests__/object-store.manager.test.ts index f7ffe0bc042..43a9e92125c 100644 --- a/packages/core/src/binary-data/__tests__/object-store.manager.test.ts +++ b/packages/core/src/binary-data/__tests__/object-store.manager.test.ts @@ -1,9 +1,9 @@ +import type { MetadataResponseHeaders } from '@n8n/blob-storage'; +import { ObjectStoreService } from '@n8n/blob-storage/object-store'; import fs from 'node:fs/promises'; import { Readable } from 'node:stream'; import { mock } from 'vitest-mock-extended'; -import { ObjectStoreService } from '@/binary-data/object-store/object-store.service.ee'; -import type { MetadataResponseHeaders } from '@/binary-data/object-store/types'; import { ObjectStoreManager } from '@/binary-data/object-store.manager'; import { mockInstance, toFileId, toStream } from '@test/utils'; diff --git a/packages/core/src/binary-data/azure-blob.manager.ts b/packages/core/src/binary-data/azure-blob.manager.ts index 8c5fbfa19cf..33b9636b018 100644 --- a/packages/core/src/binary-data/azure-blob.manager.ts +++ b/packages/core/src/binary-data/azure-blob.manager.ts @@ -1,10 +1,10 @@ import { binaryToBuffer } from '@n8n/backend-network'; +import { AzureBlobService } from '@n8n/blob-storage/azure-blob'; import { Service } from '@n8n/di'; import fs from 'node:fs/promises'; import type { Readable } from 'node:stream'; import { v4 as uuid } from 'uuid'; -import { AzureBlobService } from './azure-blob/azure-blob.service.ee'; import type { BinaryData } from './types'; @Service() diff --git a/packages/core/src/binary-data/index.ts b/packages/core/src/binary-data/index.ts index cd37a0abe0b..29dce4bb4a8 100644 --- a/packages/core/src/binary-data/index.ts +++ b/packages/core/src/binary-data/index.ts @@ -1,7 +1,4 @@ export * from './binary-data.service'; export { BinaryDataConfig } from './binary-data.config'; export type * from './types'; -// type-only: the runtime classes stay behind dynamic imports to avoid eagerly loading the S3/Azure SDKs -export type { AzureBlobService } from './azure-blob/azure-blob.service.ee'; -export type { ObjectStoreService } from './object-store/object-store.service.ee'; export { isStoredMode as isValidNonDefaultMode, FileLocation } from './utils'; diff --git a/packages/core/src/binary-data/object-store.manager.ts b/packages/core/src/binary-data/object-store.manager.ts index 12553b7dd6a..fa3d8d31356 100644 --- a/packages/core/src/binary-data/object-store.manager.ts +++ b/packages/core/src/binary-data/object-store.manager.ts @@ -1,10 +1,10 @@ import { binaryToBuffer } from '@n8n/backend-network'; +import { ObjectStoreService } from '@n8n/blob-storage/object-store'; import { Service } from '@n8n/di'; import fs from 'node:fs/promises'; import type { Readable } from 'node:stream'; import { v4 as uuid } from 'uuid'; -import { ObjectStoreService } from './object-store/object-store.service.ee'; import type { BinaryData } from './types'; @Service() diff --git a/packages/core/src/binary-data/types.ts b/packages/core/src/binary-data/types.ts index 3bb0cea5451..d0b3caecfba 100644 --- a/packages/core/src/binary-data/types.ts +++ b/packages/core/src/binary-data/types.ts @@ -1,3 +1,4 @@ +import type { BlobMetadata, PreWriteBlobMetadata } from '@n8n/blob-storage'; import type { Readable } from 'stream'; import type { BINARY_DATA_MODES } from './binary-data.config'; @@ -24,15 +25,11 @@ export namespace BinaryData { */ export type StoredMode = Exclude; - export type Metadata = { - fileName?: string; - mimeType?: string; - fileSize: number; - }; + export type Metadata = BlobMetadata; export type WriteResult = { fileId: string; fileSize: number }; - export type PreWriteMetadata = Omit; + export type PreWriteMetadata = PreWriteBlobMetadata; export type FileLocation = | { type: 'execution'; workflowId: string; executionId: string } diff --git a/packages/core/src/binary-data/utils.ts b/packages/core/src/binary-data/utils.ts index fbef8d69dab..cdd67ed5b84 100644 --- a/packages/core/src/binary-data/utils.ts +++ b/packages/core/src/binary-data/utils.ts @@ -1,6 +1,3 @@ -import { UnexpectedError } from 'n8n-workflow'; -import { Transform } from 'node:stream'; - import type { BinaryData } from './types'; export { assertDir, exists } from '@n8n/backend-common'; @@ -11,69 +8,6 @@ export function isStoredMode(mode: string): mode is BinaryData.StoredMode { return STORED_MODES.includes(mode as BinaryData.StoredMode); } -/** - * A `Transform` that re-emits its input as chunks of exactly `chunkSize` bytes, with a possibly smaller final chunk. - * `chunkSize` must be a positive integer: values `<= 0` throws an `UnexpectedError`. - * - * Between transforms the internal queue carries at most one partial chunk (< `chunkSize` bytes). - * - * Wire the upstream source into the chunker with `node:stream.pipeline()`, not plain `.pipe()`. - * `pipeline()` propagates errors from upstream to the chunker - * (so consumers see them) and propagates destroy from the chunker to upstream - * (so sockets don't dangle when the consumer aborts). - * `.pipe()` does neither. - */ -export function createFixedSizeChunker(chunkSize: number): Transform { - if (chunkSize <= 0) { - throw new UnexpectedError(`createFixedSizeChunker requires chunkSize > 0, got ${chunkSize}`); - } - - const queue: Buffer[] = []; - let queued = 0; - - const take = (size: number): Buffer => { - const out = Buffer.allocUnsafe(size); - let written = 0; - while (written < size) { - const head = queue[0]; - const need = size - written; - if (head.length <= need) { - head.copy(out, written); - written += head.length; - queue.shift(); - } else { - head.copy(out, written, 0, need); - queue[0] = head.subarray(need); - written += need; - } - } - queued -= size; - return out; - }; - - return new Transform({ - transform(chunk: Buffer, _encoding, done) { - queue.push(chunk); - queued += chunk.length; - while (queued >= chunkSize) { - this.push(take(chunkSize)); - } - done(); - }, - flush(done) { - if (queued > 0) { - this.push(take(queued)); - } - done(); - }, - destroy(error, done) { - queue.length = 0; - queued = 0; - done(error); - }, - }); -} - export const FileLocation = { ofExecution: (workflowId: string, executionId: string): BinaryData.FileLocation => ({ type: 'execution', diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 17cf24c9a95..b637983be67 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -21,6 +21,7 @@ { "path": "../@n8n/decorators/tsconfig.build.json" }, { "path": "../@n8n/backend-common/tsconfig.build.json" }, { "path": "../@n8n/backend-network/tsconfig.build.json" }, + { "path": "../@n8n/blob-storage/tsconfig.build.json" }, { "path": "../@n8n/config/tsconfig.build.json" }, { "path": "../@n8n/constants/tsconfig.build.json" }, { "path": "../@n8n/di/tsconfig.build.json" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e761d9ee863..878c434a9fb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1590,6 +1590,76 @@ importers: specifier: catalog:typescript version: 7.0.2 + packages/@n8n/blob-storage: + dependencies: + '@aws-sdk/client-s3': + specifier: 3.808.0 + version: 3.808.0 + '@azure/identity': + specifier: 4.13.0 + version: 4.13.0 + '@azure/storage-blob': + specifier: 'catalog:' + version: 12.32.0 + '@n8n/backend-common': + specifier: workspace:* + version: link:../backend-common + '@n8n/backend-network': + specifier: workspace:* + version: link:../backend-network + '@n8n/config': + specifier: workspace:* + version: link:../config + '@n8n/di': + specifier: workspace:* + version: link:../di + '@n8n/utils': + specifier: workspace:* + version: link:../utils + lodash: + specifier: 4.18.1 + version: 4.18.1 + n8n-workflow: + specifier: workspace:* + version: link:../../workflow + reflect-metadata: + specifier: 'catalog:' + version: 0.2.2 + zod: + specifier: 3.25.67 + version: 3.25.67 + devDependencies: + '@n8n/eslint-config': + specifier: workspace:* + version: link:../eslint-config + '@n8n/typescript-config': + specifier: workspace:* + version: link:../typescript-config + '@n8n/vitest-config': + specifier: workspace:* + version: link:../vitest-config + '@types/lodash': + specifier: 'catalog:' + version: 4.17.17 + '@vitest/coverage-v8': + specifier: 'catalog:' + version: 4.1.9(@vitest/browser@4.1.9)(vitest@4.1.9) + eslint: + specifier: 'catalog:' + version: 9.29.0(jiti@2.6.1) + typescript: + specifier: catalog:typescript + version: 7.0.2 + vite: + specifier: 'catalog:' + version: 8.0.2(@types/node@20.19.41)(esbuild@0.28.1)(jiti@2.6.1)(sass-embedded@1.98.0)(sass@1.98.0)(terser@5.16.1)(tsx@4.19.3)(yaml@2.8.3) + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@20.19.41)(@vitest/browser-playwright@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@20.19.41)(esbuild@0.28.1)(jiti@2.6.1)(sass-embedded@1.98.0)(sass@1.98.0)(terser@5.16.1)(tsx@4.19.3)(yaml@2.8.3)) + vitest-mock-extended: + specifier: 'catalog:' + version: 3.1.0(typescript@7.0.2)(vitest@4.1.9) + packages/@n8n/chat-hub: dependencies: '@n8n/api-types': @@ -3923,6 +3993,9 @@ importers: '@n8n/backend-network': specifier: workspace:* version: link:../@n8n/backend-network + '@n8n/blob-storage': + specifier: workspace:* + version: link:../@n8n/blob-storage '@n8n/chat-hub': specifier: workspace:* version: link:../@n8n/chat-hub @@ -4494,15 +4567,6 @@ importers: packages/core: dependencies: - '@aws-sdk/client-s3': - specifier: 3.808.0 - version: 3.808.0 - '@azure/identity': - specifier: 4.13.0 - version: 4.13.0 - '@azure/storage-blob': - specifier: 'catalog:' - version: 12.32.0 '@langchain/core': specifier: 'catalog:' version: 1.2.0(@opentelemetry/api@1.9.0)(@opentelemetry/exporter-trace-otlp-proto@0.217.0(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@2.7.1(@opentelemetry/api@1.9.0))(openai@6.46.0(@smithy/signature-v4@5.3.5)(ws@8.21.0(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.25.67))(ws@8.21.0(bufferutil@4.0.9)(utf-8-validate@5.0.10)) @@ -4512,6 +4576,9 @@ importers: '@n8n/backend-network': specifier: workspace:* version: link:../@n8n/backend-network + '@n8n/blob-storage': + specifier: workspace:* + version: link:../@n8n/blob-storage '@n8n/client-oauth2': specifier: workspace:* version: link:../@n8n/client-oauth2 @@ -30724,7 +30791,7 @@ snapshots: istanbul-lib-report: 3.0.1 istanbul-reports: 3.2.0 magicast: 0.5.2 - obug: 2.1.1 + obug: 2.1.3 std-env: 4.0.0 tinyrainbow: 3.1.0 vitest: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@20.19.41)(@vitest/browser-playwright@4.1.9)(@vitest/coverage-v8@4.1.9)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@20.19.41)(esbuild@0.28.1)(jiti@2.6.1)(sass-embedded@1.98.0)(sass@1.98.0)(terser@5.16.1)(tsx@4.19.3)(yaml@2.8.3))