From 040dcdbfc97459c28407b32a44c0c81425f81ba0 Mon Sep 17 00:00:00 2001 From: Stephen Wright Date: Fri, 14 Nov 2025 12:47:51 +0000 Subject: [PATCH] feat: Support custom encryption keys for imports / exports (#21863) --- .../export/__tests__/entities.test.ts | 34 +++++++++++++- packages/cli/src/commands/export/entities.ts | 12 +++-- .../import/__tests__/entities.test.ts | 34 ++++++++++++-- packages/cli/src/commands/import/entities.ts | 12 ++++- .../services/__tests__/export.service.test.ts | 31 ++++++++++++- .../services/__tests__/import.service.test.ts | 36 ++++++++++++++- packages/cli/src/services/export.service.ts | 41 ++++++++++++++--- packages/cli/src/services/import.service.ts | 46 +++++++++++++++---- .../src/encryption/__tests__/cipher.test.ts | 32 +++++++++++++ packages/core/src/encryption/cipher.ts | 12 ++--- 10 files changed, 256 insertions(+), 34 deletions(-) diff --git a/packages/cli/src/commands/export/__tests__/entities.test.ts b/packages/cli/src/commands/export/__tests__/entities.test.ts index edcb4ea3a04..3b747d0ed8a 100644 --- a/packages/cli/src/commands/export/__tests__/entities.test.ts +++ b/packages/cli/src/commands/export/__tests__/entities.test.ts @@ -31,6 +31,7 @@ describe('ExportEntitiesCommand', () => { 'execution_entity', 'execution_metadata', ]), + undefined, ); }); @@ -48,7 +49,38 @@ describe('ExportEntitiesCommand', () => { }; await command.run(); - expect(mockExportService.exportEntities).toHaveBeenCalledWith('./exports', new Set()); + expect(mockExportService.exportEntities).toHaveBeenCalledWith( + './exports', + new Set(), + undefined, + ); + }); + + it('should export entities with a custom encryption key', async () => { + const command = new ExportEntitiesCommand(); + // @ts-expect-error Protected property + command.flags = { + outputDir: './exports', + keyFile: './key.txt', + }; + // @ts-expect-error Protected property + command.logger = { + info: jest.fn(), + error: jest.fn(), + }; + await command.run(); + + expect(mockExportService.exportEntities).toHaveBeenCalledWith( + './exports', + new Set([ + 'execution_annotation_tags', + 'execution_annotations', + 'execution_data', + 'execution_entity', + 'execution_metadata', + ]), + 'key.txt', + ); }); }); diff --git a/packages/cli/src/commands/export/entities.ts b/packages/cli/src/commands/export/entities.ts index 2fb2c1800e2..e770ab3595c 100644 --- a/packages/cli/src/commands/export/entities.ts +++ b/packages/cli/src/commands/export/entities.ts @@ -11,12 +11,16 @@ const flagsSchema = z.object({ .string() .describe('Output directory path') .default(safeJoinPath(__dirname, './outputs')), - includeExecutionHistoryDataTables: z + includeExecutionHistoryDataTables: z.coerce .boolean() .describe( 'Include execution history data tables, these are excluded by default as they can be very large', ) .default(false), + keyFile: z + .string() + .describe('Optional path to a file containing a custom encryption key') + .optional(), }); @Command({ @@ -27,14 +31,16 @@ const flagsSchema = z.object({ '--outputDir=./exports', '--outputDir=/path/to/backup', '--includeExecutionHistoryDataTables=true', + '--keyFile=/path/to/key.txt', + '--outputDir=./exports --keyFile=/path/to/key.txt', ], flagsSchema, }) export class ExportEntitiesCommand extends BaseCommand> { async run() { const outputDir = this.flags.outputDir; - const excludedDataTables = new Set(); + const keyFilePath = this.flags.keyFile ? safeJoinPath(this.flags.keyFile) : undefined; if (!this.flags.includeExecutionHistoryDataTables) { excludedDataTables.add('execution_annotation_tags'); @@ -44,7 +50,7 @@ export class ExportEntitiesCommand extends BaseCommand { await command.run(); // Verify service call with transaction-based approach - expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false); + expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false, undefined); }); it('should import entities with custom input directory', async () => { @@ -51,7 +51,11 @@ describe('ImportEntitiesCommand', () => { await command.run(); - expect(mockImportService.importEntities).toHaveBeenCalledWith('/custom/path', false); + expect(mockImportService.importEntities).toHaveBeenCalledWith( + '/custom/path', + false, + undefined, + ); }); it('should truncate tables when truncateTables flag is true', async () => { @@ -72,7 +76,27 @@ describe('ImportEntitiesCommand', () => { await command.run(); // Verify service call with truncation enabled - expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true); + expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true, undefined); + }); + + it('should import entities with a custom encryption key', async () => { + const command = new ImportEntitiesCommand(); + // @ts-expect-error Protected property + command.flags = { + inputDir: './outputs', + truncateTables: false, + keyFile: './key.txt', + }; + + // @ts-expect-error Protected property + command.logger = { + info: jest.fn(), + error: jest.fn(), + }; + + await command.run(); + + expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false, 'key.txt'); }); it('should handle service errors gracefully', async () => { @@ -93,7 +117,7 @@ describe('ImportEntitiesCommand', () => { await expect(command.run()).rejects.toThrow('Database connection failed'); // Verify service was called - expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false); + expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false, undefined); }); it('should handle import errors with transactions', async () => { @@ -114,7 +138,7 @@ describe('ImportEntitiesCommand', () => { await expect(command.run()).rejects.toThrow('Transaction failed'); // Verify service was called with truncation enabled - expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true); + expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true, undefined); }); }); diff --git a/packages/cli/src/commands/import/entities.ts b/packages/cli/src/commands/import/entities.ts index 1caf89029b1..6ebab60c8e9 100644 --- a/packages/cli/src/commands/import/entities.ts +++ b/packages/cli/src/commands/import/entities.ts @@ -4,13 +4,18 @@ import { Container } from '@n8n/di'; import { BaseCommand } from '../base-command'; import { ImportService } from '../../services/import.service'; +import { safeJoinPath } from '@n8n/backend-common'; const flagsSchema = z.object({ inputDir: z .string() .describe('Input directory that holds output files for import') .default('./outputs'), - truncateTables: z.boolean().describe('Truncate tables before import').default(false), + truncateTables: z.coerce.boolean().describe('Truncate tables before import').default(false), + keyFile: z + .string() + .describe('Optional path to a file containing a custom encryption key') + .optional(), }); @Command({ @@ -22,6 +27,8 @@ const flagsSchema = z.object({ '--inputDir=/path/to/backup', '--truncateTables', '--inputDir=./exports --truncateTables', + '--keyFile=/path/to/key.txt', + '--inputDir=./exports --keyFile=/path/to/key.txt', ], flagsSchema, }) @@ -29,13 +36,14 @@ export class ImportEntitiesCommand extends BaseCommand ({ rm: jest.fn(), readdir: jest.fn(), appendFile: jest.fn(), + readFile: jest.fn(), })); // Mock compression utility @@ -124,6 +125,34 @@ describe('ExportService', () => { expect(mockLogger.info).toHaveBeenCalledWith('✅ Task completed successfully! \n'); }); + it('should export entities successfully with a custom encryption key', async () => { + const outputDir = '/test/output'; + const mockEntities = [ + { id: 1, email: 'test1@example.com', firstName: 'John' }, + { id: 2, email: 'test2@example.com', firstName: 'Jane' }, + ]; + + // Mock the migrations table query to fail (table doesn't exist) + jest + .mocked(mockDataSource.query) + .mockImplementationOnce(async (query: string) => { + if (query.includes('migrations') && query.includes('COUNT')) { + throw new Error('Table not found'); + } + return []; + }) + .mockResolvedValueOnce(mockEntities) // First entity (User) + .mockResolvedValueOnce([]); // Workflow entities + jest.mocked(readdir).mockResolvedValue([]); + jest.mocked(readFile).mockResolvedValueOnce('custom-encryption-key'); + + await exportService.exportEntities(outputDir, undefined, 'custom-encryption-key'); + + expect(mockCipher.encrypt).toHaveBeenCalledWith(expect.any(String), 'custom-encryption-key'); + expect(appendFile).toHaveBeenCalledWith(expect.any(String), expect.any(String), 'utf8'); + expect(mockLogger.info).toHaveBeenCalledWith('✅ Task completed successfully! \n'); + }); + it('should handle multiple pages of data', async () => { const outputDir = '/test/output'; const mockEntities = Array.from({ length: 500 }, (_, i) => ({ diff --git a/packages/cli/src/services/__tests__/import.service.test.ts b/packages/cli/src/services/__tests__/import.service.test.ts index 9678076110a..0f6b9ae8c28 100644 --- a/packages/cli/src/services/__tests__/import.service.test.ts +++ b/packages/cli/src/services/__tests__/import.service.test.ts @@ -351,8 +351,8 @@ describe('ImportService', () => { { id: 1, name: 'Test' }, { id: 2, name: 'Test2' }, ]); - expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":1,"name":"Test"}'); - expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":2,"name":"Test2"}'); + expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":1,"name":"Test"}', undefined); + expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":2,"name":"Test2"}', undefined); }); it('should handle empty lines in JSONL file', async () => { @@ -489,6 +489,38 @@ describe('ImportService', () => { expect(readFile).toHaveBeenCalledWith('/test/input/user.jsonl', 'utf8'); expect(mockEntityManager.insert).not.toHaveBeenCalled(); }); + + it('should import entities with a custom encryption key', async () => { + const importMetadata = { + entityFiles: { + user: ['/test/input/user.jsonl'], + }, + tableNames: ['user'], + }; + + mockDataSource.driver.escapeQueryWithParameters = jest + .fn() + .mockReturnValue(['INSERT COMMAND', { data: 'data' }]); + + const mockEntities = [{ id: 1, name: 'Test User' }]; + const mockContent = JSON.stringify(mockEntities[0]); + jest.mocked(readFile).mockResolvedValue(mockContent); + + await importService.importEntitiesFromFiles( + '/test/input', + mockEntityManager, + Object.keys(importMetadata.entityFiles), + importMetadata.entityFiles, + 'custom-encryption-key', + ); + + expect(mockCipher.decrypt).toHaveBeenCalledWith( + '{"id":1,"name":"Test User"}', + 'custom-encryption-key', + ); + expect(readFile).toHaveBeenCalledWith('/test/input/user.jsonl', 'utf8'); + expect(mockEntityManager.query).toHaveBeenCalledWith('INSERT COMMAND', { data: 'data' }); + }); }); describe('disableForeignKeyConstraints', () => { diff --git a/packages/cli/src/services/export.service.ts b/packages/cli/src/services/export.service.ts index ffdb8950b2a..7b9e2e13ca8 100644 --- a/packages/cli/src/services/export.service.ts +++ b/packages/cli/src/services/export.service.ts @@ -1,5 +1,5 @@ import { Logger, safeJoinPath } from '@n8n/backend-common'; -import { mkdir, rm, readdir, appendFile } from 'fs/promises'; +import { mkdir, rm, readdir, appendFile, readFile } from 'fs/promises'; import { Service } from '@n8n/di'; @@ -34,7 +34,10 @@ export class ExportService { } } - private async exportMigrationsTable(outputDir: string): Promise { + private async exportMigrationsTable( + outputDir: string, + customEncryptionKey?: string, + ): Promise { this.logger.info('\n🔧 Exporting migrations table:'); this.logger.info('=============================='); @@ -66,7 +69,11 @@ export class ExportService { const migrationsJsonl: string = allMigrations .map((migration: unknown) => JSON.stringify(migration)) .join('\n'); - await appendFile(filePath, this.cipher.encrypt(migrationsJsonl ?? '' + '\n'), 'utf8'); + await appendFile( + filePath, + this.cipher.encrypt(migrationsJsonl ?? '' + '\n', customEncryptionKey), + 'utf8', + ); this.logger.info( ` ✅ Completed export for ${migrationsTableName}: ${allMigrations.length} entities in 1 file`, @@ -83,7 +90,11 @@ export class ExportService { return systemTablesExported; } - async exportEntities(outputDir: string, excludedTables: Set = new Set()) { + async exportEntities( + outputDir: string, + excludedTables: Set = new Set(), + keyFilePath?: string, + ) { this.logger.info('\n⚠️⚠️ This feature is currently under development. ⚠️⚠️'); validateDbTypeForExportEntities(this.dataSource.options.type); @@ -91,6 +102,20 @@ export class ExportService { this.logger.info('\n🚀 Starting entity export...'); this.logger.info(`📁 Output directory: ${outputDir}`); + // Read custom encryption key from file if provided + let customEncryptionKey: string | undefined; + if (keyFilePath) { + try { + const keyFileContent = await readFile(keyFilePath, 'utf8'); + customEncryptionKey = keyFileContent.trim(); + this.logger.info(`🔑 Using custom encryption key from: ${keyFilePath}`); + } catch (error) { + throw new Error( + `Failed to read encryption key file at ${keyFilePath}: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + } + await rm(outputDir, { recursive: true }).catch(() => {}); // Ensure output directory exists await mkdir(outputDir, { recursive: true }); @@ -106,7 +131,7 @@ export class ExportService { const pageSize = 500; const entitiesPerFile = 500; - await this.exportMigrationsTable(outputDir); + await this.exportMigrationsTable(outputDir, customEncryptionKey); for (const metadata of entityMetadatas) { // Get table name and entity name @@ -171,7 +196,11 @@ export class ExportService { const entitiesJsonl: string = pageEntities .map((entity: unknown) => JSON.stringify(entity)) .join('\n'); - await appendFile(filePath, this.cipher.encrypt(entitiesJsonl) + '\n', 'utf8'); + await appendFile( + filePath, + this.cipher.encrypt(entitiesJsonl, customEncryptionKey) + '\n', + 'utf8', + ); totalEntityCount += pageEntities.length; currentFileEntityCount += pageEntities.length; diff --git a/packages/cli/src/services/import.service.ts b/packages/cli/src/services/import.service.ts index 8f337763a6d..b24d3108199 100644 --- a/packages/cli/src/services/import.service.ts +++ b/packages/cli/src/services/import.service.ts @@ -262,15 +262,19 @@ export class ImportService { /** * Read and parse JSONL file content * @param filePath - Path to the JSONL file + * @param customEncryptionKey - Optional custom encryption key * @returns Array of parsed entity objects */ - async readEntityFile(filePath: string): Promise>> { + async readEntityFile( + filePath: string, + customEncryptionKey?: string, + ): Promise>> { const content = await readFile(filePath, 'utf8'); const entities: Record[] = []; const entitySchema = z.record(z.string(), z.unknown()); for (const block of content.split('\n')) { - const lines = this.cipher.decrypt(block).split(/\r?\n/); + const lines = this.cipher.decrypt(block, customEncryptionKey).split(/\r?\n/); for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); @@ -307,11 +311,25 @@ export class ImportService { this.logger.info('✅ Successfully decompressed entities.zip'); } - async importEntities(inputDir: string, truncateTables: boolean) { + async importEntities(inputDir: string, truncateTables: boolean, keyFilePath?: string) { validateDbTypeForImportEntities(this.dataSource.options.type); + // Read custom encryption key from file if provided + let customEncryptionKey: string | undefined; + if (keyFilePath) { + try { + const keyFileContent = await readFile(keyFilePath, 'utf8'); + customEncryptionKey = keyFileContent.trim(); + this.logger.info(`🔑 Using custom encryption key from: ${keyFilePath}`); + } catch (error) { + throw new Error( + `Failed to read encryption key file at ${keyFilePath}: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + } + await this.decompressEntitiesZip(inputDir); - await this.validateMigrations(inputDir); + await this.validateMigrations(inputDir, customEncryptionKey); await this.dataSource.transaction(async (transactionManager: EntityManager) => { await this.disableForeignKeyConstraints(transactionManager); @@ -343,7 +361,13 @@ export class ImportService { } // Import entities from the specified directory - await this.importEntitiesFromFiles(inputDir, transactionManager, entityNames, entityFiles); + await this.importEntitiesFromFiles( + inputDir, + transactionManager, + entityNames, + entityFiles, + customEncryptionKey, + ); await this.enableForeignKeyConstraints(transactionManager); }); @@ -366,6 +390,7 @@ export class ImportService { * @param transactionManager - TypeORM transaction manager * @param entityNames - Array of entity names to import * @param entityFiles - Record of entity names to their file paths + * @param customEncryptionKey - Optional custom encryption key * @returns Promise that resolves when all entities are imported */ async importEntitiesFromFiles( @@ -373,6 +398,7 @@ export class ImportService { transactionManager: EntityManager, entityNames: string[], entityFiles: Record, + customEncryptionKey?: string, ): Promise { this.logger.info(`\n🚀 Starting entity import from directory: ${inputDir}`); @@ -408,7 +434,10 @@ export class ImportService { files.map(async (filePath) => { this.logger.info(` 📁 Reading file: ${filePath}`); - const entities: Array> = await this.readEntityFile(filePath); + const entities: Array> = await this.readEntityFile( + filePath, + customEncryptionKey, + ); this.logger.info(` Found ${entities.length} entities`); await Promise.all( @@ -493,9 +522,10 @@ export class ImportService { /** * Validates that the migrations in the import data match the target database * @param inputDir - Directory containing exported entity files + * @param customEncryptionKey - Optional custom encryption key * @returns Promise that resolves if migrations match, throws error if they don't */ - async validateMigrations(inputDir: string): Promise { + async validateMigrations(inputDir: string, customEncryptionKey?: string): Promise { const migrationsFilePath = safeJoinPath(inputDir, 'migrations.jsonl'); try { @@ -510,7 +540,7 @@ export class ImportService { // Read and parse migrations from file const migrationsFileContent = await readFile(migrationsFilePath, 'utf8'); const importMigrations = this.cipher - .decrypt(migrationsFileContent) + .decrypt(migrationsFileContent, customEncryptionKey) .trim() .split('\n') .filter((line) => line.trim()) diff --git a/packages/core/src/encryption/__tests__/cipher.test.ts b/packages/core/src/encryption/__tests__/cipher.test.ts index c1e14a9be07..7c94f7ab58b 100644 --- a/packages/core/src/encryption/__tests__/cipher.test.ts +++ b/packages/core/src/encryption/__tests__/cipher.test.ts @@ -34,4 +34,36 @@ describe('Cipher', () => { expect(decrypted).toEqual(''); }); }); + + describe('getKeyAndIv', () => { + it('should generate a key and iv using instance settings encryption key', () => { + const salt = Buffer.from('test-salt'); + mockInstance(InstanceSettings, { encryptionKey: 'settings-encryption-key' }); + // Clear the cached Cipher instance to get a new one with the new mock + Container.set(Cipher, new Cipher(Container.get(InstanceSettings))); + const testCipher = Container.get(Cipher); + const bufferFromSpy = jest.spyOn(Buffer, 'from'); + // @ts-expect-error - getKeyAndIv is private + const [key, iv] = testCipher.getKeyAndIv(salt); + expect(key).toBeInstanceOf(Buffer); + expect(iv).toBeInstanceOf(Buffer); + expect(bufferFromSpy).toHaveBeenCalledWith('settings-encryption-key', 'binary'); + bufferFromSpy.mockRestore(); + }); + + it('should generate a key and iv using custom encryption key', () => { + const salt = Buffer.from('test-salt'); + mockInstance(InstanceSettings, { encryptionKey: 'settings-encryption-key' }); + // Clear the cached Cipher instance to get a new one with the new mock + Container.set(Cipher, new Cipher(Container.get(InstanceSettings))); + const testCipher = Container.get(Cipher); + const bufferFromSpy = jest.spyOn(Buffer, 'from'); + // @ts-expect-error - getKeyAndIv is private + const [key, iv] = testCipher.getKeyAndIv(salt, 'custom-encryption-key'); + expect(key).toBeInstanceOf(Buffer); + expect(iv).toBeInstanceOf(Buffer); + expect(bufferFromSpy).toHaveBeenCalledWith('custom-encryption-key', 'binary'); + bufferFromSpy.mockRestore(); + }); + }); }); diff --git a/packages/core/src/encryption/cipher.ts b/packages/core/src/encryption/cipher.ts index 248ca0317b6..8af9adc15c7 100644 --- a/packages/core/src/encryption/cipher.ts +++ b/packages/core/src/encryption/cipher.ts @@ -10,26 +10,26 @@ const RANDOM_BYTES = Buffer.from('53616c7465645f5f', 'hex'); export class Cipher { constructor(private readonly instanceSettings: InstanceSettings) {} - encrypt(data: string | object) { + encrypt(data: string | object, customEncryptionKey?: string) { const salt = randomBytes(8); - const [key, iv] = this.getKeyAndIv(salt); + const [key, iv] = this.getKeyAndIv(salt, customEncryptionKey); const cipher = createCipheriv('aes-256-cbc', key, iv); const encrypted = cipher.update(typeof data === 'string' ? data : JSON.stringify(data)); return Buffer.concat([RANDOM_BYTES, salt, encrypted, cipher.final()]).toString('base64'); } - decrypt(data: string) { + decrypt(data: string, customEncryptionKey?: string) { const input = Buffer.from(data, 'base64'); if (input.length < 16) return ''; const salt = input.subarray(8, 16); - const [key, iv] = this.getKeyAndIv(salt); + const [key, iv] = this.getKeyAndIv(salt, customEncryptionKey); const contents = input.subarray(16); const decipher = createDecipheriv('aes-256-cbc', key, iv); return Buffer.concat([decipher.update(contents), decipher.final()]).toString('utf-8'); } - private getKeyAndIv(salt: Buffer): [Buffer, Buffer] { - const { encryptionKey } = this.instanceSettings; + private getKeyAndIv(salt: Buffer, customEncryptionKey?: string): [Buffer, Buffer] { + const encryptionKey = customEncryptionKey ?? this.instanceSettings.encryptionKey; const password = Buffer.concat([Buffer.from(encryptionKey, 'binary'), salt]); const hash1 = createHash('md5').update(password).digest(); const hash2 = createHash('md5')