feat: Support custom encryption keys for imports / exports (#21863)

This commit is contained in:
Stephen Wright 2025-11-14 12:47:51 +00:00 committed by GitHub
parent 99b232ca5a
commit 040dcdbfc9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 256 additions and 34 deletions

View File

@ -31,6 +31,7 @@ describe('ExportEntitiesCommand', () => {
'execution_entity',
'execution_metadata',
]),
undefined,
);
});
@ -48,7 +49,38 @@ describe('ExportEntitiesCommand', () => {
};
await command.run();
expect(mockExportService.exportEntities).toHaveBeenCalledWith('./exports', new Set<string>());
expect(mockExportService.exportEntities).toHaveBeenCalledWith(
'./exports',
new Set<string>(),
undefined,
);
});
it('should export entities with a custom encryption key', async () => {
const command = new ExportEntitiesCommand();
// @ts-expect-error Protected property
command.flags = {
outputDir: './exports',
keyFile: './key.txt',
};
// @ts-expect-error Protected property
command.logger = {
info: jest.fn(),
error: jest.fn(),
};
await command.run();
expect(mockExportService.exportEntities).toHaveBeenCalledWith(
'./exports',
new Set<string>([
'execution_annotation_tags',
'execution_annotations',
'execution_data',
'execution_entity',
'execution_metadata',
]),
'key.txt',
);
});
});

View File

@ -11,12 +11,16 @@ const flagsSchema = z.object({
.string()
.describe('Output directory path')
.default(safeJoinPath(__dirname, './outputs')),
includeExecutionHistoryDataTables: z
includeExecutionHistoryDataTables: z.coerce
.boolean()
.describe(
'Include execution history data tables, these are excluded by default as they can be very large',
)
.default(false),
keyFile: z
.string()
.describe('Optional path to a file containing a custom encryption key')
.optional(),
});
@Command({
@ -27,14 +31,16 @@ const flagsSchema = z.object({
'--outputDir=./exports',
'--outputDir=/path/to/backup',
'--includeExecutionHistoryDataTables=true',
'--keyFile=/path/to/key.txt',
'--outputDir=./exports --keyFile=/path/to/key.txt',
],
flagsSchema,
})
export class ExportEntitiesCommand extends BaseCommand<z.infer<typeof flagsSchema>> {
async run() {
const outputDir = this.flags.outputDir;
const excludedDataTables = new Set<string>();
const keyFilePath = this.flags.keyFile ? safeJoinPath(this.flags.keyFile) : undefined;
if (!this.flags.includeExecutionHistoryDataTables) {
excludedDataTables.add('execution_annotation_tags');
@ -44,7 +50,7 @@ export class ExportEntitiesCommand extends BaseCommand<z.infer<typeof flagsSchem
excludedDataTables.add('execution_metadata');
}
await Container.get(ExportService).exportEntities(outputDir, excludedDataTables);
await Container.get(ExportService).exportEntities(outputDir, excludedDataTables, keyFilePath);
}
catch(error: Error) {

View File

@ -31,7 +31,7 @@ describe('ImportEntitiesCommand', () => {
await command.run();
// Verify service call with transaction-based approach
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false);
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false, undefined);
});
it('should import entities with custom input directory', async () => {
@ -51,7 +51,11 @@ describe('ImportEntitiesCommand', () => {
await command.run();
expect(mockImportService.importEntities).toHaveBeenCalledWith('/custom/path', false);
expect(mockImportService.importEntities).toHaveBeenCalledWith(
'/custom/path',
false,
undefined,
);
});
it('should truncate tables when truncateTables flag is true', async () => {
@ -72,7 +76,27 @@ describe('ImportEntitiesCommand', () => {
await command.run();
// Verify service call with truncation enabled
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true);
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true, undefined);
});
it('should import entities with a custom encryption key', async () => {
const command = new ImportEntitiesCommand();
// @ts-expect-error Protected property
command.flags = {
inputDir: './outputs',
truncateTables: false,
keyFile: './key.txt',
};
// @ts-expect-error Protected property
command.logger = {
info: jest.fn(),
error: jest.fn(),
};
await command.run();
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false, 'key.txt');
});
it('should handle service errors gracefully', async () => {
@ -93,7 +117,7 @@ describe('ImportEntitiesCommand', () => {
await expect(command.run()).rejects.toThrow('Database connection failed');
// Verify service was called
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false);
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', false, undefined);
});
it('should handle import errors with transactions', async () => {
@ -114,7 +138,7 @@ describe('ImportEntitiesCommand', () => {
await expect(command.run()).rejects.toThrow('Transaction failed');
// Verify service was called with truncation enabled
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true);
expect(mockImportService.importEntities).toHaveBeenCalledWith('./outputs', true, undefined);
});
});

View File

@ -4,13 +4,18 @@ import { Container } from '@n8n/di';
import { BaseCommand } from '../base-command';
import { ImportService } from '../../services/import.service';
import { safeJoinPath } from '@n8n/backend-common';
const flagsSchema = z.object({
inputDir: z
.string()
.describe('Input directory that holds output files for import')
.default('./outputs'),
truncateTables: z.boolean().describe('Truncate tables before import').default(false),
truncateTables: z.coerce.boolean().describe('Truncate tables before import').default(false),
keyFile: z
.string()
.describe('Optional path to a file containing a custom encryption key')
.optional(),
});
@Command({
@ -22,6 +27,8 @@ const flagsSchema = z.object({
'--inputDir=/path/to/backup',
'--truncateTables',
'--inputDir=./exports --truncateTables',
'--keyFile=/path/to/key.txt',
'--inputDir=./exports --keyFile=/path/to/key.txt',
],
flagsSchema,
})
@ -29,13 +36,14 @@ export class ImportEntitiesCommand extends BaseCommand<z.infer<typeof flagsSchem
async run() {
const inputDir = this.flags.inputDir;
const truncateTables = this.flags.truncateTables;
const keyFilePath = this.flags.keyFile ? safeJoinPath(this.flags.keyFile) : undefined;
this.logger.info('\n⚠ This feature is currently under development. ⚠️⚠️');
this.logger.info('\n🚀 Starting entity import...');
this.logger.info(`📁 Input directory: ${inputDir}`);
this.logger.info(`🗑️ Truncate tables: ${truncateTables}`);
await Container.get(ImportService).importEntities(inputDir, truncateTables);
await Container.get(ImportService).importEntities(inputDir, truncateTables, keyFilePath);
}
catch(error: Error) {

View File

@ -1,7 +1,7 @@
import { type Logger } from '@n8n/backend-common';
import { ExportService } from '../export.service';
import { type DataSource } from '@n8n/typeorm';
import { mkdir, rm, readdir, appendFile } from 'fs/promises';
import { mkdir, rm, readdir, appendFile, readFile } from 'fs/promises';
import { mock } from 'jest-mock-extended';
import type { Cipher } from 'n8n-core';
@ -11,6 +11,7 @@ jest.mock('fs/promises', () => ({
rm: jest.fn(),
readdir: jest.fn(),
appendFile: jest.fn(),
readFile: jest.fn(),
}));
// Mock compression utility
@ -124,6 +125,34 @@ describe('ExportService', () => {
expect(mockLogger.info).toHaveBeenCalledWith('✅ Task completed successfully! \n');
});
it('should export entities successfully with a custom encryption key', async () => {
const outputDir = '/test/output';
const mockEntities = [
{ id: 1, email: 'test1@example.com', firstName: 'John' },
{ id: 2, email: 'test2@example.com', firstName: 'Jane' },
];
// Mock the migrations table query to fail (table doesn't exist)
jest
.mocked(mockDataSource.query)
.mockImplementationOnce(async (query: string) => {
if (query.includes('migrations') && query.includes('COUNT')) {
throw new Error('Table not found');
}
return [];
})
.mockResolvedValueOnce(mockEntities) // First entity (User)
.mockResolvedValueOnce([]); // Workflow entities
jest.mocked(readdir).mockResolvedValue([]);
jest.mocked(readFile).mockResolvedValueOnce('custom-encryption-key');
await exportService.exportEntities(outputDir, undefined, 'custom-encryption-key');
expect(mockCipher.encrypt).toHaveBeenCalledWith(expect.any(String), 'custom-encryption-key');
expect(appendFile).toHaveBeenCalledWith(expect.any(String), expect.any(String), 'utf8');
expect(mockLogger.info).toHaveBeenCalledWith('✅ Task completed successfully! \n');
});
it('should handle multiple pages of data', async () => {
const outputDir = '/test/output';
const mockEntities = Array.from({ length: 500 }, (_, i) => ({

View File

@ -351,8 +351,8 @@ describe('ImportService', () => {
{ id: 1, name: 'Test' },
{ id: 2, name: 'Test2' },
]);
expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":1,"name":"Test"}');
expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":2,"name":"Test2"}');
expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":1,"name":"Test"}', undefined);
expect(mockCipher.decrypt).toHaveBeenCalledWith('{"id":2,"name":"Test2"}', undefined);
});
it('should handle empty lines in JSONL file', async () => {
@ -489,6 +489,38 @@ describe('ImportService', () => {
expect(readFile).toHaveBeenCalledWith('/test/input/user.jsonl', 'utf8');
expect(mockEntityManager.insert).not.toHaveBeenCalled();
});
it('should import entities with a custom encryption key', async () => {
const importMetadata = {
entityFiles: {
user: ['/test/input/user.jsonl'],
},
tableNames: ['user'],
};
mockDataSource.driver.escapeQueryWithParameters = jest
.fn()
.mockReturnValue(['INSERT COMMAND', { data: 'data' }]);
const mockEntities = [{ id: 1, name: 'Test User' }];
const mockContent = JSON.stringify(mockEntities[0]);
jest.mocked(readFile).mockResolvedValue(mockContent);
await importService.importEntitiesFromFiles(
'/test/input',
mockEntityManager,
Object.keys(importMetadata.entityFiles),
importMetadata.entityFiles,
'custom-encryption-key',
);
expect(mockCipher.decrypt).toHaveBeenCalledWith(
'{"id":1,"name":"Test User"}',
'custom-encryption-key',
);
expect(readFile).toHaveBeenCalledWith('/test/input/user.jsonl', 'utf8');
expect(mockEntityManager.query).toHaveBeenCalledWith('INSERT COMMAND', { data: 'data' });
});
});
describe('disableForeignKeyConstraints', () => {

View File

@ -1,5 +1,5 @@
import { Logger, safeJoinPath } from '@n8n/backend-common';
import { mkdir, rm, readdir, appendFile } from 'fs/promises';
import { mkdir, rm, readdir, appendFile, readFile } from 'fs/promises';
import { Service } from '@n8n/di';
@ -34,7 +34,10 @@ export class ExportService {
}
}
private async exportMigrationsTable(outputDir: string): Promise<number> {
private async exportMigrationsTable(
outputDir: string,
customEncryptionKey?: string,
): Promise<number> {
this.logger.info('\n🔧 Exporting migrations table:');
this.logger.info('==============================');
@ -66,7 +69,11 @@ export class ExportService {
const migrationsJsonl: string = allMigrations
.map((migration: unknown) => JSON.stringify(migration))
.join('\n');
await appendFile(filePath, this.cipher.encrypt(migrationsJsonl ?? '' + '\n'), 'utf8');
await appendFile(
filePath,
this.cipher.encrypt(migrationsJsonl ?? '' + '\n', customEncryptionKey),
'utf8',
);
this.logger.info(
` ✅ Completed export for ${migrationsTableName}: ${allMigrations.length} entities in 1 file`,
@ -83,7 +90,11 @@ export class ExportService {
return systemTablesExported;
}
async exportEntities(outputDir: string, excludedTables: Set<string> = new Set()) {
async exportEntities(
outputDir: string,
excludedTables: Set<string> = new Set(),
keyFilePath?: string,
) {
this.logger.info('\n⚠ This feature is currently under development. ⚠️⚠️');
validateDbTypeForExportEntities(this.dataSource.options.type);
@ -91,6 +102,20 @@ export class ExportService {
this.logger.info('\n🚀 Starting entity export...');
this.logger.info(`📁 Output directory: ${outputDir}`);
// Read custom encryption key from file if provided
let customEncryptionKey: string | undefined;
if (keyFilePath) {
try {
const keyFileContent = await readFile(keyFilePath, 'utf8');
customEncryptionKey = keyFileContent.trim();
this.logger.info(`🔑 Using custom encryption key from: ${keyFilePath}`);
} catch (error) {
throw new Error(
`Failed to read encryption key file at ${keyFilePath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
await rm(outputDir, { recursive: true }).catch(() => {});
// Ensure output directory exists
await mkdir(outputDir, { recursive: true });
@ -106,7 +131,7 @@ export class ExportService {
const pageSize = 500;
const entitiesPerFile = 500;
await this.exportMigrationsTable(outputDir);
await this.exportMigrationsTable(outputDir, customEncryptionKey);
for (const metadata of entityMetadatas) {
// Get table name and entity name
@ -171,7 +196,11 @@ export class ExportService {
const entitiesJsonl: string = pageEntities
.map((entity: unknown) => JSON.stringify(entity))
.join('\n');
await appendFile(filePath, this.cipher.encrypt(entitiesJsonl) + '\n', 'utf8');
await appendFile(
filePath,
this.cipher.encrypt(entitiesJsonl, customEncryptionKey) + '\n',
'utf8',
);
totalEntityCount += pageEntities.length;
currentFileEntityCount += pageEntities.length;

View File

@ -262,15 +262,19 @@ export class ImportService {
/**
* Read and parse JSONL file content
* @param filePath - Path to the JSONL file
* @param customEncryptionKey - Optional custom encryption key
* @returns Array of parsed entity objects
*/
async readEntityFile(filePath: string): Promise<Array<Record<string, unknown>>> {
async readEntityFile(
filePath: string,
customEncryptionKey?: string,
): Promise<Array<Record<string, unknown>>> {
const content = await readFile(filePath, 'utf8');
const entities: Record<string, unknown>[] = [];
const entitySchema = z.record(z.string(), z.unknown());
for (const block of content.split('\n')) {
const lines = this.cipher.decrypt(block).split(/\r?\n/);
const lines = this.cipher.decrypt(block, customEncryptionKey).split(/\r?\n/);
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
@ -307,11 +311,25 @@ export class ImportService {
this.logger.info('✅ Successfully decompressed entities.zip');
}
async importEntities(inputDir: string, truncateTables: boolean) {
async importEntities(inputDir: string, truncateTables: boolean, keyFilePath?: string) {
validateDbTypeForImportEntities(this.dataSource.options.type);
// Read custom encryption key from file if provided
let customEncryptionKey: string | undefined;
if (keyFilePath) {
try {
const keyFileContent = await readFile(keyFilePath, 'utf8');
customEncryptionKey = keyFileContent.trim();
this.logger.info(`🔑 Using custom encryption key from: ${keyFilePath}`);
} catch (error) {
throw new Error(
`Failed to read encryption key file at ${keyFilePath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
);
}
}
await this.decompressEntitiesZip(inputDir);
await this.validateMigrations(inputDir);
await this.validateMigrations(inputDir, customEncryptionKey);
await this.dataSource.transaction(async (transactionManager: EntityManager) => {
await this.disableForeignKeyConstraints(transactionManager);
@ -343,7 +361,13 @@ export class ImportService {
}
// Import entities from the specified directory
await this.importEntitiesFromFiles(inputDir, transactionManager, entityNames, entityFiles);
await this.importEntitiesFromFiles(
inputDir,
transactionManager,
entityNames,
entityFiles,
customEncryptionKey,
);
await this.enableForeignKeyConstraints(transactionManager);
});
@ -366,6 +390,7 @@ export class ImportService {
* @param transactionManager - TypeORM transaction manager
* @param entityNames - Array of entity names to import
* @param entityFiles - Record of entity names to their file paths
* @param customEncryptionKey - Optional custom encryption key
* @returns Promise that resolves when all entities are imported
*/
async importEntitiesFromFiles(
@ -373,6 +398,7 @@ export class ImportService {
transactionManager: EntityManager,
entityNames: string[],
entityFiles: Record<string, string[]>,
customEncryptionKey?: string,
): Promise<void> {
this.logger.info(`\n🚀 Starting entity import from directory: ${inputDir}`);
@ -408,7 +434,10 @@ export class ImportService {
files.map(async (filePath) => {
this.logger.info(` 📁 Reading file: ${filePath}`);
const entities: Array<Record<string, unknown>> = await this.readEntityFile(filePath);
const entities: Array<Record<string, unknown>> = await this.readEntityFile(
filePath,
customEncryptionKey,
);
this.logger.info(` Found ${entities.length} entities`);
await Promise.all(
@ -493,9 +522,10 @@ export class ImportService {
/**
* Validates that the migrations in the import data match the target database
* @param inputDir - Directory containing exported entity files
* @param customEncryptionKey - Optional custom encryption key
* @returns Promise that resolves if migrations match, throws error if they don't
*/
async validateMigrations(inputDir: string): Promise<void> {
async validateMigrations(inputDir: string, customEncryptionKey?: string): Promise<void> {
const migrationsFilePath = safeJoinPath(inputDir, 'migrations.jsonl');
try {
@ -510,7 +540,7 @@ export class ImportService {
// Read and parse migrations from file
const migrationsFileContent = await readFile(migrationsFilePath, 'utf8');
const importMigrations = this.cipher
.decrypt(migrationsFileContent)
.decrypt(migrationsFileContent, customEncryptionKey)
.trim()
.split('\n')
.filter((line) => line.trim())

View File

@ -34,4 +34,36 @@ describe('Cipher', () => {
expect(decrypted).toEqual('');
});
});
describe('getKeyAndIv', () => {
it('should generate a key and iv using instance settings encryption key', () => {
const salt = Buffer.from('test-salt');
mockInstance(InstanceSettings, { encryptionKey: 'settings-encryption-key' });
// Clear the cached Cipher instance to get a new one with the new mock
Container.set(Cipher, new Cipher(Container.get(InstanceSettings)));
const testCipher = Container.get(Cipher);
const bufferFromSpy = jest.spyOn(Buffer, 'from');
// @ts-expect-error - getKeyAndIv is private
const [key, iv] = testCipher.getKeyAndIv(salt);
expect(key).toBeInstanceOf(Buffer);
expect(iv).toBeInstanceOf(Buffer);
expect(bufferFromSpy).toHaveBeenCalledWith('settings-encryption-key', 'binary');
bufferFromSpy.mockRestore();
});
it('should generate a key and iv using custom encryption key', () => {
const salt = Buffer.from('test-salt');
mockInstance(InstanceSettings, { encryptionKey: 'settings-encryption-key' });
// Clear the cached Cipher instance to get a new one with the new mock
Container.set(Cipher, new Cipher(Container.get(InstanceSettings)));
const testCipher = Container.get(Cipher);
const bufferFromSpy = jest.spyOn(Buffer, 'from');
// @ts-expect-error - getKeyAndIv is private
const [key, iv] = testCipher.getKeyAndIv(salt, 'custom-encryption-key');
expect(key).toBeInstanceOf(Buffer);
expect(iv).toBeInstanceOf(Buffer);
expect(bufferFromSpy).toHaveBeenCalledWith('custom-encryption-key', 'binary');
bufferFromSpy.mockRestore();
});
});
});

View File

@ -10,26 +10,26 @@ const RANDOM_BYTES = Buffer.from('53616c7465645f5f', 'hex');
export class Cipher {
constructor(private readonly instanceSettings: InstanceSettings) {}
encrypt(data: string | object) {
encrypt(data: string | object, customEncryptionKey?: string) {
const salt = randomBytes(8);
const [key, iv] = this.getKeyAndIv(salt);
const [key, iv] = this.getKeyAndIv(salt, customEncryptionKey);
const cipher = createCipheriv('aes-256-cbc', key, iv);
const encrypted = cipher.update(typeof data === 'string' ? data : JSON.stringify(data));
return Buffer.concat([RANDOM_BYTES, salt, encrypted, cipher.final()]).toString('base64');
}
decrypt(data: string) {
decrypt(data: string, customEncryptionKey?: string) {
const input = Buffer.from(data, 'base64');
if (input.length < 16) return '';
const salt = input.subarray(8, 16);
const [key, iv] = this.getKeyAndIv(salt);
const [key, iv] = this.getKeyAndIv(salt, customEncryptionKey);
const contents = input.subarray(16);
const decipher = createDecipheriv('aes-256-cbc', key, iv);
return Buffer.concat([decipher.update(contents), decipher.final()]).toString('utf-8');
}
private getKeyAndIv(salt: Buffer): [Buffer, Buffer] {
const { encryptionKey } = this.instanceSettings;
private getKeyAndIv(salt: Buffer, customEncryptionKey?: string): [Buffer, Buffer] {
const encryptionKey = customEncryptionKey ?? this.instanceSettings.encryptionKey;
const password = Buffer.concat([Buffer.from(encryptionKey, 'binary'), salt]);
const hash1 = createHash('md5').update(password).digest();
const hash2 = createHash('md5')