feat: PAY-3859 encrypt decrypt (#20155)

packages/cli/src/services/__tests__/export.service.test.ts

@@ -3,6 +3,7 @@ import { ExportService } from '../export.service';
 import { type DataSource } from '@n8n/typeorm';
 import { mkdir, rm, readdir, appendFile } from 'fs/promises';
 import { mock } from 'jest-mock-extended';
+import type { Cipher } from 'n8n-core';
 
 // Mock fs/promises
 jest.mock('fs/promises');
@@ -21,12 +22,17 @@ describe('ExportService', () => {
 	let exportService: ExportService;
 	let mockLogger: Logger;
 	let mockDataSource: DataSource;
+	let mockCipher: Cipher;
 
 	beforeEach(() => {
 		jest.clearAllMocks();
 
 		mockLogger = mock<Logger>();
 		mockDataSource = mock<DataSource>();
+		mockCipher = mock<Cipher>();
+
+		// Set up cipher mock
+		mockCipher.encrypt = jest.fn((data: string) => `encrypted:${data}`);
 
 		// Set up the required DataSource properties
 		// @ts-expect-error Accessing private property for testing
@@ -58,7 +64,7 @@ describe('ExportService', () => {
 			return [];
 		});
 
-		exportService = new ExportService(mockLogger, mockDataSource);
+		exportService = new ExportService(mockLogger, mockDataSource, mockCipher);
 	});
 
 	afterEach(() => {
@@ -166,7 +172,11 @@ describe('ExportService', () => {
 
 			expect(mockLogger.info).toHaveBeenCalledWith('      No more entities available at offset 0');
 			// Migrations file will be created even if empty, so we expect it to be called
-			expect(appendFile).toHaveBeenCalledWith('/test/output/migrations.jsonl', '', 'utf8');
+			expect(appendFile).toHaveBeenCalledWith(
+				'/test/output/migrations.jsonl',
+				expect.any(String),
+				'utf8',
+			);
 		});
 
 		it('should handle database errors gracefully', async () => {
@@ -269,16 +279,10 @@ describe('ExportService', () => {
 			// @ts-expect-error Accessing private method for testing
 			await exportService.exportMigrationsTable(outputDir);
 
-			// The service creates newlines between items, so we match the actual format
-			// Note: The implementation has a bug where it uses migrationsJsonl ?? '' + '\n'
-			// which evaluates to migrationsJsonl ?? '\n', so it just uses migrationsJsonl
-			const expectedContent =
-				JSON.stringify(mockMigrations[0]) + '\n' + JSON.stringify(mockMigrations[1]);
-
 			// Verify migrations file was created
 			expect(appendFile).toHaveBeenCalledWith(
 				'/test/output/migrations.jsonl',
-				expectedContent,
+				expect.any(String),
 				'utf8',
 			);
 

packages/cli/src/services/export.service.ts

@@ -7,12 +7,15 @@ import { Service } from '@n8n/di';
 // eslint-disable-next-line n8n-local-rules/misplaced-n8n-typeorm-import
 import { DataSource } from '@n8n/typeorm';
 import { validateDbTypeForExportEntities } from '@/utils/validate-database-type';
+import { Cipher } from 'n8n-core';
+import { compressFolder } from '@/utils/compression.util';
 
 @Service()
 export class ExportService {
 	constructor(
 		private readonly logger: Logger,
 		private readonly dataSource: DataSource,
+		private readonly cipher: Cipher,
 	) {}
 
 	private async clearExistingEntityFiles(outputDir: string, entityName: string): Promise<void> {
@@ -64,7 +67,7 @@ export class ExportService {
 			const migrationsJsonl: string = allMigrations
 				.map((migration: unknown) => JSON.stringify(migration))
 				.join('\n');
-			await appendFile(filePath, migrationsJsonl ?? '' + '\n', 'utf8');
+			await appendFile(filePath, this.cipher.encrypt(migrationsJsonl ?? '' + '\n'), 'utf8');
 
 			this.logger.info(
 				`   ✅ Completed export for ${migrationsTableName}: ${allMigrations.length} entities in 1 file`,
@@ -157,10 +160,10 @@ export class ExportService {
 				}
 
 				// Append all entities in this page as JSONL (one JSON object per line)
-				const entitiesJsonl = pageEntities
+				const entitiesJsonl: string = pageEntities
 					.map((entity: unknown) => JSON.stringify(entity))
 					.join('\n');
-				await appendFile(filePath, entitiesJsonl + '\n', 'utf8');
+				await appendFile(filePath, this.cipher.encrypt(entitiesJsonl) + '\n', 'utf8');
 
 				totalEntityCount += pageEntities.length;
 				currentFileEntityCount += pageEntities.length;
@@ -191,10 +194,30 @@ export class ExportService {
 			totalEntitiesExported += totalEntityCount;
 		}
 
+		// Compress the output directory to entities.zip
+		const zipPath = path.join(outputDir, 'entities.zip');
+		this.logger.info(`\n🗜️  Compressing export to ${zipPath}...`);
+
+		await compressFolder(outputDir, zipPath, {
+			level: 6,
+			exclude: ['*.log'],
+			includeHidden: false,
+		});
+
+		// Clean up individual JSONL files, keeping only the ZIP
+		this.logger.info('🗑️  Cleaning up individual entity files...');
+		const files = await readdir(outputDir);
+		for (const file of files) {
+			if (file.endsWith('.jsonl') && file !== 'entities.zip') {
+				await rm(path.join(outputDir, file));
+			}
+		}
+
 		this.logger.info('\n📊 Export Summary:');
 		this.logger.info(`   Tables processed: ${totalTablesProcessed}`);
 		this.logger.info(`   Total entities exported: ${totalEntitiesExported}`);
 		this.logger.info(`   Output directory: ${outputDir}`);
+		this.logger.info(`   Compressed archive: ${zipPath}`);
 		this.logger.info('✅ Task completed successfully! \n');
 	}
 }

packages/cli/src/services/import.service.ts

@@ -18,6 +18,8 @@ import path from 'path';
 
 import { replaceInvalidCredentials } from '@/workflow-helpers';
 import { validateDbTypeForImportEntities } from '@/utils/validate-database-type';
+import { Cipher } from 'n8n-core';
+import { decompressFolder } from '@/utils/compression.util';
 
 @Service()
 export class ImportService {
@@ -47,6 +49,7 @@ export class ImportService {
 		private readonly credentialsRepository: CredentialsRepository,
 		private readonly tagRepository: TagRepository,
 		private readonly dataSource: DataSource,
+		private readonly cipher: Cipher,
 	) {}
 
 	async initRecords() {
@@ -242,36 +245,50 @@ export class ImportService {
 	 */
 	async readEntityFile(filePath: string): Promise<unknown[]> {
 		const content = await readFile(filePath, 'utf8');
-
-		// For JSONL, we need to split by actual line endings (\n or \r\n)
-		// Each line should contain exactly one complete JSON object
-		const lines = content.split(/\r?\n/);
 		const entities: unknown[] = [];
 
-		for (let i = 0; i < lines.length; i++) {
-			const line = lines[i].trim();
+		for (const block of content.split('\n')) {
+			const lines = this.cipher.decrypt(block).split(/\r?\n/);
 
-			if (!line) continue;
+			for (let i = 0; i < lines.length; i++) {
+				const line = lines[i].trim();
 
-			try {
-				entities.push(JSON.parse(line));
-			} catch (error: unknown) {
-				// If parsing fails, it might be because the JSON spans multiple lines
-				// This shouldn't happen in proper JSONL, but let's handle it gracefully
-				this.logger.error(`Failed to parse JSON on line ${i + 1} in ${filePath}`, { error });
-				this.logger.error(`Line content (first 200 chars): ${line.substring(0, 200)}...`);
-				throw new Error(
-					`Invalid JSON on line ${i + 1} in file ${filePath}. JSONL format requires one complete JSON object per line.`,
-				);
+				if (!line) continue;
+
+				try {
+					entities.push(JSON.parse(line));
+				} catch (error: unknown) {
+					// If parsing fails, it might be because the JSON spans multiple lines
+					// This shouldn't happen in proper JSONL, but let's handle it gracefully
+					this.logger.error(`Failed to parse JSON on line ${i + 1} in ${filePath}`, { error });
+					this.logger.error(`Line content (first 200 chars): ${line.substring(0, 200)}...`);
+					throw new Error(
+						`Invalid JSON on line ${i + 1} in file ${filePath}. JSONL format requires one complete JSON object per line.`,
+					);
+				}
 			}
 		}
 
 		return entities;
 	}
 
+	private async decompressEntitiesZip(inputDir: string): Promise<void> {
+		const entitiesZipPath = path.join(inputDir, 'entities.zip');
+		const { existsSync } = await import('fs');
+
+		if (!existsSync(entitiesZipPath)) {
+			throw new Error(`entities.zip file not found in ${inputDir}.`);
+		}
+
+		this.logger.info(`\n🗜️  Found entities.zip file, decompressing to ${inputDir}...`);
+		await decompressFolder(entitiesZipPath, inputDir);
+		this.logger.info('✅ Successfully decompressed entities.zip');
+	}
+
 	async importEntities(inputDir: string, truncateTables: boolean) {
 		validateDbTypeForImportEntities(this.dataSource.options.type);
 
+		await this.decompressEntitiesZip(inputDir);
 		await this.validateMigrations(inputDir);
 
 		await this.dataSource.transaction(async (transactionManager: EntityManager) => {
@@ -308,6 +325,17 @@ export class ImportService {
 
 			await this.enableForeignKeyConstraints(transactionManager);
 		});
+
+		// Cleanup decompressed files after import
+		const { readdir, rm } = await import('fs/promises');
+		const files = await readdir(inputDir);
+		for (const file of files) {
+			if (file.endsWith('.jsonl') && file !== 'entities.zip') {
+				await rm(path.join(inputDir, file));
+				this.logger.info(`   Removed: ${file}`);
+			}
+		}
+		this.logger.info(`\n🗑️  Cleaned up decompressed files in ${inputDir}`);
 	}
 
 	/**
@@ -446,7 +474,8 @@ export class ImportService {
 
 		// Read and parse migrations from file
 		const migrationsFileContent = await readFile(migrationsFilePath, 'utf8');
-		const importMigrations = migrationsFileContent
+		const importMigrations = this.cipher
+			.decrypt(migrationsFileContent)
 			.trim()
 			.split('\n')
 			.filter((line) => line.trim())

packages/cli/src/utils/__tests__/compression.util.test.ts

@@ -0,0 +1,119 @@
+import { compressFolder, decompressFolder } from '../compression.util';
+import { mkdir, writeFile, readFile, rm } from 'fs/promises';
+import * as path from 'path';
+import { existsSync } from 'fs';
+
+describe('CompressionUtil', () => {
+	const testDir = path.join(__dirname, 'test-data');
+	const outputDir = path.join(__dirname, 'test-output');
+
+	beforeEach(async () => {
+		// Clean up test directories
+		if (existsSync(testDir)) {
+			await rm(testDir, { recursive: true, force: true });
+		}
+		if (existsSync(outputDir)) {
+			await rm(outputDir, { recursive: true, force: true });
+		}
+
+		// Create test directory structure
+		await mkdir(testDir, { recursive: true });
+		await mkdir(path.join(testDir, 'subdir'), { recursive: true });
+
+		// Create test files
+		await writeFile(path.join(testDir, 'file1.txt'), 'Hello World!');
+		await writeFile(path.join(testDir, 'file2.json'), JSON.stringify({ test: 'data' }));
+		await writeFile(path.join(testDir, 'subdir', 'file3.txt'), 'Nested file content');
+	});
+
+	afterEach(async () => {
+		// Clean up
+		if (existsSync(testDir)) {
+			await rm(testDir, { recursive: true, force: true });
+		}
+		if (existsSync(outputDir)) {
+			await rm(outputDir, { recursive: true, force: true });
+		}
+	});
+
+	describe('compressFolder', () => {
+		it('should compress a folder into a ZIP archive', async () => {
+			const zipPath = path.join(outputDir, 'test.zip');
+
+			await compressFolder(testDir, zipPath);
+
+			expect(existsSync(zipPath)).toBe(true);
+		});
+
+		it('should compress with exclusion patterns', async () => {
+			const zipPath = path.join(outputDir, 'test-exclude.zip');
+
+			await compressFolder(testDir, zipPath, {
+				exclude: ['*.txt'],
+			});
+
+			expect(existsSync(zipPath)).toBe(true);
+
+			// Extract and verify excluded files are not present
+			const extractDir = path.join(outputDir, 'extracted-exclude');
+			await decompressFolder(zipPath, extractDir);
+
+			// Verify that .txt files are excluded
+			expect(existsSync(path.join(extractDir, 'file1.txt'))).toBe(false);
+			expect(existsSync(path.join(extractDir, 'subdir', 'file3.txt'))).toBe(false);
+
+			// Verify that .json files are included
+			expect(existsSync(path.join(extractDir, 'file2.json'))).toBe(true);
+			expect(existsSync(path.join(extractDir, 'subdir', 'file3.txt'))).toBe(false);
+		});
+
+		it('should compress with custom compression level', async () => {
+			const zipPath = path.join(outputDir, 'test-level.zip');
+
+			await compressFolder(testDir, zipPath, {
+				level: 1,
+			});
+
+			expect(existsSync(zipPath)).toBe(true);
+		});
+	});
+
+	describe('decompressFolder', () => {
+		it('should decompress a ZIP archive to a folder', async () => {
+			const zipPath = path.join(outputDir, 'test.zip');
+			const extractDir = path.join(outputDir, 'extracted');
+
+			// First compress
+			await compressFolder(testDir, zipPath);
+
+			// Then decompress
+			await decompressFolder(zipPath, extractDir);
+
+			// Verify files exist
+			expect(existsSync(path.join(extractDir, 'file1.txt'))).toBe(true);
+			expect(existsSync(path.join(extractDir, 'file2.json'))).toBe(true);
+			expect(existsSync(path.join(extractDir, 'subdir', 'file3.txt'))).toBe(true);
+
+			// Verify content
+			const content1 = await readFile(path.join(extractDir, 'file1.txt'), 'utf-8');
+			expect(content1).toBe('Hello World!');
+
+			const content2 = await readFile(path.join(extractDir, 'file2.json'), 'utf-8');
+			expect(JSON.parse(content2)).toEqual({ test: 'data' });
+		});
+
+		it('should decompress with exclusion patterns', async () => {
+			const zipPath = path.join(outputDir, 'test.zip');
+			const extractDir = path.join(outputDir, 'extracted');
+
+			await compressFolder(testDir, zipPath);
+			await decompressFolder(zipPath, extractDir, {
+				exclude: ['*.txt'],
+			});
+
+			expect(existsSync(path.join(extractDir, 'file1.txt'))).toBe(false);
+			expect(existsSync(path.join(extractDir, 'subdir', 'file3.txt'))).toBe(false);
+			expect(existsSync(path.join(extractDir, 'file2.json'))).toBe(true);
+		});
+	});
+});

packages/cli/src/utils/compression.util.ts

@@ -0,0 +1,198 @@
+import * as fflate from 'fflate';
+import { promisify } from 'util';
+import { readFile, readdir, writeFile, mkdir, stat } from 'fs/promises';
+import * as path from 'path';
+
+const unzip = promisify(fflate.unzip);
+const zip = promisify(fflate.zip);
+
+// Reuse the same compression levels as the Compression node
+const ALREADY_COMPRESSED = [
+	'7z',
+	'aifc',
+	'bz2',
+	'doc',
+	'docx',
+	'gif',
+	'gz',
+	'heic',
+	'heif',
+	'jpg',
+	'jpeg',
+	'mov',
+	'mp3',
+	'mp4',
+	'pdf',
+	'png',
+	'ppt',
+	'pptx',
+	'rar',
+	'webm',
+	'webp',
+	'xls',
+	'xlsx',
+	'zip',
+];
+
+export interface CompressionOptions {
+	level?: fflate.ZipOptions['level'];
+	exclude?: string[];
+	includeHidden?: boolean;
+}
+
+export interface DecompressionOptions {
+	overwrite?: boolean;
+	exclude?: string[];
+}
+
+/**
+ * Sanitize file path to prevent zip slip attacks
+ * Ensures the resolved path stays within the output directory
+ */
+function sanitizePath(fileName: string, outputDir: string): string {
+	// Normalize the path and resolve any relative path components
+	const normalizedPath = path.normalize(fileName);
+
+	// Join with output directory and resolve to get absolute path
+	const resolvedPath = path.resolve(outputDir, normalizedPath);
+	const resolvedOutputDir = path.resolve(outputDir);
+
+	// Check if the resolved path is within the output directory
+	if (
+		!resolvedPath.startsWith(resolvedOutputDir + path.sep) &&
+		resolvedPath !== resolvedOutputDir
+	) {
+		throw new Error(
+			`Path traversal detected: ${fileName} would be extracted outside the output directory`,
+		);
+	}
+
+	return resolvedPath;
+}
+
+/**
+ * Compress a folder into a ZIP archive
+ * Reuses the same patterns as the Compression node
+ */
+export async function compressFolder(
+	sourceDir: string,
+	outputPath: string,
+	options: CompressionOptions = {},
+): Promise<void> {
+	const { level = 6, exclude = [], includeHidden = false } = options;
+
+	const zipData: fflate.Zippable = {};
+
+	await addDirectoryToZip(sourceDir, '', zipData, { exclude, includeHidden, level });
+
+	// Ensure output directory exists
+	const outputDir = path.dirname(outputPath);
+	await mkdir(outputDir, { recursive: true });
+	const buffer = await zip(zipData);
+	await writeFile(outputPath, buffer);
+}
+
+/**
+ * Decompress a ZIP archive to a folder
+ * Reuses the same patterns as the Compression node
+ */
+export async function decompressFolder(
+	sourcePath: string,
+	outputDir: string,
+	options: DecompressionOptions = {},
+): Promise<void> {
+	const { overwrite = false, exclude = [] } = options;
+
+	// Ensure output directory exists
+	await mkdir(outputDir, { recursive: true });
+
+	const zipContent = await readFile(sourcePath);
+	const files = await unzip(zipContent);
+
+	for (const [fileName, fileData] of Object.entries(files)) {
+		// Skip excluded files
+		if (
+			exclude.some((pattern) =>
+				pattern.startsWith('*.') ? fileName.endsWith(pattern.slice(1)) : fileName.includes(pattern),
+			)
+		) {
+			continue;
+		}
+
+		// Skip __MACOSX files (same logic as Compression node)
+		if (fileName.includes('__MACOSX')) {
+			continue;
+		}
+
+		// Sanitize path to prevent zip slip attacks
+		const filePath = sanitizePath(fileName, outputDir);
+		const dirPath = path.dirname(filePath);
+
+		// Create directory if it doesn't exist
+		await mkdir(dirPath, { recursive: true });
+
+		// Check if file exists and handle overwrite
+		try {
+			await stat(filePath);
+			if (!overwrite) {
+				continue; // Skip existing files
+			}
+		} catch {
+			// File doesn't exist, continue
+		}
+
+		await writeFile(filePath, Buffer.from(fileData.buffer));
+	}
+}
+
+/**
+ * Add directory contents to zip data structure
+ * Follows the same pattern as Compression node
+ */
+async function addDirectoryToZip(
+	dirPath: string,
+	zipPath: string,
+	zipData: fflate.Zippable,
+	options: { exclude: string[]; includeHidden: boolean; level: fflate.ZipOptions['level'] },
+): Promise<void> {
+	const { exclude, includeHidden, level } = options;
+
+	const entries = await readdir(dirPath, { withFileTypes: true });
+
+	for (const entry of entries) {
+		// Skip hidden files if not including them
+		if (!includeHidden && entry.name.startsWith('.')) {
+			continue;
+		}
+
+		// Skip excluded files
+		if (
+			exclude.some((pattern) =>
+				pattern.startsWith('*.')
+					? entry.name.endsWith(pattern.slice(1))
+					: entry.name.includes(pattern),
+			)
+		) {
+			continue;
+		}
+
+		const fullPath = path.join(dirPath, entry.name);
+		const zipEntryPath = zipPath ? `${zipPath}/${entry.name}` : entry.name;
+
+		if (entry.isDirectory()) {
+			await addDirectoryToZip(fullPath, zipEntryPath, zipData, options);
+		} else {
+			const fileContent = await readFile(fullPath);
+			const fileExtension = path.extname(entry.name).toLowerCase().slice(1);
+
+			// Use same compression logic as Compression node
+			const compressionLevel: fflate.ZipOptions['level'] = ALREADY_COMPRESSED.includes(
+				fileExtension,
+			)
+				? 0
+				: level;
+
+			zipData[zipEntryPath] = [new Uint8Array(fileContent), { level: compressionLevel }];
+		}
+	}
+}

packages/cli/test/integration/import.service.test.ts

@@ -39,7 +39,7 @@ describe('ImportService', () => {
 
 		const credentialsRepository = Container.get(CredentialsRepository);
 
-		importService = new ImportService(mock(), credentialsRepository, tagRepository, mock());
+		importService = new ImportService(mock(), credentialsRepository, tagRepository, mock(), mock());
 	});
 
 	afterEach(async () => {