feat(core): Always load the encryption module and seed keys safely on startup (#37416)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/@n8n/db/src/repositories/deployment-key.repository.ts

@@ -2,6 +2,7 @@ import { Service } from '@n8n/di';
 import { DataSource, Repository } from '@n8n/typeorm';
 
 import { DeploymentKey } from '../entities/deployment-key';
+import { DbLock, DbLockService } from '../services/db-lock.service';
 
 export type DeploymentKeySortField = 'createdAt' | 'updatedAt' | 'status';
 export type DeploymentKeySortDirection = 'ASC' | 'DESC';
@@ -18,10 +19,38 @@ export type ListDeploymentKeysOptions = {
 
 @Service()
 export class DeploymentKeyRepository extends Repository<DeploymentKey> {
-	constructor(dataSource: DataSource) {
+	constructor(
+		dataSource: DataSource,
+		private readonly dbLockService: DbLockService,
+	) {
 		super(DeploymentKey, dataSource.manager);
 	}
 
+	/**
+	 * Seeds the legacy aes-256-cbc data-encryption row exactly once. The
+	 * check and insert run inside a `DbLock` critical section, so mains
+	 * starting concurrently cannot create duplicate rows.
+	 */
+	async seedLegacyCbcKey(encryptedValue: string): Promise<void> {
+		await this.dbLockService.withLock(DbLock.DATA_ENCRYPTION_KEY_SEED, async (tx) => {
+			const repo = tx.getRepository(DeploymentKey);
+			const existing = await repo.findOne({
+				where: { type: 'data_encryption', algorithm: 'aes-256-cbc' },
+			});
+			if (existing) return;
+			// a create()-built entity instance, so the @BeforeInsert id hook runs
+			// (a plain object literal would skip it and violate the NOT NULL id)
+			await repo.save(
+				repo.create({
+					type: 'data_encryption',
+					value: encryptedValue,
+					algorithm: 'aes-256-cbc',
+					status: 'inactive',
+				}),
+			);
+		});
+	}
+
 	async findActiveByType(type: string): Promise<DeploymentKey | null> {
 		return await this.findOne({ where: { type, status: 'active' } });
 	}

packages/@n8n/db/src/services/db-lock.service.ts

@@ -20,6 +20,7 @@ export const enum DbLock {
 	MIGRATIONS = 1005,
 	EVAL_COLLECTION_RERUN = 1006,
 	INSTANCE_AI_SETTINGS = 1007,
+	DATA_ENCRYPTION_KEY_SEED = 1008,
 	/** Reserved for integration tests — never use in production code */
 	TEST = 9999,
 }

packages/cli/src/modules/encryption-key-manager/__tests__/encryption-bootstrap.service.integration.test.ts

@@ -12,6 +12,7 @@ beforeAll(async () => {
 		encryptionKey: INSTANCE_ENCRYPTION_KEY,
 		n8nFolder: '/tmp/n8n-test',
 		instanceType: 'main',
+		canSeedDeploymentState: true,
 	});
 	await testDb.init();
 });
@@ -60,4 +61,33 @@ describe('EncryptionBootstrapService (integration)', () => {
 		expect(cbcKeys).toHaveLength(1);
 		expect(gcmKeys).toHaveLength(1);
 	});
+
+	it('is race-safe — concurrent bootstraps never create duplicate keys (H7)', async () => {
+		const service = Container.get(EncryptionBootstrapService);
+
+		await Promise.all([...Array(5)].map(async () => await service.run()));
+
+		const all = await Container.get(DeploymentKeyRepository).find({
+			where: { type: 'data_encryption' },
+		});
+		const cbcKeys = all.filter((k) => k.algorithm === 'aes-256-cbc');
+		const gcmKeys = all.filter((k) => k.algorithm === 'aes-256-gcm' && k.status === 'active');
+		expect(cbcKeys).toHaveLength(1);
+		expect(gcmKeys).toHaveLength(1);
+	});
+
+	it('serializes concurrent seed calls in the repository critical section', async () => {
+		// Bypass the service-level fast-path check and hammer the repository
+		// directly: the DbLock critical section must let exactly one insert in.
+		const repository = Container.get(DeploymentKeyRepository);
+
+		await Promise.all(
+			[...Array(5)].map(async (_, i) => await repository.seedLegacyCbcKey(`value-${i}`)),
+		);
+
+		const rows = await repository.find({
+			where: { type: 'data_encryption', algorithm: 'aes-256-cbc' },
+		});
+		expect(rows).toHaveLength(1);
+	});
 });

packages/cli/src/modules/encryption-key-manager/__tests__/encryption-bootstrap.service.test.ts

@@ -14,10 +14,17 @@ describe('EncryptionBootstrapService', () => {
 		keyManager.bootstrapGcmKey.mockResolvedValue(undefined);
 	});
 
-	const createService = (instanceType: InstanceSettings['instanceType'] = 'main') =>
+	const createService = (
+		instanceType: InstanceSettings['instanceType'] = 'main',
+		canSeedDeploymentState = true,
+	) =>
 		new EncryptionBootstrapService(
 			keyManager,
-			mockInstance(InstanceSettings, { encryptionKey: 'test-instance-key', instanceType }),
+			mockInstance(InstanceSettings, {
+				encryptionKey: 'test-instance-key',
+				instanceType,
+				canSeedDeploymentState,
+			}),
 			encryptionKeyProxy,
 			mockLogger(),
 		);
@@ -51,6 +58,14 @@ describe('EncryptionBootstrapService', () => {
 		}
 	});
 
+	it('skips key creation when the process may not seed deployment state, but still sets the provider', async () => {
+		await createService('main', false).run();
+
+		expect(keyManager.bootstrapLegacyCbcKey).not.toHaveBeenCalled();
+		expect(keyManager.bootstrapGcmKey).not.toHaveBeenCalled();
+		expect(encryptionKeyProxy.setProvider).toHaveBeenCalledWith(keyManager);
+	});
+
 	it('bootstraps CBC before GCM', async () => {
 		const order: string[] = [];
 		keyManager.bootstrapLegacyCbcKey.mockImplementation(async () => {
@@ -64,4 +79,27 @@ describe('EncryptionBootstrapService', () => {
 
 		expect(order).toEqual(['cbc', 'gcm']);
 	});
+
+	describe('seeding failure', () => {
+		const seedError = new Error('no write access');
+
+		it('does not crash the instance while the rotation flag is off, and still sets the provider', async () => {
+			keyManager.bootstrapLegacyCbcKey.mockRejectedValue(seedError);
+
+			await expect(createService().run()).resolves.toBeUndefined();
+
+			expect(encryptionKeyProxy.setProvider).toHaveBeenCalledWith(keyManager);
+		});
+
+		it('rethrows while the rotation flag is on, because the keys are load-bearing', async () => {
+			process.env.N8N_ENV_FEAT_ENCRYPTION_KEY_ROTATION = 'true';
+			try {
+				keyManager.bootstrapGcmKey.mockRejectedValue(seedError);
+
+				await expect(createService().run()).rejects.toThrow('no write access');
+			} finally {
+				delete process.env.N8N_ENV_FEAT_ENCRYPTION_KEY_ROTATION;
+			}
+		});
+	});
 });

packages/cli/src/modules/encryption-key-manager/__tests__/key-manager.service.test.ts

@@ -160,26 +160,20 @@ describe('KeyManagerService', () => {
 			expect(repository.findOne).toHaveBeenCalledWith({
 				where: { type: 'data_encryption', algorithm: 'aes-256-cbc' },
 			});
-			expect(repository.save).not.toHaveBeenCalled();
+			expect(repository.seedLegacyCbcKey).not.toHaveBeenCalled();
 		});
 
-		it('encrypts the instance key and inserts as inactive when no CBC key exists', async () => {
+		it('encrypts the instance key and seeds it when no CBC key exists', async () => {
 			repository.findOne.mockResolvedValue(null);
-			const entity = makeKey({ algorithm: 'aes-256-cbc', status: 'inactive' });
-			repository.create.mockReturnValue(entity);
-			repository.save.mockResolvedValue(entity);
 			cipher.encryptDEKWithInstanceKey.mockReturnValue('encrypted-instance-key');
 
 			await Container.get(KeyManagerService).bootstrapLegacyCbcKey('instance-key');
 
 			expect(cipher.encryptDEKWithInstanceKey).toHaveBeenCalledWith('instance-key');
-			expect(repository.create).toHaveBeenCalledWith({
-				type: 'data_encryption',
-				value: 'encrypted-instance-key',
-				algorithm: 'aes-256-cbc',
-				status: 'inactive',
-			});
-			expect(repository.save).toHaveBeenCalledWith(entity);
+			// The repository seeds inside a DbLock critical section, which keeps
+			// concurrent startups from creating duplicate rows.
+			expect(repository.seedLegacyCbcKey).toHaveBeenCalledWith('encrypted-instance-key');
+			expect(repository.save).not.toHaveBeenCalled();
 		});
 	});
 

packages/cli/src/modules/encryption-key-manager/encryption-bootstrap.service.ts

@@ -1,8 +1,10 @@
 import { Logger } from '@n8n/backend-common';
 import { Service } from '@n8n/di';
+import { ensureError } from '@n8n/utils/errors/ensure-error';
 import { EncryptionKeyProxy, InstanceSettings } from 'n8n-core';
 
 import { KeyManagerService } from './key-manager.service';
+import { isKeyRotationEnabled } from './key-rotation-flag';
 
 @Service()
 export class EncryptionBootstrapService {
@@ -16,11 +18,28 @@ export class EncryptionBootstrapService {
 	}
 
 	async run(): Promise<void> {
-		if (this.instanceSettings.instanceType === 'main') {
-			await this.keyManager.bootstrapLegacyCbcKey(this.instanceSettings.encryptionKey);
-			await this.keyManager.bootstrapGcmKey();
+		// Seeding is deployment-wide state: only a main that is allowed to seed
+		// creates it. One-off CLI commands (`canSeedDeploymentState` false) and
+		// workers/webhooks skip seeding but still get the read-path provider.
+		if (this.instanceSettings.instanceType === 'main' && this.canSeed) {
+			try {
+				await this.keyManager.bootstrapLegacyCbcKey(this.instanceSettings.encryptionKey);
+				await this.keyManager.bootstrapGcmKey();
+			} catch (error) {
+				// While the rotation write path is off, nothing reads these keys yet,
+				// so a failed seed (e.g. restricted DB credentials) must not take the
+				// instance down. Once rotation is on, the keys are load-bearing.
+				if (isKeyRotationEnabled()) throw error;
+				this.logger.warn('Encryption key seeding failed, will retry on next startup', {
+					error: ensureError(error),
+				});
+			}
 		}
 		this.encryptionKeyProxy.setProvider(this.keyManager);
 		this.logger.debug('Encryption key bootstrap complete');
 	}
+
+	private get canSeed(): boolean {
+		return this.instanceSettings.canSeedDeploymentState;
+	}
 }

packages/cli/src/modules/encryption-key-manager/encryption-key-manager.module.ts

@@ -3,20 +3,22 @@ import { BackendModule } from '@n8n/decorators';
 import { Container } from '@n8n/di';
 import { InstanceSettings } from 'n8n-core';
 
-function isKeyRotationApiEnabled(): boolean {
-	return process.env.N8N_ENV_FEAT_ENCRYPTION_KEY_ROTATION === 'true';
-}
-
 @BackendModule({ name: 'encryption-key-manager' })
 export class EncryptionKeyManagerModule implements ModuleInterface {
 	async init() {
-		if (isKeyRotationApiEnabled()) {
-			await import('./key-manager.service.js');
-			if (Container.get(InstanceSettings).instanceType === 'main') {
-				await import('./encryption-key.controller.js');
-			}
-			const { EncryptionBootstrapService } = await import('./encryption-bootstrap.service.js');
-			await Container.get(EncryptionBootstrapService).run();
+		// Loading and seeding run on every instance, independent of the rotation
+		// flag: the whole fleet must hold the keys before any instance turns the
+		// flag on and starts using them.
+		await import('./key-manager.service.js');
+		const { isKeyRotationEnabled } = await import('./key-rotation-flag.js');
+
+		// The management API (list + rotate) stays behind the flag: rotating keys
+		// only makes sense once the rotation write path is enabled.
+		if (isKeyRotationEnabled() && Container.get(InstanceSettings).instanceType === 'main') {
+			await import('./encryption-key.controller.js');
 		}
+
+		const { EncryptionBootstrapService } = await import('./encryption-bootstrap.service.js');
+		await Container.get(EncryptionBootstrapService).run();
 	}
 }

packages/cli/src/modules/encryption-key-manager/key-manager.service.ts

@@ -81,6 +81,9 @@ export class KeyManagerService {
 	/**
 	 * Seeds an inactive aes-256-cbc key from the instance encryption key if none exists.
 	 * The value is wrapped with the instance key via AES-256-GCM before storage.
+	 * Race-safe across concurrent mains: the repository runs the check-and-insert
+	 * inside a `DbLock` critical section. The check here is only a fast path to
+	 * skip the lock on every startup after the first.
 	 */
 	async bootstrapLegacyCbcKey(instanceEncryptionKey: string): Promise<void> {
 		const existing = await this.deploymentKeyRepository.findOne({
@@ -89,13 +92,7 @@ export class KeyManagerService {
 		if (existing) return;
 
 		const encryptedValue = this.cipher.encryptDEKWithInstanceKey(instanceEncryptionKey);
-		const entity = this.deploymentKeyRepository.create({
-			type: 'data_encryption',
-			value: encryptedValue,
-			algorithm: 'aes-256-cbc',
-			status: 'inactive',
-		});
-		await this.deploymentKeyRepository.save(entity);
+		await this.deploymentKeyRepository.seedLegacyCbcKey(encryptedValue);
 	}
 
 	/**

packages/cli/src/modules/encryption-key-manager/key-rotation-flag.ts

@@ -0,0 +1,12 @@
+/**
+ * Whether the encryption-key-rotation paths and management API are enabled.
+ * It also decides whether a failed key seed is fatal on startup (see
+ * `EncryptionBootstrapService`). Module load, key seeding, and the provider
+ * wiring are unconditional (see `EncryptionKeyManagerModule`) so the keys
+ * exist fleet-wide, but the cipher still checks this flag per instance before
+ * using them to encrypt or decrypt — moving that decision out of the cipher
+ * is a separate step.
+ */
+export function isKeyRotationEnabled(): boolean {
+	return process.env.N8N_ENV_FEAT_ENCRYPTION_KEY_ROTATION === 'true';
+}

packages/core/src/instance-settings/__tests__/instance-settings.test.ts

@@ -374,6 +374,16 @@ describe('InstanceSettings', () => {
 				expect(settings.hmacSignatureSecret).toEqual(derivedHmac);
 			});
 
+			it('should expose the seeding permission as canSeedDeploymentState', async () => {
+				expect(settings.canSeedDeploymentState).toBe(true);
+
+				await settings.initialize(mockRepo, { canSeed: false });
+				expect(settings.canSeedDeploymentState).toBe(false);
+
+				await settings.initialize(mockRepo, { canSeed: true });
+				expect(settings.canSeedDeploymentState).toBe(true);
+			});
+
 			it('should still adopt env vars and existing DB rows', async () => {
 				process.env.N8N_INSTANCE_ID = 'env-pinned-id';
 				mockRepo.findActiveByType.mockImplementation(async (type: string) =>

packages/core/src/instance-settings/instance-settings.ts

@@ -69,6 +69,16 @@ export class InstanceSettings {
 
 	hmacSignatureSecret: string;
 
+	/**
+	 * Whether this process may create deployment-wide state, e.g. seed
+	 * deployment keys. Server processes (`start`, `worker`, `webhook`) may;
+	 * a one-off CLI command must not pin state for the whole deployment and
+	 * may run with restricted DB credentials. Set by `initialize()` from the
+	 * command's `seedsInstanceIdentity`; defaults to true for processes that
+	 * never call `initialize()` (e.g. tests).
+	 */
+	canSeedDeploymentState = true;
+
 	readonly instanceType: InstanceType;
 
 	constructor(
@@ -110,6 +120,7 @@ export class InstanceSettings {
 		},
 		{ canSeed = true }: { canSeed?: boolean } = {},
 	): Promise<void> {
+		this.canSeedDeploymentState = canSeed;
 		await this.initSecret(
 			repo,
 			'instance.id',