n8n/packages/cli/src/load-nodes-and-credentials.ts

import { inTest, isContainedWithin, Logger, ModuleRegistry } from '@n8n/backend-common';
import { GlobalConfig } from '@n8n/config';
import { Container, Service } from '@n8n/di';
import { isWindowsFilePath } from '@n8n/utils/files/is-windows-file-path';
import type ParcelWatcher from '@parcel/watcher';
import glob from 'fast-glob';
import fsPromises from 'fs/promises';
import type { Class, OutputSchemaLookup, Types } from 'n8n-core';
import {
	CUSTOM_EXTENSION_ENV,
	DirectoryLoader,
	ErrorReporter,
	InstanceSettings,
	CustomDirectoryLoader,
	PackageDirectoryLoader,
	LazyPackageDirectoryLoader,
	UnrecognizedCredentialTypeError,
	UnrecognizedNodeTypeError,
	ExecutionContextHookRegistry,
	CUSTOM_NODES_PACKAGE_NAME,
	resolveOutputSchemaPath,
	loadOutputSchema,
	OUTPUT_PARSER_SCHEMA_VARIANT,
} from 'n8n-core';
import type {
	KnownNodesAndCredentials,
	INodeTypeDescription,
	LoadedClass,
	ICredentialType,
	INodeType,
	IVersionedNodeType,
	INodeProperties,
	LoadedNodesAndCredentials,
	NodeLoader,
} from 'n8n-workflow';
import { ensureError } from '@n8n/utils/errors/ensure-error';
import { injectDomainRestrictionFields, UnexpectedError, UserError } from 'n8n-workflow';
import path from 'path';
import picocolors from 'picocolors';

import { CUSTOM_API_CALL_KEY, CUSTOM_API_CALL_NAME, CLI_DIR, inE2ETests } from '@/constants';
import { createAiTools, createHitlTools } from '@/tool-generation';

@Service()
export class LoadNodesAndCredentials {
	private known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };

	// This contains the actually loaded objects, and their source paths
	loaded: LoadedNodesAndCredentials = { nodes: {}, credentials: {} };

	// For nodes, this only contains the descriptions, loaded from either the
	// actual file, or the lazy loaded json
	types: Types = { nodes: [], credentials: [] };

	loaders: Record<string, NodeLoader> = {};

	excludeNodes = this.globalConfig.nodes.exclude;

	includeNodes = this.globalConfig.nodes.include;

	private postProcessors: Array<() => Promise<void>> = [];

	constructor(
		private readonly logger: Logger,
		private readonly errorReporter: ErrorReporter,
		private readonly instanceSettings: InstanceSettings,
		private readonly globalConfig: GlobalConfig,
		private readonly moduleRegistry: ModuleRegistry,
		private readonly executionContextHookRegistry: ExecutionContextHookRegistry,
	) {}

	async init() {
		if (inTest) throw new UnexpectedError('Not available in tests');

		// Make sure the imported modules can resolve dependencies fine.
		// Preserve any existing NODE_PATH (e.g. set by Docker ENV for global npm packages)
		// so that the task runner subprocess can also resolve externally installed modules.
		const delimiter = process.platform === 'win32' ? ';' : ':';
		process.env.NODE_PATH = [module.paths.join(delimiter), process.env.NODE_PATH]
			.filter(Boolean)
			.join(delimiter);

		// @ts-ignore
		// eslint-disable-next-line @typescript-eslint/no-unsafe-call
		module.constructor._initPaths();

		if (!inE2ETests) {
			this.excludeNodes = this.excludeNodes ?? [];
			this.excludeNodes.push('n8n-nodes-base.e2eTest');
		}

		if (process.env.N8N_ENV_FEAT_DYNAMIC_CREDENTIALS !== 'true') {
			this.excludeNodes = this.excludeNodes ?? [];
			this.excludeNodes.push('n8n-nodes-base.dynamicCredentialCheck');
		}

		// Load nodes from `n8n-nodes-base`
		const basePathsToScan = [
			// In case "n8n" package is in same node_modules folder.
			path.join(CLI_DIR, '..'),
			// In case "n8n" package is the root and the packages are
			// in the "node_modules" folder underneath it.
			path.join(CLI_DIR, 'node_modules'),
		];

		for (const nodeModulesDir of basePathsToScan) {
			await this.loadNodesFromNodeModules(nodeModulesDir, 'n8n-nodes-base');
			await this.loadNodesFromNodeModules(nodeModulesDir, '@n8n/n8n-nodes-langchain');
		}

		await this.loadNodesFromCustomDirectories();

		for (const loader of this.moduleRegistry.nodeLoaders) {
			if (loader.packageName in this.loaders) {
				throw new UnexpectedError(
					picocolors.red(`Node loader ${loader.packageName} is already registered.`),
				);
			}
			try {
				await loader.loadAll();
				this.loaders[loader.packageName] = loader;
			} catch (error) {
				this.logger.error(`Failed to load package "${loader.packageName}"`, {
					error: ensureError(error),
				});
				this.errorReporter.error(error, { extra: { packageName: loader.packageName } });
			}
		}

		await this.postProcessLoaders();
	}

	addPostProcessor(fn: () => Promise<void>) {
		this.postProcessors.push(fn);
	}

	releaseTypes() {
		this.types = { nodes: [], credentials: [] };
		for (const loader of Object.values(this.loaders)) {
			loader.releaseTypes();
		}
	}

	/**
	 * Returns the current node and credential types.
	 * If types have been released from memory, re-runs postProcessLoaders to
	 * repopulate them first, then releases after snapshotting.
	 *
	 * WARNING: Holding types in memory is very consuming. Use sparingly and only
	 * where the caller genuinely needs its own copy (e.g. the AI workflow builder
	 * service or the frontend service writing static JSON files).
	 */
	async collectTypes(): Promise<Types> {
		const needsReload = this.types.nodes.length === 0 && this.types.credentials.length === 0;
		if (needsReload) {
			await this.postProcessLoaders();
		}
		const types: Types = {
			nodes: this.types.nodes,
			credentials: this.types.credentials,
		};
		if (needsReload) {
			this.releaseTypes();
		}
		return types;
	}

	isKnownNode(type: string) {
		return type in this.known.nodes;
	}

	get loadedCredentials() {
		return this.loaded.credentials;
	}

	get loadedNodes() {
		return this.loaded.nodes;
	}

	get knownCredentials() {
		return this.known.credentials;
	}

	get knownNodes() {
		return this.known.nodes;
	}

	private async loadNodesFromNodeModules(
		nodeModulesDir: string,
		packageName: string,
	): Promise<void> {
		const installedPackagePaths = await glob(packageName, {
			cwd: nodeModulesDir,
			onlyDirectories: true,
			deep: 1,
		});

		for (const packagePath of installedPackagePaths) {
			try {
				await this.runDirectoryLoader(
					LazyPackageDirectoryLoader,
					path.join(nodeModulesDir, packagePath),
				);
			} catch (error) {
				this.logger.error((error as Error).message);
				this.errorReporter.error(error);
			}
		}
	}

	/**
	 * Resolves the node icon file path when loaded from /icons/${packageName}/${iconPath}.
	 *
	 * Using N8N_CUSTOM_EXTENSIONS, nodes can be loaded from any directory outside of CWD='$N8N_USER_FOLDER/.n8n/'.
	 * Custom nodes are loaded by custom-directory-loader.ts using an absolute path, different from the default package-directory-loader.ts.
	 * The icon loading logic for custom nodes seems a bit broken, because icons are resolved by absolute paths encoded in URLs.
	 * Examples when served from `/icons/${packageName}/${iconPath}`:
	 * - '/icons/CUSTOM//home/node/.n8n-custom'
	 * - '/icons/CUSTOM/C:/User/name/.n8n-custom'
	 *
	 * resolveIcon() has a special path.resolve() strategy for custom nodes considering:
	 * - An absolute Linux file path is encoded in the URL using '//'.
	 * - '//' in URLs can be normalized to '/' by proxies, load balancers, etc.
	 * - A Windows file path starts with drive letters like 'C:' not '/'.
	 *
	 * @todo Instead of fixing the broken custom node loading strategy here, make custom-directory-loader.ts also use relative paths.
	 * Besides having different icon loading strategies, encoding an absolute path in URLs seems a security risk.
	 */
	resolveIcon(packageName: string, url: string): string | undefined {
		const isCustom = packageName === CUSTOM_NODES_PACKAGE_NAME;
		const loader = this.loaders[packageName];
		if (!loader || !(loader instanceof DirectoryLoader)) {
			return undefined;
		}

		const resolvePath = (iconPath: string) => {
			return path.resolve(loader.directory, iconPath);
		};

		const resolvePathCustom = (path: string) => {
			if (isWindowsFilePath(path)) return path;
			return path.startsWith('/') ? path : '/' + path;
		};

		const pathPrefix = `/icons/${packageName}/`;
		const urlFilePath = url.substring(pathPrefix.length);
		if (isCustom && !isWindowsFilePath(urlFilePath)) {
			const relativeFilePath = resolvePath(urlFilePath);
			if (isContainedWithin(loader.directory, relativeFilePath)) {
				return relativeFilePath;
			}
		}

		const filePath = isCustom ? resolvePathCustom(urlFilePath) : resolvePath(urlFilePath);
		return isContainedWithin(loader.directory, filePath) ? filePath : undefined;
	}

	resolveSchema({
		node,
		version,
		resource,
		operation,
	}: {
		node: string;
		version: string;
		resource?: string;
		operation?: string;
	}): string | undefined {
		const nodePath = this.known.nodes[node]?.sourcePath;
		if (!nodePath) {
			return undefined;
		}

		return resolveOutputSchemaPath({
			nodeDir: path.dirname(nodePath),
			version,
			resource,
			operation,
		});
	}

	/**
	 * Schema lookup for mock/pin-data generation: parsed `__schema__` content
	 * with version fallback (same major first, then older, then newer — see the
	 * n8n-core resolver), resolved through `known.nodes` so it works for
	 * community nodes and production installs alike.
	 */
	createOutputSchemaLookup(): OutputSchemaLookup {
		return ({ type, typeVersion, resource, operation, hasOutputParser }) => {
			const nodePath = this.known.nodes[type]?.sourcePath;
			if (!nodePath) return undefined;

			return loadOutputSchema({
				nodeDir: path.dirname(nodePath),
				version: typeVersion,
				resource,
				operation,
				versionFallback: true,
				variant: hasOutputParser ? OUTPUT_PARSER_SCHEMA_VARIANT : undefined,
			});
		};
	}

	getCustomDirectories(): string[] {
		const customDirectories = [this.instanceSettings.customExtensionDir];

		if (process.env[CUSTOM_EXTENSION_ENV] !== undefined) {
			const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV].split(';');
			customDirectories.push(...customExtensionFolders);
		}

		return customDirectories;
	}

	private async loadNodesFromCustomDirectories(): Promise<void> {
		for (const directory of this.getCustomDirectories()) {
			await this.runDirectoryLoader(CustomDirectoryLoader, directory);
		}
	}

	async loadPackage(packageName: string) {
		const finalNodeUnpackedPath = path.join(
			this.instanceSettings.nodesDownloadDir,
			'node_modules',
			packageName,
		);
		return await this.runDirectoryLoader(PackageDirectoryLoader, finalNodeUnpackedPath);
	}

	async unloadPackage(packageName: string) {
		if (packageName in this.loaders) {
			this.loaders[packageName].reset();
			delete this.loaders[packageName];
		}
	}

	/**
	 * Whether any of the node's credential types may be used to
	 * make a request from a node other than itself.
	 */
	private supportsProxyAuth(description: INodeTypeDescription) {
		if (!description.credentials) return false;

		return description.credentials.some(({ name }) => {
			const credType = this.types.credentials.find((t) => t.name === name);
			if (!credType) {
				this.logger.warn(
					`Failed to load Custom API options for the node "${description.name}": Unknown credential name "${name}"`,
				);
				return false;
			}
			if (credType.authenticate !== undefined) return true;

			return (
				Array.isArray(credType.extends) &&
				credType.extends.some((parentType) =>
					['oAuth2Api', 'googleOAuth2Api', 'oAuth1Api'].includes(parentType),
				)
			);
		});
	}

	/**
	 * Inject a `Custom API Call` option into `resource` and `operation`
	 * parameters in a latest-version node that supports proxy auth.
	 */
	private injectCustomApiCallOptions() {
		this.types.nodes.forEach((node: INodeTypeDescription) => {
			const isLatestVersion =
				node.defaultVersion === undefined || node.defaultVersion === node.version;

			if (isLatestVersion) {
				if (!this.supportsProxyAuth(node)) return;

				node.properties.forEach((p) => {
					if (
						['resource', 'operation'].includes(p.name) &&
						Array.isArray(p.options) &&
						p.options[p.options.length - 1].name !== CUSTOM_API_CALL_NAME
					) {
						p.options.push({
							name: CUSTOM_API_CALL_NAME,
							value: CUSTOM_API_CALL_KEY,
						});
					}
				});
			}
		});
	}

	private shouldInjectContextEstablishmentHooks() {
		return process.env.N8N_ENV_FEAT_DYNAMIC_CREDENTIALS === 'true';
	}

	private injectContextEstablishmentHooks() {
		// Check if the feature is enabled via environment variable
		const isEnabled = this.shouldInjectContextEstablishmentHooks();

		if (!isEnabled) {
			this.logger.debug('Context establishment hooks feature is disabled');
			return;
		}

		const triggerNodes = this.types.nodes.filter((node: INodeTypeDescription) =>
			node.group.includes('trigger'),
		);

		this.logger.debug(
			`Injecting context establishment hooks for ${triggerNodes.length} trigger nodes`,
		);

		triggerNodes.forEach(this.augmentNodeTypeDescription);
	}

	private augmentNodeTypeDescription = (node: INodeTypeDescription) => {
		const hooks = this.executionContextHookRegistry.getHookForTriggerType(node.name);

		if (hooks.length > 0) {
			this.logger.debug(`Found ${hooks.length} hooks for trigger node: ${node.name}`);
		}

		// Only inject hook properties if there are applicable hooks
		if (hooks.length === 0) return;

		// This prevents double-injection if the function is called multiple times on the same node
		if (node.properties.some((p) => p.name === 'executionsHooksVersion')) return;

		// Create a fixedCollection with multipleValues for multiple hook selection
		// Each hook becomes a separate item that can be added multiple times
		const allHookValues: INodeProperties[] = [
			{
				displayName: 'User Identifier',
				name: 'hookName',
				type: 'options',
				noDataExpression: true,
				options: hooks.map((hook) => {
					const displayName = hook.hookDescription.displayName ?? hook.hookDescription.name;
					return {
						name: displayName,
						value: hook.hookDescription.name,
						description: `Use ${displayName} hook`,
					};
				}),
				// No default - force user to explicitly select a hook
				// This ensures hookName is always serialized in the workflow JSON
				default: '',
				description:
					'Configure how n8n extracts the identity token of the user triggering a webhook. It is used to run each execution with the correct user.',
				required: true,
			},
		];

		// Add all hook-specific options with display conditions
		for (const hook of hooks) {
			const hookOptions = hook.hookDescription.options ?? [];
			if (hookOptions.length > 0) {
				for (const hookOption of hookOptions) {
					// Add display condition to show only when this specific hook is selected
					const enhancedOption: INodeProperties = {
						...hookOption,
						displayOptions: {
							...hookOption.displayOptions,
							show: {
								...hookOption.displayOptions?.show,
								hookName: [hook.hookDescription.name],
							},
						},
					};
					allHookValues.push(enhancedOption);
				}
			}
		}

		// Create a hidden version property to track the hooks format version
		const executionsHooksVersion: INodeProperties = {
			displayName: 'Executions Hooks Version',
			name: 'executionsHooksVersion',
			type: 'hidden',
			default: 1,
		};

		// Create the main context establishment hooks property as a fixedCollection
		const contextHooksProperty: INodeProperties = {
			displayName: 'Identify user for end-user credentials',
			name: 'contextEstablishmentHooks',
			type: 'fixedCollection',
			placeholder: 'Add User Identifier',
			default: {},
			typeOptions: {
				multipleValues: true,
				hideEmptyMessage: true,
			},
			options: [
				{
					name: 'hooks',
					displayName: 'Hooks',
					values: allHookValues,
				},
			],
			description:
				'Configure how n8n extracts the identity token of the user triggering a webhook. It is used to run each execution with the correct user.',
		};

		// Create a notice that always appears after the hooks collection
		const contextHooksNotice: INodeProperties = {
			displayName:
				'Configure how n8n extracts the identity token of the user triggering a webhook. It is used to run each execution with the correct user.',
			name: 'contextHooksNotice',
			type: 'notice',
			default: '',
		};

		let index = node.properties.findIndex((p) => p.name === 'options');
		if (index === -1) {
			index = node.properties.length;
		}

		node.properties.splice(index, 0, contextHooksNotice);
		node.properties.splice(index, 0, contextHooksProperty);
		node.properties.splice(index, 0, executionsHooksVersion);
	};

	/**
	 * Run a loader of source files of nodes and credentials in a directory.
	 */
	private async runDirectoryLoader<T extends DirectoryLoader>(
		constructor: Class<T, ConstructorParameters<typeof DirectoryLoader>>,
		dir: string,
	) {
		const loader = new constructor(dir, this.excludeNodes, this.includeNodes);
		if (loader instanceof PackageDirectoryLoader && loader.packageName in this.loaders) {
			throw new UserError(
				picocolors.red(
					`nodes package ${loader.packageName} is already loaded.\n Please delete this second copy at path ${dir}`,
				),
			);
		}
		await loader.loadAll();
		this.loaders[loader.packageName] = loader;
		return loader;
	}

	async postProcessLoaders() {
		this.known = { nodes: {}, credentials: {} };
		this.loaded = { nodes: {}, credentials: {} };
		this.types = { nodes: [], credentials: [] };

		for (const loader of Object.values(this.loaders)) {
			// Reload types if they were released from memory
			await loader.ensureTypesLoaded();

			// list of node & credential types that will be sent to the frontend
			const { known, types, packageName } = loader;
			this.types.nodes = this.types.nodes.concat(
				types.nodes.map(({ name, ...rest }) => ({
					...rest,
					name: `${packageName}.${name}`,
				})),
			);

			const processedCredentials = types.credentials.map((credential) => ({
				...credential,
				properties: injectDomainRestrictionFields(credential),
				supportedNodes:
					loader instanceof PackageDirectoryLoader
						? credential.supportedNodes?.map((nodeName) => `${loader.packageName}.${nodeName}`)
						: undefined,
			}));

			this.types.credentials = this.types.credentials.concat(processedCredentials);

			// Add domain restriction fields to loaded credentials
			for (const credentialTypeName in known.credentials) {
				const credentialType = loader.getCredential(credentialTypeName);
				credentialType.type.properties = injectDomainRestrictionFields(credentialType.type);
			}

			for (const type in known.nodes) {
				const { className, sourcePath } = known.nodes[type];
				this.known.nodes[`${packageName}.${type}`] = {
					className,
					sourcePath: loader.resolveSourcePath(sourcePath),
				};
			}

			for (const type in known.credentials) {
				const {
					className,
					sourcePath,
					supportedNodes,
					extends: extendsArr,
				} = known.credentials[type];
				this.known.credentials[type] = {
					className,
					sourcePath: loader.resolveSourcePath(sourcePath),
					supportedNodes:
						loader instanceof PackageDirectoryLoader
							? supportedNodes?.map((nodeName) => `${loader.packageName}.${nodeName}`)
							: undefined,
					extends: extendsArr,
				};
			}
		}

		createAiTools(this.types, this.known);
		createHitlTools(this.types, this.known);

		this.injectCustomApiCallOptions();

		this.injectContextEstablishmentHooks();

		for (const postProcessor of this.postProcessors) {
			await postProcessor();
		}
	}

	recognizesNode(fullNodeType: string): boolean {
		const [packageName, nodeType] = fullNodeType.split('.');
		const { loaders } = this;
		const loader = loaders[packageName];
		return !!loader && nodeType in loader.known.nodes;
	}

	getNode(fullNodeType: string): LoadedClass<INodeType | IVersionedNodeType> {
		const [packageName, nodeType] = fullNodeType.split('.');
		const { loaders } = this;
		const loader = loaders[packageName];
		if (!loader) {
			throw new UnrecognizedNodeTypeError(packageName, nodeType);
		}
		const loadedNode = loader.getNode(nodeType);
		if (
			this.shouldInjectContextEstablishmentHooks() &&
			'properties' in loadedNode.type.description
		) {
			this.augmentNodeTypeDescription(loadedNode.type.description);
		}
		return loadedNode;
	}

	getCredential(credentialType: string): LoadedClass<ICredentialType> {
		const { loadedCredentials } = this;

		for (const loader of Object.values(this.loaders)) {
			if (credentialType in loader.known.credentials) {
				const loaded = loader.getCredential(credentialType);
				loadedCredentials[credentialType] = loaded;
			}
		}

		if (credentialType in loadedCredentials) {
			return loadedCredentials[credentialType];
		}

		throw new UnrecognizedCredentialTypeError(credentialType);
	}

	async setupHotReload() {
		const { default: debounce } = await import('lodash/debounce.js');

		const { subscribe } = await import('@parcel/watcher');

		const { Push } = await import('@/push/index.js');
		const push = Container.get(Push);

		for (const loader of Object.values(this.loaders)) {
			if (!(loader instanceof DirectoryLoader)) continue;
			const { directory } = loader;
			try {
				await fsPromises.access(directory);
			} catch {
				// If directory doesn't exist, there is nothing to watch
				continue;
			}

			const reloader = debounce(async () => {
				this.logger.info(`Hot reload triggered for ${loader.packageName}`);
				try {
					loader.reset();
					await loader.loadAll();
					await this.postProcessLoaders();
					push.broadcast({ type: 'nodeDescriptionUpdated', data: {} });
				} catch (error) {
					this.logger.error(`Hot reload failed for ${loader.packageName}`);
				}
			}, 100);

			// For lazy loaded packages, we need to watch the dist directory
			const watchPaths = loader.isLazyLoaded ? [path.join(directory, 'dist')] : [directory];
			const customNodesRoot = path.join(directory, 'node_modules');

			if (loader.packageName === CUSTOM_NODES_PACKAGE_NAME) {
				const customNodeEntries = await fsPromises.readdir(customNodesRoot, {
					withFileTypes: true,
				});

				// Custom nodes are usually symlinked using npm link. Resolve symlinks to support file watching
				const realCustomNodesPaths = await Promise.all(
					customNodeEntries
						.filter(
							(entry) =>
								(entry.isDirectory() || entry.isSymbolicLink()) && !entry.name.startsWith('.'),
						)
						.map(
							async (entry) =>
								await fsPromises.realpath(path.join(customNodesRoot, entry.name)).catch(() => null),
						),
				);

				watchPaths.push.apply(
					watchPaths,
					realCustomNodesPaths.filter((path): path is string => !!path),
				);
			}

			this.logger.debug('Watching node folders for hot reload', {
				loader: loader.packageName,
				paths: watchPaths,
			});

			for (const watchPath of watchPaths) {
				const onFileEvent: ParcelWatcher.SubscribeCallback = async (_error, events) => {
					if (events.some((event) => event.type !== 'delete')) {
						const modules = Object.keys(require.cache).filter((module) =>
							module.startsWith(watchPath),
						);

						for (const module of modules) {
							delete require.cache[module];
						}
						await reloader();
					}
				};

				// Ignore nested node_modules folders
				const ignore = ['**/node_modules/**/node_modules/**'];

				await subscribe(watchPath, onFileEvent, { ignore });
			}
		}
	}
}