github-mirror/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-05-12 16:10:30 +02:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

perf(core): Optimize generate-node-defs with parallel writes and hash-based skip (#25626)

Browse Source 
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Mutasem Aldmour 2026-02-11 14:50:50 +01:00 committed by GitHub
parent 20934363db
commit a9929f653e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 278 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/@n8n/workflow-sdk/package.json
View File

@ -14,6 +14,9 @@
},
"./dist/generate-types/generate-types": {
"default": "./dist/generate-types/generate-types.js"
},
"./dist/generate-types/generate-node-defs-cli": {
"default": "./dist/generate-types/generate-node-defs-cli.js"
}
},
"scripts": {

217
packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.test.ts
View File

@ -8,7 +8,32 @@ import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { generateNodeDefinitions } from './generate-node-defs-cli';
import { computeInputHash, generateNodeDefinitions } from './generate-node-defs-cli';
const MINIMAL_NODE = {
name: 'n8n-nodes-base.testCliNode',
displayName: 'Test CLI Node',
description: 'A node for CLI testing',
group: ['transform'],
version: 1,
properties: [
{
name: 'value',
displayName: 'Value',
type: 'string',
default: '',
},
],
inputs: ['main'],
outputs: ['main'],
};
async function setupNodesJson(dir: string, nodes: unknown[]): Promise<string> {
await fs.promises.mkdir(dir, { recursive: true });
const nodesJsonPath = path.join(dir, 'nodes.json');
await fs.promises.writeFile(nodesJsonPath, JSON.stringify(nodes));
return nodesJsonPath;
}
describe('generate-node-defs-cli', () => {
const tempDir = path.join(os.tmpdir(), `n8n-cli-test-${Date.now()}`);
@ -18,36 +43,11 @@ describe('generate-node-defs-cli', () => {
});
it('should read nodes.json from provided path and generate to output dir', async () => {
// Set up a temp dir with a nodes.json containing a minimal node
const nodesJsonDir = path.join(tempDir, 'input');
const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input'), [MINIMAL_NODE]);
const outputDir = path.join(tempDir, 'output');
await fs.promises.mkdir(nodesJsonDir, { recursive: true });
const minimalNodes = [
{
name: 'n8n-nodes-base.testCliNode',
displayName: 'Test CLI Node',
description: 'A node for CLI testing',
group: ['transform'],
version: 1,
properties: [
{
name: 'value',
displayName: 'Value',
type: 'string',
default: '',
},
],
inputs: ['main'],
outputs: ['main'],
},
];
const nodesJsonPath = path.join(nodesJsonDir, 'nodes.json');
await fs.promises.writeFile(nodesJsonPath, JSON.stringify(minimalNodes));
await generateNodeDefinitions({ nodesJsonPath, outputDir });
// Verify output has .ts and .schema.js files
const tsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'testCliNode', 'v1.ts');
expect(fs.existsSync(tsFile)).toBe(true);
@ -62,31 +62,11 @@ describe('generate-node-defs-cli', () => {
});
it('should prefix un-dotted node names with packageName', async () => {
const nodesJsonDir = path.join(tempDir, 'input-prefix');
const unprefixedNode = { ...MINIMAL_NODE, name: 'myNode' };
const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-prefix'), [
unprefixedNode,
]);
const outputDir = path.join(tempDir, 'output-prefix');
await fs.promises.mkdir(nodesJsonDir, { recursive: true });
const minimalNodes = [
{
name: 'myNode',
displayName: 'My Node',
description: 'A node without package prefix',
group: ['transform'],
version: 1,
properties: [
{
name: 'value',
displayName: 'Value',
type: 'string',
default: '',
},
],
inputs: ['main'],
outputs: ['main'],
},
];
const nodesJsonPath = path.join(nodesJsonDir, 'nodes.json');
await fs.promises.writeFile(nodesJsonPath, JSON.stringify(minimalNodes));
await generateNodeDefinitions({
nodesJsonPath,
@ -94,7 +74,6 @@ describe('generate-node-defs-cli', () => {
packageName: 'n8n-nodes-base',
});
// Should be generated under the package name directory
const tsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'myNode', 'v1.ts');
expect(fs.existsSync(tsFile)).toBe(true);
@ -110,4 +89,138 @@ describe('generate-node-defs-cli', () => {
generateNodeDefinitions({ nodesJsonPath: nonExistentPath, outputDir }),
).rejects.toThrow();
});
describe('hash-based skip', () => {
it('should skip generation on second run with unchanged nodes.json', async () => {
const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-hash'), [MINIMAL_NODE]);
const outputDir = path.join(tempDir, 'output-hash');
// First run: generates files
await generateNodeDefinitions({ nodesJsonPath, outputDir });
const hashFile = path.join(outputDir, '.nodes-hash');
expect(fs.existsSync(hashFile)).toBe(true);
// Record file modification time
const tsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'testCliNode', 'v1.ts');
const firstMtime = (await fs.promises.stat(tsFile)).mtimeMs;
// Small delay to ensure mtime would differ if file were rewritten
await new Promise((resolve) => setTimeout(resolve, 50));
// Second run: should skip (files unchanged)
const consoleSpy = jest.spyOn(console, 'log');
await generateNodeDefinitions({ nodesJsonPath, outputDir });
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('up to date'));
consoleSpy.mockRestore();
// File should not have been rewritten
const secondMtime = (await fs.promises.stat(tsFile)).mtimeMs;
expect(secondMtime).toBe(firstMtime);
});
it('should regenerate when nodes.json content changes', async () => {
const inputDir = path.join(tempDir, 'input-hash-change');
const nodesJsonPath = await setupNodesJson(inputDir, [MINIMAL_NODE]);
const outputDir = path.join(tempDir, 'output-hash-change');
// First run
await generateNodeDefinitions({ nodesJsonPath, outputDir });
const hashBefore = await fs.promises.readFile(path.join(outputDir, '.nodes-hash'), 'utf-8');
// Modify nodes.json
const modifiedNode = {
...MINIMAL_NODE,
name: 'n8n-nodes-base.modifiedNode',
displayName: 'Modified Node',
};
await fs.promises.writeFile(nodesJsonPath, JSON.stringify([modifiedNode]));
// Second run: should regenerate
await generateNodeDefinitions({ nodesJsonPath, outputDir });
const hashAfter = await fs.promises.readFile(path.join(outputDir, '.nodes-hash'), 'utf-8');
expect(hashAfter).not.toBe(hashBefore);
// New node files should exist
const newTsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'modifiedNode', 'v1.ts');
expect(fs.existsSync(newTsFile)).toBe(true);
});
it('should regenerate when hash sentinel file is missing', async () => {
const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-hash-missing'), [
MINIMAL_NODE,
]);
const outputDir = path.join(tempDir, 'output-hash-missing');
// First run
await generateNodeDefinitions({ nodesJsonPath, outputDir });
const hashFile = path.join(outputDir, '.nodes-hash');
expect(fs.existsSync(hashFile)).toBe(true);
// Delete the hash sentinel
await fs.promises.unlink(hashFile);
// Should regenerate (not skip)
const consoleSpy = jest.spyOn(console, 'log');
await generateNodeDefinitions({ nodesJsonPath, outputDir });
const logCalls = consoleSpy.mock.calls.map((c) => String(c[0]));
expect(logCalls.some((msg) => msg.includes('Generated node definitions'))).toBe(true);
consoleSpy.mockRestore();
// Hash sentinel should be recreated
expect(fs.existsSync(hashFile)).toBe(true);
});
});
describe('computeInputHash', () => {
it('should produce different hashes for different content', () => {
const hash1 = computeInputHash('content-a', '0.1.0');
const hash2 = computeInputHash('content-b', '0.1.0');
expect(hash1).not.toBe(hash2);
});
it('should produce different hashes for different SDK versions', () => {
const hash1 = computeInputHash('same-content', '0.1.0');
const hash2 = computeInputHash('same-content', '0.2.0');
expect(hash1).not.toBe(hash2);
});
it('should produce consistent hashes for same inputs', () => {
const hash1 = computeInputHash('content', '0.1.0');
const hash2 = computeInputHash('content', '0.1.0');
expect(hash1).toBe(hash2);
});
});
describe('parallel writes', () => {
it('should produce correct file count and content with multiple nodes', async () => {
const nodes = Array.from({ length: 5 }, (_, i) => ({
...MINIMAL_NODE,
name: `n8n-nodes-base.parallelNode${i}`,
displayName: `Parallel Node ${i}`,
}));
const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-parallel'), nodes);
const outputDir = path.join(tempDir, 'output-parallel');
await generateNodeDefinitions({ nodesJsonPath, outputDir });
// Each node should have v1.ts, v1.schema.js, and index.ts (3 files per node)
// Plus a root index.ts
for (let i = 0; i < 5; i++) {
const nodeDir = path.join(outputDir, 'nodes', 'n8n-nodes-base', `parallelNode${i}`);
expect(fs.existsSync(path.join(nodeDir, 'v1.ts'))).toBe(true);
expect(fs.existsSync(path.join(nodeDir, 'v1.schema.js'))).toBe(true);
expect(fs.existsSync(path.join(nodeDir, 'index.ts'))).toBe(true);
// Verify content is non-empty
const tsContent = await fs.promises.readFile(path.join(nodeDir, 'v1.ts'), 'utf-8');
expect(tsContent.length).toBeGreaterThan(0);
}
// Root index should exist
expect(fs.existsSync(path.join(outputDir, 'index.ts'))).toBe(true);
});
});
});

51
packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.ts
View File

@ -4,11 +4,16 @@
* Reads dist/types/nodes.json from CWD and generates to dist/node-definitions/.
* Used as a post-build step in node packages (nodes-base, nodes-langchain).
*
* Features:
* - Hash-based skip: computes SHA-256 of nodes.json + SDK version, skips if unchanged
* - Parallel writes: files are written in batches for I/O performance
*
* Usage:
* n8n-generate-node-defs
* # or: npx tsx generate-node-defs-cli.ts
*/
import { createHash } from 'crypto';
import * as fs from 'fs';
import { jsonParse } from 'n8n-workflow';
import * as path from 'path';
@ -16,14 +21,39 @@ import * as path from 'path';
import type { NodeTypeDescription } from './generate-types';
import { orchestrateGeneration } from './generate-types';
/** Name of the sentinel file storing the content hash */
const HASH_SENTINEL_FILE = '.nodes-hash';
export interface GenerateNodeDefinitionsOptions {
nodesJsonPath: string;
outputDir: string;
packageName?: string;
}
/**
* Compute a SHA-256 hash of the nodes.json content and SDK package version.
* Including the SDK version ensures regeneration when generation logic changes.
*/
export function computeInputHash(content: string, sdkVersion: string): string {
return createHash('sha256').update(content).update(sdkVersion).digest('hex');
}
/**
* Read the SDK package version from its package.json.
*/
function getSdkVersion(): string {
const sdkPackageJsonPath = path.join(__dirname, '..', '..', 'package.json');
try {
const pkg = JSON.parse(fs.readFileSync(sdkPackageJsonPath, 'utf-8')) as { version: string };
return pkg.version;
} catch {
return 'unknown';
}
}
/**
* Generate node definitions from a nodes.json file.
* Skips generation if the input hash matches the stored sentinel.
* Testable core logic extracted from CLI entry point.
*/
export async function generateNodeDefinitions(
@ -36,6 +66,22 @@ export async function generateNodeDefinitions(
}
const content = await fs.promises.readFile(nodesJsonPath, 'utf-8');
// Hash-based skip: check if output is already up to date
const sdkVersion = getSdkVersion();
const inputHash = computeInputHash(content, sdkVersion);
const hashFilePath = path.join(outputDir, HASH_SENTINEL_FILE);
try {
const existingHash = await fs.promises.readFile(hashFilePath, 'utf-8');
if (existingHash.trim() === inputHash) {
console.log('Node definitions up to date (hash match), skipping generation.');
return;
}
} catch {
// Hash file doesn't exist — proceed with generation
}
const nodes = jsonParse<NodeTypeDescription[]>(content);
if (packageName) {
@ -47,6 +93,11 @@ export async function generateNodeDefinitions(
}
const result = await orchestrateGeneration({ nodes, outputDir });
// Write hash sentinel after successful generation
await fs.promises.mkdir(outputDir, { recursive: true });
await fs.promises.writeFile(hashFilePath, inputHash);
console.log(`Generated node definitions for ${result.nodeCount} nodes in ${outputDir}`);
}

90
packages/@n8n/workflow-sdk/src/generate-types/generate-types.ts
View File

@ -3778,20 +3778,18 @@ function convertNodeToToolVariant(node: NodeTypeDescription): NodeTypeDescriptio
// Main Entry Point
// =============================================================================
/** Batch size for parallel file writes to avoid file descriptor exhaustion */
const WRITE_BATCH_SIZE = 100;
/**
* Write files from a plan map to disk
* Creates directories as needed
* Write files in parallel batches to stay within OS file descriptor limits.
*/
async function writePlanToDisk(baseDir: string, plan: Map<string, string>): Promise<number> {
let fileCount = 0;
for (const [relativePath, content] of plan) {
const fullPath = path.join(baseDir, relativePath);
const dir = path.dirname(fullPath);
await fs.promises.mkdir(dir, { recursive: true });
await fs.promises.writeFile(fullPath, content);
fileCount++;
async function writeFilesInBatches(files: Array<{ path: string; content: string }>): Promise<void> {
for (let i = 0; i < files.length; i += WRITE_BATCH_SIZE) {
const batch = files.slice(i, i + WRITE_BATCH_SIZE);
// eslint-disable-next-line @typescript-eslint/promise-function-async
await Promise.all(batch.map((f) => fs.promises.writeFile(f.path, f.content)));
}
return fileCount;
}
/**
@ -3817,23 +3815,24 @@ async function generateVersionSpecificFiles(
nodesByName: Map<string, NodeTypeDescription[]>,
): Promise<NodeTypeDescription[]> {
const allNodes: NodeTypeDescription[] = [];
let generatedFiles = 0;
let generatedDirs = 0;
let splitVersions = 0;
let flatVersions = 0;
// Phase 1: Pure computation — generate all file contents in memory
const allDirs = new Set<string>();
const allWrites: Array<{ path: string; content: string }> = [];
// Track split plans that need their own directory creation + writes
const splitPlans: Array<{ baseDir: string; plan: Map<string, string> }> = [];
for (const [nodeName, nodes] of nodesByName) {
try {
// Create directory for this node
const nodeDir = path.join(packageDir, nodeName);
await fs.promises.mkdir(nodeDir, { recursive: true });
generatedDirs++;
// Collect all individual versions from all node entries and map them to their source node
// This allows us to generate a file per individual version (e.g., v3, v31, v32, v33, v34)
// Accumulate writes locally — only commit to allWrites on success
const nodeWrites: Array<{ path: string; content: string }> = [];
const nodeDirs = new Set<string>([nodeDir]);
const nodeSplitPlans: Array<{ baseDir: string; plan: Map<string, string> }> = [];
const versionToNode = new Map<number, NodeTypeDescription>();
const allVersions: number[] = [];
// Track which versions use split structure
const splitVersionsSet = new Set<number>();
for (const node of nodes) {
@ -3846,48 +3845,56 @@ async function generateVersionSpecificFiles(
}
}
// Generate files for each individual version
for (const version of allVersions) {
const sourceNode = versionToNode.get(version)!;
const fileName = versionToFileName(version);
// Check if this node uses a discriminator pattern that benefits from splitting
if (hasDiscriminatorPattern(sourceNode)) {
// Generate split structure in a version directory
const versionDir = path.join(nodeDir, fileName);
const plan = planSplitVersionFiles(sourceNode, version);
const count = await writePlanToDisk(versionDir, plan);
generatedFiles += count;
nodeSplitPlans.push({ baseDir: versionDir, plan });
splitVersionsSet.add(version);
splitVersions++;
} else {
// Generate flat type file
const content = generateSingleVersionTypeFile(sourceNode, version);
const filePath = path.join(nodeDir, `${fileName}.ts`);
await fs.promises.writeFile(filePath, content);
generatedFiles++;
flatVersions++;
nodeWrites.push({ path: path.join(nodeDir, `${fileName}.ts`), content });
// Generate corresponding Zod schema file (CommonJS JavaScript for runtime loading)
const schemaContent = generateSingleVersionSchemaFile(sourceNode, version);
const schemaFilePath = path.join(nodeDir, `${fileName}.schema.js`);
await fs.promises.writeFile(schemaFilePath, schemaContent);
generatedFiles++;
nodeWrites.push({
path: path.join(nodeDir, `${fileName}.schema.js`),
content: schemaContent,
});
}
}
// Generate index.ts that re-exports all individual versions
// Pass splitVersionsSet so it knows which versions are directories vs files
const indexContent = generateVersionIndexFile(nodes[0], allVersions, splitVersionsSet);
await fs.promises.writeFile(path.join(nodeDir, 'index.ts'), indexContent);
nodeWrites.push({ path: path.join(nodeDir, 'index.ts'), content: indexContent });
// Add first node to allNodes for main index generation
// Commit: all generation succeeded for this node, merge into global arrays
allWrites.push(...nodeWrites);
for (const d of nodeDirs) allDirs.add(d);
splitPlans.push(...nodeSplitPlans);
allNodes.push(nodes[0]);
} catch (error) {
console.error(` Error generating ${nodeName}:`, error);
}
}
// Collect directories from split plans
for (const { baseDir, plan } of splitPlans) {
for (const [relativePath, content] of plan) {
const fullPath = path.join(baseDir, relativePath);
allDirs.add(path.dirname(fullPath));
allWrites.push({ path: fullPath, content });
}
}
// Phase 2: Create all directories in parallel
// eslint-disable-next-line @typescript-eslint/promise-function-async
await Promise.all([...allDirs].map((d) => fs.promises.mkdir(d, { recursive: true })));
// Phase 3: Write all files in parallel batches
await writeFilesInBatches(allWrites);
return allNodes;
}
@ -3937,9 +3944,10 @@ export async function orchestrateGeneration(options: GenerationOptions): Promise
const allNodes: NodeTypeDescription[] = [];
// Generate files for each package
// Generate files for each package, cleaning stale output first
for (const [packageName, nodesByName] of nodesByPackage) {
const packageDir = path.join(outputDir, 'nodes', packageName);
await fs.promises.rm(packageDir, { recursive: true, force: true });
const packageNodes = await generateVersionSpecificFiles(packageDir, packageName, nodesByName);
allNodes.push(...packageNodes);
}

32
packages/core/bin/generate-node-defs
View File

@ -3,34 +3,22 @@
const path = require('path');
const fs = require('fs');
const { orchestrateGeneration } = require('@n8n/workflow-sdk/dist/generate-types/generate-types');
const {
generateNodeDefinitions,
} = require('@n8n/workflow-sdk/dist/generate-types/generate-node-defs-cli');
const cwd = process.cwd();
const nodesJsonPath = path.join(cwd, 'dist', 'types', 'nodes.json');
const outputDir = path.join(cwd, 'dist', 'node-definitions');
if (!fs.existsSync(nodesJsonPath)) {
console.error(`nodes.json not found at ${nodesJsonPath}`);
process.exit(1);
}
const packageJsonPath = path.join(cwd, 'package.json');
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
(async () => {
const content = await fs.promises.readFile(nodesJsonPath, 'utf-8');
const nodes = JSON.parse(content);
const packageJsonPath = path.join(cwd, 'package.json');
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8'));
const packageName = packageJson.name;
for (const node of nodes) {
if (!node.name.includes('.')) {
node.name = `${packageName}.${node.name}`;
}
}
const result = await orchestrateGeneration({ nodes, outputDir });
console.log(`Generated node definitions for ${result.nodeCount} nodes in ${outputDir}`);
})().catch((error) => {
generateNodeDefinitions({
nodesJsonPath,
outputDir,
packageName: packageJson.name,
}).catch((error) => {
console.error('Node definition generation failed:', error);
process.exit(1);
});