diff --git a/packages/@n8n/workflow-sdk/package.json b/packages/@n8n/workflow-sdk/package.json index caa6f50232e..47d0a0d9293 100644 --- a/packages/@n8n/workflow-sdk/package.json +++ b/packages/@n8n/workflow-sdk/package.json @@ -14,6 +14,9 @@ }, "./dist/generate-types/generate-types": { "default": "./dist/generate-types/generate-types.js" + }, + "./dist/generate-types/generate-node-defs-cli": { + "default": "./dist/generate-types/generate-node-defs-cli.js" } }, "scripts": { diff --git a/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.test.ts b/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.test.ts index 0bc9b937104..7a0b6c89369 100644 --- a/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.test.ts +++ b/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.test.ts @@ -8,7 +8,32 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { generateNodeDefinitions } from './generate-node-defs-cli'; +import { computeInputHash, generateNodeDefinitions } from './generate-node-defs-cli'; + +const MINIMAL_NODE = { + name: 'n8n-nodes-base.testCliNode', + displayName: 'Test CLI Node', + description: 'A node for CLI testing', + group: ['transform'], + version: 1, + properties: [ + { + name: 'value', + displayName: 'Value', + type: 'string', + default: '', + }, + ], + inputs: ['main'], + outputs: ['main'], +}; + +async function setupNodesJson(dir: string, nodes: unknown[]): Promise { + await fs.promises.mkdir(dir, { recursive: true }); + const nodesJsonPath = path.join(dir, 'nodes.json'); + await fs.promises.writeFile(nodesJsonPath, JSON.stringify(nodes)); + return nodesJsonPath; +} describe('generate-node-defs-cli', () => { const tempDir = path.join(os.tmpdir(), `n8n-cli-test-${Date.now()}`); @@ -18,36 +43,11 @@ describe('generate-node-defs-cli', () => { }); it('should read nodes.json from provided path and generate to output dir', async () => { - // Set up a temp dir with a nodes.json containing a minimal node - const nodesJsonDir = path.join(tempDir, 'input'); + const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input'), [MINIMAL_NODE]); const outputDir = path.join(tempDir, 'output'); - await fs.promises.mkdir(nodesJsonDir, { recursive: true }); - - const minimalNodes = [ - { - name: 'n8n-nodes-base.testCliNode', - displayName: 'Test CLI Node', - description: 'A node for CLI testing', - group: ['transform'], - version: 1, - properties: [ - { - name: 'value', - displayName: 'Value', - type: 'string', - default: '', - }, - ], - inputs: ['main'], - outputs: ['main'], - }, - ]; - const nodesJsonPath = path.join(nodesJsonDir, 'nodes.json'); - await fs.promises.writeFile(nodesJsonPath, JSON.stringify(minimalNodes)); await generateNodeDefinitions({ nodesJsonPath, outputDir }); - // Verify output has .ts and .schema.js files const tsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'testCliNode', 'v1.ts'); expect(fs.existsSync(tsFile)).toBe(true); @@ -62,31 +62,11 @@ describe('generate-node-defs-cli', () => { }); it('should prefix un-dotted node names with packageName', async () => { - const nodesJsonDir = path.join(tempDir, 'input-prefix'); + const unprefixedNode = { ...MINIMAL_NODE, name: 'myNode' }; + const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-prefix'), [ + unprefixedNode, + ]); const outputDir = path.join(tempDir, 'output-prefix'); - await fs.promises.mkdir(nodesJsonDir, { recursive: true }); - - const minimalNodes = [ - { - name: 'myNode', - displayName: 'My Node', - description: 'A node without package prefix', - group: ['transform'], - version: 1, - properties: [ - { - name: 'value', - displayName: 'Value', - type: 'string', - default: '', - }, - ], - inputs: ['main'], - outputs: ['main'], - }, - ]; - const nodesJsonPath = path.join(nodesJsonDir, 'nodes.json'); - await fs.promises.writeFile(nodesJsonPath, JSON.stringify(minimalNodes)); await generateNodeDefinitions({ nodesJsonPath, @@ -94,7 +74,6 @@ describe('generate-node-defs-cli', () => { packageName: 'n8n-nodes-base', }); - // Should be generated under the package name directory const tsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'myNode', 'v1.ts'); expect(fs.existsSync(tsFile)).toBe(true); @@ -110,4 +89,138 @@ describe('generate-node-defs-cli', () => { generateNodeDefinitions({ nodesJsonPath: nonExistentPath, outputDir }), ).rejects.toThrow(); }); + + describe('hash-based skip', () => { + it('should skip generation on second run with unchanged nodes.json', async () => { + const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-hash'), [MINIMAL_NODE]); + const outputDir = path.join(tempDir, 'output-hash'); + + // First run: generates files + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + const hashFile = path.join(outputDir, '.nodes-hash'); + expect(fs.existsSync(hashFile)).toBe(true); + + // Record file modification time + const tsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'testCliNode', 'v1.ts'); + const firstMtime = (await fs.promises.stat(tsFile)).mtimeMs; + + // Small delay to ensure mtime would differ if file were rewritten + await new Promise((resolve) => setTimeout(resolve, 50)); + + // Second run: should skip (files unchanged) + const consoleSpy = jest.spyOn(console, 'log'); + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + + expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('up to date')); + consoleSpy.mockRestore(); + + // File should not have been rewritten + const secondMtime = (await fs.promises.stat(tsFile)).mtimeMs; + expect(secondMtime).toBe(firstMtime); + }); + + it('should regenerate when nodes.json content changes', async () => { + const inputDir = path.join(tempDir, 'input-hash-change'); + const nodesJsonPath = await setupNodesJson(inputDir, [MINIMAL_NODE]); + const outputDir = path.join(tempDir, 'output-hash-change'); + + // First run + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + const hashBefore = await fs.promises.readFile(path.join(outputDir, '.nodes-hash'), 'utf-8'); + + // Modify nodes.json + const modifiedNode = { + ...MINIMAL_NODE, + name: 'n8n-nodes-base.modifiedNode', + displayName: 'Modified Node', + }; + await fs.promises.writeFile(nodesJsonPath, JSON.stringify([modifiedNode])); + + // Second run: should regenerate + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + const hashAfter = await fs.promises.readFile(path.join(outputDir, '.nodes-hash'), 'utf-8'); + + expect(hashAfter).not.toBe(hashBefore); + + // New node files should exist + const newTsFile = path.join(outputDir, 'nodes', 'n8n-nodes-base', 'modifiedNode', 'v1.ts'); + expect(fs.existsSync(newTsFile)).toBe(true); + }); + + it('should regenerate when hash sentinel file is missing', async () => { + const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-hash-missing'), [ + MINIMAL_NODE, + ]); + const outputDir = path.join(tempDir, 'output-hash-missing'); + + // First run + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + const hashFile = path.join(outputDir, '.nodes-hash'); + expect(fs.existsSync(hashFile)).toBe(true); + + // Delete the hash sentinel + await fs.promises.unlink(hashFile); + + // Should regenerate (not skip) + const consoleSpy = jest.spyOn(console, 'log'); + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + + const logCalls = consoleSpy.mock.calls.map((c) => String(c[0])); + expect(logCalls.some((msg) => msg.includes('Generated node definitions'))).toBe(true); + consoleSpy.mockRestore(); + + // Hash sentinel should be recreated + expect(fs.existsSync(hashFile)).toBe(true); + }); + }); + + describe('computeInputHash', () => { + it('should produce different hashes for different content', () => { + const hash1 = computeInputHash('content-a', '0.1.0'); + const hash2 = computeInputHash('content-b', '0.1.0'); + expect(hash1).not.toBe(hash2); + }); + + it('should produce different hashes for different SDK versions', () => { + const hash1 = computeInputHash('same-content', '0.1.0'); + const hash2 = computeInputHash('same-content', '0.2.0'); + expect(hash1).not.toBe(hash2); + }); + + it('should produce consistent hashes for same inputs', () => { + const hash1 = computeInputHash('content', '0.1.0'); + const hash2 = computeInputHash('content', '0.1.0'); + expect(hash1).toBe(hash2); + }); + }); + + describe('parallel writes', () => { + it('should produce correct file count and content with multiple nodes', async () => { + const nodes = Array.from({ length: 5 }, (_, i) => ({ + ...MINIMAL_NODE, + name: `n8n-nodes-base.parallelNode${i}`, + displayName: `Parallel Node ${i}`, + })); + const nodesJsonPath = await setupNodesJson(path.join(tempDir, 'input-parallel'), nodes); + const outputDir = path.join(tempDir, 'output-parallel'); + + await generateNodeDefinitions({ nodesJsonPath, outputDir }); + + // Each node should have v1.ts, v1.schema.js, and index.ts (3 files per node) + // Plus a root index.ts + for (let i = 0; i < 5; i++) { + const nodeDir = path.join(outputDir, 'nodes', 'n8n-nodes-base', `parallelNode${i}`); + expect(fs.existsSync(path.join(nodeDir, 'v1.ts'))).toBe(true); + expect(fs.existsSync(path.join(nodeDir, 'v1.schema.js'))).toBe(true); + expect(fs.existsSync(path.join(nodeDir, 'index.ts'))).toBe(true); + + // Verify content is non-empty + const tsContent = await fs.promises.readFile(path.join(nodeDir, 'v1.ts'), 'utf-8'); + expect(tsContent.length).toBeGreaterThan(0); + } + + // Root index should exist + expect(fs.existsSync(path.join(outputDir, 'index.ts'))).toBe(true); + }); + }); }); diff --git a/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.ts b/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.ts index d409ae4d5b5..a5c872854eb 100644 --- a/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.ts +++ b/packages/@n8n/workflow-sdk/src/generate-types/generate-node-defs-cli.ts @@ -4,11 +4,16 @@ * Reads dist/types/nodes.json from CWD and generates to dist/node-definitions/. * Used as a post-build step in node packages (nodes-base, nodes-langchain). * + * Features: + * - Hash-based skip: computes SHA-256 of nodes.json + SDK version, skips if unchanged + * - Parallel writes: files are written in batches for I/O performance + * * Usage: * n8n-generate-node-defs * # or: npx tsx generate-node-defs-cli.ts */ +import { createHash } from 'crypto'; import * as fs from 'fs'; import { jsonParse } from 'n8n-workflow'; import * as path from 'path'; @@ -16,14 +21,39 @@ import * as path from 'path'; import type { NodeTypeDescription } from './generate-types'; import { orchestrateGeneration } from './generate-types'; +/** Name of the sentinel file storing the content hash */ +const HASH_SENTINEL_FILE = '.nodes-hash'; + export interface GenerateNodeDefinitionsOptions { nodesJsonPath: string; outputDir: string; packageName?: string; } +/** + * Compute a SHA-256 hash of the nodes.json content and SDK package version. + * Including the SDK version ensures regeneration when generation logic changes. + */ +export function computeInputHash(content: string, sdkVersion: string): string { + return createHash('sha256').update(content).update(sdkVersion).digest('hex'); +} + +/** + * Read the SDK package version from its package.json. + */ +function getSdkVersion(): string { + const sdkPackageJsonPath = path.join(__dirname, '..', '..', 'package.json'); + try { + const pkg = JSON.parse(fs.readFileSync(sdkPackageJsonPath, 'utf-8')) as { version: string }; + return pkg.version; + } catch { + return 'unknown'; + } +} + /** * Generate node definitions from a nodes.json file. + * Skips generation if the input hash matches the stored sentinel. * Testable core logic extracted from CLI entry point. */ export async function generateNodeDefinitions( @@ -36,6 +66,22 @@ export async function generateNodeDefinitions( } const content = await fs.promises.readFile(nodesJsonPath, 'utf-8'); + + // Hash-based skip: check if output is already up to date + const sdkVersion = getSdkVersion(); + const inputHash = computeInputHash(content, sdkVersion); + const hashFilePath = path.join(outputDir, HASH_SENTINEL_FILE); + + try { + const existingHash = await fs.promises.readFile(hashFilePath, 'utf-8'); + if (existingHash.trim() === inputHash) { + console.log('Node definitions up to date (hash match), skipping generation.'); + return; + } + } catch { + // Hash file doesn't exist — proceed with generation + } + const nodes = jsonParse(content); if (packageName) { @@ -47,6 +93,11 @@ export async function generateNodeDefinitions( } const result = await orchestrateGeneration({ nodes, outputDir }); + + // Write hash sentinel after successful generation + await fs.promises.mkdir(outputDir, { recursive: true }); + await fs.promises.writeFile(hashFilePath, inputHash); + console.log(`Generated node definitions for ${result.nodeCount} nodes in ${outputDir}`); } diff --git a/packages/@n8n/workflow-sdk/src/generate-types/generate-types.ts b/packages/@n8n/workflow-sdk/src/generate-types/generate-types.ts index 9de732a448e..d44ba1215a1 100644 --- a/packages/@n8n/workflow-sdk/src/generate-types/generate-types.ts +++ b/packages/@n8n/workflow-sdk/src/generate-types/generate-types.ts @@ -3778,20 +3778,18 @@ function convertNodeToToolVariant(node: NodeTypeDescription): NodeTypeDescriptio // Main Entry Point // ============================================================================= +/** Batch size for parallel file writes to avoid file descriptor exhaustion */ +const WRITE_BATCH_SIZE = 100; + /** - * Write files from a plan map to disk - * Creates directories as needed + * Write files in parallel batches to stay within OS file descriptor limits. */ -async function writePlanToDisk(baseDir: string, plan: Map): Promise { - let fileCount = 0; - for (const [relativePath, content] of plan) { - const fullPath = path.join(baseDir, relativePath); - const dir = path.dirname(fullPath); - await fs.promises.mkdir(dir, { recursive: true }); - await fs.promises.writeFile(fullPath, content); - fileCount++; +async function writeFilesInBatches(files: Array<{ path: string; content: string }>): Promise { + for (let i = 0; i < files.length; i += WRITE_BATCH_SIZE) { + const batch = files.slice(i, i + WRITE_BATCH_SIZE); + // eslint-disable-next-line @typescript-eslint/promise-function-async + await Promise.all(batch.map((f) => fs.promises.writeFile(f.path, f.content))); } - return fileCount; } /** @@ -3817,23 +3815,24 @@ async function generateVersionSpecificFiles( nodesByName: Map, ): Promise { const allNodes: NodeTypeDescription[] = []; - let generatedFiles = 0; - let generatedDirs = 0; - let splitVersions = 0; - let flatVersions = 0; + + // Phase 1: Pure computation — generate all file contents in memory + const allDirs = new Set(); + const allWrites: Array<{ path: string; content: string }> = []; + // Track split plans that need their own directory creation + writes + const splitPlans: Array<{ baseDir: string; plan: Map }> = []; for (const [nodeName, nodes] of nodesByName) { try { - // Create directory for this node const nodeDir = path.join(packageDir, nodeName); - await fs.promises.mkdir(nodeDir, { recursive: true }); - generatedDirs++; - // Collect all individual versions from all node entries and map them to their source node - // This allows us to generate a file per individual version (e.g., v3, v31, v32, v33, v34) + // Accumulate writes locally — only commit to allWrites on success + const nodeWrites: Array<{ path: string; content: string }> = []; + const nodeDirs = new Set([nodeDir]); + const nodeSplitPlans: Array<{ baseDir: string; plan: Map }> = []; + const versionToNode = new Map(); const allVersions: number[] = []; - // Track which versions use split structure const splitVersionsSet = new Set(); for (const node of nodes) { @@ -3846,48 +3845,56 @@ async function generateVersionSpecificFiles( } } - // Generate files for each individual version for (const version of allVersions) { const sourceNode = versionToNode.get(version)!; const fileName = versionToFileName(version); - // Check if this node uses a discriminator pattern that benefits from splitting if (hasDiscriminatorPattern(sourceNode)) { - // Generate split structure in a version directory const versionDir = path.join(nodeDir, fileName); const plan = planSplitVersionFiles(sourceNode, version); - const count = await writePlanToDisk(versionDir, plan); - generatedFiles += count; + nodeSplitPlans.push({ baseDir: versionDir, plan }); splitVersionsSet.add(version); - splitVersions++; } else { - // Generate flat type file const content = generateSingleVersionTypeFile(sourceNode, version); - const filePath = path.join(nodeDir, `${fileName}.ts`); - await fs.promises.writeFile(filePath, content); - generatedFiles++; - flatVersions++; + nodeWrites.push({ path: path.join(nodeDir, `${fileName}.ts`), content }); - // Generate corresponding Zod schema file (CommonJS JavaScript for runtime loading) const schemaContent = generateSingleVersionSchemaFile(sourceNode, version); - const schemaFilePath = path.join(nodeDir, `${fileName}.schema.js`); - await fs.promises.writeFile(schemaFilePath, schemaContent); - generatedFiles++; + nodeWrites.push({ + path: path.join(nodeDir, `${fileName}.schema.js`), + content: schemaContent, + }); } } - // Generate index.ts that re-exports all individual versions - // Pass splitVersionsSet so it knows which versions are directories vs files const indexContent = generateVersionIndexFile(nodes[0], allVersions, splitVersionsSet); - await fs.promises.writeFile(path.join(nodeDir, 'index.ts'), indexContent); + nodeWrites.push({ path: path.join(nodeDir, 'index.ts'), content: indexContent }); - // Add first node to allNodes for main index generation + // Commit: all generation succeeded for this node, merge into global arrays + allWrites.push(...nodeWrites); + for (const d of nodeDirs) allDirs.add(d); + splitPlans.push(...nodeSplitPlans); allNodes.push(nodes[0]); } catch (error) { console.error(` Error generating ${nodeName}:`, error); } } + // Collect directories from split plans + for (const { baseDir, plan } of splitPlans) { + for (const [relativePath, content] of plan) { + const fullPath = path.join(baseDir, relativePath); + allDirs.add(path.dirname(fullPath)); + allWrites.push({ path: fullPath, content }); + } + } + + // Phase 2: Create all directories in parallel + // eslint-disable-next-line @typescript-eslint/promise-function-async + await Promise.all([...allDirs].map((d) => fs.promises.mkdir(d, { recursive: true }))); + + // Phase 3: Write all files in parallel batches + await writeFilesInBatches(allWrites); + return allNodes; } @@ -3937,9 +3944,10 @@ export async function orchestrateGeneration(options: GenerationOptions): Promise const allNodes: NodeTypeDescription[] = []; - // Generate files for each package + // Generate files for each package, cleaning stale output first for (const [packageName, nodesByName] of nodesByPackage) { const packageDir = path.join(outputDir, 'nodes', packageName); + await fs.promises.rm(packageDir, { recursive: true, force: true }); const packageNodes = await generateVersionSpecificFiles(packageDir, packageName, nodesByName); allNodes.push(...packageNodes); } diff --git a/packages/core/bin/generate-node-defs b/packages/core/bin/generate-node-defs index c1479cce724..c9dd0ac00fa 100755 --- a/packages/core/bin/generate-node-defs +++ b/packages/core/bin/generate-node-defs @@ -3,34 +3,22 @@ const path = require('path'); const fs = require('fs'); -const { orchestrateGeneration } = require('@n8n/workflow-sdk/dist/generate-types/generate-types'); +const { + generateNodeDefinitions, +} = require('@n8n/workflow-sdk/dist/generate-types/generate-node-defs-cli'); const cwd = process.cwd(); const nodesJsonPath = path.join(cwd, 'dist', 'types', 'nodes.json'); const outputDir = path.join(cwd, 'dist', 'node-definitions'); -if (!fs.existsSync(nodesJsonPath)) { - console.error(`nodes.json not found at ${nodesJsonPath}`); - process.exit(1); -} +const packageJsonPath = path.join(cwd, 'package.json'); +const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')); -(async () => { - const content = await fs.promises.readFile(nodesJsonPath, 'utf-8'); - const nodes = JSON.parse(content); - - const packageJsonPath = path.join(cwd, 'package.json'); - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')); - const packageName = packageJson.name; - - for (const node of nodes) { - if (!node.name.includes('.')) { - node.name = `${packageName}.${node.name}`; - } - } - - const result = await orchestrateGeneration({ nodes, outputDir }); - console.log(`Generated node definitions for ${result.nodeCount} nodes in ${outputDir}`); -})().catch((error) => { +generateNodeDefinitions({ + nodesJsonPath, + outputDir, + packageName: packageJson.name, +}).catch((error) => { console.error('Node definition generation failed:', error); process.exit(1); });