build: Install Kafka binding on fresh clone (#36502)

.github/scripts/docker/kafka-native-smoke-check.mjs

@@ -14,7 +14,9 @@ const REQUIRED_FEATURES = ['ssl', 'sasl_scram'];
 const COMPRESSION_CODECS = ['gzip', 'snappy', 'lz4', 'zstd'];
 
 const n8nInstallDir = process.env.N8N_INSTALL_DIR || '/usr/local/lib/node_modules/n8n';
-const nodesBasePackageJson = path.join(n8nInstallDir, 'node_modules/n8n-nodes-base/package.json');
+const nodesBasePackageJson =
+	process.env.NODES_BASE_PACKAGE_JSON ||
+	path.join(n8nInstallDir, 'node_modules/n8n-nodes-base/package.json');
 // n8n-nodes-base is a pnpm symlink; resolve it so require() walks up from its real
 // location in the pnpm virtual store, where its dependencies actually live.
 const require = createRequire(realpathSync(nodesBasePackageJson));

.github/workflows/test-unit-reusable.yml

@@ -54,6 +54,13 @@ jobs:
         with:
           node-version: ${{ inputs.nodeVersion }}
 
+      # Confluent publishes prebuilds up to Node 24 only.
+      - name: Verify Kafka native binding installed
+        if: startsWith(inputs.nodeVersion, '24.')
+        env:
+          NODES_BASE_PACKAGE_JSON: packages/nodes-base/package.json
+        run: node .github/scripts/docker/kafka-native-smoke-check.mjs
+
       # cli is the only backend package wired into janitor scoping today, so
       # the workspace bulk run excludes it and a dedicated step invokes the
       # scoped variant. This avoids turbo silently no-op'ing `test:unit:changed`

package.json

@@ -105,6 +105,7 @@
   },
   "pnpm": {
     "onlyBuiltDependencies": [
+      "@confluentinc/kafka-javascript",
       "@vscode/ripgrep",
       "isolated-vm",
       "sqlite3"
@@ -237,7 +238,8 @@
       "assert@2.1.0": "patches/assert@2.1.0.patch",
       "lodash@4.18.1": "patches/lodash@4.18.1.patch",
       "@langchain/openai@1.4.4": "patches/@langchain__openai@1.4.4.patch",
-      "@langchain/google-common@2.1.24": "patches/@langchain__google-common@2.1.24.patch"
+      "@langchain/google-common@2.1.24": "patches/@langchain__google-common@2.1.24.patch",
+      "@confluentinc/kafka-javascript@1.9.1": "patches/@confluentinc__kafka-javascript@1.9.1.patch"
     }
   }
 }

patches/@confluentinc__kafka-javascript@1.9.1.patch

@@ -0,0 +1,13 @@
+diff --git a/package.json b/package.json
+index ed972a91dca9ad78b47f1d8d295df58cdef269ff..d8f7499f85c1432cbf62bdbbe01fc12680e8e31c 100644
+--- a/package.json
++++ b/package.json
+@@ -10,7 +10,7 @@
+     "configure": "node-gyp configure",
+     "build": "node-gyp build",
+     "test": "make test",
+-    "install": "node-pre-gyp install --fallback-to-build",
++    "install": "node-pre-gyp install || echo '[@confluentinc/kafka-javascript] no prebuilt binding for this Node version or platform (see the node-pre-gyp error above), so the Kafka node will not run. Use Node 24 for local Kafka work.'",
+     "install-from-source": "npm ci && node-pre-gyp install --build-from-source=@confluentinc/kafka-javascript --fallback-to-build",
+     "prepack": "node ./ci/prepublish.js",
+     "test:types": "tsc -p ."

pnpm-lock.yaml

@@ -816,6 +816,9 @@ overrides:
   '@zone-eu/mailsplit@<5.4.15': 5.4.15
 
 patchedDependencies:
+  '@confluentinc/kafka-javascript@1.9.1':
+    hash: c597d3e3d9d6d0d33588b7dc1e88d55f270ba155721aad464bd7fb03e28f5003
+    path: patches/@confluentinc__kafka-javascript@1.9.1.patch
   '@langchain/google-common@2.1.24':
     hash: b50436a6c7b0a82d54472dff331ba1b42ab8b816995f83295eef2ee4b482a87d
     path: patches/@langchain__google-common@2.1.24.patch
@@ -6439,7 +6442,7 @@ importers:
         version: 3.808.0
       '@confluentinc/kafka-javascript':
         specifier: 'catalog:'
-        version: 1.9.1(encoding@0.1.13)
+        version: 1.9.1(patch_hash=c597d3e3d9d6d0d33588b7dc1e88d55f270ba155721aad464bd7fb03e28f5003)(encoding@0.1.13)
       '@e965/xlsx':
         specifier: 'catalog:'
         version: 0.20.3
@@ -26037,7 +26040,7 @@ snapshots:
     dependencies:
       commander: 13.1.0
 
-  '@confluentinc/kafka-javascript@1.9.1(encoding@0.1.13)':
+  '@confluentinc/kafka-javascript@1.9.1(patch_hash=c597d3e3d9d6d0d33588b7dc1e88d55f270ba155721aad464bd7fb03e28f5003)(encoding@0.1.13)':
     dependencies:
       '@mapbox/node-pre-gyp': 2.0.3(encoding@0.1.13)
       bindings: 1.5.0

scripts/build-n8n.mjs

@@ -35,7 +35,12 @@ const config = {
 };
 
 // Define backend patches to keep during deployment
-const PATCHES_TO_KEEP = ['pdfjs-dist', 'pkce-challenge', 'bull'];
+const PATCHES_TO_KEEP = [
+	'pdfjs-dist',
+	'pkce-challenge',
+	'bull',
+	'@confluentinc/kafka-javascript',
+];
 
 // #endregion ===== Configuration =====