ci: Optimize GHCR cleanup script with early termination and parallel pagination (#24892)

.github/scripts/cleanup-ghcr-images.mjs

@@ -2,70 +2,178 @@
 /**
  * Cleanup GHCR images for n8n CI
  *
- * Usage:
- *   node cleanup-ghcr-images.mjs --tag <tag>     # Delete specific tag
- *   node cleanup-ghcr-images.mjs --pr <number>   # Delete all pr-{number}-* tags
- *   node cleanup-ghcr-images.mjs --stale <days>  # Delete pr-* images older than N days
+ * Modes:
+ *   --tag <tag>     Delete exact tag (merge queue cleanup - single image)
+ *   --pr <number>   Delete all pr-{number}-* tags (PR cleanup - all runs for a PR)
+ *   --stale <days>  Delete pr-* images older than N days (weekly scheduled cleanup)
+ *
+ * Context:
+ *   - PR runs use --pr to clean all images from failed/retried commits
+ *   - Merge queue runs use --tag since PR number isn't available (image tagged pr--{run_id})
+ *   - Weekly cron uses --stale to catch any orphaned images
  */
-import { execSync } from 'node:child_process';
+import { exec } from 'node:child_process';
+import { promisify } from 'node:util';
 
+const execAsync = promisify(exec);
 const ORG = 'n8n-io';
 const PACKAGES = ['n8n', 'runners'];
 const [mode, rawValue] = process.argv.slice(2);
+
 if (!['--tag', '--pr', '--stale'].includes(mode) || !rawValue) {
-	console.error('Usage: cleanup-ghcr-images.mjs --tag|--pr|--stale <value>');
+	console.error('Usage: cleanup-ghcr-images.mjs --tag <tag> | --pr <number> | --stale <days>');
 	process.exit(1);
 }
+
 const value = mode === '--stale' ? parseInt(rawValue, 10) : rawValue;
-if (mode === '--stale' && (!Number.isFinite(value) || value <= 0)) {
-	console.error('Error: --stale requires a positive number');
+if (mode === '--stale' && (isNaN(value) || value <= 0)) {
+	console.error('Error: --stale requires a positive number of days');
 	process.exit(1);
 }
 
-let hasErrors = false;
+async function ghApi(path) {
+	const { stdout } = await execAsync(
+		`gh api "/orgs/${ORG}/packages/container/${path}"`,
+	);
+	return JSON.parse(stdout);
+}
 
-function ghApi(path, del = false) {
-	const method = del ? '--method DELETE ' : '';
+async function ghDelete(path) {
+	await execAsync(`gh api --method DELETE "/orgs/${ORG}/packages/container/${path}"`);
+}
+
+async function fetchPage(pkg, page) {
 	try {
-		const out = execSync(
-			`gh api ${method}"/orgs/${ORG}/packages/container/${path}" -H "Accept: application/vnd.github+json"`,
-			{ encoding: 'utf8', stdio: 'pipe' },
-		);
-		return del ? null : JSON.parse(out);
-	} catch {
-		if (del) hasErrors = true;
-		return null;
+		return await ghApi(`${pkg}/versions?per_page=100&page=${page}`);
+	} catch (err) {
+		if (err.code === 1 && err.stderr?.includes('404')) return [];
+		throw new Error(`Failed to fetch ${pkg} page ${page}: ${err.message}`);
 	}
 }
 
-function getVersions(pkg) {
+const isPrImage = (v, prNum) => {
+	const tags = v.metadata?.container?.tags || [];
+	return prNum ? tags.some((t) => t.startsWith(`pr-${prNum}-`)) : tags.some((t) => t.startsWith('pr-'));
+};
+
+const isStale = (v, days) => {
+	const cutoff = Date.now() - days * 86400000;
+	return isPrImage(v) && new Date(v.created_at) < cutoff;
+};
+
+async function getVersionsForTag(pkg, tag) {
+	const batch = await fetchPage(pkg, 1);
+	const match = batch.find((v) => v.metadata?.container?.tags?.includes(tag));
+	return match ? [match] : [];
+}
+
+async function getVersionsForPr(pkg, prNumber) {
 	const versions = [];
+	let emptyPages = 0;
+
 	for (let page = 1; ; page++) {
-		const batch = ghApi(`${pkg}/versions?per_page=100&page=${page}`);
-		if (!batch?.length) break;
-		versions.push(...batch);
+		const batch = await fetchPage(pkg, page);
+		if (!batch.length) break;
+
+		const matches = batch.filter((v) => isPrImage(v, prNumber));
+		if (matches.length) {
+			versions.push(...matches);
+			emptyPages = 0;
+		} else if (++emptyPages >= 3) {
+			console.log(`  Early termination after ${page} pages`);
+			break;
+		}
 		if (batch.length < 100) break;
 	}
 	return versions;
 }
 
-function shouldDelete(v) {
-	const tags = v.metadata?.container?.tags || [];
-	if (mode === '--tag') return tags.includes(value);
-	if (mode === '--pr') return tags.some((t) => t.startsWith(`pr-${value}-`));
-	if (mode === '--stale') {
-		const cutoff = Date.now() - value * 86400000;
-		return tags.some((t) => t.startsWith('pr-')) && new Date(v.created_at) < cutoff;
+async function getVersionsForStale(pkg, days) {
+	const versions = [];
+	const cutoff = Date.now() - days * 86400000;
+	// Use 2x cutoff as safety window for early termination
+	const earlyExitCutoff = Date.now() - days * 2 * 86400000;
+	let pagesWithoutPrImages = 0;
+
+	const firstPage = await fetchPage(pkg, 1);
+	if (!firstPage.length) return [];
+
+	for (const v of firstPage) {
+		if (isStale(v, days)) versions.push(v);
 	}
-	return false;
+	if (firstPage.length < 100) return versions;
+
+	for (let page = 2; ; page += 10) {
+		const batches = await Promise.all(
+			Array.from({ length: 10 }, (_, i) => fetchPage(pkg, page + i)),
+		);
+		let done = false;
+		for (const batch of batches) {
+			if (!batch.length || batch.length < 100) done = true;
+
+			let hasPrImages = false;
+			for (const v of batch) {
+				if (isPrImage(v)) {
+					hasPrImages = true;
+					if (new Date(v.created_at) < cutoff) versions.push(v);
+				}
+			}
+
+			// Early termination: if we've gone through pages without finding
+			// any PR images and all items are older than 2x cutoff, we're past
+			// the PR image window
+			if (!hasPrImages) {
+				pagesWithoutPrImages++;
+				const oldestInBatch = batch[batch.length - 1];
+				if (
+					pagesWithoutPrImages >= 3 &&
+					oldestInBatch &&
+					new Date(oldestInBatch.created_at) < earlyExitCutoff
+				) {
+					console.log(`  Early termination at page ${page + batches.indexOf(batch)}`);
+					done = true;
+				}
+			} else {
+				pagesWithoutPrImages = 0;
+			}
+
+			if (!batch.length || done) break;
+		}
+		if (done) break;
+	}
+	return versions;
 }
 
+let hasErrors = false;
+
 for (const pkg of PACKAGES) {
-	const toDelete = getVersions(pkg).filter(shouldDelete);
-	if (!toDelete.length) console.log(`No matching images found for ${pkg}`);
+	console.log(`Processing ${pkg}...`);
+	let consecutiveErrors = 0;
+
+	const toDelete =
+		mode === '--tag'
+			? await getVersionsForTag(pkg, value)
+			: mode === '--pr'
+				? await getVersionsForPr(pkg, value)
+				: await getVersionsForStale(pkg, value);
+
+	if (!toDelete.length) {
+		console.log(`  No matching images found`);
+		continue;
+	}
+
 	for (const v of toDelete) {
-		ghApi(`${pkg}/versions/${v.id}`, true);
-		console.log(`Deleted ${pkg}:${v.metadata.container.tags.join(',')}`);
+		try {
+			await ghDelete(`${pkg}/versions/${v.id}`);
+			console.log(`  Deleted ${v.metadata.container.tags.join(',')}`);
+			consecutiveErrors = 0;
+		} catch (err) {
+			console.error(`  Failed to delete ${v.id}: ${err.message}`);
+			hasErrors = true;
+			if (++consecutiveErrors >= 3) {
+				throw new Error('Too many consecutive delete failures, aborting');
+			}
+		}
 	}
 }
 

.github/workflows/test-e2e-ci-reusable.yml

@@ -123,7 +123,7 @@ jobs:
     name: 'Cleanup Docker Image'
     needs: [multi-main-e2e, multi-main-isolated]
     if: ${{ !failure() && !cancelled() && !github.event.pull_request.head.repo.fork }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     permissions:
       packages: write
       contents: read
@@ -137,4 +137,11 @@ jobs:
       - name: Delete images from GHCR
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: node .github/scripts/cleanup-ghcr-images.mjs --tag pr-${{ github.event.pull_request.number }}-${{ github.run_id }}
+        run: |
+          if [ -n "${{ github.event.pull_request.number }}" ]; then
+            echo "PR context: cleaning all images for PR ${{ github.event.pull_request.number }}"
+            node .github/scripts/cleanup-ghcr-images.mjs --pr ${{ github.event.pull_request.number }}
+          else
+            echo "Merge queue context: cleaning image pr--${{ github.run_id }}"
+            node .github/scripts/cleanup-ghcr-images.mjs --tag pr--${{ github.run_id }}
+          fi

.github/workflows/util-cleanup-pr-images.yml

@@ -1,19 +1,14 @@
 name: 'Util: Cleanup PR Docker Images'
 
 on:
-  pull_request:
-    types: [closed]
   schedule:
     # Weekly cleanup: Sunday at 3 AM UTC
     - cron: '0 3 * * 0'
 
 jobs:
   cleanup:
-    name: 'Delete PR images'
-    # Skip for fork PRs (they don't have GHCR images)
-    # Always run on schedule
-    if: ${{ github.event_name == 'schedule' || !github.event.pull_request.head.repo.fork }}
-    runs-on: ubuntu-latest
+    name: 'Delete stale PR images'
+    runs-on: ubuntu-slim
     permissions:
       packages: write
       contents: read
@@ -21,20 +16,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          # On PR close, the PR branch may be deleted - use default branch
-          # On schedule, use the default ref
-          ref: ${{ github.event.repository.default_branch || github.ref }}
           sparse-checkout: .github/scripts
           sparse-checkout-cone-mode: false
 
-      - name: Delete images for closed PR
-        if: ${{ github.event_name == 'pull_request' }}
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: node .github/scripts/cleanup-ghcr-images.mjs --pr ${{ github.event.pull_request.number }}
-
       - name: Delete stale PR images
-        if: ${{ github.event_name == 'schedule' }}
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: node .github/scripts/cleanup-ghcr-images.mjs --stale 7