diff --git a/.github/scripts/cleanup-ghcr-images.mjs b/.github/scripts/cleanup-ghcr-images.mjs index 1babf64c5c0..17cc2f40ade 100644 --- a/.github/scripts/cleanup-ghcr-images.mjs +++ b/.github/scripts/cleanup-ghcr-images.mjs @@ -2,70 +2,178 @@ /** * Cleanup GHCR images for n8n CI * - * Usage: - * node cleanup-ghcr-images.mjs --tag # Delete specific tag - * node cleanup-ghcr-images.mjs --pr # Delete all pr-{number}-* tags - * node cleanup-ghcr-images.mjs --stale # Delete pr-* images older than N days + * Modes: + * --tag Delete exact tag (merge queue cleanup - single image) + * --pr Delete all pr-{number}-* tags (PR cleanup - all runs for a PR) + * --stale Delete pr-* images older than N days (weekly scheduled cleanup) + * + * Context: + * - PR runs use --pr to clean all images from failed/retried commits + * - Merge queue runs use --tag since PR number isn't available (image tagged pr--{run_id}) + * - Weekly cron uses --stale to catch any orphaned images */ -import { execSync } from 'node:child_process'; +import { exec } from 'node:child_process'; +import { promisify } from 'node:util'; +const execAsync = promisify(exec); const ORG = 'n8n-io'; const PACKAGES = ['n8n', 'runners']; const [mode, rawValue] = process.argv.slice(2); + if (!['--tag', '--pr', '--stale'].includes(mode) || !rawValue) { - console.error('Usage: cleanup-ghcr-images.mjs --tag|--pr|--stale '); + console.error('Usage: cleanup-ghcr-images.mjs --tag | --pr | --stale '); process.exit(1); } + const value = mode === '--stale' ? parseInt(rawValue, 10) : rawValue; -if (mode === '--stale' && (!Number.isFinite(value) || value <= 0)) { - console.error('Error: --stale requires a positive number'); +if (mode === '--stale' && (isNaN(value) || value <= 0)) { + console.error('Error: --stale requires a positive number of days'); process.exit(1); } -let hasErrors = false; +async function ghApi(path) { + const { stdout } = await execAsync( + `gh api "/orgs/${ORG}/packages/container/${path}"`, + ); + return JSON.parse(stdout); +} -function ghApi(path, del = false) { - const method = del ? '--method DELETE ' : ''; +async function ghDelete(path) { + await execAsync(`gh api --method DELETE "/orgs/${ORG}/packages/container/${path}"`); +} + +async function fetchPage(pkg, page) { try { - const out = execSync( - `gh api ${method}"/orgs/${ORG}/packages/container/${path}" -H "Accept: application/vnd.github+json"`, - { encoding: 'utf8', stdio: 'pipe' }, - ); - return del ? null : JSON.parse(out); - } catch { - if (del) hasErrors = true; - return null; + return await ghApi(`${pkg}/versions?per_page=100&page=${page}`); + } catch (err) { + if (err.code === 1 && err.stderr?.includes('404')) return []; + throw new Error(`Failed to fetch ${pkg} page ${page}: ${err.message}`); } } -function getVersions(pkg) { +const isPrImage = (v, prNum) => { + const tags = v.metadata?.container?.tags || []; + return prNum ? tags.some((t) => t.startsWith(`pr-${prNum}-`)) : tags.some((t) => t.startsWith('pr-')); +}; + +const isStale = (v, days) => { + const cutoff = Date.now() - days * 86400000; + return isPrImage(v) && new Date(v.created_at) < cutoff; +}; + +async function getVersionsForTag(pkg, tag) { + const batch = await fetchPage(pkg, 1); + const match = batch.find((v) => v.metadata?.container?.tags?.includes(tag)); + return match ? [match] : []; +} + +async function getVersionsForPr(pkg, prNumber) { const versions = []; + let emptyPages = 0; + for (let page = 1; ; page++) { - const batch = ghApi(`${pkg}/versions?per_page=100&page=${page}`); - if (!batch?.length) break; - versions.push(...batch); + const batch = await fetchPage(pkg, page); + if (!batch.length) break; + + const matches = batch.filter((v) => isPrImage(v, prNumber)); + if (matches.length) { + versions.push(...matches); + emptyPages = 0; + } else if (++emptyPages >= 3) { + console.log(` Early termination after ${page} pages`); + break; + } if (batch.length < 100) break; } return versions; } -function shouldDelete(v) { - const tags = v.metadata?.container?.tags || []; - if (mode === '--tag') return tags.includes(value); - if (mode === '--pr') return tags.some((t) => t.startsWith(`pr-${value}-`)); - if (mode === '--stale') { - const cutoff = Date.now() - value * 86400000; - return tags.some((t) => t.startsWith('pr-')) && new Date(v.created_at) < cutoff; +async function getVersionsForStale(pkg, days) { + const versions = []; + const cutoff = Date.now() - days * 86400000; + // Use 2x cutoff as safety window for early termination + const earlyExitCutoff = Date.now() - days * 2 * 86400000; + let pagesWithoutPrImages = 0; + + const firstPage = await fetchPage(pkg, 1); + if (!firstPage.length) return []; + + for (const v of firstPage) { + if (isStale(v, days)) versions.push(v); } - return false; + if (firstPage.length < 100) return versions; + + for (let page = 2; ; page += 10) { + const batches = await Promise.all( + Array.from({ length: 10 }, (_, i) => fetchPage(pkg, page + i)), + ); + let done = false; + for (const batch of batches) { + if (!batch.length || batch.length < 100) done = true; + + let hasPrImages = false; + for (const v of batch) { + if (isPrImage(v)) { + hasPrImages = true; + if (new Date(v.created_at) < cutoff) versions.push(v); + } + } + + // Early termination: if we've gone through pages without finding + // any PR images and all items are older than 2x cutoff, we're past + // the PR image window + if (!hasPrImages) { + pagesWithoutPrImages++; + const oldestInBatch = batch[batch.length - 1]; + if ( + pagesWithoutPrImages >= 3 && + oldestInBatch && + new Date(oldestInBatch.created_at) < earlyExitCutoff + ) { + console.log(` Early termination at page ${page + batches.indexOf(batch)}`); + done = true; + } + } else { + pagesWithoutPrImages = 0; + } + + if (!batch.length || done) break; + } + if (done) break; + } + return versions; } +let hasErrors = false; + for (const pkg of PACKAGES) { - const toDelete = getVersions(pkg).filter(shouldDelete); - if (!toDelete.length) console.log(`No matching images found for ${pkg}`); + console.log(`Processing ${pkg}...`); + let consecutiveErrors = 0; + + const toDelete = + mode === '--tag' + ? await getVersionsForTag(pkg, value) + : mode === '--pr' + ? await getVersionsForPr(pkg, value) + : await getVersionsForStale(pkg, value); + + if (!toDelete.length) { + console.log(` No matching images found`); + continue; + } + for (const v of toDelete) { - ghApi(`${pkg}/versions/${v.id}`, true); - console.log(`Deleted ${pkg}:${v.metadata.container.tags.join(',')}`); + try { + await ghDelete(`${pkg}/versions/${v.id}`); + console.log(` Deleted ${v.metadata.container.tags.join(',')}`); + consecutiveErrors = 0; + } catch (err) { + console.error(` Failed to delete ${v.id}: ${err.message}`); + hasErrors = true; + if (++consecutiveErrors >= 3) { + throw new Error('Too many consecutive delete failures, aborting'); + } + } } } diff --git a/.github/workflows/test-e2e-ci-reusable.yml b/.github/workflows/test-e2e-ci-reusable.yml index 7287e2befdb..72904af10b5 100644 --- a/.github/workflows/test-e2e-ci-reusable.yml +++ b/.github/workflows/test-e2e-ci-reusable.yml @@ -123,7 +123,7 @@ jobs: name: 'Cleanup Docker Image' needs: [multi-main-e2e, multi-main-isolated] if: ${{ !failure() && !cancelled() && !github.event.pull_request.head.repo.fork }} - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: packages: write contents: read @@ -137,4 +137,11 @@ jobs: - name: Delete images from GHCR env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: node .github/scripts/cleanup-ghcr-images.mjs --tag pr-${{ github.event.pull_request.number }}-${{ github.run_id }} + run: | + if [ -n "${{ github.event.pull_request.number }}" ]; then + echo "PR context: cleaning all images for PR ${{ github.event.pull_request.number }}" + node .github/scripts/cleanup-ghcr-images.mjs --pr ${{ github.event.pull_request.number }} + else + echo "Merge queue context: cleaning image pr--${{ github.run_id }}" + node .github/scripts/cleanup-ghcr-images.mjs --tag pr--${{ github.run_id }} + fi diff --git a/.github/workflows/util-cleanup-pr-images.yml b/.github/workflows/util-cleanup-pr-images.yml index 3402c0c7032..7f87fe6c9e9 100644 --- a/.github/workflows/util-cleanup-pr-images.yml +++ b/.github/workflows/util-cleanup-pr-images.yml @@ -1,19 +1,14 @@ name: 'Util: Cleanup PR Docker Images' on: - pull_request: - types: [closed] schedule: # Weekly cleanup: Sunday at 3 AM UTC - cron: '0 3 * * 0' jobs: cleanup: - name: 'Delete PR images' - # Skip for fork PRs (they don't have GHCR images) - # Always run on schedule - if: ${{ github.event_name == 'schedule' || !github.event.pull_request.head.repo.fork }} - runs-on: ubuntu-latest + name: 'Delete stale PR images' + runs-on: ubuntu-slim permissions: packages: write contents: read @@ -21,20 +16,10 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - # On PR close, the PR branch may be deleted - use default branch - # On schedule, use the default ref - ref: ${{ github.event.repository.default_branch || github.ref }} sparse-checkout: .github/scripts sparse-checkout-cone-mode: false - - name: Delete images for closed PR - if: ${{ github.event_name == 'pull_request' }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: node .github/scripts/cleanup-ghcr-images.mjs --pr ${{ github.event.pull_request.number }} - - name: Delete stale PR images - if: ${{ github.event_name == 'schedule' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: node .github/scripts/cleanup-ghcr-images.mjs --stale 7