From d54c23e4755488f8c4bf05e8eab6b4b73a3f2de3 Mon Sep 17 00:00:00 2001 From: David Parry Date: Tue, 24 Mar 2026 14:09:40 +1100 Subject: [PATCH] fix: build amd64 and arm64 images in parallel via matrix Each platform now gets its own runner with QEMU, running concurrently. The single-job QEMU approach built architectures sequentially. Restores the digest+merge pattern to combine parallel builds into a single multi-arch manifest. --- .github/workflows/build-disk-collector.yml | 104 +++++++++++++------ .github/workflows/build-primary-image.yml | 106 ++++++++++++++------ .github/workflows/build-sidecar-updater.yml | 104 +++++++++++++------ 3 files changed, 220 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build-disk-collector.yml b/.github/workflows/build-disk-collector.yml index 4c7f06b..9018a22 100644 --- a/.github/workflows/build-disk-collector.yml +++ b/.github/workflows/build-disk-collector.yml @@ -16,11 +16,13 @@ on: default: false jobs: - check_authorization: - name: Check authorization to publish new Docker image + setup: + name: Resolve version and check authorization runs-on: ubuntu-latest outputs: isAuthorized: ${{ steps.check-auth.outputs.is_authorized }} + version: ${{ steps.vars.outputs.version }} + tag_latest: ${{ steps.vars.outputs.tag_latest }} steps: - name: check-auth id: check-auth @@ -30,19 +32,7 @@ jobs: else echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT fi - - build: - name: Build and push multi-arch disk-collector image - needs: check_authorization - if: needs.check_authorization.outputs.isAuthorized == 'true' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Resolve version and tags + - name: Resolve version and tag_latest id: vars run: | if [ "${{ github.event_name }}" = "release" ]; then @@ -53,18 +43,24 @@ jobs: VERSION="${{ inputs.version }}" TAG_LATEST=${{ inputs.tag_latest }} fi - - TAGS="ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:${VERSION} - ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:v${VERSION}" - if [ "$TAG_LATEST" = "true" ]; then - TAGS="$TAGS - ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:latest" - fi - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "tags<> $GITHUB_OUTPUT - echo "$TAGS" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + echo "tag_latest=$TAG_LATEST" >> $GITHUB_OUTPUT + + build: + name: Build disk-collector image (${{ matrix.platform }}) + needs: setup + if: needs.setup.outputs.isAuthorized == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: [linux/amd64, linux/arm64] + steps: + - name: Checkout code + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -75,13 +71,59 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 with: context: install/sidecar-disk-collector - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.vars.outputs.tags }} + platforms: ${{ matrix.platform }} + outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector,push-by-digest=true,name-canonical=true,push=true + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Create and push multi-arch manifest + needs: [setup, build] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create manifest list and push + run: | + VERSION="${{ needs.setup.outputs.version }}" + TAGS="-t ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:${VERSION}" + TAGS="$TAGS -t ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:v${VERSION}" + if [ "${{ needs.setup.outputs.tag_latest }}" = "true" ]; then + TAGS="$TAGS -t ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:latest" + fi + docker buildx imagetools create $TAGS \ + $(printf 'ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector@sha256:%s ' *) + working-directory: /tmp/digests - name: Inspect image run: | - docker buildx imagetools inspect ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:${{ steps.vars.outputs.version }} + docker buildx imagetools inspect ghcr.io/${{ github.repository_owner }}/project-nomad-disk-collector:${{ needs.setup.outputs.version }} diff --git a/.github/workflows/build-primary-image.yml b/.github/workflows/build-primary-image.yml index 1d776ab..8a04082 100644 --- a/.github/workflows/build-primary-image.yml +++ b/.github/workflows/build-primary-image.yml @@ -16,11 +16,13 @@ on: default: false jobs: - check_authorization: - name: Check authorization to publish new Docker image + setup: + name: Resolve version and check authorization runs-on: ubuntu-latest outputs: isAuthorized: ${{ steps.check-auth.outputs.is_authorized }} + version: ${{ steps.vars.outputs.version }} + tag_latest: ${{ steps.vars.outputs.tag_latest }} steps: - name: check-auth id: check-auth @@ -30,19 +32,7 @@ jobs: else echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT fi - - build: - name: Build and push multi-arch primary image - needs: check_authorization - if: needs.check_authorization.outputs.isAuthorized == 'true' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Resolve version and tags + - name: Resolve version and tag_latest id: vars run: | if [ "${{ github.event_name }}" = "release" ]; then @@ -53,18 +43,24 @@ jobs: VERSION="${{ inputs.version }}" TAG_LATEST=${{ inputs.tag_latest }} fi - - TAGS="ghcr.io/${{ github.repository_owner }}/project-nomad:${VERSION} - ghcr.io/${{ github.repository_owner }}/project-nomad:v${VERSION}" - if [ "$TAG_LATEST" = "true" ]; then - TAGS="$TAGS - ghcr.io/${{ github.repository_owner }}/project-nomad:latest" - fi - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "tags<> $GITHUB_OUTPUT - echo "$TAGS" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + echo "tag_latest=$TAG_LATEST" >> $GITHUB_OUTPUT + + build: + name: Build primary image (${{ matrix.platform }}) + needs: setup + if: needs.setup.outputs.isAuthorized == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: [linux/amd64, linux/arm64] + steps: + - name: Checkout code + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -75,16 +71,62 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 with: - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.vars.outputs.tags }} + platforms: ${{ matrix.platform }} + outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/project-nomad,push-by-digest=true,name-canonical=true,push=true build-args: | - VERSION=${{ steps.vars.outputs.version }} + VERSION=${{ needs.setup.outputs.version }} BUILD_DATE=${{ github.event.release.created_at }} VCS_REF=${{ github.sha }} + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Create and push multi-arch manifest + needs: [setup, build] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create manifest list and push + run: | + VERSION="${{ needs.setup.outputs.version }}" + TAGS="-t ghcr.io/${{ github.repository_owner }}/project-nomad:${VERSION}" + TAGS="$TAGS -t ghcr.io/${{ github.repository_owner }}/project-nomad:v${VERSION}" + if [ "${{ needs.setup.outputs.tag_latest }}" = "true" ]; then + TAGS="$TAGS -t ghcr.io/${{ github.repository_owner }}/project-nomad:latest" + fi + docker buildx imagetools create $TAGS \ + $(printf 'ghcr.io/${{ github.repository_owner }}/project-nomad@sha256:%s ' *) + working-directory: /tmp/digests - name: Inspect image run: | - docker buildx imagetools inspect ghcr.io/${{ github.repository_owner }}/project-nomad:${{ steps.vars.outputs.version }} + docker buildx imagetools inspect ghcr.io/${{ github.repository_owner }}/project-nomad:${{ needs.setup.outputs.version }} diff --git a/.github/workflows/build-sidecar-updater.yml b/.github/workflows/build-sidecar-updater.yml index 3cd6c07..2555db2 100644 --- a/.github/workflows/build-sidecar-updater.yml +++ b/.github/workflows/build-sidecar-updater.yml @@ -16,11 +16,13 @@ on: default: false jobs: - check_authorization: - name: Check authorization to publish new Docker image + setup: + name: Resolve version and check authorization runs-on: ubuntu-latest outputs: isAuthorized: ${{ steps.check-auth.outputs.is_authorized }} + version: ${{ steps.vars.outputs.version }} + tag_latest: ${{ steps.vars.outputs.tag_latest }} steps: - name: check-auth id: check-auth @@ -30,19 +32,7 @@ jobs: else echo "is_authorized=${{ contains(secrets.DEPLOYMENT_AUTHORIZED_USERS, github.triggering_actor) }}" >> $GITHUB_OUTPUT fi - - build: - name: Build and push multi-arch sidecar-updater image - needs: check_authorization - if: needs.check_authorization.outputs.isAuthorized == 'true' - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Resolve version and tags + - name: Resolve version and tag_latest id: vars run: | if [ "${{ github.event_name }}" = "release" ]; then @@ -53,18 +43,24 @@ jobs: VERSION="${{ inputs.version }}" TAG_LATEST=${{ inputs.tag_latest }} fi - - TAGS="ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:${VERSION} - ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:v${VERSION}" - if [ "$TAG_LATEST" = "true" ]; then - TAGS="$TAGS - ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:latest" - fi - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "tags<> $GITHUB_OUTPUT - echo "$TAGS" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + echo "tag_latest=$TAG_LATEST" >> $GITHUB_OUTPUT + + build: + name: Build sidecar-updater image (${{ matrix.platform }}) + needs: setup + if: needs.setup.outputs.isAuthorized == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + platform: [linux/amd64, linux/arm64] + steps: + - name: Checkout code + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -75,13 +71,59 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 with: context: install/sidecar-updater - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.vars.outputs.tags }} + platforms: ${{ matrix.platform }} + outputs: type=image,name=ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater,push-by-digest=true,name-canonical=true,push=true + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Create and push multi-arch manifest + needs: [setup, build] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create manifest list and push + run: | + VERSION="${{ needs.setup.outputs.version }}" + TAGS="-t ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:${VERSION}" + TAGS="$TAGS -t ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:v${VERSION}" + if [ "${{ needs.setup.outputs.tag_latest }}" = "true" ]; then + TAGS="$TAGS -t ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:latest" + fi + docker buildx imagetools create $TAGS \ + $(printf 'ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater@sha256:%s ' *) + working-directory: /tmp/digests - name: Inspect image run: | - docker buildx imagetools inspect ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:${{ steps.vars.outputs.version }} + docker buildx imagetools inspect ghcr.io/${{ github.repository_owner }}/project-nomad-sidecar-updater:${{ needs.setup.outputs.version }}