Added parallel docker builds and caching for go build in the container

Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com>

.github/workflows/build_publish_beta_docker_image.yml

@@ -1,77 +1,261 @@
-name: Docker beta build
+---
+# Github Actions release for rclone
+# -*- compile-command: "yamllint -f build_publish_beta_docker_image.yml" -*-
 
+name: Build & Push Docker Images
+
+# Trigger the workflow on push or pull request
 on:
-    push:
+  push:
-      branches:
+    branches:
-        - master
+      - '**'
+    tags:
+      - '**'
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      manual:
+        description: Manual run (bypass default conditions)
+        type: boolean
+        default: true
+
 jobs:
-    build:
+  build-image:
-        if: github.repository == 'rclone/rclone'
+    if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
-        runs-on: ubuntu-latest
+    timeout-minutes: 60
-        name: Build image job
+    strategy:
-        steps:
+      fail-fast: false
-            - name: Free some space
+      matrix:
-              shell: bash
+        include:
-              run: |
+          - platform: linux/amd64
-                df -h .
+            runs-on: ubuntu-latest
-                # Remove android SDK
+          - platform: linux/386
-                sudo rm -rf /usr/local/lib/android || true
+            runs-on: ubuntu-latest
-                # Remove .net runtime
+          - platform: linux/arm64
-                sudo rm -rf /usr/share/dotnet || true
+            runs-on: ubuntu-latest
-                df -h .
+          - platform: linux/arm/v7
-            - name: Checkout master
+            runs-on: ubuntu-latest
-              uses: actions/checkout@v4
+          - platform: linux/arm/v6
-              with:
+            runs-on: ubuntu-latest
-                fetch-depth: 0
+
-            - name: Login to Docker Hub
+    name: Build Docker Image for ${{ matrix.platform }}
-              uses: docker/login-action@v3
+    runs-on: ${{ matrix.runs-on }}
-              with:
+
-                username: ${{ secrets.DOCKERHUB_USERNAME }}
+    steps:
-                password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Free Space
-            - name: Extract metadata (tags, labels) for Docker
+        shell: bash
-              id: meta
+        run: |
-              uses: docker/metadata-action@v5
+          df -h .
-              with:
+          # Remove android SDK
-                images: ghcr.io/${{ github.repository }}
+          sudo rm -rf /usr/local/lib/android || true
-            - name: Set up QEMU
+          # Remove .net runtime
-              uses: docker/setup-qemu-action@v3
+          sudo rm -rf /usr/share/dotnet || true
-            - name: Set up Docker Buildx
+          df -h .
-              uses: docker/setup-buildx-action@v3
+
-            - name: Login to GitHub Container Registry
+      - name: Checkout Repository
-              uses: docker/login-action@v3
+        uses: actions/checkout@v4
-              with:
+        with:
-                registry: ghcr.io
+          fetch-depth: 0
-                # This is the user that triggered the Workflow. In this case, it will
+
-                # either be the user whom created the Release or manually triggered
+      - name: Set REPO_NAME Variable
-                # the workflow_dispatch.
+        run: |
-                username: ${{ github.actor }}
+          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
-                # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by
+
-                # GitHub Actions at the start of a workflow run to identify the job.
+      - name: Set PLATFORM Variable
-                # This is used to authenticate against GitHub Container Registry.
+        run: |
-                # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
+          platform=${{ matrix.platform }}
-                # for more detailed information.
+          echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV
-                password: ${{ secrets.GITHUB_TOKEN }}
+
-            - name: Show disk usage
+      - name: Extract Metadata (tags, labels) for Docker
-              shell: bash
+        id: meta
-              run: |
+        uses: docker/metadata-action@v5
-                df -h .
+        env:
-            - name: Build and publish image
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages)
-              uses: docker/build-push-action@v6
+        with:
-              with:
+          images: |
-                file: Dockerfile
+            ghcr.io/${{ env.REPO_NAME }}
-                context: .
+          labels: |
-                push: true # push the image to ghcr
+            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
-                tags: |
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
-                  ghcr.io/rclone/rclone:beta
+            org.opencontainers.image.authors=rclone <https://github.com/rclone>
-                  rclone/rclone:beta
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
-                labels: ${{ steps.meta.outputs.labels }}
+            org.opencontainers.image.revision=${{ github.sha }}
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
+          tags: |
-                cache-from: type=gha, scope=${{ github.workflow }}
+            type=sha
-                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
+            type=ref,event=pr
-                provenance: false
+            type=ref,event=branch
-              # Eventually cache will need to be cleared if builds more frequent than once a week
+            type=semver,pattern={{version}}
-              # https://github.com/docker/build-push-action/issues/252
+            type=semver,pattern={{major}}
-            - name: Show disk usage
+            type=semver,pattern={{major}}.{{minor}}
-              shell: bash
+            type=raw,value=beta,enable={{is_default_branch}}
-              run: |
+
-                df -h .
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Load Go Build Cache for Docker
+        id: go-cache
+        uses: actions/cache@v4
+        with:
+          # Cache only the go builds, the module download is cached via the docker layer caching
+          path: |
+            go-build-cache
+          key: ${{ runner.os }}-go-cache-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-cache-${{ env.PLATFORM }}-
+
+      - name: Inject Go Build Cache into Docker
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          cache-map: |
+            {
+              "go-build-cache": "/root/.cache/go-build"
+            }
+          skip-extraction: ${{ steps.cache.outputs.go-cache-hit }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          # This is the user that triggered the Workflow. In this case, it will
+          # either be the user whom created the Release or manually triggered
+          # the workflow_dispatch.
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and Publish Image Digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          file: Dockerfile
+          context: .
+          provenance: false
+          # don't specify 'tags' here (error "get can't push tagged ref by digest")
+          # tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
+          platforms: ${{ matrix.platform }}
+          outputs: |
+            type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:buildcache-${{ env.PLATFORM }}
+          cache-to: |
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:buildcache-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd
+
+      - name: Export Image Digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload Image Digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM }}
+          path: /tmp/digests/*
+          retention-days: 1
+          if-no-files-found: error
+
+  merge-image:
+    runs-on: ubuntu-latest
+    needs:
+      - build-image
+
+    steps:
+      - name: Download Image Digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Set REPO_NAME Variable
+        run: |
+          echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV}
+
+      - name: Extract Metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: index
+        with:
+          images: |
+            ${{ env.REPO_NAME }}
+            ghcr.io/${{ env.REPO_NAME }}
+          labels: |
+            org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.authors=rclone <https://github.com/rclone>
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=beta,enable={{is_default_branch}}
+
+      - name: Extract Tags
+        shell: python
+        run: |
+          import json, os
+
+          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
+          metadata = json.loads(metadata_json)
+
+          tags = [f"--tag '{tag}'" for tag in metadata["tags"]]
+          tags_string = " ".join(tags)
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"TAGS={tags_string}\n")
+
+      - name: Extract Annotations
+        shell: python
+        run: |
+          import json, os
+
+          metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON']
+          metadata = json.loads(metadata_json)
+
+          annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]]
+          annotations_string = " ".join(annotations)
+
+          with open(os.environ['GITHUB_ENV'], 'a') as env:
+              env.write(f"ANNOTATIONS={annotations_string}\n")
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          # This is the user that triggered the Workflow. In this case, it will
+          # either be the user whom created the Release or manually triggered
+          # the workflow_dispatch.
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create & Push Manifest List
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create \
+          ${{ env.TAGS }} \
+          ${{ env.ANNOTATIONS }} \
+          $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *)
+
+      - name: Inspect and Run Multi-Platform Image
+        run: |
+          docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }}
+          docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version

.github/workflows/build_publish_release_docker_image.yml

@@ -1,58 +1,14 @@
-name: Docker release build
+---
+# Github Actions release for rclone
+# -*- compile-command: "yamllint -f parsable build_publish_release_docker_image.yml" -*-
+
+name: Release Build for Docker Plugin
 
 on:
   release:
     types: [published]
 
 jobs:
-    build:
-        if: github.repository == 'rclone/rclone'
-        runs-on: ubuntu-latest
-        name: Build image job
-        steps:
-            - name: Free some space
-              shell: bash
-              run: |
-                df -h .
-                # Remove android SDK
-                sudo rm -rf /usr/local/lib/android || true
-                # Remove .net runtime
-                sudo rm -rf /usr/share/dotnet || true
-                df -h .
-            - name: Checkout master
-              uses: actions/checkout@v4
-              with:
-                fetch-depth: 0
-            - name: Get actual patch version
-              id: actual_patch_version
-              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
-            - name: Get actual minor version
-              id: actual_minor_version
-              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
-            - name: Get actual major version
-              id: actual_major_version
-              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
-            - name: Set up QEMU
-              uses: docker/setup-qemu-action@v3
-            - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v3
-            - name: Login to Docker Hub
-              uses: docker/login-action@v3
-              with:
-                username: ${{ secrets.DOCKER_HUB_USER }}
-                password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-            - name: Build and publish image
-              uses: docker/build-push-action@v6
-              with:
-                file: Dockerfile
-                context: .
-                platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                push: true
-                tags: |
-                  rclone/rclone:latest
-                  rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }}
-                  rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }}
-                  rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
 
     build_docker_volume_plugin:
         if: github.repository == 'rclone/rclone'