From 3936454543614c62a5c40ff55bd229b8932e7eb1 Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:43:18 +0530 Subject: [PATCH] Added parallel docker builds and caching for go build in the container Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- .../build_publish_beta_docker_image.yml | 77 ----- .../workflows/build_publish_docker_image.yml | 284 ++++++++++++++++++ .../workflows/build_publish_docker_plugin.yml | 45 +++ .../build_publish_release_docker_image.yml | 89 ------ 4 files changed, 329 insertions(+), 166 deletions(-) delete mode 100644 .github/workflows/build_publish_beta_docker_image.yml create mode 100644 .github/workflows/build_publish_docker_image.yml create mode 100644 .github/workflows/build_publish_docker_plugin.yml delete mode 100644 .github/workflows/build_publish_release_docker_image.yml diff --git a/.github/workflows/build_publish_beta_docker_image.yml b/.github/workflows/build_publish_beta_docker_image.yml deleted file mode 100644 index fa838c6ff..000000000 --- a/.github/workflows/build_publish_beta_docker_image.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Docker beta build - -on: - push: - branches: - - master -jobs: - build: - if: github.repository == 'rclone/rclone' - runs-on: ubuntu-latest - name: Build image job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - # This is the user that triggered the Workflow. In this case, it will - # either be the user whom created the Release or manually triggered - # the workflow_dispatch. - username: ${{ github.actor }} - # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by - # GitHub Actions at the start of a workflow run to identify the job. - # This is used to authenticate against GitHub Container Registry. - # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret - # for more detailed information. - password: ${{ secrets.GITHUB_TOKEN }} - - name: Show disk usage - shell: bash - run: | - df -h . - - name: Build and publish image - uses: docker/build-push-action@v6 - with: - file: Dockerfile - context: . - push: true # push the image to ghcr - tags: | - ghcr.io/rclone/rclone:beta - rclone/rclone:beta - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 - cache-from: type=gha, scope=${{ github.workflow }} - cache-to: type=gha, mode=max, scope=${{ github.workflow }} - provenance: false - # Eventually cache will need to be cleared if builds more frequent than once a week - # https://github.com/docker/build-push-action/issues/252 - - name: Show disk usage - shell: bash - run: | - df -h . diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml new file mode 100644 index 000000000..6719fbf29 --- /dev/null +++ b/.github/workflows/build_publish_docker_image.yml @@ -0,0 +1,284 @@ +--- +# Github Actions release for rclone +# -*- compile-command: "yamllint -f build_publish_docker_image.yml" -*- + +name: Build & Push Docker Images + +# Trigger the workflow on push or pull request +on: + push: + branches: + - '**' + tags: + - '**' + pull_request: + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +jobs: + build-image: + if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runs-on: ubuntu-latest + - platform: linux/386 + runs-on: ubuntu-latest + - platform: linux/arm64 + runs-on: ubuntu-latest + - platform: linux/arm/v7 + runs-on: ubuntu-latest + - platform: linux/arm/v6 + runs-on: ubuntu-latest + + name: Build Docker Image for ${{ matrix.platform }} + runs-on: ${{ matrix.runs-on }} + + steps: + - name: Free Space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set REPO_NAME Variable + run: | + echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} + + - name: Set PLATFORM Variable + run: | + platform=${{ matrix.platform }} + echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV + + - name: Set CACHE_NAME Variable + shell: python + run: | + import os, re + + def slugify(input_string, max_length=63): + slug = input_string.lower() + slug = re.sub(r'[^a-z0-9 -]', ' ', slug) + slug = slug.strip() + slug = re.sub(r'\s+', '-', slug) + slug = re.sub(r'-+', '-', slug) + slug = slug[:max_length] + slug = re.sub(r'[-]+$', '', slug) + return slug + + ref_name_slug = "cache" + + if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request": + ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME']) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"CACHE_NAME={ref_name_slug}\n") + + - name: Extract Metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages) + with: + images: | + ghcr.io/${{ env.REPO_NAME }} + labels: | + org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone + org.opencontainers.image.vendor=${{ github.repository_owner }} + org.opencontainers.image.authors=rclone + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + type=sha + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=beta,enable={{is_default_branch}} + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load Go Build Cache for Docker + id: go-cache + uses: actions/cache@v4 + with: + # Cache only the go builds, the module download is cached via the docker layer caching + path: | + go-build-cache + key: ${{ runner.os }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + + - name: Inject Go Build Cache into Docker + uses: reproducible-containers/buildkit-cache-dance@v3 + with: + cache-map: | + { + "go-build-cache": "/root/.cache/go-build" + } + skip-extraction: ${{ steps.cache.outputs.go-cache-hit }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + # This is the user that triggered the Workflow. In this case, it will + # either be the user whom created the Release or manually triggered + # the workflow_dispatch. + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Publish Image Digest + id: build + uses: docker/build-push-action@v6 + with: + file: Dockerfile + context: . + provenance: false + # don't specify 'tags' here (error "get can't push tagged ref by digest") + # tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + platforms: ${{ matrix.platform }} + outputs: | + type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: | + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + cache-to: | + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd + + - name: Export Image Digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload Image Digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM }} + path: /tmp/digests/* + retention-days: 1 + if-no-files-found: error + + merge-image: + runs-on: ubuntu-latest + needs: + - build-image + + steps: + - name: Download Image Digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set REPO_NAME Variable + run: | + echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} + + - name: Extract Metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + with: + images: | + ${{ env.REPO_NAME }} + ghcr.io/${{ env.REPO_NAME }} + labels: | + org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone + org.opencontainers.image.vendor=${{ github.repository_owner }} + org.opencontainers.image.authors=rclone + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + type=sha + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=beta,enable={{is_default_branch}} + + - name: Extract Tags + shell: python + run: | + import json, os + + metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON'] + metadata = json.loads(metadata_json) + + tags = [f"--tag '{tag}'" for tag in metadata["tags"]] + tags_string = " ".join(tags) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"TAGS={tags_string}\n") + + - name: Extract Annotations + shell: python + run: | + import json, os + + metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON'] + metadata = json.loads(metadata_json) + + annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]] + annotations_string = " ".join(annotations) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"ANNOTATIONS={annotations_string}\n") + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + # This is the user that triggered the Workflow. In this case, it will + # either be the user whom created the Release or manually triggered + # the workflow_dispatch. + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create & Push Manifest List + working-directory: /tmp/digests + run: | + docker buildx imagetools create \ + ${{ env.TAGS }} \ + ${{ env.ANNOTATIONS }} \ + $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *) + + - name: Inspect and Run Multi-Platform Image + run: | + docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} + docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} + docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version diff --git a/.github/workflows/build_publish_docker_plugin.yml b/.github/workflows/build_publish_docker_plugin.yml new file mode 100644 index 000000000..4056cb04b --- /dev/null +++ b/.github/workflows/build_publish_docker_plugin.yml @@ -0,0 +1,45 @@ +--- +# Github Actions release for rclone +# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*- + +name: Release Build for Docker Plugin + +on: + release: + types: [published] + +jobs: + + build_docker_volume_plugin: + if: github.repository == 'rclone/rclone' + needs: build + runs-on: ubuntu-latest + name: Build docker plugin job + steps: + - name: Free some space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + - name: Checkout master + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build and publish docker plugin + shell: bash + run: | + VER=${GITHUB_REF#refs/tags/} + PLUGIN_USER=rclone + docker login --username ${{ secrets.DOCKER_HUB_USER }} \ + --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" + for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do + export PLUGIN_USER PLUGIN_ARCH + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} + done + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} diff --git a/.github/workflows/build_publish_release_docker_image.yml b/.github/workflows/build_publish_release_docker_image.yml deleted file mode 100644 index b69285a7e..000000000 --- a/.github/workflows/build_publish_release_docker_image.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Docker release build - -on: - release: - types: [published] - -jobs: - build: - if: github.repository == 'rclone/rclone' - runs-on: ubuntu-latest - name: Build image job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get actual patch version - id: actual_patch_version - run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g') - - name: Get actual minor version - id: actual_minor_version - run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2) - - name: Get actual major version - id: actual_major_version - run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1) - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Build and publish image - uses: docker/build-push-action@v6 - with: - file: Dockerfile - context: . - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 - push: true - tags: | - rclone/rclone:latest - rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }} - rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }} - rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }} - - build_docker_volume_plugin: - if: github.repository == 'rclone/rclone' - needs: build - runs-on: ubuntu-latest - name: Build docker plugin job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Build and publish docker plugin - shell: bash - run: | - VER=${GITHUB_REF#refs/tags/} - PLUGIN_USER=rclone - docker login --username ${{ secrets.DOCKER_HUB_USER }} \ - --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" - for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do - export PLUGIN_USER PLUGIN_ARCH - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} - done - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v}