add stable release promotion workflow

.github/workflows/node-sdk.yml

@@ -2,7 +2,7 @@ name: Build/Release Node SDK
 
 on:
   release:
-    types: [ published ]
+    types: [ released ]
   pull_request:
     branches: [ main ]
 

.github/workflows/promote-downstreams.yml

@@ -0,0 +1,108 @@
+name: Promote Downstream Releases
+
+on: 
+  # may be triggered manually on a release tag that represents a prerelease to promote it to a release in the downstream package repositories and Docker Hub
+  workflow_dispatch:
+  # automatically trigger if an existing GitHub release is marked "latest"
+  release:
+    types: [released]  # this release event activity type excludes prereleases
+
+# cancel older, redundant runs of same workflow on same branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  parse_version:
+    name: Require Stable Release Semver
+    runs-on: ubuntu-24.04
+    outputs:
+      version: ${{ steps.parse.outputs.version }}
+    steps:
+      - name: Parse Release Version
+        id: parse
+        shell: bash
+        run: |
+          if [[ "${GITHUB_REF_NAME}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "GITHUB_REF_NAME=${GITHUB_REF_NAME} is a stable release semver ref"
+            echo "version=${GITHUB_REF_NAME#v}" | tee -a $GITHUB_OUTPUT
+          else
+            echo "GITHUB_REF_NAME=${GITHUB_REF_NAME} is not a stable release semver ref" >&2
+            exit 1
+          fi
+
+  promote_docker:
+    name: Tag Container Image ${{ matrix.image.repo }}:latest
+    needs: parse_version
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: true
+      matrix:
+        image:
+          - repo: ${{ vars.ZROK_CONTAINER_IMAGE_REPO || 'docker.io/openziti/zrok' }}
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKER_HUB_API_USER || secrets.DOCKER_HUB_API_USER }}
+          password: ${{ secrets.DOCKER_HUB_API_TOKEN }}
+
+      - name: Tag Latest
+        shell: bash
+        run: >
+          docker buildx imagetools create --tag
+          ${{ matrix.image.repo }}:latest
+          ${{ matrix.image.repo }}:${{ needs.parse_version.outputs.version }}
+
+  promote_artifactory:
+    name: Promote ${{ matrix.package_name }}-${{ matrix.arch.rpm }}.${{ matrix.packager }}
+    needs: parse_version
+    strategy:
+      fail-fast: true
+      matrix:
+        package_name:
+          - zrok
+          - zrok-share
+        arch:
+          - deb: amd64
+            rpm: x86_64
+          - deb: arm64
+            rpm: aarch64
+          - deb: armhf
+            rpm: armv7hl
+        packager:
+          - rpm
+          - deb
+    runs-on: ubuntu-24.04
+    env:
+      ZITI_DEB_TEST_REPO: ${{ vars.ZITI_DEB_TEST_REPO || 'zitipax-openziti-deb-test' }}
+      ZITI_RPM_TEST_REPO: ${{ vars.ZITI_RPM_TEST_REPO || 'zitipax-openziti-rpm-test' }}
+      ZITI_DEB_PROD_REPO: ${{ vars.ZITI_DEB_PROD_REPO || 'zitipax-openziti-deb-stable' }}
+      ZITI_RPM_PROD_REPO: ${{ vars.ZITI_RPM_PROD_REPO || 'zitipax-openziti-rpm-stable' }}
+    steps:
+      - name: Configure jFrog CLI
+        uses: jfrog/setup-jfrog-cli@v4
+        env:
+          JF_ENV_1: ${{ secrets.ZITI_ARTIFACTORY_CLI_CONFIG_PACKAGE_UPLOAD }}
+
+      - name: Copy RPM from test repo to stable repo with jFrog CLI
+        if: matrix.packager == 'rpm'
+        shell: bash
+        run: >
+          jf rt copy
+          --recursive=false
+          --flat=true
+          --fail-no-op=true
+          ${{ env.ZITI_RPM_TEST_REPO }}/redhat/${{ matrix.arch.rpm }}/${{ matrix.package_name }}-${{ needs.parse_version.outputs.version }}-1.${{ matrix.arch.rpm }}.rpm
+          ${{ env.ZITI_RPM_PROD_REPO }}/redhat/${{ matrix.arch.rpm }}/
+
+      - name: Copy DEB from test repo to stable repo with jFrog CLI
+        if: matrix.packager == 'deb'
+        shell: bash
+        run: >
+          jf rt copy
+          --recursive=false
+          --flat=true
+          --fail-no-op=true
+          ${{ env.ZITI_DEB_TEST_REPO }}/pool/${{ matrix.package_name }}/${{ matrix.arch.deb }}/${{ matrix.package_name }}_${{ needs.parse_version.outputs.version }}-1_${{ matrix.arch.deb }}.deb
+          ${{ env.ZITI_DEB_PROD_REPO }}/pool/${{ matrix.package_name }}/${{ matrix.arch.deb }}/

.github/workflows/publish-docker-images.yml

@@ -1,55 +1,40 @@
 name: Publish Docker Images
 
 on:
-  workflow_dispatch:
+  workflow_call:
     inputs:
       zrok-version:
-        description: zrok release tag to publish as a Docker image
+        description: Image tag to publish for zrok container images
         type: string
         required: true
-  release:
-    types:
-      - released  # excludes "prereleased" which is included in "published" to
-                  # avoid prematurely releasing semver tagged container images
 
 jobs:
   publish-docker-images:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     env:
-      RELEASE_REF: ${{ inputs.zrok-version || github.ref }}
+      RELEASE_REF: ${{ inputs.zrok-version }}
     steps:
       # compose the semver string without leading "refs/tags" or "v" so we can predict the
       # release artifact filename
       - name: Set zrok Version Semver from Tag Ref
         id: semver
         run: |
-          zrok_semver=${RELEASE_REF#refs/tags/}
+          echo "zrok_semver=${RELEASE_REF#v}" | tee -a $GITHUB_OUTPUT
-          echo "zrok_semver=${zrok_semver#v}" | tee -a $GITHUB_OUTPUT
 
       - name: Checkout Workspace
         uses: actions/checkout@v4
 
-      - name: Create the Release Arch Dirs
-        run: |
-          for TGZ in dist/{amd,arm}64/linux/; do
-            mkdir -pv ${TGZ}
-          done
-
       - name: Download Linux AMD64 Release Artifact
-        uses: dsaltares/fetch-gh-release-asset@1.1.2
+        uses: actions/download-artifact@v4
         with:
-          version: tags/v${{ steps.semver.outputs.zrok_semver }}
+          name: release-builds-linux-amd64
-          file: zrok.*_linux_amd64.tar.gz
+          path: dist/amd64/linux
-          regex: true
-          target: dist/amd64/linux/zrok_linux_amd64.tar.gz
 
       - name: Download Linux ARM64 Release Artifact
-        uses: dsaltares/fetch-gh-release-asset@1.1.2
+        uses: actions/download-artifact@v4
         with:
-          version: tags/v${{ steps.semver.outputs.zrok_semver }}
+          name: release-builds-linux-arm64
-          file: zrok.*_linux_arm64.tar.gz
+          path: dist/arm64/linux
-          regex: true
-          target: dist/arm64/linux/zrok_linux_arm64.tar.gz
 
       - name: Unpack the Release Artifacts
         run: |
@@ -79,7 +64,6 @@ jobs:
         id: tagprep_cli
         run: |
           DOCKER_TAGS="${ZROK_CONTAINER_IMAGE_REPO}:${ZROK_CONTAINER_IMAGE_TAG}"
-          DOCKER_TAGS+=",${ZROK_CONTAINER_IMAGE_REPO}:latest"
           echo "DOCKER_TAGS=${DOCKER_TAGS}" | tee -a $GITHUB_OUTPUT
 
       # this is the CLI image with the Linux binary for each

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Release
+name: Pre-Release
 
 on:
   push:
@@ -10,16 +10,37 @@ permissions:
   contents: write
 
 env:
-  ZITI_DEB_PROD_REPO: ${{ vars.ZITI_DEB_PROD_REPO || 'zitipax-openziti-deb-stable' }}
+  ZITI_DEB_TEST_REPO: ${{ vars.ZITI_DEB_TEST_REPO || 'zitipax-openziti-deb-test' }}
-  ZITI_RPM_PROD_REPO: ${{ vars.ZITI_RPM_PROD_REPO || 'zitipax-openziti-rpm-stable' }}
+  ZITI_RPM_TEST_REPO: ${{ vars.ZITI_RPM_TEST_REPO || 'zitipax-openziti-rpm-test' }}
   JFROG_CLI_VERSION: ${{ vars.JFROG_CLI_VERSION || '2.50.4' }}
 
 jobs:
-  build-linux-amd64:
+  build-linux:
-    runs-on: ubuntu-20.04
+    name: build-linux-${{ matrix.arch.goreleaser }}
+    needs:
+      - build-darwin
+      - build-windows
+    runs-on: ubuntu-22.04  # oldest Docker host runner for broadest kernel, syscall, ABI support
+    container: openziti/ziti-builder:v2  # v2 is based on ubuntu focal 20.04 for broadest glibc support
+    strategy:
+      matrix:
+        arch:
+          - goreleaser: amd64
+            deb: amd64
+            rpm: x86_64
+            gcc: gcc g++
+          - goreleaser: arm64
+            deb: arm64
+            rpm: aarch64
+            gcc: gcc-aarch64-linux-gnu
+          - goreleaser: armhf
+            deb: armhf
+            rpm: armv7hl
+            gcc: gcc-arm-linux-gnueabihf
     steps:
-      - run: sudo apt update
+      # skipped because ziti-builder image provides the multi-arch, multi-lib build toolchain
-      - run: sudo apt-get install gcc-multilib g++-multilib
+      # - run: apt-get update
+      # - run: apt-get --yes --quiet install ${{ matrix.arch.gcc }}
 
       - uses: actions/checkout@v4
         with:
@@ -48,13 +69,13 @@ jobs:
         with:
           distribution: goreleaser
           version: '~> v2'
-          args: release --skip=publish --config .goreleaser-linux-amd64.yml
+          args: release --skip=publish --config .goreleaser-linux-${{ matrix.arch.goreleaser }}.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - uses: actions/upload-artifact@v4
         with:
-          name: release-builds-linux-amd64
+          name: release-builds-linux-${{ matrix.arch.goreleaser }}
           path: ./dist/*.gz
 
       - name: Configure jFrog CLI
@@ -64,165 +85,61 @@ jobs:
         env:
           JF_ENV_1: ${{ secrets.ZITI_ARTIFACTORY_CLI_CONFIG_PACKAGE_UPLOAD }}
 
-      - name: Upload RPM to Artifactory release repo
+      - name: Upload RPM to Artifactory pre-release repo
         shell: bash
+        env:
+          ARCH: ${{ matrix.arch.rpm }}
         run: |
-          for RPM in ./dist/*.rpm; do
+          set -o xtrace
-            echo "INFO: Uploading $RPM"
+          shopt -s nullglob
-            jf rt upload --recursive=false --flat=true \
+          for PAX in zrok{,-share}; do
-              "$RPM" \
+            _pattern="./dist/${PAX}-${GITHUB_REF_NAME#v}-*.${ARCH}.rpm"
-              ${{ env.ZITI_RPM_PROD_REPO }}/redhat/x86_64/
+            if ! compgen -G "$_pattern" > /dev/null; then
+              echo "ERROR: No RPM files found matching pattern '${_pattern}'" >&2
+              exit 1
+            fi
+            _rpms=( $_pattern )
+            if [[ ${#_rpms[@]} -ne 1 ]]; then
+              echo "ERROR: Failed to find exactly one RPM file, got: '${_rpms[@]}'" >&2
+              exit 1
+            fi
+
+            echo "INFO: Uploading ${_rpms[0]}"
+            jf rt upload \
+              --recursive=false \
+              --flat=true \
+              --fail-no-op=true \
+              "${_rpms[0]}" \
+              ${{ env.ZITI_RPM_TEST_REPO }}/redhat/${ARCH}/
           done
 
-      - name: Upload DEB to Artifactory release repo
+      - name: Upload DEB to Artifactory pre-release repo
         shell: bash
+        env:
+          ARCH: ${{ matrix.arch.deb }}
         run: |
-          for DEB in ./dist/*.deb; do
+          set -o xtrace
-            echo "INFO: Uploading $DEB"
+          shopt -s nullglob
-            jf rt upload --recursive=false --flat=true \
+          for PAX in zrok{,-share}; do
-            --deb=debian/main/amd64 \
+            _pattern="./dist/${PAX}_${GITHUB_REF_NAME#v}-*_${ARCH}.deb"
-            "$DEB" \
+            if ! compgen -G "$_pattern" > /dev/null; then
-            ${{ env.ZITI_DEB_PROD_REPO }}/pool/zrok/amd64/
+              echo "ERROR: No DEB files found matching pattern '${_pattern}'" >&2
-          done
+              exit 1
+            fi
+            _debs=( $_pattern )
+            if [[ ${#_debs[@]} -ne 1 ]]; then
+              echo "ERROR: Failed to match exactly one DEB file, got: '${_debs[@]}'" >&2
+              exit 1
+            fi
 
-  build-linux-arm64:
+            echo "INFO: Uploading ${_debs[0]}"
-    runs-on: ubuntu-20.04
+            jf rt upload \
-    steps:
+              --recursive=false \
-      - run: sudo apt update
+              --flat=true \
-      - run: sudo apt-get install gcc-aarch64-linux-gnu
+              --fail-no-op=true \
-
+              --deb=debian/main/${ARCH} \
-      - uses: actions/checkout@v4
+              "${_debs[0]}" \
-        with:
+              ${{ env.ZITI_DEB_TEST_REPO }}/pool/${PAX}/${ARCH}/
-          fetch-depth: 0
-
-      - run: git fetch --force --tags
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: ./go.mod
-          cache: true
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 18.x
-
-      - run: npm install
-        working-directory: ui
-
-      - run: npm run build
-        working-directory: ui
-        env:
-          CI: "true"
-
-      - uses: goreleaser/goreleaser-action@v6
-        with:
-          distribution: goreleaser
-          version: '~> v2'
-          args: release --skip=publish --config .goreleaser-linux-arm64.yml
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: release-builds-linux-arm64
-          path: ./dist/*.gz
-
-      - name: Configure jFrog CLI
-        uses: jfrog/setup-jfrog-cli@v4
-        with:
-          version: ${{ env.JFROG_CLI_VERSION }}
-        env:
-          JF_ENV_1: ${{ secrets.ZITI_ARTIFACTORY_CLI_CONFIG_PACKAGE_UPLOAD }}
-
-      - name: Upload RPM to Artifactory release repo
-        shell: bash
-        run: |
-          for RPM in ./dist/*.rpm; do
-            echo "INFO: Uploading $RPM"
-            jf rt upload --recursive=false --flat=true \
-              "$RPM" \
-              ${{ env.ZITI_RPM_PROD_REPO }}/redhat/aarch64/
-          done
-
-      - name: Upload DEB to Artifactory release repo
-        shell: bash
-        run: |
-          for DEB in ./dist/*.deb; do
-            echo "INFO: Uploading $DEB"
-            jf rt upload --recursive=false --flat=true \
-            --deb=debian/main/arm64 \
-            "$DEB" \
-            ${{ env.ZITI_DEB_PROD_REPO }}/pool/zrok/arm64/
-          done
-
-  build-linux-arm:
-    runs-on: ubuntu-20.04
-    steps:
-      - run: sudo apt update
-      - run: sudo apt-get install gcc-arm-linux-gnueabihf
-
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - run: git fetch --force --tags
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version-file: ./go.mod
-          cache: true
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 18.x
-
-      - run: npm install
-        working-directory: ui
-
-      - run: npm run build
-        working-directory: ui
-        env:
-          CI: "true"
-
-      - uses: goreleaser/goreleaser-action@v6
-        with:
-          distribution: goreleaser
-          version: '~> v2'
-          args: release --skip=publish --config .goreleaser-linux-armhf.yml
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: release-builds-linux-arm
-          path: ./dist/*.gz
-
-      - name: Configure jFrog CLI
-        uses: jfrog/setup-jfrog-cli@v4
-        with:
-          version: ${{ env.JFROG_CLI_VERSION }}
-        env:
-          JF_ENV_1: ${{ secrets.ZITI_ARTIFACTORY_CLI_CONFIG_PACKAGE_UPLOAD }}
-
-      - name: Upload RPM to Artifactory release repo
-        shell: bash
-        run: |
-          for RPM in ./dist/*.rpm; do
-            echo "INFO: Uploading $RPM"
-            jf rt upload --recursive=false --flat=true \
-              "$RPM" \
-              ${{ env.ZITI_RPM_PROD_REPO }}/redhat/armv7/
-          done
-
-      - name: Upload DEB to Artifactory release repo
-        shell: bash
-        run: |
-          for DEB in ./dist/*.deb; do
-            echo "INFO: Uploading $DEB"
-            jf rt upload --recursive=false --flat=true \
-            --deb=debian/main/armv7 \
-            "$DEB" \
-            ${{ env.ZITI_DEB_PROD_REPO }}/pool/zrok/armv7/
           done
 
   build-darwin:
@@ -316,23 +233,20 @@ jobs:
           name: release-builds-windows
           path: ./dist/*.gz
 
-  publish-release:
+  call-publish-docker-images:
-    # allow skipped but not failed
+    name: Publish Release Docker Images
-    if: ${{
-          !cancelled()
-          && (needs.build-linux-amd64.result == 'success')
-          && (needs.build-linux-arm.result   == 'success')
-          && (needs.build-linux-arm64.result == 'success')
-          && (needs.build-darwin.result      == 'success' || needs.build-darwin.result      == 'skipped')
-          && (needs.build-windows.result     == 'success' || needs.build-windows.result     == 'skipped')
-      }}
     needs:
-      - build-linux-amd64
+      - build-linux
-      - build-linux-arm
-      - build-linux-arm64
       - build-darwin
       - build-windows
+    uses: ./.github/workflows/publish-docker-images.yml
+    secrets: inherit
+    with:
+      zrok-version: ${{ github.ref_name }}
+
+  draft-release:
     runs-on: ubuntu-latest
+    needs: call-publish-docker-images
     steps:
       - uses: actions/checkout@v4
 

CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## v0.4.49
+
+FIX: Pre-releases are no longer uploaded to the stable Linux package repo.
+
+CHANGE: Pre-releases are uploaded to the pre-release Linux package repo and Docker Hub for testing. [RELEASING.md](./RELEASING.md) describes releaser steps and the events they trigger.
+
+CHANGE: Linux release binaries are now built on the ziti-builder container image based on Ubuntu Focal 20.04 to preserve backward compatibility as the ubuntu-20.04 GitHub runner is end of life.
+
 ## v0.4.48
 
 FEATURE: The controller configuration now supports a `disable_auto_migration` boolean in the `store` stanza. When set to `true`, the controller will not attempt to auto-migrate (or otherwise validate the migration state) of the underlying database. Leaving `disable_auto_migration` out, or setting it to false will retain the default behavior of auto-migrating when starting the zrok controller. The `zrok admin migrate` command will still perform a migration regardless of how this setting is configured in the controller configuration (https://github.com/openziti/zrok/issues/866)

RELEASING.md

@@ -3,15 +3,25 @@
 
 ## Manual Steps
 
-1. Create a semver Git tag on main starting with a 'v' character.
+> [!NOTE]
-1. Push the tag to GitHub.
+> Each trigger is outlined separately, but some may occur simultaneously, e.g., when a draft release is published as stable rather than first publishing it as a pre-release, or a pre-release is promoted to stable and marked as latest at the same time.
-1. Wait for automated steps to complete.
-1. In GitHub Releases, edit the draft release as needed and finalize.
 
-## Automated Steps
+1. Push a tag to GitHub like `v*.*.*` to trigger **the pre-release workflow**. Wait for this workflow to complete before marking the release stable (`isPrerelease: false`).
+    1. Linux packages are uploaded to Artifactory as pre-releases.
+    1. Docker images are uploaded to Docker Hub as pre-releases.
+    1. A release is drafted in GitHub.
+1. Edit the draft and publish the release as a pre-release (`isPrerelease: true`).
+    1. The one-time GitHub "published" event fires, and binaries are available in GitHub.
+1. Edit the pre-release to mark it as a stable release (`isPrerelease: false`).
+    1. The one-time GitHub "released" event fires.
+    1. Linux packages are promoted to "stable" in Artifactory.
+    1. Docker images are promoted to `:latest` in Docker Hub.
+    1. Homebrew formulae are built and released.
+    1. Python wheels are built and released to PyPi.
+    1. Node.js packages are built and released to NPM.
+1. Edit the stable release to mark it as latest.
+    1. https://docs.zrok.io/docs/guides/install/ always serves the "latest" stable version via GitHub binary download URLs.
 
-1. The Release workflow is triggered by creating the Git tag and
+## Rolling Back Downstreams
-    1. uploads Linux packages to Artifactory and
+
-    1. drafts a release in GitHub Releases.
+The concepts, tools, and procedures for managing existing downstream artifacts in Artifactory and Docker Hub are identical for zrok and ziti. Here's the [RELEASING.md document for ziti](https://github.com/openziti/ziti/blob/main/RELEASING.md#rolling-back-downstreams).
-1. The Publish Container Images workflow is triggered by the Release event and
-    1. pushes Docker images to Docker Hub.