chore: switch to cargo dist for releases (#2085)

* chore: switch to cargo dist for releases

From https://axo.dev

cargo-dist handles building releases far better than we can, and do so
for several large projects now.

We will need to change our install script to use the cargo-dist
installer.

Historically, we have used the system package manager wherever possible.
Once switched to the new installer, this will no longer be the case. If
the user wishes to use their package manager, and Atuin is maintained
there, then they can choose to do so.

This way, we can ensure that users are running a known build, can easily
uninstall (just delete the atuin dir), easily update, etc. Builds will
use our lockfile, and can have their checksum verified. Later, I'd like
to introduce build signing.

As Axo are focused on release engineering, they will likely have
resolved many more issues than we have - libc versions, etc.

I'm not particularly happy with our response of "just use your package
manager", as many users seem to have difficulty there. It's unclear what
our installer has done, as this behaviour varies massively across
systems. It's also unclear how some package maintainers may have patched
things

I'm hoping that some better release tooling will lead to more confidence
in the process, and therefore more frequent releases.

Uninstall clarity: #111, #372, #640, #1485, #1546, #2049, #1529

* config

* add protobuf

* test build

* use native arm mac

* lol

* add toolchain

* use 1.78, 2vcpu

* nix flake update

* 1.77
This commit is contained in:
Ellie Huxtable 2024-06-05 13:25:01 +01:00 committed by GitHub
parent 9943c53a70
commit 7ec002ba44
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 317 additions and 227 deletions

View File

@ -1,223 +0,0 @@
name: Release
env:
MIN_SUPPORTED_RUST_VERSION: "1.51.0"
CICD_INTERMEDIATES_DIR: "_cicd-intermediates"
on:
workflow_dispatch:
inputs:
tag:
description: 'Tag to build release binaries for'
required: true
type: string
push:
tags:
- "v*"
release:
types: [created]
jobs:
build:
name: ${{ matrix.job.os }} (${{ matrix.job.target }})
runs-on: ${{ matrix.job.os }}
strategy:
fail-fast: false
matrix:
job:
- { os: ubuntu-20.04 , target: x86_64-unknown-linux-musl }
- { os: ubuntu-20.04 , target: x86_64-unknown-linux-gnu }
- { os: ubuntu-20.04 , target: aarch64-unknown-linux-gnu }
- { os: macos-12 , target: x86_64-apple-darwin }
- { os: macos-14 , target: aarch64-apple-darwin }
steps:
- name: Check for release
id: is-release
shell: bash
run: |
unset IS_RELEASE ; if [[ $GITHUB_REF =~ ^refs/tags/v[0-9].* ]]; then IS_RELEASE='true' ; fi
echo "IS_RELEASE=${IS_RELEASE}" >> $GITHUB_OUTPUT
- name: Checkout source code
if: steps.is-release.outputs.IS_RELEASE
uses: actions/checkout@v3
- name: Checkout source code
if: ${{ !steps.is-release.outputs.IS_RELEASE }}
uses: actions/checkout@v3
with:
ref: ${{ inputs.tag }}
- name: Install prerequisites
shell: bash
run: |
case ${{ matrix.job.target }} in
arm-unknown-linux-gnueabihf) sudo apt-get -y update ; sudo apt-get -y install gcc-arm-linux-gnueabihf ;;
aarch64-unknown-linux-gnu) sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu qemu-user;;
x86_64-unknown-linux-musl) sudo apt-get -y update ; sudo apt-get -y install musl-tools ;;
esac
- name: Extract crate information
shell: bash
run: |
echo "PROJECT_NAME=$(sed -n 's/^name = "\(.*\)"/\1/p' crates/atuin/Cargo.toml)" >> $GITHUB_ENV
echo "PROJECT_VERSION=$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n1)" >> $GITHUB_ENV
echo "PROJECT_MAINTAINER=$(sed -n 's/^authors = \["\(.*\)"\]/\1/p' Cargo.toml)" >> $GITHUB_ENV
echo "PROJECT_HOMEPAGE=$(sed -n 's/^homepage = "\(.*\)"/\1/p' Cargo.toml)" >> $GITHUB_ENV
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: stable
targets: ${{ matrix.job.target }}
override: true
profile: minimal # minimal component installation (ie, no documentation)
- name: Install Protoc
uses: arduino/setup-protoc@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Show version information (Rust, cargo, GCC)
shell: bash
run: |
gcc --version || true
rustup -V
rustup toolchain list
rustup default
cargo -V
rustc -V
- name: Build
run: |
case ${{ matrix.job.target }} in
aarch64-unknown-linux-gnu) export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc";;
esac;
cargo build --locked --release --target=${{ matrix.job.target }}
- name: Strip debug information from executable
id: strip
shell: bash
run: |
# Figure out suffix of binary
EXE_suffix=""
case ${{ matrix.job.target }} in
*-pc-windows-*) EXE_suffix=".exe" ;;
esac;
# Figure out what strip tool to use if any
# musl builds use the default strip
STRIP="strip"
case ${{ matrix.job.target }} in
arm-unknown-linux-gnueabihf) STRIP="arm-linux-gnueabihf-strip" ;;
aarch64-unknown-linux-gnu) STRIP="aarch64-linux-gnu-strip" ;;
*-pc-windows-msvc) STRIP="" ;;
esac;
# Setup paths
BIN_DIR="${{ env.CICD_INTERMEDIATES_DIR }}/stripped-release-bin/"
mkdir -p "${BIN_DIR}"
BIN_NAME="${{ env.PROJECT_NAME }}${EXE_suffix}"
BIN_PATH="${BIN_DIR}${BIN_NAME}"
# Copy the release build binary to the result location
cp "target/${{ matrix.job.target }}/release/${BIN_NAME}" "${BIN_DIR}"
# Also strip if possible
if [ -n "${STRIP}" ]; then
"${STRIP}" "${BIN_PATH}"
fi
# Let subsequent steps know where to find the (stripped) bin
echo "BIN_PATH=${BIN_PATH}" >> "$GITHUB_OUTPUT"
echo "BIN_NAME=${BIN_NAME}" >> "$GITHUB_OUTPUT"
- name: Create tarball
id: package
shell: bash
run: |
PKG_suffix=".tar.gz" ; case ${{ matrix.job.target }} in *-pc-windows-*) PKG_suffix=".zip" ;; esac;
PKG_BASENAME=${PROJECT_NAME}-v${PROJECT_VERSION}-${{ matrix.job.target }}
PKG_NAME=${PKG_BASENAME}${PKG_suffix}
echo "PKG_NAME=${PKG_NAME}" >> "$GITHUB_OUTPUT"
PKG_STAGING="${{ env.CICD_INTERMEDIATES_DIR }}/package"
ARCHIVE_DIR="${PKG_STAGING}/${PKG_BASENAME}/"
mkdir -p "${ARCHIVE_DIR}"
mkdir -p "${ARCHIVE_DIR}/completions"
# Binary
cp "${{ steps.strip.outputs.BIN_PATH }}" "$ARCHIVE_DIR"
# README, LICENSE and CHANGELOG files
cp "README.md" "LICENSE" "$ARCHIVE_DIR"
QEMU_PREFIX=""
case ${{ matrix.job.target }} in
aarch64-unknown-linux-gnu) QEMU_PREFIX="qemu-aarch64 -L /usr/aarch64-linux-gnu" ;;
esac;
# Shell completions
for sh in 'bash' 'fish' 'zsh' 'nushell'; do
$QEMU_PREFIX "${{ steps.strip.outputs.BIN_PATH }}" gen-completions -s $sh -o "${ARCHIVE_DIR}/completions"
done
# base compressed package
pushd "${PKG_STAGING}/" >/dev/null
case ${{ matrix.job.target }} in
*-pc-windows-*) 7z -y a "${PKG_NAME}" "${PKG_BASENAME}"/* | tail -2 ;;
*) tar czf "${PKG_NAME}" "${PKG_BASENAME}"/* ;;
esac;
popd >/dev/null
# Let subsequent steps know where to find the compressed package
echo "PKG_PATH=${PKG_STAGING}/${PKG_NAME}" >> "$GITHUB_OUTPUT"
- name: "Artifact upload: tarball"
uses: actions/upload-artifact@master
with:
name: ${{ steps.package.outputs.PKG_NAME }}
path: ${{ steps.package.outputs.PKG_PATH }}
- name: Create Debian package
id: debian-package
shell: bash
if: startsWith(matrix.job.os, 'ubuntu') && endsWith(matrix.job.target, 'gnu')
run: |
cargo install cargo-deb
cargo deb --deb-revision="" -p atuin
case ${{ matrix.job.target }} in
aarch64-*-linux-*) DPKG_ARCH=arm64 ;;
arm-*-linux-*hf) DPKG_ARCH=armhf ;;
i686-*-linux-*) DPKG_ARCH=i686 ;;
x86_64-*-linux-*) DPKG_ARCH=amd64 ;;
*) DPKG_ARCH=notset ;;
esac;
DPKG_NAME="${PROJECT_NAME}_${PROJECT_VERSION}_${DPKG_ARCH}.deb"
DPKG_PATH="target/debian/${PKG_BASENAME}.deb"
DPKG_PATH="target/debian/${DPKG_NAME}"
echo DPKG_NAME=${DPKG_NAME} >> $GITHUB_OUTPUT
echo DPKG_PATH=${DPKG_PATH} >> $GITHUB_OUTPUT
- name: "Artifact upload: Debian package"
uses: actions/upload-artifact@master
if: steps.debian-package.outputs.DPKG_NAME
with:
name: ${{ steps.debian-package.outputs.DPKG_NAME }}
path: ${{ steps.debian-package.outputs.DPKG_PATH }}
- name: Publish archives and packages
uses: softprops/action-gh-release@v1
if: steps.is-release.outputs.IS_RELEASE
with:
files: |
${{ steps.package.outputs.PKG_PATH }}
${{ steps.debian-package.outputs.DPKG_PATH }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

271
.github/workflows/release.yml vendored Normal file
View File

@ -0,0 +1,271 @@
# Copyright 2022-2024, axodotdev
# SPDX-License-Identifier: MIT or Apache-2.0
#
# CI that:
#
# * checks for a Git Tag that looks like a release
# * builds artifacts with cargo-dist (archives, installers, hashes)
# * uploads those artifacts to temporary workflow zip
# * on success, uploads the artifacts to a GitHub Release
#
# Note that the GitHub Release will be created with a generated
# title/body based on your changelogs.
name: Release
permissions:
contents: write
# This task will run whenever you push a git tag that looks like a version
# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
#
# If PACKAGE_NAME is specified, then the announcement will be for that
# package (erroring out if it doesn't have the given version or isn't cargo-dist-able).
#
# If PACKAGE_NAME isn't specified, then the announcement will be for all
# (cargo-dist-able) packages in the workspace with that version (this mode is
# intended for workspaces with only one dist-able package, or with all dist-able
# packages versioned/released in lockstep).
#
# If you push multiple tags at once, separate instances of this workflow will
# spin up, creating an independent announcement for each one. However, GitHub
# will hard limit this to 3 tags per commit, as it will assume more tags is a
# mistake.
#
# If there's a prerelease-style suffix to the version, then the release(s)
# will be marked as a prerelease.
on:
pull_request:
push:
tags:
- '**[0-9]+.[0-9]+.[0-9]+*'
jobs:
# Run 'cargo dist plan' (or host) to determine what tasks we need to do
plan:
runs-on: "ubuntu-20.04"
outputs:
val: ${{ steps.plan.outputs.manifest }}
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
publishing: ${{ !github.event.pull_request }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Install cargo-dist
# we specify bash to get pipefail; it guards against the `curl` command
# failing. otherwise `sh` won't catch that `curl` returned non-0
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.15.0/cargo-dist-installer.sh | sh"
# sure would be cool if github gave us proper conditionals...
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
# functionality based on whether this is a pull_request, and whether it's from a fork.
# (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
# but also really annoying to build CI around when it needs secrets to work right.)
- id: plan
run: |
cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
echo "cargo dist ran successfully"
cat plan-dist-manifest.json
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v4
with:
name: artifacts-plan-dist-manifest
path: plan-dist-manifest.json
# Build and packages all the platform-specific things
build-local-artifacts:
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
# Let the initial task tell us to not run (currently very blunt)
needs:
- plan
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
strategy:
fail-fast: false
# Target platforms/runners are computed by cargo-dist in create-release.
# Each member of the matrix has the following arguments:
#
# - runner: the github runner
# - dist-args: cli flags to pass to cargo dist
# - install-dist: expression to run to install cargo-dist on the runner
#
# Typically there will be:
# - 1 "global" task that builds universal installers
# - N "local" tasks that build each platform's binaries and platform-specific installers
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
runs-on: ${{ matrix.runner }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
steps:
- name: enable windows longpaths
run: |
git config --global core.longpaths true
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: swatinem/rust-cache@v2
with:
key: ${{ join(matrix.targets, '-') }}
- name: Install cargo-dist
run: ${{ matrix.install_dist }}
# Get the dist-manifest
- name: Fetch local artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- name: Install dependencies
run: |
${{ matrix.packages_install }}
- name: Build artifacts
run: |
# Actually do builds and make zips and whatnot
cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
echo "cargo dist ran successfully"
- id: cargo-dist
name: Post-build
# We force bash here just because github makes it really hard to get values up
# to "real" actions without writing to env-vars, and writing to env-vars has
# inconsistent syntax between shell and powershell.
shell: bash
run: |
# Parse out what we just built and upload it to scratch storage
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v4
with:
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Build and package all the platform-agnostic(ish) things
build-global-artifacts:
needs:
- plan
- build-local-artifacts
runs-on: "ubuntu-20.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Install cargo-dist
shell: bash
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.15.0/cargo-dist-installer.sh | sh"
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
- name: Fetch local artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- id: cargo-dist
shell: bash
run: |
cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
echo "cargo dist ran successfully"
# Parse out what we just built and upload it to scratch storage
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
- name: "Upload artifacts"
uses: actions/upload-artifact@v4
with:
name: artifacts-build-global
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Determines if we should publish/announce
host:
needs:
- plan
- build-local-artifacts
- build-global-artifacts
# Only run if we're "publishing", and only if local and global didn't fail (skipped is fine)
if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: "ubuntu-20.04"
outputs:
val: ${{ steps.host.outputs.manifest }}
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Install cargo-dist
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.15.0/cargo-dist-installer.sh | sh"
# Fetch artifacts from scratch-storage
- name: Fetch artifacts
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
# This is a harmless no-op for GitHub Releases, hosting for that happens in "announce"
- id: host
shell: bash
run: |
cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
echo "artifacts uploaded and released successfully"
cat dist-manifest.json
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
- name: "Upload dist-manifest.json"
uses: actions/upload-artifact@v4
with:
# Overwrite the previous copy
name: artifacts-dist-manifest
path: dist-manifest.json
# Create a GitHub Release while uploading all files to it
announce:
needs:
- plan
- host
# use "always() && ..." to allow us to wait for all publish jobs while
# still allowing individual publish jobs to skip themselves (for prereleases).
# "host" however must run to completion, no skipping allowed!
if: ${{ always() && needs.host.result == 'success' }}
runs-on: "ubuntu-20.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: "Download GitHub Artifacts"
uses: actions/download-artifact@v4
with:
pattern: artifacts-*
path: artifacts
merge-multiple: true
- name: Cleanup
run: |
# Remove the granular manifests
rm -f artifacts/*-dist-manifest.json
- name: Create GitHub Release
uses: ncipollo/release-action@v1
with:
tag: ${{ needs.plan.outputs.tag }}
name: ${{ fromJson(needs.host.outputs.val).announcement_title }}
body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }}
prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }}
artifacts: "artifacts/*"

View File

@ -7,7 +7,7 @@ exclude = ["ui/backend"]
[workspace.package]
version = "18.2.0"
authors = ["Ellie Huxtable <ellie@elliehuxtable.com>"]
rust-version = "1.74"
rust-version = "1.77"
license = "MIT"
homepage = "https://atuin.sh"
repository = "https://github.com/atuinsh/atuin"
@ -55,3 +55,43 @@ default-features = false
[workspace.dependencies.sqlx]
version = "0.7"
features = ["runtime-tokio-rustls", "time", "postgres", "uuid"]
# Config for 'cargo dist'
[workspace.metadata.dist]
# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
cargo-dist-version = "0.15.0"
# CI backends to support
ci = "github"
# The installers to generate for each app
installers = ["shell"]
# Target platforms to build apps for (Rust target-triple syntax)
targets = [
"aarch64-apple-darwin",
"aarch64-unknown-linux-gnu",
"aarch64-unknown-linux-musl",
"x86_64-apple-darwin",
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
]
# Publish jobs to run in CI
pr-run-mode = "upload"
# Whether to install an updater program
install-updater = true
# The archive format to use for non-windows builds (defaults .tar.xz)
unix-archive = ".tar.gz"
# The profile that 'cargo dist' will build with
[profile.dist]
inherits = "release"
lto = "thin"
[workspace.metadata.dist.github-custom-runners]
aarch64-apple-darwin = "macos-14"
aarch64-unknown-linux-gnu = "buildjet-2vcpu-ubuntu-2204-arm"
aarch64-unknown-linux-musl = "buildjet-2vcpu-ubuntu-2204-arm"
[workspace.metadata.dist.dependencies.homebrew]
protobuf = '*'
[workspace.metadata.dist.dependencies.apt]
protobuf-compiler = '*'

View File

@ -36,11 +36,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1717112898,
"narHash": "sha256-7R2ZvOnvd9h8fDd65p0JnB7wXfUvreox3xFdYWd1BnY=",
"lastModified": 1717399147,
"narHash": "sha256-eCWaE/q1VItpFAxxLVt171MdtDcjEnwi6QB/yuF73JU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "6132b0f6e344ce2fe34fc051b72fb46e34f668e0",
"rev": "4a4ecb0ab415c9fccfb005567a215e6a9564cdf5",
"type": "github"
},
"original": {

2
rust-toolchain.toml Normal file
View File

@ -0,0 +1,2 @@
[toolchain]
channel = "1.77"