From 9fa223eaaf0eda4e5f76621dc6d457417afb8b54 Mon Sep 17 00:00:00 2001 From: Remo Senekowitsch Date: Wed, 12 Jun 2024 17:45:38 +0200 Subject: [PATCH] chore(build): compile protobufs with protox (#2122) * chore(build): compile protobufs with protox protox is a pure-rust implementation of the protobuf compiler. Therefore, it can be managed by cargo. This removes the implicit dependency on protoc being available in the environment for the build. * fix(build): replace copypasta in build script The paths passed to `compile` aren't actually used by the build. `skip_protoc_run` prevents that. That's why a clean build succeeds even with this mistake. However, the paths are passed to a `cargo:rerun-if-changed` directive. So this mistake would've caused a failed incremental build if the protobuf definitions were changed. --- .github/workflows/rust.yml | 30 ---------- CONTRIBUTING.md | 1 - Cargo.lock | 102 ++++++++++++++++++++++++++++++++- Cargo.toml | 6 -- Cross.toml | 4 -- Dockerfile | 2 +- README.md | 3 - atuin.nix | 5 +- crates/atuin-daemon/Cargo.toml | 1 + crates/atuin-daemon/build.rs | 19 +++++- 10 files changed, 118 insertions(+), 55 deletions(-) delete mode 100644 Cross.toml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6e734c99..0f4734a0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,11 +24,6 @@ jobs: with: toolchain: stable - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 with: path: | @@ -65,11 +60,6 @@ jobs: with: tool: cross - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 with: path: | @@ -105,11 +95,6 @@ jobs: with: toolchain: stable - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: taiki-e/install-action@v2 name: Install nextest with: @@ -140,11 +125,6 @@ jobs: with: toolchain: stable - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 with: path: | @@ -189,11 +169,6 @@ jobs: with: toolchain: stable - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: taiki-e/install-action@v2 name: Install nextest with: @@ -224,11 +199,6 @@ jobs: toolchain: stable components: clippy - - name: Install Protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/cache@v3 with: path: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 31111a56..096049fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,6 @@ Thank you so much for considering contributing to Atuin! We really appreciate it Development dependencies 1. A rust toolchain ([rustup](https://rustup.rs) recommended) -2. [Protobuf compiler](https://grpc.io/docs/protoc-installation/) We commit to supporting the latest stable version of Rust - nothing more, nothing less, no nightly. diff --git a/Cargo.lock b/Cargo.lock index 58b65a3a..fd6ecbac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,6 +326,7 @@ dependencies = [ "listenfd", "prost", "prost-types", + "protox", "rand", "time", "tokio", @@ -1896,7 +1897,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afd0f0bff60c0e845844b6ee665e07990541ef3b70d8cd21861cf85b69fbef4" dependencies = [ - "logos", + "logos 0.14.0", "time", ] @@ -2020,13 +2021,36 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive 0.13.0", +] + [[package]] name = "logos" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "161971eb88a0da7ae0c333e1063467c5b5727e7fb6b710b8db4814eade3a42e8" dependencies = [ - "logos-derive", + "logos-derive 0.14.0", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 2.0.66", ] [[package]] @@ -2044,13 +2068,22 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen 0.13.0", +] + [[package]] name = "logos-derive" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c2a69b3eb68d5bd595107c9ee58d7e07fe2bb5e360cc85b0f084dedac80de0a" dependencies = [ - "logos-codegen", + "logos-codegen 0.14.0", ] [[package]] @@ -2175,6 +2208,29 @@ dependencies = [ "sketches-ddsketch", ] +[[package]] +name = "miette" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" +dependencies = [ + "cfg-if", + "miette-derive", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf09caffaac8068c346b6df2a7fc27a177fd20b39421a39ce0a211bde679a6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "mime" version = "0.3.17" @@ -2673,6 +2729,19 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "prost-reflect" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5eec97d5d34bdd17ad2db2219aabf46b054c6c41bd5529767c9ce55be5898f" +dependencies = [ + "logos 0.14.0", + "miette", + "once_cell", + "prost", + "prost-types", +] + [[package]] name = "prost-types" version = "0.12.6" @@ -2682,6 +2751,33 @@ dependencies = [ "prost", ] +[[package]] +name = "protox" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a29b3c5596eb23a849deba860b53ffd468199d9ad5fe4402a7d55379e16aa2d2" +dependencies = [ + "bytes", + "miette", + "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", +] + +[[package]] +name = "protox-parse" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033b939d76d358f7c32120c86c71f515bae45e64f2bde455200356557276276c" +dependencies = [ + "logos 0.13.0", + "miette", + "prost-types", + "thiserror", +] + [[package]] name = "quanta" version = "0.11.1" diff --git a/Cargo.toml b/Cargo.toml index 8c3df8ba..033efaa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,9 +90,3 @@ lto = "thin" aarch64-apple-darwin = "macos-14" aarch64-unknown-linux-gnu = "buildjet-2vcpu-ubuntu-2204-arm" aarch64-unknown-linux-musl = "buildjet-2vcpu-ubuntu-2204-arm" - -[workspace.metadata.dist.dependencies.homebrew] -protobuf = '*' - -[workspace.metadata.dist.dependencies.apt] -protobuf-compiler = '*' diff --git a/Cross.toml b/Cross.toml deleted file mode 100644 index a81b3d80..00000000 --- a/Cross.toml +++ /dev/null @@ -1,4 +0,0 @@ -[build] -pre-build = [ - "apt update && apt install -y protobuf-compiler" -] diff --git a/Dockerfile b/Dockerfile index b73a4b42..12a197c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder # Ensure working C compile setup (not installed by default in arm64 images) -RUN apt update && apt install build-essential protobuf-compiler -y +RUN apt update && apt install build-essential -y COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json diff --git a/README.md b/README.md index 1bff59ec..b3c8f709 100644 --- a/README.md +++ b/README.md @@ -174,9 +174,6 @@ cargo install atuin And then follow [the shell setup](#shell-plugin) -#### Daemon Build Dependencies -[Protoc](https://grpc.io/docs/protoc-installation/) is required to build the daemon, which is enabled by default. This is because we use protobufs for the message serialization - ### Homebrew ``` diff --git a/atuin.nix b/atuin.nix index 43450b8e..c02a82de 100644 --- a/atuin.nix +++ b/atuin.nix @@ -13,7 +13,6 @@ Security, SystemConfiguration, AppKit, - protobuf, }: rustPlatform.buildRustPackage { name = "atuin"; @@ -28,9 +27,7 @@ rustPlatform.buildRustPackage { nativeBuildInputs = [installShellFiles]; - buildInputs = lib.optionals stdenv.isDarwin [libiconv Security SystemConfiguration AppKit protobuf]; - - env.PROTOC = lib.getExe' protobuf "protoc"; + buildInputs = lib.optionals stdenv.isDarwin [libiconv Security SystemConfiguration AppKit]; postInstall = '' installShellCompletion --cmd atuin \ diff --git a/crates/atuin-daemon/Cargo.toml b/crates/atuin-daemon/Cargo.toml index 3133bcdf..0f3ccb1a 100644 --- a/crates/atuin-daemon/Cargo.toml +++ b/crates/atuin-daemon/Cargo.toml @@ -38,4 +38,5 @@ rand.workspace = true listenfd = "1.0.1" [build-dependencies] +protox = "0.6.0" tonic-build = "0.11" diff --git a/crates/atuin-daemon/build.rs b/crates/atuin-daemon/build.rs index 95118f6f..37721249 100644 --- a/crates/atuin-daemon/build.rs +++ b/crates/atuin-daemon/build.rs @@ -1,4 +1,17 @@ -fn main() -> Result<(), Box> { - tonic_build::compile_protos("./proto/history.proto")?; - Ok(()) +use std::{env, fs, path::PathBuf}; + +use protox::prost::Message; + +fn main() -> std::io::Result<()> { + let file_descriptors = protox::compile(["history.proto"], ["./proto/"]).unwrap(); + + let file_descriptor_path = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR not set")) + .join("file_descriptor_set.bin"); + fs::write(&file_descriptor_path, file_descriptors.encode_to_vec()).unwrap(); + + tonic_build::configure() + .build_server(true) + .file_descriptor_set_path(&file_descriptor_path) + .skip_protoc_run() + .compile(&["history.proto"], &["."]) }