devex: manage devtools in a project-local subdirectory + cleanup README (#829)

.circleci/config.yml

@@ -15,16 +15,6 @@ commands:
               echo "$line" >> $BASH_ENV
             fi
 
-  invoke-lazy-sh:
-    parameters:
-      subcommand:
-        type: string
-    steps:
-      - run:
-          environment:
-            TERM: xterm
-          command: ./lazy.sh <<parameters.subcommand>>
-
   apt-update-and-install-common-deps:
     steps:
       - run: sudo apt-get update
@@ -34,18 +24,12 @@ commands:
       # The need for this was required for cimg/go:1.12, but let's future proof this here and now.
       - run: sudo apt-get install -y git ca-certificates
 
-  install-godep:
-    steps:
-      - apt-update-and-install-common-deps
-      - invoke-lazy-sh:
-          subcommand: godep
 
   install-docdep:
     steps:
       - apt-update-and-install-common-deps
       - run: sudo apt install python3 python3-pip libgirepository1.0-dev
-      - invoke-lazy-sh:
+      - run: pip3 install -r docs/requirements.txt
-          subcommand: docdep
 
   docs-publish-sh:
     parameters:
@@ -195,12 +179,25 @@ jobs:
 
       - go/load-cache:
           key: quickcheck-<<parameters.goversion>>
-      - install-godep
+      - run: make build/install
       - run: go mod download
       - run: cd build && go mod download
       - go/save-cache:
           key: quickcheck-<<parameters.goversion>>
 
+      # ensure all code has been generated
+      - run: make generate
+      - run: |
+          if output=$(git status --porcelain) && [ -z "$output" ]; then
+            echo "Working directory clean"
+          else
+            echo "Uncommitted changes"
+            echo ""
+            echo "$output"
+            exit 1
+          fi
+
+      # other checks
       - run: make formatcheck
       - run: make zrepl-bin test-platform-bin
       - run: make vet

Makefile

@@ -28,7 +28,6 @@ GO_MOD_READONLY := -mod=readonly
 GO_EXTRA_BUILDFLAGS :=
 GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
-GOLANGCI_LINT := golangci-lint
 GOCOVMERGE := gocovmerge
 RELEASE_GOVERSION ?= go1.23.1
 STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION))
@@ -71,7 +70,7 @@ release: ensure-release-toolchain
 	$(MAKE) noarch
 
 release-docker: $(ARTIFACTDIR) release-docker-mkcachemount
-	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
+	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build/build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile
 	docker build -t zrepl_release --pull -f $(ARTIFACTDIR)/build.Dockerfile .
 	docker run --rm -i \
 		$(_RELEASE_DOCKER_CACHEMOUNT) \
@@ -244,8 +243,8 @@ _run_make_foreach_target_tuple:
 ##################### REGULAR TARGETS #####################
 .PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin
 
-lint:
+lint: build/install
-	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
+	$(GO_ENV_VARS) build/install/gobin/golangci-lint run ./...
 
 vet:
 	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
@@ -323,10 +322,27 @@ cover-full:
 # not part of the build, must do that manually
 .PHONY: generate formatcheck format
 
-generate:
+build/install:
-	protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto
+	rm -rf build/install.tmp
-	protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto
+	mkdir build/install.tmp
-	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+	-echo "installing protoc"
+	mkdir build/install.tmp/protoc
+	bash -x build/get_protoc.bash build/install.tmp/protoc
+
+	-echo "installing go tools"
+	build/go_install_tools.bash build/install.tmp/gobin
+
+	mv build/install.tmp build/install
+
+
+generate: build/install
+	# TODO: would be nice to run with a pure path here
+	PATH="$(CURDIR)/build/install/gobin:$(CURDIR)/build/install/protoc/bin:$$PATH" && \
+		build/install/protoc/bin/protoc -I=internal/replication/logic/pdu --go_out=internal/replication/logic/pdu --go-grpc_out=internal/replication/logic/pdu internal/replication/logic/pdu/pdu.proto && \
+		build/install/protoc/bin/protoc -I=internal/rpc/grpcclientidentity/example --go_out=internal/rpc/grpcclientidentity/example/pdu --go-grpc_out=internal/rpc/grpcclientidentity/example/pdu internal/rpc/grpcclientidentity/example/grpcauth.proto && \
+		$(GO) generate $(GO_BUILDFLAGS) -x ./... && \
+		true
 
 GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
 FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'

README.md

@@ -31,19 +31,35 @@ zrepl is a one-stop ZFS backup & replication solution.
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
 
-## Building, Releasing, Downstream-Packaging
+## Development
-
-This section provides an overview of the zrepl build & release process.
-Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
-
-### Overview
 
 zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
 The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
 
-Install **build dependencies** using  `./lazy.sh devsetup`.
+### Building
-`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+
-If you just want to install the Go dependencies, run `./lazy.sh godep`.
+#### Go Code
+
+Dependencies:
+
+* Go 1.22 or newer
+* GNU Make
+* Git
+* wget (`make generate`)
+* unzip (`make generate`)
+
+Some Go code is **generated**, and generated code is committed to the source tree.
+Therefore, building does not require having code generation tools set up.
+When making changes that require code to be (re-)generated, run `make generate`.
+I downloads and installs pinned versions of the code generation tools into `./build/install`.
+There is a CI check that ensures Git state is clean, i.e., code generation has been done by a PR and is deterministic.
+
+#### Docs
+
+Set up a Python environment that has `docs/requirements.txt` installed via `pip`.
+Use a  [venv](https://docs.python.org/3/library/venv.html) to avoid global state.
+
+### Testing
 
 The **test suite** is split into pure **Go tests** (`make test-go`) and **platform tests** that interact with ZFS and thus generally **require root privileges** (`sudo make test-platform`).
 Platform tests run on their own pool with the name `zreplplatformtest`, which is created using the file vdev in `/tmp`.
@@ -51,33 +67,25 @@ Platform tests run on their own pool with the name `zreplplatformtest`, which is
 For a full **code coverage** profile, run `make test-go COVER=1 && sudo make test-platform && make cover-merge`.
 An HTML report can be generated using `make cover-html`.
 
-**Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
+### Circle CI
 
-### Build & Release Process
+We use CircleCI for automated build & test pre- and post-merge.
 
-**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
-It provides phony targets for
-* local development (building, running tests, etc)
-* building a release in Docker (used by the CI & release management)
-* building .deb and .rpm packages out of the release artifacts.
-
-**Build tooling & dependencies** are documented as code in `lazy.sh`.
-Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
-
-**We use CircleCI for continuous integration**.
 There are two workflows:
 
 * `ci` runs for every commit / branch / tag pushed to GitHub.
   It is supposed to run very fast (<5min and provides quick feedback to developers).
   It runs formatting checks, lints and tests on the most important OSes / architectures.
-  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
 
 * `release` runs
   * on manual triggers through the CircleCI API (in order to produce a release)
   * periodically on `master`
-  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
 
-**Releases** are issued via Git tags + GitHub Releases feature.
+Artifacts are stored in CircleCI.
+
+### Releasing
+
+Releases are issued via Git tags + GitHub Releases feature.
 The procedure to issue a release is as follows:
 * Issue the source release:
   * Git tag the release on the `master` branch.
@@ -88,14 +96,16 @@ The procedure to issue a release is as follows:
   * Download the artifacts to the release manager's machine.
   * Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
   * Issue the GitHub release.
-  * Add the .rpm and .deb files to the official zrepl repos, publish those.
+  * Add the .rpm and .deb files to the official zrepl repos.
+    * Code for management of these repos: https://github.com/zrepl/package-repo-ops (private repo at this time)
 
 **Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
 The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
 Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
 
-### Additional Notes to Distro Package Maintainers
+## Notes to Distro Package Maintainers
 
+* The `Makefile` in this project is not suitable for builds in distros.
 * Run the platform tests (Docs -> Usage -> Platform Tests) **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
 * Ship a default config that adheres to your distro's `hier` and logging system.
 * Ship a service manager file and _please_ try to upstream it to this repository.
@@ -105,31 +115,3 @@ Downstream packagers can read the changelog to determine whether they want to pu
   This is how `zrepl version` knows what version number to show.
   Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
 * Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
-
-
-## Contributing Code
-
-* Open an issue when starting to hack on a new feature
-* Commits should reference the issue they are related to
-* Docs improvements not documenting new features do not require an issue.
-
-### Breaking Changes
-
-Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
-
-### Glossary & Naming Inconsistencies
-
-In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
-However, we need a word for *filesystem* & *ZVOL* but not a snapshot, bookmark, etc.
-
-Toward the user, the following terminology is used:
-
-* **filesystem**: a ZFS filesystem or a ZVOL
-* **filesystem version**: a ZFS snapshot or a bookmark
-
-Sadly, the zrepl implementation is inconsistent in its use of these words:
-variables and types are often named *dataset* when they in fact refer to a *filesystem*.
-
-There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
-
-However, new contributions & patches should fix naming without further notice in the commit message.

build.Dockerfile

@@ -1,36 +0,0 @@
-FROM !SUBSTITUTED_BY_MAKEFILE
-
-RUN apt-get update && apt-get install -y \
-    python3-pip \
-    python3-venv \
-    unzip \
-    gawk
-
-ADD build.installprotoc.bash ./
-RUN bash build.installprotoc.bash
-
-# setup venv
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-ADD lazy.sh /tmp/lazy.sh
-ADD docs/requirements.txt /tmp/requirements.txt
-ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
-RUN /tmp/lazy.sh docdep
-
-# prepare volume mount of git checkout to /zrepl
-RUN mkdir -p /src/github.com/zrepl/zrepl
-RUN mkdir -p /.cache && chmod -R 0777 /.cache
-
-# $GOPATH is /go
-# Go 1.12 doesn't use modules within GOPATH, but 1.13 and later do
-# => store source outside of GOPATH
-WORKDIR /src
-
-# Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
-ADD build/ /tmp/build
-RUN /tmp/lazy.sh godep
-
-RUN chmod -R 0777 /go
-

build/.gitignore

@@ -0,0 +1 @@
+install/

build/build.Dockerfile

@@ -0,0 +1,22 @@
+FROM !SUBSTITUTED_BY_MAKEFILE
+
+RUN apt-get update && apt-get install -y \
+    python3-pip \
+    python3-venv \
+    unzip \
+    gawk
+
+# setup venv for docs
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+ADD docs/requirements.txt /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt
+
+# Go toolchain uses xdg-cache
+RUN mkdir -p /.cache && chmod -R 0777 /.cache
+
+# Go devtools are managed by Makefile
+
+WORKDIR /src
+

build/get_protoc.bash

@@ -2,6 +2,8 @@
 set -euo pipefail
 set -x
 
+cd "$1"
+
 MACH=$(uname -m)
 MACH="${MACH/aarch64/aarch_64}"
 
@@ -13,7 +15,7 @@ if [ -e "$FILENAME" ]; then
     exit 1
 fi
 
-wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
+wget --continue https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/"$FILENAME"
 
 stat "$FILENAME"
 
@@ -22,4 +24,4 @@ d622619dcbfb5ecb281cfb92c1a74d6a0f42e752d9a2774b197f475f7ab1c8c4  protoc-28.0-li
 b2e187c8b9f2d97cd3ecae4926d1bb2cbebe3ab768e7c987cbc86bb17f319358  protoc-28.0-linux-x86_64.zip
 EOF
 
-unzip -d /usr/local "$FILENAME"
+unzip -d . "$FILENAME"

build/go_install_host_tool.source

@@ -0,0 +1,6 @@
+export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
+source <(go env)
+# build tools for the host platform
+export GOOS="$GOHOSTOS"
+export GOARCH="$GOHOSTARCH"
+# TODO GOARM=$GOHOSTARM?

build/go_install_tools.bash

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+set -x
+
+OUTDIR="$(readlink -f "$1")"
+if [ -e "$OUTDIR" ]; then
+    echo "$OUTDIR" already exists 1>&2
+    exit 1
+fi
+
+# go install command below will install tools to $GOBIN
+export GOBIN="$OUTDIR"
+
+cd "$(dirname "$0")"
+
+source ./go_install_host_tool.source
+
+cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'

docs/installation.rst

@@ -11,10 +11,9 @@ Installation
 
 .. toctree::
 
-    installation/user-privileges
     installation/packages
     installation/apt-repos
     installation/rpm-repos
-    installation/compile-from-source
+    installation/user-privileges
     installation/freebsd-jail-with-iocage
     installation/what-next

docs/installation/compile-from-source.rst

@@ -1,44 +0,0 @@
-.. _binary releases: https://github.com/zrepl/zrepl/releases
-
-.. _installation-compile-from-source:
-
-Compile From Source
-~~~~~~~~~~~~~~~~~~~
-
-Producing a release requires **Go 1.11** or newer and **Python 3** + **pip3** + ``docs/requirements.txt`` for the Sphinx documentation.
-A tutorial to install Go is available over at `golang.org <https://golang.org/doc/install>`_.
-Python and pip3 should probably be installed via your distro's package manager.
-
-::
-
-   cd to/your/zrepl/checkout
-   python3 -m venv3
-   source venv3/bin/activate
-   ./lazy.sh devsetup
-   make release
-   # build artifacts are available in ./artifacts/release
-
-The Python venv is used for the documentation build dependencies.
-If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
-
-Alternatively, you can use the Docker build process:
-it is used to produce the official zrepl `binary releases`_
-and serves as a reference for build dependencies and procedure:
-
-::
-
-    cd to/your/zrepl/checkout
-    # make sure your user has access to the docker socket
-    make release-docker
-    # if you want .deb or .rpm packages, invoke the follwoing
-    # targets _after_ you invoked release-docker
-    make deb-docker
-    make rpm-docker
-    # build artifacts are available in ./artifacts/release
-    # packages are available in ./artifacts
-
-
-.. NOTE::
-
-    It is your job to install the built binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
-    Otherwise, the examples in the :ref:`quick-start guides <quickstart-toc>` may need to be adjusted.

docs/installation/packages.rst

@@ -16,6 +16,9 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - OS / Distro
       - Install Command
       - Link
+    * - any
+      - Statically linked binaries.
+      - `Official GitHub releases <binary releases_>`_
     * - FreeBSD
       - ``pkg install zrepl``
       - `<https://www.freshports.org/sysutils/zrepl/>`_
@@ -30,7 +33,7 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - Arch Linux
       - ``yay install zrepl``
       - Available on `AUR <https://aur.archlinux.org/packages/zrepl>`_
-    * - Fedora, CentOS, RHEL, OpenSUSE
+    * - Fedora / RHEL / OpenSUSE
       - ``dnf install zrepl``
       - :ref:`RPM repository config <installation-rpm-repos>`
     * - Debian + Ubuntu
@@ -42,6 +45,6 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - Void Linux
       - ``xbps-install zrepl``
       - Available since `a88a2a4 <https://github.com/void-linux/void-packages/commit/a88a2a4d7bf56072dadf61ab56b8424e39155890>`_
-    * - Others
+    * - any
-      -
+      - Build from source
-      - Use `binary releases`_ or build from source.
+      - :repomasterlink:`README.md`.

internal/platformtest/tests/gen/wrapper.bash

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+source ../../../build/go_install_host_tool.source
+
+GOOS=$GOHOSTOS GOARCH=$GOHOSTARCH go run ./gen ./

internal/platformtest/tests/tests.go

@@ -13,4 +13,4 @@ func (c Case) String() string {
 	return runtime.FuncForPC(reflect.ValueOf(c).Pointer()).Name()
 }
 
-//go:generate go run ./gen github.com/zrepl/zrepl/platformtest/tests
+//go:generate ./gen/wrapper.bash

lazy.sh

@@ -1,95 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-
-if tty -s; then
-    bold=$(tput bold)
-    normal=$(tput sgr0)
-else
-    bold=""
-    normal=""
-fi
-
-step() {
-    echo "${bold}$1${normal}"
-}
-
-godep() {
-    step "install build dependencies (versions pinned in build/go.mod and build/tools.go)"
-
-    if ! type go >/dev/null; then
-        step "go binary not installed or not in \$PATH" 1>&2
-        exit 1
-    fi
-
-    if [ -z "$GOPATH" ]; then
-        step "Your GOPATH is not configured correctly" 1>&2
-        exit 1
-    fi
-
-    if ! (echo "$PATH" | grep "${GOPATH}/bin" > /dev/null); then
-        step "GOPATH/bin is not in your PATH (it should be towards the start of it)"
-        exit 1
-    fi
-
-    pushd "$(dirname "${BASH_SOURCE[0]}")"/build
-    set -x
-    export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
-    source <(go env)
-    export GOOS="$GOHOSTOS"
-    export GOARCH="$GOHOSTARCH"
-    # TODO GOARM=$GOHOSTARM?
-    cat tools.go | grep _ | awk -F'"' '{print $2}' | tee | xargs -tI '{}' go install '{}'
-    set +x
-    popd
-    if ! type stringer || ! type protoc-gen-go || ! type protoc-gen-go-grpc || ! type enumer || ! type goimports || ! type golangci-lint || ! type gocovmerge; then
-        echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
-        exit 1
-    fi
-}
-
-docdep() {
-    if ! type pip3; then
-        step "pip3 binary not installed or not in \$PATH" 1>&2
-        exit 1
-    fi
-    step "Installing doc build dependencies"
-    # shellcheck disable=SC2155
-    local reqpath="$(dirname "${BASH_SOURCE[0]}")/docs/requirements.txt"
-    if [ -n "$ZREPL_LAZY_DOCS_REQPATH" ]; then
-        reqpath="$ZREPL_LAZY_DOCS_REQPATH"
-    fi
-    pip3 install -r "$reqpath"
-}
-
-release() {
-    step "Making release"
-    make release
-}
-
-# shellcheck disable=SC2198
-if [ -z "$@" ]; then
-    step "No command specified, exiting"
-    exit 1
-fi
-
-for cmd in "$@"; do
-    case "$cmd" in
-        godep|docdep|release|docs)
-            eval $cmd
-            continue
-            ;;
-        devsetup)
-            step "Installing development dependencies"
-            godep
-            docdep
-            step "Development dependencies installed"
-            continue
-            ;;
-        *)
-            step "Invalid command ${cmd}, exiting"
-            exit 1
-            ;;
-    esac
-done
-