From 08769a8752de5eb933ef3e3a8c2b9c9efb5d3cd7 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 12:57:58 +0000 Subject: [PATCH 01/13] fix: accidental use of wrong logging package --- daemon/job/active.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/job/active.go b/daemon/job/active.go index b857aea..29f3060 100644 --- a/daemon/job/active.go +++ b/daemon/job/active.go @@ -482,7 +482,7 @@ func (j *ActiveSide) do(ctx context.Context) { go func() { select { case <-reset.Wait(ctx): - log.Info("reset received, cancelling current invocation") + GetLogger(ctx).Info("reset received, cancelling current invocation") cancelThisRun() case <-ctx.Done(): } From def510abfdc67996ec489fa2a9f89c1e051e0655 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 12:58:26 +0000 Subject: [PATCH 02/13] chore: require go 1.22/1.23, upgrade protobuf, upgrade all deps Go upgrade: - Go 1.23 is current => use that for release builds - Go 1.22 is less than one year old, it's desirable to support it. - The [`Go Toolchains`](https://go.dev/doc/toolchain) stuff is available in both of these (would also be in Go 1.21). That is quite nice stuff, but required some changes to how we versions we use in CircleCI and the `release-docker` Makefile target. Protobuf upgrade: - Go to protobuf GH release website - Download latest locally - run `sha256sum` - replace existing pinned hashes - `make generate` Deps upgrade: - `go get -t -u all` - repository moves aren't handled well automatically, fix manually - repeat until no changes --- .circleci/config.yml | 41 +- Makefile | 7 +- build.installprotoc.bash | 8 +- build/go.mod | 222 +- build/go.sum | 2517 ++--------------- client/status/status_dump.go | 2 +- client/status/status_interactive.go | 2 +- client/status/status_legacy.go | 2 +- daemon/hooks/stepstatus_enumer.go | 1 - daemon/job/active.go | 1 - daemon/job/activesidestate_enumer.go | 1 - .../trace_unique_concurrent_task_namer.go | 2 +- ...trace_unique_concurrent_task_namer_test.go | 2 +- daemon/pruner/state_enumer.go | 1 - ...holdercreationencryptionproperty_enumer.go | 1 - endpoint/replicationguaranteekind_enumer.go | 1 - go.mod | 84 +- go.sum | 241 +- replication/driver/errorclass_enumer.go | 1 - ...initialreplicationautoresolution_enumer.go | 1 - replication/logic/pdu/pdu.pb.go | 58 +- replication/logic/pdu/pdu_grpc.pb.go | 72 +- .../base2bufpool/nofitbehavior_enumer.go | 1 - .../example/pdu/grpcauth.pb.go | 15 +- .../example/pdu/grpcauth_grpc.pb.go | 36 +- ...placeholdercreateencryptionvalue_enumer.go | 1 - zfs/propertysource_enumer.go | 1 - 27 files changed, 769 insertions(+), 2553 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2c9911..a9d1c20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ version: 2.1 orbs: - # NB: 1.7.2 is not the Go version, but the Orb version + # NB: this is not the Go version, but the Orb version # https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache - go: circleci/go@1.7.2 + go: circleci/go@1.11.0 commands: setup-home-local-bin: steps: @@ -99,10 +99,6 @@ parameters: type: boolean default: false - release_docker_baseimage_tag: - type: string - default: "1.21" - workflows: version: 2 @@ -110,20 +106,20 @@ workflows: when: << pipeline.parameters.do_ci >> jobs: - quickcheck-docs - - quickcheck-go: &quickcheck-go-smoketest - name: quickcheck-go-amd64-linux-1.21 - goversion: &latest-go-release "1.21" + - quickcheck-go: + name: quickcheck-go-amd64-linux-1.23.1 + goversion: &latest-go-release "1.23.1" goos: linux goarch: amd64 - - test-go-on-latest-go-release: + - test-go: goversion: *latest-go-release - quickcheck-go: requires: - - quickcheck-go-amd64-linux-1.21 #quickcheck-go-smoketest.name - matrix: &quickcheck-go-matrix + - quickcheck-go-amd64-linux-1.23.1 #quickcheck-go-smoketest.name + matrix: alias: quickcheck-go-matrix parameters: - goversion: [*latest-go-release, "1.20"] + goversion: [*latest-go-release, "1.22.7"] goos: ["linux", "freebsd"] goarch: ["amd64", "arm64"] exclude: @@ -138,6 +134,7 @@ workflows: goos: ["linux"] goarch: ["amd64"] requires: + - test-go - quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >> release: @@ -167,7 +164,7 @@ workflows: jobs: quickcheck-docs: docker: - - image: cimg/base:2023.09 + - image: cimg/base:2024.09 steps: - checkout - install-docdep @@ -186,10 +183,12 @@ jobs: goarch: type: string docker: - - image: cimg/go:<> + # any toolchain >= 1.22 will auto-download GOTOOLCHAIN + - image: &cimg_with_modern_go cimg/go:1.22 environment: GOOS: <> GOARCH: <> + GOTOOLCHAIN: "go<>" steps: - checkout @@ -237,12 +236,14 @@ jobs: - run: sudo zfs version - run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH" - test-go-on-latest-go-release: + test-go: parameters: goversion: type: string docker: - - image: cimg/go:<> + - image: *cimg_with_modern_go + environment: + GOTOOLCHAIN: "go<>" steps: - checkout - go/load-cache: @@ -256,7 +257,7 @@ jobs: image: ubuntu-2004:202201-02 steps: - checkout - - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<> + - run: make release-docker - persist_to_workspace: root: . paths: [.] @@ -286,7 +287,7 @@ jobs: release-upload: docker: - - image: cimg/base:2020.08 + - image: cimg/base:2024.09 steps: - attach_workspace: at: . @@ -296,7 +297,7 @@ jobs: publish-zrepl-github-io: docker: - - image: cimg/base:2023.09 + - image: cimg/base:2024.09 steps: - checkout - install-docdep diff --git a/Makefile b/Makefile index 6e3e7bd..fcc4b8a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"') GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"') GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"') GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"') -GO_ENV_VARS := GO111MODULE=on CGO_ENABLED=0 +GO_ENV_VARS := CGO_ENABLED=0 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)" GO_MOD_READONLY := -mod=readonly GO_EXTRA_BUILDFLAGS := @@ -30,8 +30,8 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS) GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS) GOLANGCI_LINT := golangci-lint GOCOVMERGE := gocovmerge -RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.21 -RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG) +RELEASE_DOCKER_TOOLCHAIN ?= 1.22.7 +RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_TOOLCHAIN) ifneq ($(GOARM),) ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM) @@ -59,6 +59,7 @@ release: clean $(MAKE) noarch release-docker: $(ARTIFACTDIR) + # upstream docker image sets GOTOOLCHAIN=local sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile . docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \ diff --git a/build.installprotoc.bash b/build.installprotoc.bash index c39c74a..6320ee7 100644 --- a/build.installprotoc.bash +++ b/build.installprotoc.bash @@ -5,7 +5,7 @@ set -x MACH=$(uname -m) MACH="${MACH/aarch64/aarch_64}" -VERSION=3.6.1 +VERSION=28.0 FILENAME=protoc-"$VERSION"-linux-"$MACH".zip if [ -e "$FILENAME" ]; then @@ -18,8 +18,8 @@ wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/" stat "$FILENAME" sha256sum -c --ignore-missing < Date: Sun, 8 Sep 2024 13:02:47 +0000 Subject: [PATCH 03/13] chore: newer staticcheck complains about useless fmt.Sprintf --- rpc/versionhandshake/versionhandshake.go | 34 ++++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/rpc/versionhandshake/versionhandshake.go b/rpc/versionhandshake/versionhandshake.go index 0fb2160..2919cea 100644 --- a/rpc/versionhandshake/versionhandshake.go +++ b/rpc/versionhandshake/versionhandshake.go @@ -91,8 +91,8 @@ func (e HandshakeError) Timeout() bool { return false } -func hsErr(format string, args ...interface{}) *HandshakeError { - return &HandshakeError{msg: fmt.Sprintf(format, args...)} +func hsErr(msg string) *HandshakeError { + return &HandshakeError{msg: msg} } func hsIOErr(err error, format string, args ...interface{}) *HandshakeError { @@ -115,10 +115,10 @@ func (m *HandshakeMessage) Encode() ([]byte, error) { var extensions strings.Builder for i, ext := range m.Extensions { if strings.ContainsAny(ext, "\n") { - return nil, hsErr("Extension #%d contains forbidden newline character", i) + return nil, hsErr(fmt.Sprintf("Extension #%d contains forbidden newline character", i)) } if !utf8.ValidString(ext) { - return nil, hsErr("Extension #%d is not valid UTF-8", i) + return nil, hsErr(fmt.Sprintf("Extension #%d is not valid UTF-8", i)) } extensions.WriteString(ext) extensions.WriteString("\n") @@ -143,8 +143,8 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { return hsErr("could not parse handshake message length") } if followLen > maxLen { - return hsErr("handshake message length exceeds max length (%d vs %d)", - followLen, maxLen) + return hsErr(fmt.Sprintf("handshake message length exceeds max length (%d vs %d)", + followLen, maxLen)) } var buf bytes.Buffer @@ -159,15 +159,15 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { n, err = fmt.Fscanf(&buf, "ZREPL_ZFS_REPLICATION PROTOVERSION=%04d EXTENSIONS=%4d\n", &protoVersion, &extensionCount) if n != 2 || err != nil { - return hsErr("could not parse handshake message: %s", err) + return hsErr(fmt.Sprintf("could not parse handshake message: %s", err)) } if protoVersion < 1 { - return hsErr("invalid protocol version %q", protoVersion) + return hsErr(fmt.Sprintf("invalid protocol version %q", protoVersion)) } m.ProtocolVersion = protoVersion if extensionCount < 0 { - return hsErr("invalid extension count %q", extensionCount) + return hsErr(fmt.Sprintf("invalid extension count %q", extensionCount)) } if extensionCount == 0 { if buf.Len() != 0 { @@ -178,7 +178,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { } s := buf.String() if strings.Count(s, "\n") != extensionCount { - return hsErr("inconsistent extension count: found %d, header says %d", len(m.Extensions), extensionCount) + return hsErr(fmt.Sprintf("inconsistent extension count: found %d, header says %d", len(m.Extensions), extensionCount)) } exts := strings.Split(s, "\n") if exts[len(exts)-1] != "" { @@ -203,12 +203,12 @@ func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) (rErr *H } hsb, err := ours.Encode() if err != nil { - return hsErr("could not encode protocol banner: %s", err) + return hsErr(fmt.Sprintf("could not encode protocol banner: %s", err)) } err = conn.SetDeadline(deadline) if err != nil { - return hsErr("could not set deadline for protocol banner handshake: %s", err) + return hsErr(fmt.Sprintf("could not set deadline for protocol banner handshake: %s", err)) } defer func() { if rErr != nil { @@ -216,22 +216,22 @@ func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) (rErr *H } err := conn.SetDeadline(time.Time{}) if err != nil { - rErr = hsErr("could not reset deadline after protocol banner handshake: %s", err) + rErr = hsErr(fmt.Sprintf("could not reset deadline after protocol banner handshake: %s", err)) } }() _, err = io.Copy(conn, bytes.NewBuffer(hsb)) if err != nil { - return hsErr("could not send protocol banner: %s", err) + return hsErr(fmt.Sprintf("could not send protocol banner: %s", err)) } theirs := HandshakeMessage{} if err := theirs.DecodeReader(conn, HandshakeMessageMaxLen); err != nil { - return hsErr("could not decode protocol banner: %s", err) + return hsErr(fmt.Sprintf("could not decode protocol banner: %s", err)) } if theirs.ProtocolVersion != ours.ProtocolVersion { - return hsErr("protocol versions do not match: ours is %d, theirs is %d", - ours.ProtocolVersion, theirs.ProtocolVersion) + return hsErr(fmt.Sprintf("protocol versions do not match: ours is %d, theirs is %d", + ours.ProtocolVersion, theirs.ProtocolVersion)) } // ignore extensions, we don't use them From 48c5b600243c9fad2397666e549cb1f9973567b7 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 13:11:55 +0000 Subject: [PATCH 04/13] chore: grpc.DialContext has been deprecated --- .../grpchelper/authlistener_grpc_adaptor_wrapper.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go b/rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go index 91358d5..382ecc2 100644 --- a/rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go +++ b/rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go @@ -3,7 +3,6 @@ package grpchelper import ( - "context" "time" "google.golang.org/grpc" @@ -38,7 +37,7 @@ func ClientConn(cn transport.Connecter, log Logger) *grpc.ClientConn { cred := grpc.WithTransportCredentials(grpcclientidentity.NewTransportCredentials(log)) // we use context.Background without a timeout here because we don't set grpc.WithBlock // => docs: "In the non-blocking case, the ctx does not act against the connection. It only controls the setup steps." - cc, err := grpc.DialContext(context.Background(), "doesn't matter done by dialer", dialerOption, cred, ka) + cc, err := grpc.NewClient("passthrough://doesntmatterdonebydialer", dialerOption, cred, ka) if err != nil { log.WithError(err).Error("cannot create gRPC client conn (non-blocking)") // It's ok to panic here: the we call grpc.DialContext without the From 740ab4b1b27e64f233ed83cab59d379d30202cb4 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 13:15:26 +0000 Subject: [PATCH 05/13] chore: io/ioutil has been deprecated --- config/config.go | 3 +-- platformtest/platformtest_ops.go | 3 +-- platformtest/tests/gen/gen.go | 3 +-- platformtest/tests/sendStream.go | 5 ++--- rpc/dataconn/frameconn/frameconn.go | 3 +-- .../timeoutconn/internal/wireevaluator/wireevaluator.go | 3 +-- .../internal/wireevaluator/wireevaluator_closewrite.go | 5 ++--- tlsconf/tlsconf.go | 3 +-- zfs/zfs_pipe_linux.go | 3 +-- 9 files changed, 11 insertions(+), 20 deletions(-) diff --git a/config/config.go b/config/config.go index f4b84c0..7810e15 100644 --- a/config/config.go +++ b/config/config.go @@ -2,7 +2,6 @@ package config import ( "fmt" - "io/ioutil" "log/syslog" "os" "time" @@ -671,7 +670,7 @@ func ParseConfig(path string) (i *Config, err error) { var bytes []byte - if bytes, err = ioutil.ReadFile(path); err != nil { + if bytes, err = os.ReadFile(path); err != nil { return } diff --git a/platformtest/platformtest_ops.go b/platformtest/platformtest_ops.go index c4f1773..9c1e166 100644 --- a/platformtest/platformtest_ops.go +++ b/platformtest/platformtest_ops.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "fmt" - "io/ioutil" "os" "os/exec" "strings" @@ -64,7 +63,7 @@ func (o *FSOp) Run(ctx context.Context, e Execer) error { if o.Encrypted { const passphraseFilePath = "/tmp/zreplplatformtest.encryption.passphrase" const passphrase = "foobar2342" - err := ioutil.WriteFile(passphraseFilePath, []byte(passphrase), 0600) + err := os.WriteFile(passphraseFilePath, []byte(passphrase), 0600) if err != nil { panic(err) } diff --git a/platformtest/tests/gen/gen.go b/platformtest/tests/gen/gen.go index 4d75871..3147297 100644 --- a/platformtest/tests/gen/gen.go +++ b/platformtest/tests/gen/gen.go @@ -6,7 +6,6 @@ import ( "go/format" "go/parser" "go/token" - "io/ioutil" "os" "sort" "strings" @@ -132,7 +131,7 @@ var Cases = []Case { formatted, err := format.Source(buf.Bytes()) check(err) - err = ioutil.WriteFile("generated_cases.go", formatted, 0664) + err = os.WriteFile("generated_cases.go", formatted, 0664) check(err) } diff --git a/platformtest/tests/sendStream.go b/platformtest/tests/sendStream.go index 7fbe5e7..019d5c8 100644 --- a/platformtest/tests/sendStream.go +++ b/platformtest/tests/sendStream.go @@ -3,7 +3,6 @@ package tests import ( "fmt" "io" - "io/ioutil" "os" "os/exec" "path" @@ -102,7 +101,7 @@ func SendStreamCloseAfterEOFRead(ctx *platformtest.Context) { sendStream := sendStreamTest(ctx) - _, err := io.Copy(ioutil.Discard, sendStream) + _, err := io.Copy(io.Discard, sendStream) require.NoError(ctx, err) var buf [128]byte @@ -122,7 +121,7 @@ func SendStreamMultipleCloseAfterEOF(ctx *platformtest.Context) { sendStream := sendStreamTest(ctx) - _, err := io.Copy(ioutil.Discard, sendStream) + _, err := io.Copy(io.Discard, sendStream) require.NoError(ctx, err) var buf [128]byte diff --git a/rpc/dataconn/frameconn/frameconn.go b/rpc/dataconn/frameconn/frameconn.go index dccf417..96ec273 100644 --- a/rpc/dataconn/frameconn/frameconn.go +++ b/rpc/dataconn/frameconn/frameconn.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "sync" "syscall" @@ -354,7 +353,7 @@ func (c *Conn) Shutdown(deadline time.Time) error { // TODO DoS mitigation by reading limited number of bytes // see discussion above why this is non-trivial defer prometheus.NewTimer(prom.ShutdownDrainSeconds).ObserveDuration() - n, _ := io.Copy(ioutil.Discard, c.nc) + n, _ := io.Copy(io.Discard, c.nc) prom.ShutdownDrainBytesRead.Observe(float64(n)) return closeWire("close") diff --git a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go index 29b0462..c786dba 100644 --- a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go +++ b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator.go @@ -5,7 +5,6 @@ import ( "context" "flag" "fmt" - "io/ioutil" "os" "path" @@ -47,7 +46,7 @@ func main() { flag.StringVar(&args.testCase, "testcase", "", "") flag.Parse() - bytes, err := ioutil.ReadFile(args.configPath) + bytes, err := os.ReadFile(args.configPath) noerror(err) err = yaml.UnmarshalStrict(bytes, &conf) noerror(err) diff --git a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go index e2dddbd..670486e 100644 --- a/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go +++ b/rpc/dataconn/timeoutconn/internal/wireevaluator/wireevaluator_closewrite.go @@ -3,7 +3,6 @@ package main import ( "bytes" "io" - "io/ioutil" "log" "github.com/zrepl/zrepl/transport" @@ -83,7 +82,7 @@ func (CloseWrite) receiver(wire transport.Wire) { // consume half the test data, then detect an error, send it and CloseWrite r := io.LimitReader(wire, int64(5*len(closeWriteTestSendData)/3)) - _, err := io.Copy(ioutil.Discard, r) + _, err := io.Copy(io.Discard, r) noerror(err) var errBuf bytes.Buffer @@ -95,7 +94,7 @@ func (CloseWrite) receiver(wire transport.Wire) { noerror(err) // drain wire, as documented in transport.Wire, this is the only way we know the client closed the conn - _, err = io.Copy(ioutil.Discard, wire) + _, err = io.Copy(io.Discard, wire) if err != nil { // io.Copy masks io.EOF to nil, and we expect io.EOF from the client's Close() call log.Panicf("unexpected error returned from reading conn: %s", err) diff --git a/tlsconf/tlsconf.go b/tlsconf/tlsconf.go index 71e81c0..85b8dc9 100644 --- a/tlsconf/tlsconf.go +++ b/tlsconf/tlsconf.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "os" "time" @@ -14,7 +13,7 @@ import ( func ParseCAFile(certfile string) (*x509.CertPool, error) { pool := x509.NewCertPool() - pem, err := ioutil.ReadFile(certfile) + pem, err := os.ReadFile(certfile) if err != nil { return nil, err } diff --git a/zfs/zfs_pipe_linux.go b/zfs/zfs_pipe_linux.go index 104997c..e52d5e1 100644 --- a/zfs/zfs_pipe_linux.go +++ b/zfs/zfs_pipe_linux.go @@ -3,7 +3,6 @@ package zfs import ( "errors" "fmt" - "io/ioutil" "os" "strconv" "strings" @@ -20,7 +19,7 @@ func getPipeCapacityHint(envvar string) int { // https://github.com/zrepl/zrepl/issues/424#issuecomment-800370928 // https://bugzilla.kernel.org/show_bug.cgi?id=212295 if _, err := os.Stat("/proc/sys/fs/pipe-max-size"); err == nil { - if dat, err := ioutil.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil { + if dat, err := os.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil { if capacity, err = strconv.ParseInt(strings.TrimSpace(string(dat)), 10, 64); err != nil { capacity = 1 << 25 } From 0ab92d48614eacdc237b2f3173fac0d66c66d418 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 15:12:31 +0000 Subject: [PATCH 06/13] build: avoid compiling platformtest test list generator This also fixes a deprecation warning. --- .circleci/config.yml | 2 -- Makefile | 7 ++----- platformtest/tests/gen/gen.go | 7 ++----- platformtest/tests/tests.go | 2 +- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a9d1c20..490abfa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,12 +202,10 @@ jobs: key: quickcheck-<> - run: make formatcheck - - run: make generate-platform-test-list - run: make zrepl-bin test-platform-bin - run: make vet - run: make lint - - run: rm -f artifacts/generate-platform-test-list - store_artifacts: path: artifacts - persist_to_workspace: diff --git a/Makefile b/Makefile index fcc4b8a..08405be 100644 --- a/Makefile +++ b/Makefile @@ -228,7 +228,7 @@ _run_make_foreach_target_tuple: $(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos GOARCH=amd64 ##################### REGULAR TARGETS ##################### -.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list +.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin lint: $(GO_ENV_VARS) $(GOLANGCI_LINT) run ./... @@ -252,9 +252,6 @@ endif zrepl-bin: $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)" -generate-platform-test-list: - $(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen - COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE) cover-platform-bin: $(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \ @@ -312,7 +309,7 @@ cover-full: # not part of the build, must do that manually .PHONY: generate formatcheck format -generate: generate-platform-test-list +generate: protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto $(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./... diff --git a/platformtest/tests/gen/gen.go b/platformtest/tests/gen/gen.go index 3147297..dd319f4 100644 --- a/platformtest/tests/gen/gen.go +++ b/platformtest/tests/gen/gen.go @@ -21,7 +21,6 @@ func check(err error) { } type platformtestFuncDeclFinder struct { - pkg *packages.Package testFuncs []*ast.FuncDecl } @@ -78,7 +77,7 @@ func main() { pkgs, err := packages.Load( &packages.Config{ - Mode: packages.LoadFiles, + Mode: packages.NeedFiles, Tests: false, }, os.Args[1], @@ -97,9 +96,7 @@ func main() { s := token.NewFileSet() a, err := parser.ParseFile(s, f, nil, parser.AllErrors) check(err) - finder := &platformtestFuncDeclFinder{ - pkg: p, - } + finder := &platformtestFuncDeclFinder{} ast.Walk(finder, a) tests = append(tests, finder.testFuncs...) } diff --git a/platformtest/tests/tests.go b/platformtest/tests/tests.go index 4b16441..9db0326 100644 --- a/platformtest/tests/tests.go +++ b/platformtest/tests/tests.go @@ -13,4 +13,4 @@ func (c Case) String() string { return runtime.FuncForPC(reflect.ValueOf(c).Pointer()).Name() } -//go:generate ../../artifacts/generate-platform-test-list github.com/zrepl/zrepl/platformtest/tests +//go:generate go run ./gen github.com/zrepl/zrepl/platformtest/tests From 3cb1865909fb3de00359c27186ce5dc86b921c6e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 15:34:39 +0000 Subject: [PATCH 07/13] chore: trace spans: use crypto/rand for generating them math/rand.Read is deprecated in newer Go versions. Also, it appears that crypto/rand is faster when used from multiple goroutines: https://gist.github.com/problame/0699acd6f99db4163f26f0b8a61569f3 --- daemon/daemon.go | 7 ------- daemon/logging/trace/trace_genID.go | 11 ++++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 75333e0..f83b758 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -3,7 +3,6 @@ package daemon import ( "context" "fmt" - "math/rand" "os" "os/signal" "strings" @@ -39,12 +38,6 @@ func Run(ctx context.Context, conf *config.Config) error { cancel() }() - // The math/rand package is used presently for generating trace IDs, we - // seed it with the current time and pid so that the IDs are mostly - // unique. - rand.Seed(time.Now().UnixNano()) - rand.Seed(int64(os.Getpid())) - outlets, err := logging.OutletsFromConfig(*conf.Global.Logging) if err != nil { return errors.Wrap(err, "cannot build logging from config") diff --git a/daemon/logging/trace/trace_genID.go b/daemon/logging/trace/trace_genID.go index 908e3bd..f8ac7bf 100644 --- a/daemon/logging/trace/trace_genID.go +++ b/daemon/logging/trace/trace_genID.go @@ -1,8 +1,8 @@ package trace import ( + "crypto/rand" "encoding/base64" - "math/rand" "strings" "github.com/zrepl/zrepl/util/envconst" @@ -20,12 +20,9 @@ func genID() string { var out strings.Builder enc := base64.NewEncoder(base64.RawStdEncoding, &out) buf := make([]byte, genIdNumBytes) - for i := 0; i < len(buf); { - n, err := rand.Read(buf[i:]) - if err != nil { - panic(err) - } - i += n + _, err := rand.Read(buf) + if err != nil { + panic(err) } n, err := enc.Write(buf[:]) if err != nil || n != len(buf) { From 5a8f0b9a2441f723e4068e49dfb5c58be21e76cf Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 21:56:07 +0000 Subject: [PATCH 08/13] build: make release: check toolchain GOVERSION matches expecations (and refactor/extend Makefile a bit) --- Makefile | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 08405be..7d55740 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,10 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS) GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS) GOLANGCI_LINT := golangci-lint GOCOVMERGE := gocovmerge -RELEASE_DOCKER_TOOLCHAIN ?= 1.22.7 -RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_TOOLCHAIN) +RELEASE_GOVERSION ?= go1.23.1 +STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION)) +RELEASE_DOCKER_BASEIMAGE ?= golang:$(STRIPPED_GOVERSION) +RELEASE_DOCKER_CACHEMOUNT := ifneq ($(GOARM),) ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM) @@ -39,34 +41,46 @@ else ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH) endif -.PHONY: printvars -printvars: - @echo GOOS=$(GOOS) - @echo GOARCH=$(GOARCH) - @echo GOARM=$(GOARM) +ifneq ($(RELEASE_DOCKER_CACHEMOUNT),) + _RELEASE_DOCKER_CACHEMOUNT := -v $(RELEASE_DOCKER_CACHEMOUNT)/mod:/go/pkg/mod -v $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache:/root/.cache/go-build +.PHONY: release-docker-mkcachemount +release-docker-mkcachemount: + mkdir -p $(RELEASE_DOCKER_CACHEMOUNT) + mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/mod + mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache +else + _RELEASE_DOCKER_CACHEMOUNT := +.PHONY: release-docker-mkcachemount +release-docker-mkcachemount: + # nothing to do +endif ##################### PRODUCING A RELEASE ############# -.PHONY: release wrapup-and-checksum check-git-clean sign clean +.PHONY: release wrapup-and-checksum check-git-clean sign clean ensure-release-toolchain -release: clean - # no cross-platform support for target test - $(MAKE) test-go +ensure-release-toolchain: + # ensure the toolchain is actually the one we expect + test $(RELEASE_GOVERSION) = "$$($(GO_ENV_VARS) $(GO) env GOVERSION)" + +release: ensure-release-toolchain $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet" $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint" $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin" $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin" $(MAKE) noarch -release-docker: $(ARTIFACTDIR) - # upstream docker image sets GOTOOLCHAIN=local - sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile - docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile . - docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \ +release-docker: $(ARTIFACTDIR) release-docker-mkcachemount + sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile + docker build -t zrepl_release --pull -f $(ARTIFACTDIR)/build.Dockerfile . + docker run --rm -i \ + $(_RELEASE_DOCKER_CACHEMOUNT) \ + -v $(CURDIR):/src -u $$(id -u):$$(id -g) \ zrepl_release \ make release \ GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \ - ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) + ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) \ + RELEASE_GOVERSION=$(RELEASE_GOVERSION) debs-docker: $(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb From e390aa0c5a74bf2569b90346b15c6195f6cfde78 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 22:03:27 +0000 Subject: [PATCH 09/13] build: circleci: update VM image used for release builds Doesn't matter much because everything happens inside Docker. --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 490abfa..84dd2bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -252,7 +252,7 @@ jobs: release-build: machine: - image: ubuntu-2004:202201-02 + image: &release-vm-image "ubuntu-2404:current" steps: - checkout - run: make release-docker @@ -261,7 +261,7 @@ jobs: paths: [.] release-deb: machine: - image: ubuntu-2004:202201-02 + image: *release-vm-image steps: - attach_workspace: at: . @@ -273,7 +273,7 @@ jobs: release-rpm: machine: - image: ubuntu-2004:202201-02 + image: *release-vm-image steps: - attach_workspace: at: . From 7b6adab6b156ae8be63d62726f9f138b7517b93a Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 22:36:15 +0000 Subject: [PATCH 10/13] build: circleci: only archive artifacts/release --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84dd2bc..3329652 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -291,7 +291,7 @@ jobs: at: . - run: make wrapup-and-checksum - store_artifacts: - path: artifacts + path: artifacts/release publish-zrepl-github-io: docker: From 82adb2b9f5191aab45332ac50cd6e1ff1729f173 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 22:38:24 +0000 Subject: [PATCH 11/13] build: circleci: remove obsolete script The binary packaging workflow has long since been moved to this repo (I don't think the external workflow work ever completed). --- .../trigger_debian_binary_packaging_workflow.bash | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100755 .circleci/trigger_debian_binary_packaging_workflow.bash diff --git a/.circleci/trigger_debian_binary_packaging_workflow.bash b/.circleci/trigger_debian_binary_packaging_workflow.bash deleted file mode 100755 index f71f08a..0000000 --- a/.circleci/trigger_debian_binary_packaging_workflow.bash +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -COMMIT="$1" -GO_VERSION="$2" - -curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \ - -H 'Accept: application/vnd.github.v3+json' \ - -H "Authorization: token $GITHUB_ACCESS_TOKEN" \ - --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}' From b8f55a97ba4666ab59cfec5430045344050dcfed Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 22:43:58 +0000 Subject: [PATCH 12/13] build: circleci: use `large` class for release-build job --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3329652..c3dd13c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -253,6 +253,7 @@ jobs: release-build: machine: image: &release-vm-image "ubuntu-2404:current" + resource_class: large steps: - checkout - run: make release-docker From 3df5e223bef965ddaf4965288b941070c98ea1e1 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 8 Sep 2024 22:55:10 +0000 Subject: [PATCH 13/13] take notes for changelog --- docs/changelog.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9d12265..ebad298 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,6 +40,16 @@ Those changes will likely come with some breakage in the config. However, I want to avoid breaking **use cases** that are satisfied by the current design. There will be beta/RC releases to give users a chance to evaluate. +0.7.0 (unreleased) +------------------ + +INCOMPLETE LIST OF CHANGES + +* |maint| Update to Go 1.23, require Go 1.22. Adopt `go.mod` toolchain aka `GOTOOLCHAIN` env var. +* |maint| Fix deprecations exposed by the toolchain update. +* |maint| Long-overdue update of all our dependencies & address deprecations. +* |maint| The `make release-docker` in CircleCI produces executables that are bit-identical to my personal machine. + 0.6.1 -----