diff --git a/.circleci/config.yml b/.circleci/config.yml index e2c9911..c3dd13c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ version: 2.1 orbs: - # NB: 1.7.2 is not the Go version, but the Orb version + # NB: this is not the Go version, but the Orb version # https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache - go: circleci/go@1.7.2 + go: circleci/go@1.11.0 commands: setup-home-local-bin: steps: @@ -99,10 +99,6 @@ parameters: type: boolean default: false - release_docker_baseimage_tag: - type: string - default: "1.21" - workflows: version: 2 @@ -110,20 +106,20 @@ workflows: when: << pipeline.parameters.do_ci >> jobs: - quickcheck-docs - - quickcheck-go: &quickcheck-go-smoketest - name: quickcheck-go-amd64-linux-1.21 - goversion: &latest-go-release "1.21" + - quickcheck-go: + name: quickcheck-go-amd64-linux-1.23.1 + goversion: &latest-go-release "1.23.1" goos: linux goarch: amd64 - - test-go-on-latest-go-release: + - test-go: goversion: *latest-go-release - quickcheck-go: requires: - - quickcheck-go-amd64-linux-1.21 #quickcheck-go-smoketest.name - matrix: &quickcheck-go-matrix + - quickcheck-go-amd64-linux-1.23.1 #quickcheck-go-smoketest.name + matrix: alias: quickcheck-go-matrix parameters: - goversion: [*latest-go-release, "1.20"] + goversion: [*latest-go-release, "1.22.7"] goos: ["linux", "freebsd"] goarch: ["amd64", "arm64"] exclude: @@ -138,6 +134,7 @@ workflows: goos: ["linux"] goarch: ["amd64"] requires: + - test-go - quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >> release: @@ -167,7 +164,7 @@ workflows: jobs: quickcheck-docs: docker: - - image: cimg/base:2023.09 + - image: cimg/base:2024.09 steps: - checkout - install-docdep @@ -186,10 +183,12 @@ jobs: goarch: type: string docker: - - image: cimg/go:<> + # any toolchain >= 1.22 will auto-download GOTOOLCHAIN + - image: &cimg_with_modern_go cimg/go:1.22 environment: GOOS: <> GOARCH: <> + GOTOOLCHAIN: "go<>" steps: - checkout @@ -203,12 +202,10 @@ jobs: key: quickcheck-<> - run: make formatcheck - - run: make generate-platform-test-list - run: make zrepl-bin test-platform-bin - run: make vet - run: make lint - - run: rm -f artifacts/generate-platform-test-list - store_artifacts: path: artifacts - persist_to_workspace: @@ -237,12 +234,14 @@ jobs: - run: sudo zfs version - run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH" - test-go-on-latest-go-release: + test-go: parameters: goversion: type: string docker: - - image: cimg/go:<> + - image: *cimg_with_modern_go + environment: + GOTOOLCHAIN: "go<>" steps: - checkout - go/load-cache: @@ -253,16 +252,17 @@ jobs: release-build: machine: - image: ubuntu-2004:202201-02 + image: &release-vm-image "ubuntu-2404:current" + resource_class: large steps: - checkout - - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<> + - run: make release-docker - persist_to_workspace: root: . paths: [.] release-deb: machine: - image: ubuntu-2004:202201-02 + image: *release-vm-image steps: - attach_workspace: at: . @@ -274,7 +274,7 @@ jobs: release-rpm: machine: - image: ubuntu-2004:202201-02 + image: *release-vm-image steps: - attach_workspace: at: . @@ -286,17 +286,17 @@ jobs: release-upload: docker: - - image: cimg/base:2020.08 + - image: cimg/base:2024.09 steps: - attach_workspace: at: . - run: make wrapup-and-checksum - store_artifacts: - path: artifacts + path: artifacts/release publish-zrepl-github-io: docker: - - image: cimg/base:2023.09 + - image: cimg/base:2024.09 steps: - checkout - install-docdep diff --git a/.circleci/trigger_debian_binary_packaging_workflow.bash b/.circleci/trigger_debian_binary_packaging_workflow.bash deleted file mode 100755 index f71f08a..0000000 --- a/.circleci/trigger_debian_binary_packaging_workflow.bash +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -COMMIT="$1" -GO_VERSION="$2" - -curl -v -X POST https://api.github.com/repos/zrepl/debian-binary-packaging/dispatches \ - -H 'Accept: application/vnd.github.v3+json' \ - -H "Authorization: token $GITHUB_ACCESS_TOKEN" \ - --data '{"event_type": "push", "client_payload": { "zrepl_main_repo_commit": "'"$COMMIT"'", "go_version": "'"$GO_VERSION"'" }}' diff --git a/Makefile b/Makefile index 6e3e7bd..7d55740 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"') GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"') GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"') GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"') -GO_ENV_VARS := GO111MODULE=on CGO_ENABLED=0 +GO_ENV_VARS := CGO_ENABLED=0 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)" GO_MOD_READONLY := -mod=readonly GO_EXTRA_BUILDFLAGS := @@ -30,8 +30,10 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS) GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS) GOLANGCI_LINT := golangci-lint GOCOVMERGE := gocovmerge -RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.21 -RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG) +RELEASE_GOVERSION ?= go1.23.1 +STRIPPED_GOVERSION := $(subst go,,$(RELEASE_GOVERSION)) +RELEASE_DOCKER_BASEIMAGE ?= golang:$(STRIPPED_GOVERSION) +RELEASE_DOCKER_CACHEMOUNT := ifneq ($(GOARM),) ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM) @@ -39,33 +41,46 @@ else ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH) endif -.PHONY: printvars -printvars: - @echo GOOS=$(GOOS) - @echo GOARCH=$(GOARCH) - @echo GOARM=$(GOARM) +ifneq ($(RELEASE_DOCKER_CACHEMOUNT),) + _RELEASE_DOCKER_CACHEMOUNT := -v $(RELEASE_DOCKER_CACHEMOUNT)/mod:/go/pkg/mod -v $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache:/root/.cache/go-build +.PHONY: release-docker-mkcachemount +release-docker-mkcachemount: + mkdir -p $(RELEASE_DOCKER_CACHEMOUNT) + mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/mod + mkdir -p $(RELEASE_DOCKER_CACHEMOUNT)/xdg-cache +else + _RELEASE_DOCKER_CACHEMOUNT := +.PHONY: release-docker-mkcachemount +release-docker-mkcachemount: + # nothing to do +endif ##################### PRODUCING A RELEASE ############# -.PHONY: release wrapup-and-checksum check-git-clean sign clean +.PHONY: release wrapup-and-checksum check-git-clean sign clean ensure-release-toolchain -release: clean - # no cross-platform support for target test - $(MAKE) test-go +ensure-release-toolchain: + # ensure the toolchain is actually the one we expect + test $(RELEASE_GOVERSION) = "$$($(GO_ENV_VARS) $(GO) env GOVERSION)" + +release: ensure-release-toolchain $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet" $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint" $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin" $(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin" $(MAKE) noarch -release-docker: $(ARTIFACTDIR) - sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile - docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile . - docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \ +release-docker: $(ARTIFACTDIR) release-docker-mkcachemount + sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > $(ARTIFACTDIR)/build.Dockerfile + docker build -t zrepl_release --pull -f $(ARTIFACTDIR)/build.Dockerfile . + docker run --rm -i \ + $(_RELEASE_DOCKER_CACHEMOUNT) \ + -v $(CURDIR):/src -u $$(id -u):$$(id -g) \ zrepl_release \ make release \ GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \ - ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) + ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE) \ + RELEASE_GOVERSION=$(RELEASE_GOVERSION) debs-docker: $(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb @@ -227,7 +242,7 @@ _run_make_foreach_target_tuple: $(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos GOARCH=amd64 ##################### REGULAR TARGETS ##################### -.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list +.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin lint: $(GO_ENV_VARS) $(GOLANGCI_LINT) run ./... @@ -251,9 +266,6 @@ endif zrepl-bin: $(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)" -generate-platform-test-list: - $(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen - COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE) cover-platform-bin: $(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \ @@ -311,7 +323,7 @@ cover-full: # not part of the build, must do that manually .PHONY: generate formatcheck format -generate: generate-platform-test-list +generate: protoc -I=replication/logic/pdu --go_out=replication/logic/pdu --go-grpc_out=replication/logic/pdu replication/logic/pdu/pdu.proto protoc -I=rpc/grpcclientidentity/example --go_out=rpc/grpcclientidentity/example/pdu --go-grpc_out=rpc/grpcclientidentity/example/pdu rpc/grpcclientidentity/example/grpcauth.proto $(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./... diff --git a/build.installprotoc.bash b/build.installprotoc.bash index c39c74a..6320ee7 100644 --- a/build.installprotoc.bash +++ b/build.installprotoc.bash @@ -5,7 +5,7 @@ set -x MACH=$(uname -m) MACH="${MACH/aarch64/aarch_64}" -VERSION=3.6.1 +VERSION=28.0 FILENAME=protoc-"$VERSION"-linux-"$MACH".zip if [ -e "$FILENAME" ]; then @@ -18,8 +18,8 @@ wget https://github.com/protocolbuffers/protobuf/releases/download/v"$VERSION"/" stat "$FILENAME" sha256sum -c --ignore-missing < docs: "In the non-blocking case, the ctx does not act against the connection. It only controls the setup steps." - cc, err := grpc.DialContext(context.Background(), "doesn't matter done by dialer", dialerOption, cred, ka) + cc, err := grpc.NewClient("passthrough://doesntmatterdonebydialer", dialerOption, cred, ka) if err != nil { log.WithError(err).Error("cannot create gRPC client conn (non-blocking)") // It's ok to panic here: the we call grpc.DialContext without the diff --git a/rpc/versionhandshake/versionhandshake.go b/rpc/versionhandshake/versionhandshake.go index 0fb2160..2919cea 100644 --- a/rpc/versionhandshake/versionhandshake.go +++ b/rpc/versionhandshake/versionhandshake.go @@ -91,8 +91,8 @@ func (e HandshakeError) Timeout() bool { return false } -func hsErr(format string, args ...interface{}) *HandshakeError { - return &HandshakeError{msg: fmt.Sprintf(format, args...)} +func hsErr(msg string) *HandshakeError { + return &HandshakeError{msg: msg} } func hsIOErr(err error, format string, args ...interface{}) *HandshakeError { @@ -115,10 +115,10 @@ func (m *HandshakeMessage) Encode() ([]byte, error) { var extensions strings.Builder for i, ext := range m.Extensions { if strings.ContainsAny(ext, "\n") { - return nil, hsErr("Extension #%d contains forbidden newline character", i) + return nil, hsErr(fmt.Sprintf("Extension #%d contains forbidden newline character", i)) } if !utf8.ValidString(ext) { - return nil, hsErr("Extension #%d is not valid UTF-8", i) + return nil, hsErr(fmt.Sprintf("Extension #%d is not valid UTF-8", i)) } extensions.WriteString(ext) extensions.WriteString("\n") @@ -143,8 +143,8 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { return hsErr("could not parse handshake message length") } if followLen > maxLen { - return hsErr("handshake message length exceeds max length (%d vs %d)", - followLen, maxLen) + return hsErr(fmt.Sprintf("handshake message length exceeds max length (%d vs %d)", + followLen, maxLen)) } var buf bytes.Buffer @@ -159,15 +159,15 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { n, err = fmt.Fscanf(&buf, "ZREPL_ZFS_REPLICATION PROTOVERSION=%04d EXTENSIONS=%4d\n", &protoVersion, &extensionCount) if n != 2 || err != nil { - return hsErr("could not parse handshake message: %s", err) + return hsErr(fmt.Sprintf("could not parse handshake message: %s", err)) } if protoVersion < 1 { - return hsErr("invalid protocol version %q", protoVersion) + return hsErr(fmt.Sprintf("invalid protocol version %q", protoVersion)) } m.ProtocolVersion = protoVersion if extensionCount < 0 { - return hsErr("invalid extension count %q", extensionCount) + return hsErr(fmt.Sprintf("invalid extension count %q", extensionCount)) } if extensionCount == 0 { if buf.Len() != 0 { @@ -178,7 +178,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error { } s := buf.String() if strings.Count(s, "\n") != extensionCount { - return hsErr("inconsistent extension count: found %d, header says %d", len(m.Extensions), extensionCount) + return hsErr(fmt.Sprintf("inconsistent extension count: found %d, header says %d", len(m.Extensions), extensionCount)) } exts := strings.Split(s, "\n") if exts[len(exts)-1] != "" { @@ -203,12 +203,12 @@ func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) (rErr *H } hsb, err := ours.Encode() if err != nil { - return hsErr("could not encode protocol banner: %s", err) + return hsErr(fmt.Sprintf("could not encode protocol banner: %s", err)) } err = conn.SetDeadline(deadline) if err != nil { - return hsErr("could not set deadline for protocol banner handshake: %s", err) + return hsErr(fmt.Sprintf("could not set deadline for protocol banner handshake: %s", err)) } defer func() { if rErr != nil { @@ -216,22 +216,22 @@ func DoHandshakeVersion(conn net.Conn, deadline time.Time, version int) (rErr *H } err := conn.SetDeadline(time.Time{}) if err != nil { - rErr = hsErr("could not reset deadline after protocol banner handshake: %s", err) + rErr = hsErr(fmt.Sprintf("could not reset deadline after protocol banner handshake: %s", err)) } }() _, err = io.Copy(conn, bytes.NewBuffer(hsb)) if err != nil { - return hsErr("could not send protocol banner: %s", err) + return hsErr(fmt.Sprintf("could not send protocol banner: %s", err)) } theirs := HandshakeMessage{} if err := theirs.DecodeReader(conn, HandshakeMessageMaxLen); err != nil { - return hsErr("could not decode protocol banner: %s", err) + return hsErr(fmt.Sprintf("could not decode protocol banner: %s", err)) } if theirs.ProtocolVersion != ours.ProtocolVersion { - return hsErr("protocol versions do not match: ours is %d, theirs is %d", - ours.ProtocolVersion, theirs.ProtocolVersion) + return hsErr(fmt.Sprintf("protocol versions do not match: ours is %d, theirs is %d", + ours.ProtocolVersion, theirs.ProtocolVersion)) } // ignore extensions, we don't use them diff --git a/tlsconf/tlsconf.go b/tlsconf/tlsconf.go index 71e81c0..85b8dc9 100644 --- a/tlsconf/tlsconf.go +++ b/tlsconf/tlsconf.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "os" "time" @@ -14,7 +13,7 @@ import ( func ParseCAFile(certfile string) (*x509.CertPool, error) { pool := x509.NewCertPool() - pem, err := ioutil.ReadFile(certfile) + pem, err := os.ReadFile(certfile) if err != nil { return nil, err } diff --git a/zfs/filesystemplaceholdercreateencryptionvalue_enumer.go b/zfs/filesystemplaceholdercreateencryptionvalue_enumer.go index 0771d2f..9d0e5b4 100644 --- a/zfs/filesystemplaceholdercreateencryptionvalue_enumer.go +++ b/zfs/filesystemplaceholdercreateencryptionvalue_enumer.go @@ -1,6 +1,5 @@ // Code generated by "enumer -type=FilesystemPlaceholderCreateEncryptionValue -trimprefix=FilesystemPlaceholderCreateEncryption"; DO NOT EDIT. -// package zfs import ( diff --git a/zfs/propertysource_enumer.go b/zfs/propertysource_enumer.go index 4f93d76..22816a8 100644 --- a/zfs/propertysource_enumer.go +++ b/zfs/propertysource_enumer.go @@ -1,6 +1,5 @@ // Code generated by "enumer -type=PropertySource -trimprefix=Source"; DO NOT EDIT. -// package zfs import ( diff --git a/zfs/zfs_pipe_linux.go b/zfs/zfs_pipe_linux.go index 104997c..e52d5e1 100644 --- a/zfs/zfs_pipe_linux.go +++ b/zfs/zfs_pipe_linux.go @@ -3,7 +3,6 @@ package zfs import ( "errors" "fmt" - "io/ioutil" "os" "strconv" "strings" @@ -20,7 +19,7 @@ func getPipeCapacityHint(envvar string) int { // https://github.com/zrepl/zrepl/issues/424#issuecomment-800370928 // https://bugzilla.kernel.org/show_bug.cgi?id=212295 if _, err := os.Stat("/proc/sys/fs/pipe-max-size"); err == nil { - if dat, err := ioutil.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil { + if dat, err := os.ReadFile("/proc/sys/fs/pipe-max-size"); err == nil { if capacity, err = strconv.ParseInt(strings.TrimSpace(string(dat)), 10, 64); err != nil { capacity = 1 << 25 }