build: tooling to use CircleCI artifacts for releasing

Also, include RPMs and DEBs in the sha256sum.txt

.circleci/config.yml

@@ -283,6 +283,7 @@ jobs:
     steps:
       - attach_workspace:
           at: .
+      - run: make wrapup-and-checksum
       - store_artifacts:
           path: artifacts
 

.circleci/download_artifacts.py

@@ -0,0 +1,84 @@
+import argparse
+from pathlib import Path
+import re
+
+import requests
+
+import time
+
+import os
+import argparse
+from pathlib import Path
+
+
+circle_token = os.environ.get('CIRCLE_TOKEN')
+if not circle_token:
+    raise ValueError('CIRCLE_TOKEN environment variable not set')
+
+parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
+parser.add_argument('build_num', type=str, help='Build number')
+parser.add_argument('dst', type=Path, help='Destination directory')
+parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
+parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
+
+args = parser.parse_args()
+
+res = requests.get(
+    f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
+    headers={
+        "Circle-Token": circle_token,
+    },
+)
+res.raise_for_status()
+
+# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
+# [ {
+#   "path" : "raw-test-output/go-test-report.xml",
+#   "pretty_path" : "raw-test-output/go-test-report.xml",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
+# }, {
+#   "path" : "raw-test-output/go-test.out",
+#   "pretty_path" : "raw-test-output/go-test.out",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
+# } ]
+res = res.json()
+
+for artifact in res:
+    if not artifact["pretty_path"].startswith(args.prefix):
+        continue
+    if not re.match(args.match, artifact["pretty_path"]):
+        continue
+    stripped = artifact["pretty_path"][len(args.prefix):]
+    print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
+    artifact_rel = Path(stripped)
+    artifact_dst = args.dst / artifact_rel
+    artifact_dst.parent.mkdir(parents=True, exist_ok=True)
+
+    res = requests.get(
+        artifact["url"],
+        headers={
+            "Circle-Token": circle_token,
+        },
+        stream=True,
+    )
+    res.raise_for_status()
+
+    total_size = int(res.headers.get("Content-Length", 0))
+    block_size = 128 * 1024
+    with open(artifact_dst, "wb") as f:
+        progress = 0
+        start_time = time.time()
+        for chunk in res.iter_content(chunk_size=block_size):
+            f.write(chunk)
+            progress += len(chunk)
+            percent = progress / total_size * 100
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= 5:
+                print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
+                start_time = time.time()
+        print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
+    print("Download complete!")
+
+print("All files downloaded")

Makefile

@@ -52,12 +52,6 @@ release: clean
 	$(MAKE) test-go
 	$(MAKE) bins-all
 	$(MAKE) noarch
-	$(MAKE) wrapup-and-checksum
-	$(MAKE) check-git-clean
-ifeq (SIGN, 1)
-	$(MAKE) sign
-endif
-	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
 release-docker: $(ARTIFACTDIR)
 	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
@@ -144,7 +138,7 @@ deb-docker:
 		zrepl_debian_pkg \
 		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
 
-# expects `release` target to have run before
+# expects `release`, `deb` & `rpm` targets to have run before
 NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
 wrapup-and-checksum:
 	rm -f $(NOARCH_TARBALL)
@@ -181,16 +175,21 @@ check-git-clean:
 
 tag-release:
 	test -n "$(ZREPL_TAG_VERSION)" || exit 1
-	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+	git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
 
 sign:
-	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+	gpg -u '328A6627FA98061D!' \
 		--armor \
 		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
 
+download-circleci-release:
+	rm -rf "$(ARTIFACTDIR)"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release"
+
 ##################### BINARIES #####################
 .PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list