diff --git a/.circleci/config.yml b/.circleci/config.yml index c2f6e33..20d7baf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -283,6 +283,7 @@ jobs: steps: - attach_workspace: at: . + - run: make wrapup-and-checksum - store_artifacts: path: artifacts diff --git a/.circleci/download_artifacts.py b/.circleci/download_artifacts.py new file mode 100644 index 0000000..809d9c3 --- /dev/null +++ b/.circleci/download_artifacts.py @@ -0,0 +1,84 @@ +import argparse +from pathlib import Path +import re + +import requests + +import time + +import os +import argparse +from pathlib import Path + + +circle_token = os.environ.get('CIRCLE_TOKEN') +if not circle_token: + raise ValueError('CIRCLE_TOKEN environment variable not set') + +parser = argparse.ArgumentParser(description='Download artifacts from CircleCI') +parser.add_argument('build_num', type=str, help='Build number') +parser.add_argument('dst', type=Path, help='Destination directory') +parser.add_argument('--prefix', type=str, default='', help='Filter for prefix') +parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex') + +args = parser.parse_args() + +res = requests.get( + f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts", + headers={ + "Circle-Token": circle_token, + }, +) +res.raise_for_status() + +# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job +# [ { +# "path" : "raw-test-output/go-test-report.xml", +# "pretty_path" : "raw-test-output/go-test-report.xml", +# "node_index" : 0, +# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml" +# }, { +# "path" : "raw-test-output/go-test.out", +# "pretty_path" : "raw-test-output/go-test.out", +# "node_index" : 0, +# "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out" +# } ] +res = res.json() + +for artifact in res: + if not artifact["pretty_path"].startswith(args.prefix): + continue + if not re.match(args.match, artifact["pretty_path"]): + continue + stripped = artifact["pretty_path"][len(args.prefix):] + print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}") + artifact_rel = Path(stripped) + artifact_dst = args.dst / artifact_rel + artifact_dst.parent.mkdir(parents=True, exist_ok=True) + + res = requests.get( + artifact["url"], + headers={ + "Circle-Token": circle_token, + }, + stream=True, + ) + res.raise_for_status() + + total_size = int(res.headers.get("Content-Length", 0)) + block_size = 128 * 1024 + with open(artifact_dst, "wb") as f: + progress = 0 + start_time = time.time() + for chunk in res.iter_content(chunk_size=block_size): + f.write(chunk) + progress += len(chunk) + percent = progress / total_size * 100 + elapsed_time = time.time() - start_time + if elapsed_time >= 5: + print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r") + start_time = time.time() + print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)") + print("Download complete!") + +print("All files downloaded") diff --git a/Makefile b/Makefile index 66949f1..1df8c64 100644 --- a/Makefile +++ b/Makefile @@ -52,12 +52,6 @@ release: clean $(MAKE) test-go $(MAKE) bins-all $(MAKE) noarch - $(MAKE) wrapup-and-checksum - $(MAKE) check-git-clean -ifeq (SIGN, 1) - $(MAKE) sign -endif - @echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release" release-docker: $(ARTIFACTDIR) sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile @@ -144,7 +138,7 @@ deb-docker: zrepl_debian_pkg \ make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) -# expects `release` target to have run before +# expects `release`, `deb` & `rpm` targets to have run before NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar wrapup-and-checksum: rm -f $(NOARCH_TARBALL) @@ -181,16 +175,21 @@ check-git-clean: tag-release: test -n "$(ZREPL_TAG_VERSION)" || exit 1 - git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)" + git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)" sign: - gpg -u "89BC 5D89 C845 568B F578 B306 CDBD 8EC8 E27C A5FC" \ + gpg -u '328A6627FA98061D!' \ --armor \ --detach-sign $(ARTIFACTDIR)/release/sha512sum.txt clean: docs-clean rm -rf "$(ARTIFACTDIR)" +download-circleci-release: + rm -rf "$(ARTIFACTDIR)" + mkdir -p "$(ARTIFACTDIR)/release" + python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release" + ##################### BINARIES ##################### .PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list