From c8660b2fe3191c79202a80b4272d8749581e7c60 Mon Sep 17 00:00:00 2001 From: Ian Tangney Date: Tue, 1 Apr 2025 16:50:01 +0000 Subject: [PATCH] KASM-7103 Mirror workspaces core to quay and github --- .gitlab-ci.yml | 13 +++++++++++++ ci-scripts/gitlab-ci.template | 35 +++++++++++++++++++++++++++++++++++ ci-scripts/manifest.sh | 27 +++++++++++++++++++++++++++ ci-scripts/quay_readme.sh | 15 +++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 ci-scripts/quay_readme.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eedbda6..fdd8d5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ variables: KASM_RELEASE: "1.16.0" TEST_INSTALLER: "https://kasm-static-content.s3.amazonaws.com/kasm_release_1.16.0.a1d5b7.tar.gz" SCAN_CONTAINERS: "true" + MIRROR_ORG_NAME: "kasmtech" before_script: - export SANITIZED_BRANCH="$(echo ${CI_COMMIT_REF_NAME:0:64} | sed -r 's#^release/##' | sed 's/\//_/g')" @@ -41,6 +42,7 @@ pipeline: variables: - $README_USERNAME_RUN - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT_RUN - $REVERT_IS_ROLLING_RUN trigger: @@ -60,6 +62,17 @@ pipeline_readme: include: - artifact: gitlab-ci.yml job: template +pipeline_readme_quay: + stage: run + only: + variables: + - $QUAY_API_KEY_RUN + variables: + QUAY_API_KEY: $QUAY_API_KEY_RUN + trigger: + include: + - artifact: gitlab-ci.yml + job: template pipeline_revert: stage: run only: diff --git a/ci-scripts/gitlab-ci.template b/ci-scripts/gitlab-ci.template index e391a54..1027934 100644 --- a/ci-scripts/gitlab-ci.template +++ b/ci-scripts/gitlab-ci.template @@ -16,6 +16,7 @@ variables: TEST_INSTALLER: "{{ TEST_INSTALLER }}" DOCKER_HOST: tcp://docker:2375 DOCKER_TLS_CERTDIR: "" + MIRROR_ORG_NAME: "{{ MIRROR_ORG_NAME }}" before_script: - docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD - export SANITIZED_BRANCH="$(echo ${CI_COMMIT_REF_NAME:0:64} | sed -r 's#^release/##' | sed 's/\//_/g')" @@ -38,6 +39,7 @@ build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING tags: @@ -63,6 +65,7 @@ build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING tags: @@ -89,6 +92,7 @@ test_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -118,6 +122,7 @@ test_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -147,6 +152,7 @@ scan_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY needs: - build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }} tags: @@ -177,6 +183,7 @@ scan_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY needs: - build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }} artifacts: @@ -210,6 +217,7 @@ manifest_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -238,6 +246,7 @@ manifest_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: variables: - $README_USERNAME - $README_PASSWORD + - $QUAY_API_KEY - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -280,6 +289,32 @@ update_readmes_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: - oci-fixed-amd {% endfor %} +{% for IMAGE in multiImages %} +update_quay_readmes_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: + stage: readme + script: + - apk add bash + - bash ci-scripts/quay_readme.sh "{{ IMAGE.name1 }}" "{{ IMAGE.name2 }}" + only: + variables: + - $QUAY_API_KEY + tags: + - oci-fixed-amd +{% endfor %} + +{% for IMAGE in singleImages %} +update_quay_readmes_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: + stage: readme + script: + - apk add bash + - bash ci-scripts/quay_readme.sh "{{ IMAGE.name1 }}" "{{ IMAGE.name2 }}" + only: + variables: + - $QUAY_API_KEY + tags: + - oci-fixed-amd +{% endfor %} + ## Revert Images to specific build id ## {% for IMAGE in multiImages %} dockerhub_revert_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}: diff --git a/ci-scripts/manifest.sh b/ci-scripts/manifest.sh index a742070..bc5f381 100755 --- a/ci-scripts/manifest.sh +++ b/ci-scripts/manifest.sh @@ -1,8 +1,10 @@ #! /bin/bash +set -e # Globals FAILED="false" PUBLIC_BUILD="false" +REGISTRY_MIRRORS=("quay.io" "ghcr.io") # Ingest cli variables ## Parse input ## @@ -103,6 +105,24 @@ if [[ "${TYPE}" == "multi" ]]; then docker manifest create ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} docker manifest annotate ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8 docker manifest push --purge ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + for MIRROR in "${REGISTRY_MIRRORS[@]}"; do + docker tag \ + ${ORG_NAME}/image-cache-private:x86_64-core-${NAME1}-${NAME2}-${PULL_BRANCH}-${CI_PIPELINE_ID} \ + ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} + docker tag \ + ${ORG_NAME}/image-cache-private:aarch64-core-${NAME1}-${NAME2}-${PULL_BRANCH}-${CI_PIPELINE_ID} \ + ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} + + # Push arches to live repo + docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} + docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} + + # Manifest to meta tag + docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} || : + docker manifest create ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} + docker manifest annotate ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8 + docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + done fi # Tag images to private repo @@ -138,6 +158,13 @@ else # Push image to live repo docker push ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + for MIRROR in "${REGISTRY_MIRRORS[@]}"; do + docker tag \ + ${ORG_NAME}/image-cache-private:x86_64-core-${NAME1}-${NAME2}-${PULL_BRANCH}-${CI_PIPELINE_ID} \ + ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + + docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + done fi # Tage image to private repo diff --git a/ci-scripts/quay_readme.sh b/ci-scripts/quay_readme.sh new file mode 100644 index 0000000..433651e --- /dev/null +++ b/ci-scripts/quay_readme.sh @@ -0,0 +1,15 @@ +#! /bin/bash + +## Parse input ## +FULLNAME="core-$1-$2" + +if [[ "$1" == "$2" ]] ; then + FULLNAME="core-$1" +fi + +## Run readme updater ## +docker run -v $PWD/docs:/docs \ + -e RELEASE="$KASM_RELEASE" \ + -e QUAY_API_KEY="$QUAY_API_KEY" \ + -e QUAY_REPOSITORY="${MIRROR_ORG_NAME}/${FULLNAME}" \ + kasmweb/dockerhub-updater:develop