From 6b57b84cb658df50a9e0666a06d0c4f84447fb53 Mon Sep 17 00:00:00 2001 From: Ian Tangney Date: Thu, 27 Mar 2025 13:43:27 -0400 Subject: [PATCH] Resolve KASM-6955 "Feature/ mirror to quay github" --- .gitlab-ci.yml | 16 +------ ci-scripts/app-layer.sh | 1 + ci-scripts/gitlab-ci.template | 78 ++++++++++++++++++++++++++++------- ci-scripts/manifest.sh | 33 +++++++++++++++ ci-scripts/quay_readme.sh | 11 +++++ 5 files changed, 108 insertions(+), 31 deletions(-) create mode 100644 ci-scripts/quay_readme.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0e02ad..02cbcd6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ variables: USE_PRIVATE_IMAGES: 0 KASM_RELEASE: "1.16.0" TEST_INSTALLER: "https://kasm-static-content.s3.amazonaws.com/kasm_release_1.16.0.a1d5b7.tar.gz" + MIRROR_ORG_NAME: "kasmtech" before_script: - export SANITIZED_BRANCH="$(echo $CI_COMMIT_REF_NAME | sed -r 's#^release/##' | sed 's/\//_/g')" @@ -33,27 +34,12 @@ pipeline: stage: run except: variables: - - $README_USERNAME_RUN - - $README_PASSWORD_RUN - $DOCKERHUB_REVERT_RUN - $REVERT_IS_ROLLING_RUN trigger: include: - artifact: gitlab-ci.yml job: template -pipeline_readme: - stage: run - only: - variables: - - $README_USERNAME_RUN - - $README_PASSWORD_RUN - variables: - README_USERNAME: $README_USERNAME_RUN - README_PASSWORD: $README_PASSWORD_RUN - trigger: - include: - - artifact: gitlab-ci.yml - job: template pipeline_revert: stage: run only: diff --git a/ci-scripts/app-layer.sh b/ci-scripts/app-layer.sh index c4ecd17..c4f58f9 100644 --- a/ci-scripts/app-layer.sh +++ b/ci-scripts/app-layer.sh @@ -1,4 +1,5 @@ #! /bin/bash +set -e # Ingest cli variables ## Parse input ## diff --git a/ci-scripts/gitlab-ci.template b/ci-scripts/gitlab-ci.template index be0506e..23e98d9 100644 --- a/ci-scripts/gitlab-ci.template +++ b/ci-scripts/gitlab-ci.template @@ -17,8 +17,11 @@ variables: DOCKER_HOST: tcp://docker:2375 DOCKER_TLS_CERTDIR: "" TEST_INSTALLER: "{{ TEST_INSTALLER }}" + MIRROR_ORG_NAME: "{{ MIRROR_ORG_NAME }}" before_script: - docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD + - if [ -f "$CI_COMMIT_REF_PROTECTED" == "true" ]; then docker login --username $QUAY_USERNAME --password $QUAY_PASSWORD quay.io; fi + - if [ -f "$CI_COMMIT_REF_PROTECTED" == "true" ]; then docker login --username $GHCR_USERNAME --password $GHCR_PASSWORD ghcr.io; fi - export SANITIZED_BRANCH="$(echo $CI_COMMIT_REF_NAME | sed -r 's#^release/##' | sed 's/\//_/g')" - export BASE_TAG="{{ BASE_TAG }}" @@ -38,8 +41,9 @@ build_{{ IMAGE.name }}: {% endfor %}{% endif %} except: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING tags: @@ -63,8 +67,9 @@ build_{{ IMAGE.name }}: {% endfor %}{% endif %} except: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING tags: @@ -89,8 +94,9 @@ test_{{ IMAGE.name }}: {% endfor %}{% endif %} except: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -118,8 +124,9 @@ test_{{ IMAGE.name }}: {% endfor %}{% endif %} except: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -151,8 +158,9 @@ manifest_{{ IMAGE.name }}: {% endfor %}{% endif %} except: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -180,8 +188,9 @@ manifest_{{ IMAGE.name }}: {% endfor %}{% endif %} except: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + - $QUAY_API_KEY_RUN - $DOCKERHUB_REVERT - $REVERT_IS_ROLLING needs: @@ -204,8 +213,11 @@ update_readmes_{{ IMAGE.name }}: - bash ci-scripts/readme.sh "{{ IMAGE.name }}" only: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + variables: + README_USERNAME: $README_USERNAME_RUN + README_PASSWORD: $README_PASSWORD_RUN tags: - oci-fixed-amd {% endfor %} @@ -218,8 +230,42 @@ update_readmes_{{ IMAGE.name }}: - bash ci-scripts/readme.sh "{{ IMAGE.name }}" only: variables: - - $README_USERNAME - - $README_PASSWORD + - $README_USERNAME_RUN + - $README_PASSWORD_RUN + variables: + README_USERNAME: $README_USERNAME_RUN + README_PASSWORD: $README_PASSWORD_RUN + tags: + - oci-fixed-amd +{% endfor %} + +## Update Quay Readmes ## +{% for IMAGE in multiImages %} +update_quay_readmes_{{ IMAGE.name }}: + stage: readme + script: + - apk add bash + - bash ci-scripts/quay_readme.sh "{{ IMAGE.name }}" + only: + variables: + - $QUAY_API_KEY_RUN + variables: + QUAY_API_KEY: $QUAY_API_KEY_RUN + tags: + - oci-fixed-amd +{% endfor %} + +{% for IMAGE in singleImages %} +update_quay_readmes_{{ IMAGE.name }}: + stage: readme + script: + - apk add bash + - bash ci-scripts/quay_readme.sh "{{ IMAGE.name }}" + only: + variables: + - $QUAY_API_KEY_RUN + variables: + QUAY_API_KEY: $QUAY_API_KEY_RUN tags: - oci-fixed-amd {% endfor %} diff --git a/ci-scripts/manifest.sh b/ci-scripts/manifest.sh index b690cf6..ab7e1d1 100755 --- a/ci-scripts/manifest.sh +++ b/ci-scripts/manifest.sh @@ -1,7 +1,9 @@ #! /bin/bash +set -e # Globals FAILED="false" +REGISTRY_MIRRORS=("quay.io" "ghcr.io") # Ingest cli variables ## Parse input ## @@ -13,8 +15,10 @@ PULL_BRANCH=${SANITIZED_BRANCH} # Determine if this is a private or public build if [[ "${CI_COMMIT_REF_NAME}" == release/* ]] || [[ "${CI_COMMIT_REF_NAME}" == "develop" ]]; then + PUBLIC_BUILD="true" ENDPOINT="${NAME}" else + PUBLIC_BUILD="false" ENDPOINT="${NAME}-private" fi @@ -95,6 +99,26 @@ if [[ "${TYPE}" == "multi" ]]; then docker manifest annotate ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8 docker manifest push --purge ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + if [[ "${PUBLIC_BUILD}" == "true" ]]; then + for MIRROR in "${REGISTRY_MIRRORS[@]}"; do + docker tag \ + ${ORG_NAME}/image-cache-private:x86_64-${NAME}-${PULL_BRANCH}-${CI_PIPELINE_ID} \ + ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} + docker tag \ + ${ORG_NAME}/image-cache-private:aarch64-${NAME}-${PULL_BRANCH}-${CI_PIPELINE_ID} \ + ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} + + # Push arches to live repo + docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} + docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} + + # Manifest to meta tag + docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} || : + docker manifest create ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} + docker manifest annotate ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8 + docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + done + fi # Single arch image just pull and push else @@ -109,4 +133,13 @@ else # Push image docker push ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + if [[ "${PUBLIC_BUILD}" == "true" ]]; then + for MIRROR in "${REGISTRY_MIRRORS[@]}"; do + docker tag \ + ${ORG_NAME}/image-cache-private:x86_64-${NAME}-${PULL_BRANCH}-${CI_PIPELINE_ID} \ + ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + + docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} + done + fi fi diff --git a/ci-scripts/quay_readme.sh b/ci-scripts/quay_readme.sh new file mode 100644 index 0000000..5f32ca3 --- /dev/null +++ b/ci-scripts/quay_readme.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +## Parse input ## +NAME=$1 + +## Run readme updater ## +docker run -v $PWD/docs:/docs \ + -e RELEASE="$KASM_RELEASE" \ + -e QUAY_API_KEY="$QUAY_API_KEY" \ + -e QUAY_REPOSITORY="${MIRROR_ORG_NAME}/${NAME}" \ + kasmweb/dockerhub-updater:develop