Merge branch 'feature/KASM-7103_mirror_to_quay_github' into 'develop'

KASM-7103 Mirror workspaces core to quay and github

Closes KASM-7103

See merge request kasm-technologies/internal/workspaces-core-images!278

.gitlab-ci.yml

@@ -18,6 +18,7 @@ variables:
   KASM_RELEASE: "1.16.0"
   TEST_INSTALLER: "https://kasm-static-content.s3.amazonaws.com/kasm_release_1.16.0.a1d5b7.tar.gz"
   SCAN_CONTAINERS: "true"
+  MIRROR_ORG_NAME: "kasmtech"
 before_script:
   - export SANITIZED_BRANCH="$(echo ${CI_COMMIT_REF_NAME:0:64} | sed -r 's#^release/##' | sed 's/\//_/g')"
 
@@ -41,6 +42,7 @@ pipeline:
     variables:
       - $README_USERNAME_RUN
       - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT_RUN
       - $REVERT_IS_ROLLING_RUN
   trigger:
@@ -60,6 +62,17 @@ pipeline_readme:
     include:
       - artifact: gitlab-ci.yml
         job: template
+pipeline_readme_quay:
+  stage: run
+  only:
+    variables:
+      - $QUAY_API_KEY_RUN
+  variables:
+      QUAY_API_KEY: $QUAY_API_KEY_RUN
+  trigger:
+    include:
+      - artifact: gitlab-ci.yml
+        job: template
 pipeline_revert:
   stage: run
   only:

ci-scripts/gitlab-ci.template

@@ -16,6 +16,7 @@ variables:
   TEST_INSTALLER: "{{ TEST_INSTALLER }}"
   DOCKER_HOST: tcp://docker:2375
   DOCKER_TLS_CERTDIR: ""
+  MIRROR_ORG_NAME: "{{ MIRROR_ORG_NAME }}"
 before_script:
   - docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
   - export SANITIZED_BRANCH="$(echo ${CI_COMMIT_REF_NAME:0:64} | sed -r 's#^release/##' | sed 's/\//_/g')"
@@ -38,6 +39,7 @@ build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   tags:
@@ -63,6 +65,7 @@ build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   tags:
@@ -89,6 +92,7 @@ test_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -118,6 +122,7 @@ test_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -147,6 +152,7 @@ scan_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
   needs:
     - build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}
   tags:
@@ -177,6 +183,7 @@ scan_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
   needs:
     - build_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}
   artifacts:
@@ -210,6 +217,7 @@ manifest_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -238,6 +246,7 @@ manifest_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     variables:
       - $README_USERNAME
       - $README_PASSWORD
+      - $QUAY_API_KEY
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -280,6 +289,32 @@ update_readmes_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
     - oci-fixed-amd
 {% endfor %}
 
+{% for IMAGE in multiImages %}
+update_quay_readmes_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
+  stage: readme
+  script:
+    - apk add bash
+    - bash ci-scripts/quay_readme.sh "{{ IMAGE.name1 }}" "{{ IMAGE.name2 }}"
+  only:
+    variables:
+      - $QUAY_API_KEY
+  tags:
+    - oci-fixed-amd
+{% endfor %}
+
+{% for IMAGE in singleImages %}
+update_quay_readmes_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:
+  stage: readme
+  script:
+    - apk add bash
+    - bash ci-scripts/quay_readme.sh "{{ IMAGE.name1 }}" "{{ IMAGE.name2 }}"
+  only:
+    variables:
+      - $QUAY_API_KEY
+  tags:
+    - oci-fixed-amd
+{% endfor %}
+
 ## Revert Images to specific build id ##
 {% for IMAGE in multiImages %}
 dockerhub_revert_{{ IMAGE.name1 }}_{{ IMAGE.name2 }}:

ci-scripts/manifest.sh

@@ -1,8 +1,10 @@
 #! /bin/bash
+set -e 
 
 # Globals
 FAILED="false"
 PUBLIC_BUILD="false"
+REGISTRY_MIRRORS=("quay.io" "ghcr.io")
 
 # Ingest cli variables
 ## Parse input ##
@@ -103,6 +105,24 @@ if [[ "${TYPE}" == "multi" ]]; then
     docker manifest create ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
     docker manifest annotate ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8
     docker manifest push --purge ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+    for MIRROR in "${REGISTRY_MIRRORS[@]}"; do
+      docker tag \
+        ${ORG_NAME}/image-cache-private:x86_64-core-${NAME1}-${NAME2}-${PULL_BRANCH}-${CI_PIPELINE_ID}  \
+        ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH}
+      docker tag \
+        ${ORG_NAME}/image-cache-private:aarch64-core-${NAME1}-${NAME2}-${PULL_BRANCH}-${CI_PIPELINE_ID} \
+        ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
+
+      # Push arches to live repo
+      docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH}
+      docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
+
+      # Manifest to meta tag
+      docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} || :
+      docker manifest create ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
+      docker manifest annotate ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8
+      docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+    done
   fi
 
   # Tag images to private repo
@@ -138,6 +158,13 @@ else
 
     # Push image to live repo
     docker push ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+    for MIRROR in "${REGISTRY_MIRRORS[@]}"; do
+      docker tag \
+        ${ORG_NAME}/image-cache-private:x86_64-core-${NAME1}-${NAME2}-${PULL_BRANCH}-${CI_PIPELINE_ID} \
+        ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+
+      docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+    done
   fi
 
   # Tage image to private repo

ci-scripts/quay_readme.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+
+## Parse input ##
+FULLNAME="core-$1-$2"
+
+if [[ "$1" == "$2" ]] ; then
+  FULLNAME="core-$1"
+fi
+
+## Run readme updater ##
+docker run -v $PWD/docs:/docs \
+  -e RELEASE="$KASM_RELEASE" \
+  -e QUAY_API_KEY="$QUAY_API_KEY" \
+  -e QUAY_REPOSITORY="${MIRROR_ORG_NAME}/${FULLNAME}" \
+  kasmweb/dockerhub-updater:develop