Merge branch 'feature/KASM-6955_mirror_to_quay_github' into 'develop'

Resolve KASM-6955 "Feature/ mirror to quay github"

Closes KASM-6955

See merge request kasm-technologies/internal/workspaces-images!246

.gitlab-ci.yml

@@ -12,6 +12,7 @@ variables:
   USE_PRIVATE_IMAGES: 0
   KASM_RELEASE: "1.16.0"
   TEST_INSTALLER: "https://kasm-static-content.s3.amazonaws.com/kasm_release_1.16.0.a1d5b7.tar.gz"
+  MIRROR_ORG_NAME: "kasmtech"
 before_script:
   - export SANITIZED_BRANCH="$(echo $CI_COMMIT_REF_NAME | sed -r 's#^release/##' | sed 's/\//_/g')"
 
@@ -33,27 +34,12 @@ pipeline:
   stage: run
   except:
     variables:
-      - $README_USERNAME_RUN
-      - $README_PASSWORD_RUN
       - $DOCKERHUB_REVERT_RUN
       - $REVERT_IS_ROLLING_RUN
   trigger:
     include:
       - artifact: gitlab-ci.yml
         job: template
-pipeline_readme:
-  stage: run
-  only:
-    variables:
-      - $README_USERNAME_RUN
-      - $README_PASSWORD_RUN
-  variables:
-      README_USERNAME: $README_USERNAME_RUN
-      README_PASSWORD: $README_PASSWORD_RUN
-  trigger:
-    include:
-      - artifact: gitlab-ci.yml
-        job: template
 pipeline_revert:
   stage: run
   only:

ci-scripts/app-layer.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
+set -e
 
 # Ingest cli variables
 ## Parse input ##

ci-scripts/gitlab-ci.template

@@ -17,8 +17,11 @@ variables:
   DOCKER_HOST: tcp://docker:2375
   DOCKER_TLS_CERTDIR: ""
   TEST_INSTALLER: "{{ TEST_INSTALLER }}"
+  MIRROR_ORG_NAME: "{{ MIRROR_ORG_NAME }}"
 before_script:
   - docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+  - if [ -f "$CI_COMMIT_REF_PROTECTED" == "true" ]; then docker login --username $QUAY_USERNAME --password $QUAY_PASSWORD quay.io; fi 
+  - if [ -f "$CI_COMMIT_REF_PROTECTED" == "true" ]; then docker login --username $GHCR_USERNAME --password $GHCR_PASSWORD ghcr.io; fi 
   - export SANITIZED_BRANCH="$(echo $CI_COMMIT_REF_NAME | sed -r 's#^release/##' | sed 's/\//_/g')"
   - export BASE_TAG="{{ BASE_TAG }}"
 
@@ -38,8 +41,9 @@ build_{{ IMAGE.name }}:
       {% endfor %}{% endif %}
   except:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   tags:
@@ -63,8 +67,9 @@ build_{{ IMAGE.name }}:
       {% endfor %}{% endif %}
   except:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   tags:
@@ -89,8 +94,9 @@ test_{{ IMAGE.name }}:
       {% endfor %}{% endif %}
   except:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -118,8 +124,9 @@ test_{{ IMAGE.name }}:
       {% endfor %}{% endif %}
   except:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -151,8 +158,9 @@ manifest_{{ IMAGE.name }}:
       {% endfor %}{% endif %}
   except:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -180,8 +188,9 @@ manifest_{{ IMAGE.name }}:
       {% endfor %}{% endif %}
   except:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+      - $QUAY_API_KEY_RUN
       - $DOCKERHUB_REVERT
       - $REVERT_IS_ROLLING
   needs:
@@ -204,8 +213,11 @@ update_readmes_{{ IMAGE.name }}:
     - bash ci-scripts/readme.sh "{{ IMAGE.name }}"
   only:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+  variables:
+    README_USERNAME: $README_USERNAME_RUN
+    README_PASSWORD: $README_PASSWORD_RUN
   tags:
     - oci-fixed-amd
 {% endfor %}
@@ -218,8 +230,42 @@ update_readmes_{{ IMAGE.name }}:
     - bash ci-scripts/readme.sh "{{ IMAGE.name }}"
   only:
     variables:
-      - $README_USERNAME
+      - $README_USERNAME_RUN
-      - $README_PASSWORD
+      - $README_PASSWORD_RUN
+  variables:
+    README_USERNAME: $README_USERNAME_RUN
+    README_PASSWORD: $README_PASSWORD_RUN
+  tags:
+    - oci-fixed-amd
+{% endfor %}
+
+## Update Quay Readmes ##
+{% for IMAGE in multiImages %}
+update_quay_readmes_{{ IMAGE.name }}:
+  stage: readme
+  script:
+    - apk add bash
+    - bash ci-scripts/quay_readme.sh "{{ IMAGE.name }}"
+  only:
+    variables:
+      - $QUAY_API_KEY_RUN
+  variables:
+    QUAY_API_KEY: $QUAY_API_KEY_RUN
+  tags:
+    - oci-fixed-amd
+{% endfor %}
+
+{% for IMAGE in singleImages %}
+update_quay_readmes_{{ IMAGE.name }}:
+  stage: readme
+  script:
+    - apk add bash
+    - bash ci-scripts/quay_readme.sh "{{ IMAGE.name }}"
+  only:
+    variables:
+      - $QUAY_API_KEY_RUN
+  variables:
+    QUAY_API_KEY: $QUAY_API_KEY_RUN
   tags:
     - oci-fixed-amd
 {% endfor %}

ci-scripts/manifest.sh

@@ -1,7 +1,9 @@
 #! /bin/bash
+set -e
 
 # Globals
 FAILED="false"
+REGISTRY_MIRRORS=("quay.io" "ghcr.io")
 
 # Ingest cli variables
 ## Parse input ##
@@ -13,8 +15,10 @@ PULL_BRANCH=${SANITIZED_BRANCH}
 
 # Determine if this is a private or public build
 if [[ "${CI_COMMIT_REF_NAME}" == release/* ]] || [[ "${CI_COMMIT_REF_NAME}" == "develop" ]]; then
+  PUBLIC_BUILD="true"
   ENDPOINT="${NAME}"
 else
+  PUBLIC_BUILD="false"
   ENDPOINT="${NAME}-private"
 fi
 
@@ -95,6 +99,26 @@ if [[ "${TYPE}" == "multi" ]]; then
   docker manifest annotate ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8
   docker manifest push --purge ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
 
+  if [[ "${PUBLIC_BUILD}" == "true" ]]; then
+    for MIRROR in "${REGISTRY_MIRRORS[@]}"; do
+      docker tag \
+        ${ORG_NAME}/image-cache-private:x86_64-${NAME}-${PULL_BRANCH}-${CI_PIPELINE_ID} \
+        ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH}
+      docker tag \
+        ${ORG_NAME}/image-cache-private:aarch64-${NAME}-${PULL_BRANCH}-${CI_PIPELINE_ID} \
+        ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
+
+      # Push arches to live repo
+      docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH}
+      docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
+
+      # Manifest to meta tag
+      docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} || :
+      docker manifest create ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:x86_64-${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH}
+      docker manifest annotate ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH} ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:aarch64-${SANITIZED_BRANCH} --os linux --arch arm64 --variant v8
+      docker manifest push --purge ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+    done
+  fi
 # Single arch image just pull and push
 else
 
@@ -109,4 +133,13 @@ else
   # Push image
   docker push ${ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
 
+  if [[ "${PUBLIC_BUILD}" == "true" ]]; then
+    for MIRROR in "${REGISTRY_MIRRORS[@]}"; do
+      docker tag \
+        ${ORG_NAME}/image-cache-private:x86_64-${NAME}-${PULL_BRANCH}-${CI_PIPELINE_ID} \
+        ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+
+      docker push ${MIRROR}/${MIRROR_ORG_NAME}/${ENDPOINT}:${SANITIZED_BRANCH}
+    done
+  fi
 fi

ci-scripts/quay_readme.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+## Parse input ##
+NAME=$1
+
+## Run readme updater ##
+docker run -v $PWD/docs:/docs \
+  -e RELEASE="$KASM_RELEASE" \
+  -e QUAY_API_KEY="$QUAY_API_KEY" \
+  -e QUAY_REPOSITORY="${MIRROR_ORG_NAME}/${NAME}" \
+  kasmweb/dockerhub-updater:develop