Build system for hub.docker.com

2024-11-07 08:34:00 +01:00 · 2019-02-01 09:58:07 +01:00 · 2019-02-01 09:58:07 +01:00 · 4ef420d443
commit 4ef420d443
parent 5a09659278
9 changed files with 240 additions and 70 deletions
--- a/DOCKER_HUB.md
+++ b/DOCKER_HUB.md
@ -0,0 +1,66 @@
+# cloud.docker.com Configuration
+
+The automatic build is configured in cloud.docker.com.
+
+The following build configuration is expected:
+
+```yaml
+Source Repository: github.com/netbox-community/netbox-docker
+Build Location: Build on Docker Hub's infrastructure
+Autotest: Internal and External Pull Requests
+Repository Links: Enable for Base Image
+Build Rules:
+- Source Type: Branch
+  Source: master
+  Docker Tag: branches-main
+  Dockerfile location: Dockerfile
+- Source Type: Branch
+  Source: master
+  Docker Tag: branches-ldap
+  Dockerfile location: Dockerfile.ldap
+- Source Type: Branch
+  Source: master
+  Docker Tag: prerelease-main
+  Dockerfile location: Dockerfile
+- Source Type: Branch
+  Source: master
+  Docker Tag: prerelease-ldap
+  Dockerfile location: Dockerfile.ldap
+- Source Type: Branch
+  Source: master
+  Docker Tag: release-main
+  Dockerfile location: Dockerfile
+- Source Type: Branch
+  Source: master
+  Docker Tag: release-ldap
+  Dockerfile location: Dockerfile.ldap
+Build Environment Variables:
+# Create an app on Github and use it's OATH credentials here
+- Key: GITHUB_OAUTH_CLIENT_ID
+  Value: <secret>
+- Key: GITHUB_OAUTH_CLIENT_SECRET
+  Value: <secret>
+Build Triggers:
+- Name: Cron Trigger
+# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds
+```
+
+## Background Knowledge
+
+The build system of cloud.docker.com is not made for this kind of project.
+But we found a way to make it work, and this is how:
+
+1. The docker hub build system [allows to overwrite the scripts that get executed
+   for `build`, `test` and `push`](overwrite). See `hooks/*`.
+2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`.
+3. The `build` script runs `run_build()` from `hooks/common`.
+   This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`.
+4. The `test` script just invokes `docker-compose` commands.
+5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag.
+   This causes the `build.sh` script to not re-build the Docker image, but just the just built image.
+
+The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_).
+
+The _Dockerfile location_ configuration setting is completely ignored by the build scripts.
+
+[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands
--- a/build-all.sh
+++ b/build-all.sh
@ -12,9 +12,9 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")

 echo "⚙️ Configured builds: ${BUILDS[*]}"

-VARIANTS=("" "ldap")
+VARIANTS=("main" "ldap")

-if [ ! -z "${DEBUG}" ]; then
+if [ -n "${DEBUG}" ]; then
  export DEBUG
 fi

@ -22,67 +22,63 @@ ERROR=0

 # Don't build if not on `master` and don't build if on a pull request,
 # but build when DEBUG is not empty
-if [ ! -z "${DEBUG}" ] || \
-  ( [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] ); then
-  for VARIANT in "${VARIANTS[@]}"; do
-		export VARIANT
+for VARIANT in "${VARIANTS[@]}"; do
+  export VARIANT

-    # Checking which VARIANT to build
-    if [ -z "$VARIANT" ]; then
-      DOCKERFILE="Dockerfile"
-    else
-      DOCKERFILE="Dockerfile.${VARIANT}"
+  # Checking which VARIANT to build
+  if [ "${VARIANT}" == "main" ]; then
+    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}"
+  else
+    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}"

-      # Fail fast
-      if [ ! -f "${DOCKERFILE}" ]; then
-        echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
-        ERROR=1
+    # Fail fast
+    if [ ! -f "${DOCKERFILE}" ]; then
+      echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
+      ERROR=1

-        if [ -z "$DEBUG" ]; then
-          continue
-        else
-          echo "⚠️ Would skip this, but DEBUG is enabled."
-        fi
+      if [ -z "$DEBUG" ]; then
+        continue
+      else
+        echo "⚠️ Would skip this, but DEBUG is enabled."
      fi
    fi
+  fi

-    for BUILD in "${BUILDS[@]}"; do
-      echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
-      case $BUILD in
-        release)
-          # build the latest release
-          # shellcheck disable=SC2068
-          ./build-latest.sh $@ || ERROR=1
-          ;;
-        prerelease)
-          # build the latest pre-release
-          # shellcheck disable=SC2068
-      		PRERELEASE=true ./build-latest.sh $@ || ERROR=1
-          ;;
-        branches)
-          # build all branches
-          # shellcheck disable=SC2068
-      		./build-branches.sh $@ || ERROR=1
-          ;;
-        special)
-          # special build
-          # shellcheck disable=SC2068
-      		#SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
-          ;;
-        *)
-          echo "🚨 Unrecognized build '$BUILD'."
+  for BUILD in "${BUILDS[@]}"; do
+    echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
+    case $BUILD in
+      release)
+        # build the latest release
+        # shellcheck disable=SC2068
+        ./build-latest.sh $@ || ERROR=1
+        ;;
+      prerelease)
+        # build the latest pre-release
+        # shellcheck disable=SC2068
+        PRERELEASE=true ./build-latest.sh $@ || ERROR=1
+        ;;
+      branches)
+        # build all branches
+        # shellcheck disable=SC2068
+        ./build-branches.sh $@ || ERROR=1
+        ;;
+      special)
+        # special build
+        # shellcheck disable=SC2068
+        #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
+        echo "✅ No special builds today."
+        ;;
+      *)
+        echo "🚨 Unrecognized build '$BUILD'."

-          if [ -z "$DEBUG" ]; then
-            exit 1
-          else
-            echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
-          fi
-          ;;
-      esac
-    done
+        if [ -z "$DEBUG" ]; then
+          exit 1
+        else
+          echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
+        fi
+        ;;
+    esac
  done
-else
-  echo "❎ Not building anything."
-fi
+done

 exit $ERROR
--- a/build-branches.sh
+++ b/build-branches.sh
@ -3,7 +3,7 @@

 echo "▶️ $0 $*"

-if [ ! -z "${GITHUB_OAUTH_CLIENT_ID}" ] && [ ! -z "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
+if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
  echo "🗝 Performing authenticated Github API calls."
  GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
 else
--- a/build-latest.sh
+++ b/build-latest.sh
@ -3,7 +3,7 @@

 echo "▶️ $0 $*"

-if [ ! -z "${GITHUB_OAUTH_CLIENT_ID}" ] && [ ! -z "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
+if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
  echo "🗝 Performing authenticated Github API calls."
  GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
 else
--- a/build.sh
+++ b/build.sh
@ -6,9 +6,10 @@ echo "▶️ $0 $*"
 set -e

 if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
-  echo "Usage: ${0} <branch> [--push]"
-  echo "  branch  The branch or tag to build. Required."
-  echo "  --push  Pushes built Docker image to docker hub."
+  echo "Usage: ${0} <branch> [--push|--push-only]"
+  echo "  branch       The branch or tag to build. Required."
+  echo "  --push       Pushes built the Docker image to the registry."
+  echo "  --push-only  Does not build. Only pushes the Docker image to the registry."
  echo ""
  echo "You can use the following ENV variables to customize the build:"
  echo "  DEBUG    If defined, the script does not stop when certain checks are unsatisfied."
@ -49,8 +50,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
  echo "             ARG FROM_TAG=latest"
  echo "             FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG"
  echo "           Example: VARIANT=ldap will result in the tag 'latest-ldap' and the"
-  echo "            Dockerfile 'Dockerfile.ldap' being used."
-  echo "           Default: empty"
+  echo "           Dockerfile './Dockerfile.ldap' being used."
+  echo "           Exception: VARIANT=main will use the './Dockerfile' Dockerfile"
+  echo "           Default: main"
  echo "  HTTP_PROXY The proxy to use for http requests."
  echo "           Example: http://proxy.domain.tld:3128"
  echo "           Default: empty"
@ -95,7 +97,7 @@ esac
 DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"

 # Checking which VARIANT to build
-if [ -z "$VARIANT" ]; then
+if [ "$VARIANT" == "main" ]; then
  DOCKERFILE="Dockerfile"
 else
  DOCKERFILE="Dockerfile.${VARIANT}"
@ -117,8 +119,8 @@ DOCKER_OPTS=("${DOCKER_OPTS[@]}")

 # caching is only ok for version tags
 case "${TAG}" in
-  v*) ;;
-  *)  DOCKER_OPTS+=( "--no-cache" ) ;;
+ v*) ;;
+ *)  DOCKER_OPTS+=( "--no-cache" ) ;;
 esac

 DOCKER_OPTS+=( "--pull" )
@ -152,11 +154,13 @@ else
  DOCKER_CMD="echo docker"
 fi

-echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
-$DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
-echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
+if [ "${2}" != "--push-only" ] ; then
+  echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
+  $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
+  echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
+fi

-if [ "${2}" == "--push" ] ; then
+if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
  echo "⏫ Pushing '${DOCKER_TAG}"
  $DOCKER_CMD push "${DOCKER_TAG}"
  echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'."
--- a/hooks/build
+++ b/hooks/build
@ -0,0 +1,5 @@
+#!/bin/bash
+
+. hooks/common
+
+run_build
--- a/hooks/common
+++ b/hooks/common
@ -0,0 +1,82 @@
+#!/bin/bash
+
+ensure_jq() {
+  echo "🛠🛠🛠 Installing JQ via apt-get"
+  [ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq )
+}
+
+ensure_dockerfile_present() {
+  if [ "${VARIANT}" == "main" ]; then
+    DOCKERFILE="Dockerfile"
+  else
+    DOCKERFILE="Dockerfile.${VARIANT}"
+
+    # Fail fast
+    if [ ! -f "${DOCKERFILE}" ]; then
+      echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
+
+      if [ -z "$DEBUG" ]; then
+        exit 1
+      else
+        echo "⚠️ Would skip this, but DEBUG is enabled."
+      fi
+    fi
+
+    if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then
+      echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'."
+      echo "   This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'."
+    fi
+  fi
+}
+
+# Passes args to the scripts
+run_build() {
+  echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant"
+  case $BUILD in
+    release)
+      # build the latest release
+      # shellcheck disable=SC2068
+      ./build-latest.sh $@
+      ;;
+    prerelease)
+      # build the latest pre-release
+      # shellcheck disable=SC2068
+      PRERELEASE=true ./build-latest.sh $@
+      ;;
+    branches)
+      # build all branches
+      # shellcheck disable=SC2068
+      ./build-branches.sh $@
+      ;;
+    special)
+      # special build
+      # shellcheck disable=SC2068
+      #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@
+      echo "✅ No special builds today."
+      ;;
+    *)
+      echo "🚨 Unrecognized build '$BUILD'."
+
+      if [ -z "$DEBUG" ]; then
+        exit 1
+      else
+        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
+      fi
+      ;;
+  esac
+}
+
+echo "🤖🤖🤖 Preparing build"
+export DOCKER_ORG="index.docker.io/cimnine"
+export DOCKER_REPO=netbox-test
+export DOCKERHUB_REPO=cimnine/netbox-test
+
+# mis-using the "${DOCKER_TAG}" variable as "branch to build"
+export BUILD="${DOCKER_TAG%-*}"
+export VARIANT="${DOCKER_TAG#*-}"
+
+unset DOCKER_TAG
+
+ensure_dockerfile_present
+
+ensure_jq
--- a/hooks/push
+++ b/hooks/push
@ -0,0 +1,5 @@
+#!/bin/bash
+
+. hooks/common
+
+run_build --push-only
--- a/hooks/test
+++ b/hooks/test
@ -0,0 +1,12 @@
+#!/bin/bash
+
+. hooks/common
+
+if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then
+  echo "🐳🐳🐳 Testing"
+  docker-compose pull --parallel
+  docker-compose build
+  docker-compose run netbox ./manage.py test
+else
+  echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'."
+fi