feat: organizing infrastructure_files folder and adds new envs (#1235)

This PR aims to organize a little the files within `infrastructure_files` folder and adds some new ENV vars to the process. 1. It creates the `artifacts` folder within the `infrastructure_files` folder, the idea behind it is to split templates from artifacts created after running `./configure.sh`. It makes it easier to cp/rsync only `artifacts` content to the final server/destination. 2. Creates `NETBIRD_TURN_DOMAIN` and `TURN_DOMAIN` ENV vars. The idea behind it is to make it possible to split the management/signal server from TURN server. If `NETBIRD_TURN_DOMAIN` is not set, then, `TURN_DOMAIN` will be set as `NETBIRD_DOMAIN`. 3. Creates `*_TAG` ENVs for each component. The idea behind it is to give the users the choice to use `latest` tag as default or tie it to specific versions of each component in the stack.
2024-11-24 00:54:01 +01:00 · 2023-12-17 13:43:06 -03:00 · 2023-12-17 13:43:06 -03:00 · 56896794b3
commit 56896794b3
parent f73a2e2848
9 changed files with 71 additions and 40 deletions
--- a/.github/workflows/test-infrastructure-files.yml
+++ b/.github/workflows/test-infrastructure-files.yml
@ -62,7 +62,7 @@ jobs:
          CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false

      - name: check values
-        working-directory: infrastructure_files
+        working-directory: infrastructure_files/artifacts
        env:
          CI_NETBIRD_DOMAIN: localhost
          CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id
@ -143,7 +143,7 @@ jobs:
          docker build -t netbirdio/signal:latest .

      - name: run docker compose up
-        working-directory: infrastructure_files
+        working-directory: infrastructure_files/artifacts
        run: |
          docker-compose up -d
          sleep 5
@ -152,9 +152,9 @@ jobs:

      - name: test running containers
        run: |
-          count=$(docker compose ps --format json | jq '. | select(.Name | contains("infrastructure_files")) | .State' | grep -c running)
+          count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running)
          test $count -eq 4
-        working-directory: infrastructure_files
+        working-directory: infrastructure_files/artifacts

  test-getting-started-script:
    runs-on: ubuntu-latest
--- a/.gitignore
+++ b/.gitignore
@ -6,11 +6,11 @@ bin/
 .env
 conf.json
 http-cmds.sh
-infrastructure_files/management.json
-infrastructure_files/management-*.json
-infrastructure_files/docker-compose.yml
-infrastructure_files/openid-configuration.json
-infrastructure_files/turnserver.conf
+infrastructure_files/artifacts/management.json
+infrastructure_files/artifacts/management-*.json
+infrastructure_files/artifacts/docker-compose.yml
+infrastructure_files/artifacts/openid-configuration.json
+infrastructure_files/artifacts/turnserver.conf
 management/management
 client/client
 client/client.exe
--- a/infrastructure_files/artifacts/.gitkeep
+++ b/infrastructure_files/artifacts/.gitkeep
--- a/infrastructure_files/base.setup.env
+++ b/infrastructure_files/base.setup.env
@ -20,6 +20,9 @@ NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=${NETBIRD_MGMT_IDP_SIGNKEY_REFRESH:-false}
 NETBIRD_SIGNAL_PROTOCOL="http"
 NETBIRD_SIGNAL_PORT=${NETBIRD_SIGNAL_PORT:-10000}

+# Turn
+TURN_DOMAIN=${NETBIRD_TURN_DOMAIN:-$NETBIRD_DOMAIN}
+
 # Turn credentials
 # User
 TURN_USER=self
@ -59,8 +62,16 @@ NETBIRD_DASH_AUTH_AUDIENCE=$NETBIRD_AUTH_AUDIENCE
 # Store config
 NETBIRD_STORE_CONFIG_ENGINE=${NETBIRD_STORE_CONFIG_ENGINE:-"jsonfile"}

+# Image tags
+NETBIRD_DASHBOARD_TAG=${NETBIRD_DASHBOARD_TAG:-"latest"}
+NETBIRD_SIGNAL_TAG=${NETBIRD_SIGNAL_TAG:-"latest"}
+NETBIRD_MANAGEMENT_TAG=${NETBIRD_MANAGEMENT_TAG:-"latest"}
+COTURN_TAG=${COTURN_TAG:-"latest"}
+
+
 # exports
 export NETBIRD_DOMAIN
+export NETBIRD_TURN_DOMAIN
 export NETBIRD_AUTH_CLIENT_ID
 export NETBIRD_AUTH_CLIENT_SECRET
 export NETBIRD_AUTH_AUDIENCE
@ -79,6 +90,7 @@ export NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID
 export NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT
 export NETBIRD_AUTH_REDIRECT_URI
 export NETBIRD_AUTH_SILENT_REDIRECT_URI
+export TURN_DOMAIN
 export TURN_USER
 export TURN_PASSWORD
 export TURN_MIN_PORT
@ -104,3 +116,7 @@ export NETBIRD_AUTH_PKCE_AUDIENCE
 export NETBIRD_DASH_AUTH_USE_AUDIENCE
 export NETBIRD_DASH_AUTH_AUDIENCE
 export NETBIRD_STORE_CONFIG_ENGINE
+export NETBIRD_DASHBOARD_TAG
+export NETBIRD_SIGNAL_TAG
+export NETBIRD_MANAGEMENT_TAG
+export COTURN_TAG
--- a/infrastructure_files/configure.sh
+++ b/infrastructure_files/configure.sh
@ -54,6 +54,9 @@ if [[ "x-$TURN_PASSWORD" == "x-" ]]; then
  export TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g')
 fi

+artifacts_path="./artifacts"
+mkdir -p $artifacts_path
+
 MGMT_VOLUMENAME="${VOLUME_PREFIX}${MGMT_VOLUMESUFFIX}"
 SIGNAL_VOLUMENAME="${VOLUME_PREFIX}${SIGNAL_VOLUMESUFFIX}"
 LETSENCRYPT_VOLUMENAME="${VOLUME_PREFIX}${LETSENCRYPT_VOLUMESUFFIX}"
@ -94,13 +97,13 @@ if [[ -z "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" ]]; then
 fi

 echo "loading OpenID configuration from ${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT} to the openid-configuration.json file"
-curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o openid-configuration.json
+curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o ${artifacts_path}/openid-configuration.json

-export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' openid-configuration.json)
-export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' openid-configuration.json)
-export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' openid-configuration.json)
-export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' openid-configuration.json)
-export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT=$(jq -r '.authorization_endpoint' openid-configuration.json)
+export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT=$(jq -r '.authorization_endpoint' ${artifacts_path}/openid-configuration.json)

 if [[ ! -z "${NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID}" ]]; then
  # user enabled Device Authorization Grant feature
@ -185,17 +188,17 @@ fi
 env | grep NETBIRD

 bkp_postfix="$(date +%s)"
-if test -f 'docker-compose.yml'; then
-    cp docker-compose.yml "docker-compose.yml.bkp.${bkp_postfix}"
+if test -f "${artifacts_path}/docker-compose.yml"; then
+    cp $artifacts_path/docker-compose.yml "${artifacts_path}/docker-compose.yml.bkp.${bkp_postfix}"
 fi

-if test -f 'management.json'; then
-    cp management.json "management.json.bkp.${bkp_postfix}"
+if test -f "${artifacts_path}/management.json"; then
+    cp $artifacts_path/management.json "${artifacts_path}/management.json.bkp.${bkp_postfix}"
 fi

-if test -f 'turnserver.conf'; then
-    cp turnserver.conf "turnserver.conf.bpk.${bkp_postfix}"
+if test -f "${artifacts_path}/turnserver.conf"; then
+    cp ${artifacts_path}/turnserver.conf "${artifacts_path}/turnserver.conf.bpk.${bkp_postfix}"
 fi
-envsubst <docker-compose.yml.tmpl >docker-compose.yml
-envsubst <management.json.tmpl | jq . >management.json
-envsubst <turnserver.conf.tmpl >turnserver.conf
+envsubst <docker-compose.yml.tmpl >$artifacts_path/docker-compose.yml
+envsubst <management.json.tmpl | jq . >$artifacts_path/management.json
+envsubst <turnserver.conf.tmpl >$artifacts_path/turnserver.conf
--- a/infrastructure_files/docker-compose.yml.tmpl
+++ b/infrastructure_files/docker-compose.yml.tmpl
@ -2,7 +2,7 @@ version: "3"
 services:
  #UI dashboard
  dashboard:
-    image: wiretrustee/dashboard:latest
+    image: wiretrustee/dashboard:$NETBIRD_DASHBOARD_TAG
    restart: unless-stopped
    ports:
      - 80:80
@ -31,7 +31,7 @@ services:

  # Signal
  signal:
-    image: netbirdio/signal:latest
+    image: netbirdio/signal:$NETBIRD_SIGNAL_TAG
    restart: unless-stopped
    volumes:
      - $SIGNAL_VOLUMENAME:/var/lib/netbird
@ -43,7 +43,7 @@ services:

  # Management
  management:
-    image: netbirdio/management:latest
+    image: netbirdio/management:$NETBIRD_MANAGEMENT_TAG
    restart: unless-stopped
    depends_on:
      - dashboard
@ -65,9 +65,9 @@ services:

  # Coturn
  coturn:
-    image: coturn/coturn
+    image: coturn/coturn:$COTURN_TAG
    restart: unless-stopped
-    domainname: $NETBIRD_DOMAIN
+    domainname: $TURN_DOMAIN
    volumes:
      - ./turnserver.conf:/etc/turnserver.conf:ro
    #      - ./privkey.pem:/etc/coturn/private/privkey.pem:ro
--- a/infrastructure_files/docker-compose.yml.tmpl.traefik
+++ b/infrastructure_files/docker-compose.yml.tmpl.traefik
@ -2,7 +2,7 @@ version: "3"
 services:
  #UI dashboard
  dashboard:
-    image: wiretrustee/dashboard:latest
+    image: wiretrustee/dashboard:$NETBIRD_DASHBOARD_TAG
    restart: unless-stopped
    #ports:
    #  - 80:80
@ -35,7 +35,7 @@ services:

  # Signal
  signal:
-    image: netbirdio/signal:latest
+    image: netbirdio/signal:$NETBIRD_SIGNAL_TAG
    restart: unless-stopped
    volumes:
      - $SIGNAL_VOLUMENAME:/var/lib/netbird
@ -52,7 +52,7 @@ services:

  # Management
  management:
-    image: netbirdio/management:latest
+    image: netbirdio/management:$NETBIRD_MANAGEMENT_TAG
    restart: unless-stopped
    depends_on:
      - dashboard
@ -84,9 +84,9 @@ services:

  # Coturn
  coturn:
-    image: coturn/coturn
+    image: coturn/coturn:$COTURN_TAG
    restart: unless-stopped
-    domainname: $NETBIRD_DOMAIN
+    domainname: $TURN_DOMAIN
    volumes:
      - ./turnserver.conf:/etc/turnserver.conf:ro
    #      - ./privkey.pem:/etc/coturn/private/privkey.pem:ro
--- a/infrastructure_files/management.json.tmpl
+++ b/infrastructure_files/management.json.tmpl
@ -2,7 +2,7 @@
    "Stuns": [
        {
            "Proto": "udp",
-            "URI": "stun:$NETBIRD_DOMAIN:3478",
+            "URI": "stun:$TURN_DOMAIN:3478",
            "Username": "",
            "Password": null
        }
@ -11,7 +11,7 @@
        "Turns": [
            {
                "Proto": "udp",
-                "URI": "turn:$NETBIRD_DOMAIN:3478",
+                "URI": "turn:$TURN_DOMAIN:3478",
                "Username": "$TURN_USER",
                "Password": "$TURN_PASSWORD"
            }
--- a/infrastructure_files/setup.env.example
+++ b/infrastructure_files/setup.env.example
@ -1,8 +1,20 @@
 ## example file, you can copy this file to setup.env and update its values
 ##
+
+# Image tags
+# you can force specific tags for each component; will be set to latest if empty
+NETBIRD_DASHBOARD_TAG=""
+NETBIRD_SIGNAL_TAG=""
+NETBIRD_MANAGEMENT_TAG=""
+COTURN_TAG=""
+
 # Dashboard domain. e.g. app.mydomain.com
 NETBIRD_DOMAIN=""

+# TURN server domain. e.g. turn.mydomain.com
+# if not specified it will assume NETBIRD_DOMAIN
+NETBIRD_TURN_DOMAIN=""
+
 # -------------------------------------------
 # OIDC
 #  e.g., https://example.eu.auth0.com/.well-known/openid-configuration