feat: organizing infrastructure_files folder and adds new envs (#1235)

This PR aims to organize a little the files within `infrastructure_files` folder and adds some new ENV vars to the process.

1. It creates the `artifacts` folder within the `infrastructure_files` folder, the idea behind it is to split templates from artifacts created after running `./configure.sh`. It makes it easier to cp/rsync only `artifacts` content to the final server/destination.

2. Creates `NETBIRD_TURN_DOMAIN` and `TURN_DOMAIN` ENV vars. The idea behind it is to make it possible to split the management/signal server from TURN server. If `NETBIRD_TURN_DOMAIN` is not set, then, `TURN_DOMAIN` will be set as `NETBIRD_DOMAIN`.

3. Creates `*_TAG` ENVs for each component. The idea behind it is to give the users the choice to use `latest` tag as default or tie it to specific versions of each component in the stack.

.github/workflows/test-infrastructure-files.yml

@@ -62,7 +62,7 @@ jobs:
           CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
 
       - name: check values
-        working-directory: infrastructure_files
+        working-directory: infrastructure_files/artifacts
         env:
           CI_NETBIRD_DOMAIN: localhost
           CI_NETBIRD_AUTH_CLIENT_ID: testing.client.id
@@ -107,7 +107,7 @@ jobs:
           grep Engine management.json  | grep "$CI_NETBIRD_STORE_CONFIG_ENGINE"
           grep IdpSignKeyRefreshEnabled management.json | grep "$CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH"
           grep UseIDToken management.json | grep false
-          grep -A 1 IdpManagerConfig management.json | grep ManagerType | grep $CI_NETBIRD_MGMT_IDP 
+          grep -A 1 IdpManagerConfig management.json | grep ManagerType | grep $CI_NETBIRD_MGMT_IDP
           grep -A 3 IdpManagerConfig management.json | grep -A 1 ClientConfig | grep Issuer | grep $CI_NETBIRD_AUTH_AUTHORITY
           grep -A 4 IdpManagerConfig management.json | grep -A 2 ClientConfig | grep TokenEndpoint | grep $CI_NETBIRD_AUTH_TOKEN_ENDPOINT
           grep -A 5 IdpManagerConfig management.json | grep -A 3 ClientConfig | grep ClientID | grep $CI_NETBIRD_IDP_MGMT_CLIENT_ID
@@ -143,7 +143,7 @@ jobs:
           docker build -t netbirdio/signal:latest .
 
       - name: run docker compose up
-        working-directory: infrastructure_files
+        working-directory: infrastructure_files/artifacts
         run: |
           docker-compose up -d
           sleep 5
@@ -152,9 +152,9 @@ jobs:
 
       - name: test running containers
         run: |
-          count=$(docker compose ps --format json | jq '. | select(.Name | contains("infrastructure_files")) | .State' | grep -c running)
+          count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running)
           test $count -eq 4
-        working-directory: infrastructure_files
+        working-directory: infrastructure_files/artifacts
 
   test-getting-started-script:
     runs-on: ubuntu-latest
@@ -179,4 +179,4 @@ jobs:
       - name: test zitadel.env file gen
         run: test -f zitadel.env
       - name: test dashboard.env file gen
-        run: test -f dashboard.env
+        run: test -f dashboard.env

.gitignore

@@ -6,11 +6,11 @@ bin/
 .env
 conf.json
 http-cmds.sh
-infrastructure_files/management.json
-infrastructure_files/management-*.json
-infrastructure_files/docker-compose.yml
-infrastructure_files/openid-configuration.json
-infrastructure_files/turnserver.conf
+infrastructure_files/artifacts/management.json
+infrastructure_files/artifacts/management-*.json
+infrastructure_files/artifacts/docker-compose.yml
+infrastructure_files/artifacts/openid-configuration.json
+infrastructure_files/artifacts/turnserver.conf
 management/management
 client/client
 client/client.exe

infrastructure_files/base.setup.env

@@ -20,6 +20,9 @@ NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=${NETBIRD_MGMT_IDP_SIGNKEY_REFRESH:-false}
 NETBIRD_SIGNAL_PROTOCOL="http"
 NETBIRD_SIGNAL_PORT=${NETBIRD_SIGNAL_PORT:-10000}
 
+# Turn
+TURN_DOMAIN=${NETBIRD_TURN_DOMAIN:-$NETBIRD_DOMAIN}
+
 # Turn credentials
 # User
 TURN_USER=self
@@ -59,8 +62,16 @@ NETBIRD_DASH_AUTH_AUDIENCE=$NETBIRD_AUTH_AUDIENCE
 # Store config
 NETBIRD_STORE_CONFIG_ENGINE=${NETBIRD_STORE_CONFIG_ENGINE:-"jsonfile"}
 
+# Image tags
+NETBIRD_DASHBOARD_TAG=${NETBIRD_DASHBOARD_TAG:-"latest"}
+NETBIRD_SIGNAL_TAG=${NETBIRD_SIGNAL_TAG:-"latest"}
+NETBIRD_MANAGEMENT_TAG=${NETBIRD_MANAGEMENT_TAG:-"latest"}
+COTURN_TAG=${COTURN_TAG:-"latest"}
+
+
 # exports
 export NETBIRD_DOMAIN
+export NETBIRD_TURN_DOMAIN
 export NETBIRD_AUTH_CLIENT_ID
 export NETBIRD_AUTH_CLIENT_SECRET
 export NETBIRD_AUTH_AUDIENCE
@@ -79,6 +90,7 @@ export NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID
 export NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT
 export NETBIRD_AUTH_REDIRECT_URI
 export NETBIRD_AUTH_SILENT_REDIRECT_URI
+export TURN_DOMAIN
 export TURN_USER
 export TURN_PASSWORD
 export TURN_MIN_PORT
@@ -103,4 +115,8 @@ export NETBIRD_AUTH_PKCE_USE_ID_TOKEN
 export NETBIRD_AUTH_PKCE_AUDIENCE
 export NETBIRD_DASH_AUTH_USE_AUDIENCE
 export NETBIRD_DASH_AUTH_AUDIENCE
-export NETBIRD_STORE_CONFIG_ENGINE
+export NETBIRD_STORE_CONFIG_ENGINE
+export NETBIRD_DASHBOARD_TAG
+export NETBIRD_SIGNAL_TAG
+export NETBIRD_MANAGEMENT_TAG
+export COTURN_TAG

infrastructure_files/configure.sh

@@ -54,6 +54,9 @@ if [[ "x-$TURN_PASSWORD" == "x-" ]]; then
   export TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g')
 fi
 
+artifacts_path="./artifacts"
+mkdir -p $artifacts_path
+
 MGMT_VOLUMENAME="${VOLUME_PREFIX}${MGMT_VOLUMESUFFIX}"
 SIGNAL_VOLUMENAME="${VOLUME_PREFIX}${SIGNAL_VOLUMESUFFIX}"
 LETSENCRYPT_VOLUMENAME="${VOLUME_PREFIX}${LETSENCRYPT_VOLUMESUFFIX}"
@@ -94,13 +97,13 @@ if [[ -z "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" ]]; then
 fi
 
 echo "loading OpenID configuration from ${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT} to the openid-configuration.json file"
-curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o openid-configuration.json
+curl "${NETBIRD_AUTH_OIDC_CONFIGURATION_ENDPOINT}" -q -o ${artifacts_path}/openid-configuration.json
 
-export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' openid-configuration.json)
-export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' openid-configuration.json)
-export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' openid-configuration.json)
-export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' openid-configuration.json)
-export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT=$(jq -r '.authorization_endpoint' openid-configuration.json)
+export NETBIRD_AUTH_AUTHORITY=$(jq -r '.issuer' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_JWT_CERTS=$(jq -r '.jwks_uri' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_TOKEN_ENDPOINT=$(jq -r '.token_endpoint' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_DEVICE_AUTH_ENDPOINT=$(jq -r '.device_authorization_endpoint' ${artifacts_path}/openid-configuration.json)
+export NETBIRD_AUTH_PKCE_AUTHORIZATION_ENDPOINT=$(jq -r '.authorization_endpoint' ${artifacts_path}/openid-configuration.json)
 
 if [[ ! -z "${NETBIRD_AUTH_DEVICE_AUTH_CLIENT_ID}" ]]; then
   # user enabled Device Authorization Grant feature
@@ -185,17 +188,17 @@ fi
 env | grep NETBIRD
 
 bkp_postfix="$(date +%s)"
-if test -f 'docker-compose.yml'; then
-    cp docker-compose.yml "docker-compose.yml.bkp.${bkp_postfix}"
+if test -f "${artifacts_path}/docker-compose.yml"; then
+    cp $artifacts_path/docker-compose.yml "${artifacts_path}/docker-compose.yml.bkp.${bkp_postfix}"
 fi
 
-if test -f 'management.json'; then
-    cp management.json "management.json.bkp.${bkp_postfix}"
+if test -f "${artifacts_path}/management.json"; then
+    cp $artifacts_path/management.json "${artifacts_path}/management.json.bkp.${bkp_postfix}"
 fi
 
-if test -f 'turnserver.conf'; then
-    cp turnserver.conf "turnserver.conf.bpk.${bkp_postfix}"
+if test -f "${artifacts_path}/turnserver.conf"; then
+    cp ${artifacts_path}/turnserver.conf "${artifacts_path}/turnserver.conf.bpk.${bkp_postfix}"
 fi
-envsubst <docker-compose.yml.tmpl >docker-compose.yml
-envsubst <management.json.tmpl | jq . >management.json
-envsubst <turnserver.conf.tmpl >turnserver.conf
+envsubst <docker-compose.yml.tmpl >$artifacts_path/docker-compose.yml
+envsubst <management.json.tmpl | jq . >$artifacts_path/management.json
+envsubst <turnserver.conf.tmpl >$artifacts_path/turnserver.conf

infrastructure_files/docker-compose.yml.tmpl

@@ -2,7 +2,7 @@ version: "3"
 services:
   #UI dashboard
   dashboard:
-    image: wiretrustee/dashboard:latest
+    image: wiretrustee/dashboard:$NETBIRD_DASHBOARD_TAG
     restart: unless-stopped
     ports:
       - 80:80
@@ -31,7 +31,7 @@ services:
 
   # Signal
   signal:
-    image: netbirdio/signal:latest
+    image: netbirdio/signal:$NETBIRD_SIGNAL_TAG
     restart: unless-stopped
     volumes:
       - $SIGNAL_VOLUMENAME:/var/lib/netbird
@@ -43,7 +43,7 @@ services:
 
   # Management
   management:
-    image: netbirdio/management:latest
+    image: netbirdio/management:$NETBIRD_MANAGEMENT_TAG
     restart: unless-stopped
     depends_on:
       - dashboard
@@ -65,9 +65,9 @@ services:
 
   # Coturn
   coturn:
-    image: coturn/coturn
+    image: coturn/coturn:$COTURN_TAG
     restart: unless-stopped
-    domainname: $NETBIRD_DOMAIN
+    domainname: $TURN_DOMAIN
     volumes:
       - ./turnserver.conf:/etc/turnserver.conf:ro
     #      - ./privkey.pem:/etc/coturn/private/privkey.pem:ro

infrastructure_files/docker-compose.yml.tmpl.traefik

@@ -2,7 +2,7 @@ version: "3"
 services:
   #UI dashboard
   dashboard:
-    image: wiretrustee/dashboard:latest
+    image: wiretrustee/dashboard:$NETBIRD_DASHBOARD_TAG
     restart: unless-stopped
     #ports:
     #  - 80:80
@@ -35,7 +35,7 @@ services:
 
   # Signal
   signal:
-    image: netbirdio/signal:latest
+    image: netbirdio/signal:$NETBIRD_SIGNAL_TAG
     restart: unless-stopped
     volumes:
       - $SIGNAL_VOLUMENAME:/var/lib/netbird
@@ -52,7 +52,7 @@ services:
 
   # Management
   management:
-    image: netbirdio/management:latest
+    image: netbirdio/management:$NETBIRD_MANAGEMENT_TAG
     restart: unless-stopped
     depends_on:
       - dashboard
@@ -84,9 +84,9 @@ services:
 
   # Coturn
   coturn:
-    image: coturn/coturn
+    image: coturn/coturn:$COTURN_TAG
     restart: unless-stopped
-    domainname: $NETBIRD_DOMAIN
+    domainname: $TURN_DOMAIN
     volumes:
       - ./turnserver.conf:/etc/turnserver.conf:ro
     #      - ./privkey.pem:/etc/coturn/private/privkey.pem:ro

infrastructure_files/management.json.tmpl

@@ -2,7 +2,7 @@
     "Stuns": [
         {
             "Proto": "udp",
-            "URI": "stun:$NETBIRD_DOMAIN:3478",
+            "URI": "stun:$TURN_DOMAIN:3478",
             "Username": "",
             "Password": null
         }
@@ -11,7 +11,7 @@
         "Turns": [
             {
                 "Proto": "udp",
-                "URI": "turn:$NETBIRD_DOMAIN:3478",
+                "URI": "turn:$TURN_DOMAIN:3478",
                 "Username": "$TURN_USER",
                 "Password": "$TURN_PASSWORD"
             }

infrastructure_files/setup.env.example

@@ -1,8 +1,20 @@
 ## example file, you can copy this file to setup.env and update its values
 ##
+
+# Image tags
+# you can force specific tags for each component; will be set to latest if empty
+NETBIRD_DASHBOARD_TAG=""
+NETBIRD_SIGNAL_TAG=""
+NETBIRD_MANAGEMENT_TAG=""
+COTURN_TAG=""
+
 # Dashboard domain. e.g. app.mydomain.com
 NETBIRD_DOMAIN=""
 
+# TURN server domain. e.g. turn.mydomain.com
+# if not specified it will assume NETBIRD_DOMAIN
+NETBIRD_TURN_DOMAIN=""
+
 # -------------------------------------------
 # OIDC
 #  e.g., https://example.eu.auth0.com/.well-known/openid-configuration
@@ -70,4 +82,4 @@ NETBIRD_LETSENCRYPT_EMAIL=""
 # Disable anonymous metrics collection, see more information at https://netbird.io/docs/FAQ/metrics-collection
 NETBIRD_DISABLE_ANONYMOUS_METRICS=false
 # DNS DOMAIN configures the domain name used for peer resolution. By default it is netbird.selfhosted
-NETBIRD_MGMT_DNS_DOMAIN=netbird.selfhosted
+NETBIRD_MGMT_DNS_DOMAIN=netbird.selfhosted