From 8aa4f240c7401b4ef8feba5bdfeebbed3f73167a Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Thu, 3 Aug 2023 19:19:17 +0200 Subject: [PATCH] Add getting started script with Zitadel (#1005) add getting started script with zitadel limit tests for infrastructure file workflow limit release workflow based on relevant files --- .github/workflows/release.yml | 10 + ...inux.yml => test-infrastructure-files.yml} | 34 +- .goreleaser.yaml | 10 + README.md | 2 +- .../getting-started-with-zitadel.sh | 736 ++++++++++++++++++ infrastructure_files/zitadel.sh | 122 --- 6 files changed, 788 insertions(+), 126 deletions(-) rename .github/workflows/{test-docker-compose-linux.yml => test-infrastructure-files.yml} (86%) create mode 100644 infrastructure_files/getting-started-with-zitadel.sh delete mode 100644 infrastructure_files/zitadel.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a3c04f5d..aaae51dde 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,16 @@ on: branches: - main pull_request: + paths: + - 'go.mod' + - 'go.sum' + - '.goreleaser.yml' + - '.goreleaser_ui.yaml' + - '.goreleaser_ui_darwin.yaml' + - '.github/workflows/release.yml' + - 'release_files/**' + - '**/Dockerfile' + - '**/Dockerfile.*' env: SIGN_PIPE_VER: "v0.0.8" diff --git a/.github/workflows/test-docker-compose-linux.yml b/.github/workflows/test-infrastructure-files.yml similarity index 86% rename from .github/workflows/test-docker-compose-linux.yml rename to .github/workflows/test-infrastructure-files.yml index 3910de0f2..3861487c2 100644 --- a/.github/workflows/test-docker-compose-linux.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -1,10 +1,13 @@ -name: Test Docker Compose Linux +name: Test Infrastructure files on: push: branches: - main pull_request: + paths: + - 'infrastructure_files/**' + - '.github/workflows/test-infrastructure-files.yml' concurrency: @@ -12,7 +15,7 @@ concurrency: cancel-in-progress: true jobs: - test: + test-docker-compose: runs-on: ubuntu-latest steps: - name: Install jq @@ -35,7 +38,7 @@ jobs: ${{ runner.os }}-go- - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: cp setup.env run: cp infrastructure_files/tests/setup.env infrastructure_files/ @@ -121,3 +124,28 @@ jobs: count=$(docker compose ps --format json | jq '.[] | select(.Project | contains("infrastructure_files")) | .State' | grep -c running) test $count -eq 4 working-directory: infrastructure_files + + test-getting-started-script: + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get install -y jq + + - name: Checkout code + uses: actions/checkout@v3 + + - name: run script + run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh + + - name: test Caddy file gen + run: test -f Caddyfile + - name: test docker-compose file gen + run: test -f docker-compose.yml + - name: test management.json file gen + run: test -f management.json + - name: test turnserver.conf file gen + run: test -f turnserver.conf + - name: test zitadel.env file gen + run: test -f zitadel.env + - name: test dashboard.env file gen + run: test -f dashboard.env \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e4dca7478..7a219110a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -377,3 +377,13 @@ uploads: target: https://pkgs.wiretrustee.com/yum/{{ .Arch }}{{ if .Arm }}{{ .Arm }}{{ end }} username: dev@wiretrustee.com method: PUT + +checksum: + extra_files: + - glob: ./infrastructure_files/getting-started-with-zitadel.sh + - glob: ./release_files/install.sh + +release: + extra_files: + - glob: ./infrastructure_files/getting-started-with-zitadel.sh + - glob: ./release_files/install.sh \ No newline at end of file diff --git a/README.md b/README.md index 03e0a6c38..774004a2a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ NetBird uses [NAT traversal techniques](https://en.wikipedia.org/wiki/Interactiv - \[x] Network Activity Monitoring. **Coming soon:** -- \[ ] Mobile clients. +- \[ ] Mobile clients. ### Secure peer-to-peer VPN with SSO and MFA in minutes diff --git a/infrastructure_files/getting-started-with-zitadel.sh b/infrastructure_files/getting-started-with-zitadel.sh new file mode 100644 index 000000000..0c4e7ad40 --- /dev/null +++ b/infrastructure_files/getting-started-with-zitadel.sh @@ -0,0 +1,736 @@ +#!/bin/bash + +set -e + +handle_request_command_status() { + PARSED_RESPONSE=$1 + FUNCTION_NAME=$2 + RESPONSE=$3 + if [[ $PARSED_RESPONSE -ne 0 ]]; then + echo "ERROR calling $FUNCTION_NAME:" $(echo "$RESPONSE" | jq -r '.message') > /dev/stderr + exit 1 + fi +} + +handle_zitadel_request_response() { + PARSED_RESPONSE=$1 + FUNCTION_NAME=$2 + RESPONSE=$3 + if [[ $PARSED_RESPONSE == "null" ]]; then + echo "ERROR calling $FUNCTION_NAME:" $(echo "$RESPONSE" | jq -r '.message') > /dev/stderr + exit 1 + fi + sleep 1 +} + +check_docker_compose() { + if command -v docker-compose &> /dev/null + then + echo "docker-compose" + return + fi + if docker compose --help &> /dev/null + then + echo "docker compose" + return + fi + + echo "docker-compose is not installed or not in PATH. Please follow the steps from the official guide: https://docs.docker.com/engine/install/" > /dev/stderr + exit 1 +} + +check_jq() { + if ! command -v jq &> /dev/null + then + echo "jq is not installed or not in PATH, please install with your package manager. e.g. sudo apt install jq" > /dev/stderr + exit 1 + fi +} + +wait_crdb() { + set +e + while true; do + if $DOCKER_COMPOSE_COMMAND exec -T crdb curl -sf -o /dev/null 'http://localhost:8080/health?ready=1'; then + break + fi + echo -n " ." + sleep 5 + done + echo " done" + set -e +} + +init_crdb() { + echo -e "\nInitializing Zitadel's CockroachDB\n\n" + $DOCKER_COMPOSE_COMMAND up -d crdb + echo "" + # shellcheck disable=SC2028 + echo -n "Waiting cockroachDB to become ready " + wait_crdb + $DOCKER_COMPOSE_COMMAND exec -T crdb /bin/bash -c "cp /cockroach/certs/* /zitadel-certs/ && cockroach cert create-client --overwrite --certs-dir /zitadel-certs/ --ca-key /zitadel-certs/ca.key zitadel_user && chown -R 1000:1000 /zitadel-certs/" + handle_request_command_status $? "init_crdb failed" "" +} + +get_main_ip_address() { + if [[ "$OSTYPE" == "darwin"* ]]; then + interface=$(route -n get default | grep 'interface:' | awk '{print $2}') + ip_address=$(ifconfig "$interface" | grep 'inet ' | awk '{print $2}') + else + interface=$(ip route | grep default | awk '{print $5}' | head -n 1) + ip_address=$(ip addr show "$interface" | grep 'inet ' | awk '{print $2}' | cut -d'/' -f1) + fi + + echo "$ip_address" +} + +wait_pat() { + PAT_PATH=$1 + set +e + while true; do + if [[ -f "$PAT_PATH" ]]; then + break + fi + echo -n " ." + sleep 1 + done + echo " done" + set -e +} + +wait_api() { + INSTANCE_URL=$1 + PAT=$2 + set +e + while true; do + curl -s --fail -o /dev/null "$INSTANCE_URL/auth/v1/users/me" -H "Authorization: Bearer $PAT" + if [[ $? -eq 0 ]]; then + break + fi + echo -n " ." + sleep 1 + done + echo " done" + set -e +} + +create_new_project() { + INSTANCE_URL=$1 + PAT=$2 + PROJECT_NAME="NETBIRD" + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/projects" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{"name": "'"$PROJECT_NAME"'"}' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.id') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_new_project" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_new_application() { + INSTANCE_URL=$1 + PAT=$2 + APPLICATION_NAME=$3 + BASE_REDIRECT_URL1=$4 + BASE_REDIRECT_URL2=$5 + LOGOUT_URL=$6 + ZITADEL_DEV_MODE=$7 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/projects/$PROJECT_ID/apps/oidc" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "'"$APPLICATION_NAME"'", + "redirectUris": [ + "'"$BASE_REDIRECT_URL1"'", + "'"$BASE_REDIRECT_URL2"'" + ], + "postLogoutRedirectUris": [ + "'"$LOGOUT_URL"'" + ], + "RESPONSETypes": [ + "OIDC_RESPONSE_TYPE_CODE" + ], + "grantTypes": [ + "OIDC_GRANT_TYPE_AUTHORIZATION_CODE", + "OIDC_GRANT_TYPE_REFRESH_TOKEN" + ], + "appType": "OIDC_APP_TYPE_USER_AGENT", + "authMethodType": "OIDC_AUTH_METHOD_TYPE_NONE", + "version": "OIDC_VERSION_1_0", + "devMode": '"$ZITADEL_DEV_MODE"', + "accessTokenType": "OIDC_TOKEN_TYPE_JWT", + "accessTokenRoleAssertion": true, + "skipNativeAppSuccessPage": true + }' + ) + + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.clientId') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_new_application" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_service_user() { + INSTANCE_URL=$1 + PAT=$2 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/users/machine" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userName": "netbird-service-account", + "name": "Netbird Service Account", + "description": "Netbird Service Account for IDP management", + "accessTokenType": "ACCESS_TOKEN_TYPE_JWT" + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.userId') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_service_user" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_service_user_secret() { + INSTANCE_URL=$1 + PAT=$2 + USER_ID=$3 + + RESPONSE=$( + curl -sS -X PUT "$INSTANCE_URL/management/v1/users/$USER_ID/secret" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{}' + ) + SERVICE_USER_CLIENT_ID=$(echo "$RESPONSE" | jq -r '.clientId') + handle_zitadel_request_response "$SERVICE_USER_CLIENT_ID" "create_service_user_secret_id" "$RESPONSE" + SERVICE_USER_CLIENT_SECRET=$(echo "$RESPONSE" | jq -r '.clientSecret') + handle_zitadel_request_response "$SERVICE_USER_CLIENT_SECRET" "create_service_user_secret" "$RESPONSE" +} + +add_organization_user_manager() { + INSTANCE_URL=$1 + PAT=$2 + USER_ID=$3 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/orgs/me/members" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "'"$USER_ID"'", + "roles": [ + "ORG_USER_MANAGER" + ] + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.creationDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "add_organization_user_manager" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +create_admin_user() { + INSTANCE_URL=$1 + PAT=$2 + USERNAME=$3 + PASSWORD=$4 + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/management/v1/users/human/_import" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userName": "'"$USERNAME"'", + "profile": { + "firstName": "Zitadel", + "lastName": "Admin" + }, + "email": { + "email": "'"$USERNAME"'", + "isEmailVerified": true + }, + "password": "'"$PASSWORD"'", + "passwordChangeRequired": true + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.userId') + handle_zitadel_request_response "$PARSED_RESPONSE" "create_admin_user" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +add_instance_admin() { + INSTANCE_URL=$1 + PAT=$2 + USER_ID=$3 + + RESPONSE=$( + curl -sS -X POST "$INSTANCE_URL/admin/v1/members" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + -d '{ + "userId": "'"$USER_ID"'", + "roles": [ + "IAM_OWNER" + ] + }' + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.creationDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "add_instance_admin" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +delete_auto_service_user() { + INSTANCE_URL=$1 + PAT=$2 + + RESPONSE=$( + curl -sS -X GET "$INSTANCE_URL/auth/v1/users/me" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + ) + USER_ID=$(echo "$RESPONSE" | jq -r '.user.id') + handle_zitadel_request_response "$USER_ID" "delete_auto_service_user_get_user" "$RESPONSE" + + RESPONSE=$( + curl -sS -X DELETE "$INSTANCE_URL/admin/v1/members/$USER_ID" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.changeDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "delete_auto_service_user_remove_instance_permissions" "$RESPONSE" + + RESPONSE=$( + curl -sS -X DELETE "$INSTANCE_URL/management/v1/orgs/me/members/$USER_ID" \ + -H "Authorization: Bearer $PAT" \ + -H "Content-Type: application/json" \ + ) + PARSED_RESPONSE=$(echo "$RESPONSE" | jq -r '.details.changeDate') + handle_zitadel_request_response "$PARSED_RESPONSE" "delete_auto_service_user_remove_org_permissions" "$RESPONSE" + echo "$PARSED_RESPONSE" +} + +init_zitadel() { + echo -e "\nInitializing Zitadel with NetBird's applications\n" + INSTANCE_URL="$NETBIRD_HTTP_PROTOCOL://$NETBIRD_DOMAIN:$NETBIRD_PORT" + + TOKEN_PATH=./machinekey/zitadel-admin-sa.token + + echo -n "Waiting for Zitadel's PAT to be created " + wait_pat "$TOKEN_PATH" + echo "Reading Zitadel PAT" + PAT=$(cat $TOKEN_PATH) + if [ "$PAT" = "null" ]; then + echo "Failed requesting getting Zitadel PAT" + exit 1 + fi + + echo -n "Waiting for Zitadel to become ready " + wait_api "$INSTANCE_URL" "$PAT" + + # create the zitadel project + echo "Creating new zitadel project" + PROJECT_ID=$(create_new_project "$INSTANCE_URL" "$PAT") + + ZITADEL_DEV_MODE=false + BASE_REDIRECT_URL=$NETBIRD_HTTP_PROTOCOL://$NETBIRD_DOMAIN + if [[ $NETBIRD_HTTP_PROTOCOL == "http" ]]; then + ZITADEL_DEV_MODE=true + fi + + # create zitadel spa applications + echo "Creating new Zitadel SPA Dashboard application" + DASHBOARD_APPLICATION_CLIENT_ID=$(create_new_application "$INSTANCE_URL" "$PAT" "Dashboard" "$BASE_REDIRECT_URL/nb-auth" "$BASE_REDIRECT_URL/nb-silent-auth" "$BASE_REDIRECT_URL/" "$ZITADEL_DEV_MODE") + + echo "Creating new Zitadel SPA Cli application" + CLI_APPLICATION_CLIENT_ID=$(create_new_application "$INSTANCE_URL" "$PAT" "Cli" "http://localhost:53000/" "http://localhost:54000/" "http://localhost:53000/" "true") + + MACHINE_USER_ID=$(create_service_user "$INSTANCE_URL" "$PAT") + + SERVICE_USER_CLIENT_ID="null" + SERVICE_USER_CLIENT_SECRET="null" + + create_service_user_secret "$INSTANCE_URL" "$PAT" "$MACHINE_USER_ID" + + DATE=$(add_organization_user_manager "$INSTANCE_URL" "$PAT" "$MACHINE_USER_ID") + + ZITADEL_ADMIN_USERNAME="admin@$NETBIRD_DOMAIN" + ZITADEL_ADMIN_PASSWORD="$(openssl rand -base64 32 | sed 's/=//g')@" + + HUMAN_USER_ID=$(create_admin_user "$INSTANCE_URL" "$PAT" "$ZITADEL_ADMIN_USERNAME" "$ZITADEL_ADMIN_PASSWORD") + + DATE="null" + + DATE=$(add_instance_admin "$INSTANCE_URL" "$PAT" "$HUMAN_USER_ID") + + DATE="null" + DATE=$(delete_auto_service_user "$INSTANCE_URL" "$PAT") + if [ "$DATE" = "null" ]; then + echo "Failed deleting auto service user" + echo "Please remove it manually" + fi + + export NETBIRD_AUTH_CLIENT_ID=$DASHBOARD_APPLICATION_CLIENT_ID + export NETBIRD_AUTH_CLIENT_ID_CLI=$CLI_APPLICATION_CLIENT_ID + export NETBIRD_IDP_MGMT_CLIENT_ID=$SERVICE_USER_CLIENT_ID + export NETBIRD_IDP_MGMT_CLIENT_SECRET=$SERVICE_USER_CLIENT_SECRET + export ZITADEL_ADMIN_USERNAME + export ZITADEL_ADMIN_PASSWORD +} + +check_nb_domain() { + DOMAIN=$1 + if [ "$DOMAIN-x" == "-x" ]; then + echo "The NETBIRD_DOMAIN variable cannot be empty." > /dev/stderr + return 1 + fi + + if [ "$DOMAIN" == "netbird.example.com" ]; then + echo "The NETBIRD_DOMAIN cannot be netbird.example.com" > /dev/stderr + retrun 1 + fi + return 0 +} + +read_nb_domain() { + READ_NETBIRD_DOMAIN="" + echo -n "Enter the domain you want to use for NetBird (e.g. netbird.my-domain.com): " > /dev/stderr + read -r READ_NETBIRD_DOMAIN + if ! check_nb_domain "$READ_NETBIRD_DOMAIN"; then + read_nb_domain + fi + echo "$READ_NETBIRD_DOMAIN" +} + +initEnvironment() { + CADDY_SECURE_DOMAIN="" + ZITADEL_EXTERNALSECURE="false" + ZITADEL_TLS_MODE="disabled" + ZITADEL_MASTERKEY="$(openssl rand -base64 32 | head -c 32)" + NETBIRD_PORT=80 + NETBIRD_HTTP_PROTOCOL="http" + TURN_USER="self" + TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g') + TURN_MIN_PORT=49152 + TURN_MAX_PORT=65535 + + if ! check_nb_domain "$NETBIRD_DOMAIN"; then + NETBIRD_DOMAIN=$(read_nb_domain) + fi + + if [ "$NETBIRD_DOMAIN" == "use-ip" ]; then + NETBIRD_DOMAIN=$(get_main_ip_address) + else + ZITADEL_EXTERNALSECURE="true" + ZITADEL_TLS_MODE="external" + NETBIRD_PORT=443 + CADDY_SECURE_DOMAIN=", $NETBIRD_DOMAIN:$NETBIRD_PORT" + NETBIRD_HTTP_PROTOCOL="https" + fi + + if [[ "$OSTYPE" == "darwin"* ]]; then + ZIDATE_TOKEN_EXPIRATION_DATE=$(date -u -v+30M "+%Y-%m-%dT%H:%M:%SZ") + else + ZIDATE_TOKEN_EXPIRATION_DATE=$(date -u -d "+30 minutes" "+%Y-%m-%dT%H:%M:%SZ") + fi + + check_jq + + DOCKER_COMPOSE_COMMAND=$(check_docker_compose) + + if [ -f zitadel.env ]; then + echo "Generated files already exist, if you want to reinitialize the environment, please remove them first." + echo "You can use the following commands:" + echo " $DOCKER_COMPOSE_COMMAND down --volumes # to remove all containers and volumes" + echo " rm -f docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json" + echo "Be aware that this will remove all data from the database, and you will have to reconfigure the dashboard." + exit 1 + fi + + echo Rendering initial files... + renderDockerCompose > docker-compose.yml + renderCaddyfile > Caddyfile + renderZitadelEnv > zitadel.env + echo "" > dashboard.env + echo "" > turnserver.conf + echo "" > management.json + + mkdir -p machinekey + chmod 777 machinekey + + init_crdb + + echo -e "\nStarting Zidatel IDP for user management\n\n" + $DOCKER_COMPOSE_COMMAND up -d caddy zitadel + init_zitadel + + echo -e "\nRendering NetBird files...\n" + renderTurnServerConf > turnserver.conf + renderManagementJson > management.json + renderDashboardEnv > dashboard.env + + echo -e "\nStarting NetBird services\n" + $DOCKER_COMPOSE_COMMAND up -d + echo -e "\nDone!\n" + echo "You can access the NetBird dashboard at $NETBIRD_HTTP_PROTOCOL://$NETBIRD_DOMAIN:$NETBIRD_PORT" + echo "Login with the following credentials:" + echo "Username: $ZITADEL_ADMIN_USERNAME" | tee .env + echo "Password: $ZITADEL_ADMIN_PASSWORD" | tee -a .env +} + +renderCaddyfile() { + cat <