Update self hosting scripts (#367)

split setup.env with example and base

add setup.env to .gitignore to avoid overwrite from new versions

Added test workflow for docker-compose 
and validated configure.sh generated variables
This commit is contained in:
Maycon Santos 2022-06-24 14:50:14 +02:00 committed by GitHub
parent 06860c4c10
commit 1aafc15607
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 120 additions and 275 deletions

View File

@ -1,5 +1,6 @@
name: Test Code Darwin
on: [push,pull_request]
jobs:
test:
strategy:

View File

@ -1,5 +1,6 @@
name: Test Code Linux
on: [push,pull_request]
jobs:
test:
strategy:

View File

@ -1,5 +1,6 @@
name: Test Code Windows
on: [push,pull_request]
jobs:
pre:
runs-on: ubuntu-latest

View File

@ -0,0 +1,58 @@
name: Test Docker Compose Linux
on: [push,pull_request]
jobs:
test:
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.18.x
- name: Cache Go modules
uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Checkout code
uses: actions/checkout@v2
- name: cp setup.env
run: cp infrastructure_files/tests/setup.env infrastructure_files/
- name: run configure
working-directory: infrastructure_files
run: bash -x configure.sh
env:
CI_NETBIRD_AUTH0_DOMAIN: ${{ secrets.CI_NETBIRD_AUTH0_DOMAIN }}
CI_NETBIRD_AUTH0_CLIENT_ID: ${{ secrets.CI_NETBIRD_AUTH0_CLIENT_ID }}
CI_NETBIRD_AUTH0_AUDIENCE: testing.ci
- name: check values
working-directory: infrastructure_files
env:
CI_NETBIRD_AUTH0_DOMAIN: ${{ secrets.CI_NETBIRD_AUTH0_DOMAIN }}
CI_NETBIRD_AUTH0_CLIENT_ID: ${{ secrets.CI_NETBIRD_AUTH0_CLIENT_ID }}
CI_NETBIRD_AUTH0_AUDIENCE: testing.ci
run: |
grep AUTH0_DOMAIN docker-compose.yml | grep $CI_NETBIRD_AUTH0_DOMAIN
grep AUTH0_CLIENT_ID docker-compose.yml | grep $CI_NETBIRD_AUTH0_CLIENT_ID
grep AUTH0_AUDIENCE docker-compose.yml | grep $CI_NETBIRD_AUTH0_AUDIENCE
grep NETBIRD_MGMT_API_ENDPOINT docker-compose.yml | grep "http://localhost:33071"
grep NETBIRD_MGMT_GRPC_API_ENDPOINT docker-compose.yml | grep "http://localhost:33073"
- name: run docker compose up
working-directory: infrastructure_files
run: |
docker-compose up -d
sleep 5
- name: test running containers
run: |
count=$(docker compose ps --format json | jq '.[] | select(.Project | contains("infrastructure_files")) | .State' | grep -c running)
test $count -eq 4
working-directory: infrastructure_files

1
.gitignore vendored
View File

@ -10,3 +10,4 @@ infrastructure_files/management.json
infrastructure_files/docker-compose.yml
*.syso
client/.distfiles/
infrastructure_files/setup.env

View File

@ -1,104 +0,0 @@
### Table of contents
* [About Netbird](#about-netbird)
* [Why Wireguard with Netbird?](#why-wireguard-with-netbird)
* [Netbird vs. Traditional VPN](#netbird-vs-traditional-vpn)
* [High-level technology overview](#high-level-technology-overview)
* [Getting started](#getting-started)
### About Netbird
Netbird is an open-source VPN platform built on top of [WireGuard®](https://www.wireguard.com/) making it easy to create secure private networks for your organization or home.
It requires zero configuration effort leaving behind the hassle of opening ports, complex firewall rules, vpn gateways, and so forth.
There is no centralized VPN server with Netbird - your computers, devices, machines, and servers connect to each other directly over a fast encrypted tunnel.
It literally takes less than 5 minutes to provision a secure peer-to-peer VPN with Netbird. Check our [Quickstart Guide Video](https://www.youtube.com/watch?v=cWTsGUJAUaU) to see the setup in action.
### Why Wireguard with Netbird?
WireGuard is a modern and extremely fast VPN tunnel utilizing state-of-the-art [cryptography](https://www.wireguard.com/protocol/)
and Netbird uses Wireguard to establish a secure tunnel between machines.
Built with simplicity in mind, Wireguard ensures that traffic between two machines is encrypted and flowing, however, it requires a few things to be done beforehand.
First, in order to connect, the machines have to be configured.
On each machine, you need to generate private and public keys and prepare a WireGuard configuration file.
The configuration also includes a private IP address that should be unique per machine.
Secondly, to accept the incoming traffic, the machines have to trust each other.
The generated public keys have to be pre-shared on the machines.
This works similarly to SSH with its authorised_keys file.
Lastly, the connectivity between the machines has to be ensured.
To make machines reach one another, you are required to set a WireGuard endpoint property which indicates the IP address and port of the remote machine to connect to.
On many occasions, machines are hidden behind firewalls and NAT devices,
meaning that you may need to configure a port forwarding or open holes in your firewall to ensure the machines are reachable.
The undertakings mentioned above might not be complicated if you have just a few machines, but the complexity grows as the number of machines increases.
Netbird simplifies the setup by automatically generating private and public keys, assigning unique private IP addresses, and takes care of sharing public keys between the machines.
It is worth mentioning that the private key never leaves the machine.
So only the machine that owns the key can decrypt traffic addressed to it.
The same applies also to the relayed traffic mentioned below.
Furthermore, Netbird ensures connectivity by leveraging advanced [NAT traversal techniques](https://en.wikipedia.org/wiki/NAT_traversal)
and removing the necessity of port forwarding, opening holes in the firewall, and having a public static IP address.
In cases when a direct peer-to-peer connection isn't possible, all traffic is relayed securely between peers.
Netbird also monitors the connection health and restarts broken connections.
There are a few more things that we are working on to make secure private networks simple. A few examples are ACLs, MFA and activity monitoring.
Check out the WireGuard [Quick Start](https://www.wireguard.com/quickstart/) guide to learn more about configuring "plain" WireGuard without Netbird.
### Netbird vs. Traditional VPN
In the traditional VPN model, everything converges on a centralized, protected network where all the clients are connecting to a central VPN server.
An increasing amount of connections can easily overload the VPN server.
Even a short downtime of a server can cause expensive system disruptions, and a remote team's inability to work.
Centralized VPNs imply all the traffic going through the central server causing network delays and increased traffic usage.
Such systems require an experienced team to set up and maintain.
Configuring firewalls, setting up NATs, SSO integration, and managing access control lists can be a nightmare.
Traditional centralized VPNs are often compared to a [castle-and-moat](https://en.wikipedia.org/wiki/Moat) model
in which once accessed, user is trusted and can access critical infrastructure and resources without any restrictions.
Netbird decentralizes networks using direct point-to-point connections, as opposed to traditional models.
Consequently, network performance is increased since traffic flows directly between the machines bypassing VPN servers or gateways.
To achieve this, Netbird client applications employ signalling servers to find other machines and negotiate connections.
These are similar to the signaling servers used in [WebRTC](https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Signaling_and_video_calling#the_signaling_server)
Thanks to [NAT traversal techniques](https://en.wikipedia.org/wiki/NAT_traversal),
outlined in the [Why not just Wireguard?](#why-wireguard-with-netbird) section above,
Netbird installation doesn't require complex network and firewall configuration.
It just works, minimising the maintenance effort.
Finally, each machine or device in the Netbird network verifies incoming connections accepting only the trusted ones.
This is ensured by Wireguard's [Crypto Routing concept](https://www.wireguard.com/#cryptokey-routing).
### High-level technology overview
In essence, Netbird is an open source platform consisting of a collection of systems, responsible for handling peer-to-peer connections, tunneling and network management (IP, keys, ACLs, etc).
<p align="center">
<img src="media/high-level-dia.png" alt="high-level-dia" width="781"/>
</p>
Netbird uses open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE (WebRTC)](https://github.com/pion/ice), [Coturn](https://github.com/coturn/coturn),
and [software](https://github.com/netbirdio/netbird) developed by Netbird authors to make it all work together.
To learn more about Netbird architecture, please refer to the [architecture section](../docs/architecture.md).
### Getting Started
There are 2 ways of getting started with Netbird:
- use Cloud Managed version
- self-hosting
We recommend starting with the cloud managed version hosted at [app.netbird.io](https://app.netbird.io) - the quickest way to get familiar with the system.
See [Quickstart Guide](../docs/quickstart.md) for instructions.
If you don't want to use the managed version, check out our [Self-hosting Guide](../docs/self-hosting.md).

View File

@ -1,2 +0,0 @@
### Architecture
TODO

Binary file not shown.

Before

Width:  |  Height:  |  Size: 86 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

View File

@ -1,41 +0,0 @@
## Quickstart guide (Cloud Managed version)
Step-by-step video guide on YouTube:
[![IMAGE ALT TEXT](https://img.youtube.com/vi/cWTsGUJAUaU/0.jpg)](https://youtu.be/cWTsGUJAUaU "Netbird - secure private network in less than 5 minutes")
This guide describes how to create secure VPN and connect 2 machines peer-to-peer.
One machine is a Raspberry Pi Compute Module 4 hosted at home (Peer A), and the other one is a regular Ubuntu server running in the Data Center (Peer B).
Both machines are running Linux (Raspbian and Ubuntu respectively), but you could also use Mac or Windows operating systems.
1. Sign-up at [https://app.netbird.io/](https://app.netbird.io/)
You can use your email and password to sign-up or any available social login option (e.g., GitHub account)
<img src="media/auth.png" alt="auth" width="350"/>
2. After a successful login you will be redirected to the ```Peers``` screen which is empty because you don't have any peers yet.
Click ```Add peer``` to add a new machine.
<img src="media/empty-peers.png" alt="empty-peers" width="700"/>
3. Choose a setup key which will be used to associate your new machine with your account (in our case it is ```Default key```).
Choose your machine operating system (in our case it is ```Linux```) and proceed with the installation steps on the machine.
<img src="media/add-peer.png" alt="add-peer" width="700"/>
4. Repeat #3 for the 2nd machine.
5. Return to ```Peers``` and you should notice 2 new machines with status ```Connected```
<img src="media/peers.png" alt="peers" width="700"/>
6. To test the connection you could try pinging devices:
On Peer A:
```ping 100.64.0.2```
On Peer B:
```ping 100.64.0.1```
7. Done! You now have a secure peer-to-peer VPN configured.

View File

@ -1,106 +0,0 @@
### Self-hosting
Netbird is an open-source platform that can be self-hosted on your servers.
It relies on components developed by Netbird Authors [Management Service](https://github.com/netbirdio/netbird/tree/main/management), [Management UI Dashboard](https://github.com/netbirdio/dashboard), [Signal Service](https://github.com/netbirdio/netbird/tree/main/signal),
a 3rd party open-source STUN/TURN service [Coturn](https://github.com/coturn/coturn) and a 3rd party service [Auth0](https://auth0.com/).
All the components can be self-hosted except for the Auth0 service.
We chose Auth0 to "outsource" the user management part of the platform because we believe that implementing a proper user auth requires significant amount of time to make it right.
We focused on connectivity instead. It also offers an always free plan that should be ok for most users as its limits are high enough for most teams.
If you would like to learn more about the architecture please refer to the [Netbird Architecture section](architecture.md).
### Step-by-step video guide on YouTube:
[![IMAGE ALT TEXT](https://img.youtube.com/vi/Ofpgx5WhT0k/0.jpg)](https://youtu.be/Ofpgx5WhT0k "Netbird Self-Hosting Guide")
### Requirements
- Virtual machine offered by any cloud provider (e.g., AWS, DigitalOcean, Hetzner, Google Cloud, Azure ...).
- Any Unix OS.
- Docker Compose installed (see [Install Docker Compose](https://docs.docker.com/compose/install/)).
- Domain name pointing to the public IP address of your server.
- Netbird Open ports ```443, 33071, 33073, 10000``` (Dashboard, Management HTTP API, Management gRpc API, Signal gRpc) on your server.
- Coturn is used for relay using the STUN/TURN protocols. It requires a listening port, ```UDP 3478```, and range of ports,```UDP 49152-65535```, for dynamic relay connections. These are set as defaults in [setup file](https://github.com/netbirdio/netbird/blob/main/infrastructure_files/setup.env#L34), but can be configured to your requirements.
- Maybe a cup of coffee or tea :)
### Step-by-step guide
For this tutorial we will be using domain ```test.netbird.io``` which points to our Ubuntu 20.04 machine hosted at Hetzner.
1. Create Auth0 account at [auth0.com](https://auth0.com/).
2. Login to your server, clone Netbird repository:
```bash
git clone https://github.com/netbirdio/netbird.git netbird/
```
and switch to the ```netbird/infrastructure_files/``` folder that contains docker compose file:
```bash
cd netbird/infrastructure_files/
```
3. Prepare configuration files.
To simplify the setup we have prepared a script to substitute required properties in the [turnserver.conf.tmpl](../infrastructure_files/turnserver.conf.tmpl),[docker-compose.yml.tmpl](../infrastructure_files/docker-compose.yml.tmpl) and [management.json.tmpl](../infrastructure_files/management.json.tmpl) files.
The [setup.env](../infrastructure_files/setup.env) file contains the following properties that have to be filled:
```bash
# Dashboard domain. e.g. app.mydomain.com
NETBIRD_DOMAIN=""
# e.g. dev-24vkclam.us.auth0.com
NETBIRD_AUTH0_DOMAIN=""
# e.g. 61u3JMXRO0oOevc7gCkZLCwePQvT4lL0
NETBIRD_AUTH0_CLIENT_ID=""
# e.g. https://app.mydomain.com/ or https://app.mydomain.com,
# Make sure you used the exact same value for Identifier
# you used when creating your Auth0 API
NETBIRD_AUTH0_AUDIENCE=""
# e.g. hello@mydomain.com
NETBIRD_LETSENCRYPT_EMAIL=""
```
> Other options are available, but they are automatically updated.
Please follow the steps to get the values.
4. Configure ```NETBIRD_AUTH0_DOMAIN``` ```NETBIRD_AUTH0_CLIENT_ID``` ```NETBIRD_AUTH0_AUDIENCE``` properties.
* To obtain these, please use [Auth0 React SDK Guide](https://auth0.com/docs/quickstart/spa/react/01-login#configure-auth0) up until "Install the Auth0 React SDK".
:grey_exclamation: Use ```https://YOUR DOMAIN``` as ````Allowed Callback URLs````, ```Allowed Logout URLs```, ```Allowed Web Origins``` and ```Allowed Origins (CORS)```
* set the variables in the ```setup.env```
5. Configure ```NETBIRD_AUTH0_AUDIENCE``` property.
* Check [Auth0 Golang API Guide](https://auth0.com/docs/quickstart/backend/golang) to obtain AuthAudience.
* set the property in the ```setup.env``` file.
6. Configure ```NETBIRD_LETSENCRYPT_EMAIL``` property.
This can be any email address. [Let's Encrypt](https://letsencrypt.org/) will create an account while generating a new certificate.
7. Make sure all the properties set in the ```setup.env``` file and run:
```bash
./configure.sh
```
This will export all the properties as environment variables and generate ```docker-compose.yml``` and ```management.json``` files substituting required variables.
8. Run docker compose:
```bash
docker-compose up -d
```
9. Optionally check the logs by running:
```bash
docker-compose logs signal
docker-compose logs management
docker-compose logs coturn
docker-compose logs dashboard
10. Once the server is running, you can access the dashboard by https://$NETBIRD_DOMAIN
11. Adding a peer will require you to enter the management URL by following the steps in the page https://$NETBIRD_DOMAIN/add-peer and in the 3rd step:
```shell
sudo netbird up --setup-key <PASTE-SETUP-KEY> --management-url https://$NETBIRD_DOMAIN:33073
```

3
go.sum
View File

@ -896,12 +896,9 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXHCguynGhKjWVsIUt5cY=
golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664 h1:wEZYwx+kK+KlZ0hpvP2Ls1Xr4+RWnlzGFwPP0aiDjIU=
golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

View File

@ -1,19 +1,4 @@
# Dashboard domain and auth0 configuration
# Dashboard domain. e.g. app.mydomain.com
NETBIRD_DOMAIN=""
# e.g. dev-24vkclam.us.auth0.com
NETBIRD_AUTH0_DOMAIN=""
# e.g. 61u3JMXRO0oOevc7gCkZLCwePQvT4lL0
NETBIRD_AUTH0_CLIENT_ID=""
# e.g. https://app.mydomain.com/ or https://app.mydomain.com,
# Make sure you used the exact same value for Identifier
# you used when creating your Auth0 API
NETBIRD_AUTH0_AUDIENCE=""
# e.g. hello@mydomain.com
NETBIRD_LETSENCRYPT_EMAIL=""
## From this point, most settings are being done automatically, but you can edit if you need some customization
## Most settings are being done automatically with the sourced variables from setup.env, but you can edit if you need some customization
# Management API
@ -24,7 +9,7 @@ NETBIRD_MGMT_GRPC_API_PORT=33073
# Management API endpoint address, used by the Dashboard
NETBIRD_MGMT_API_ENDPOINT=https://$NETBIRD_DOMAIN:$NETBIRD_MGMT_API_PORT
# Management GRPC API endpoint address, used by the hosts to register
NETBIRD_MGMT_GRPC_API_ENDPOINT=https://$NETBIRD_DOMAIN:NETBIRD_MGMT_GRPC_API_PORT
NETBIRD_MGMT_GRPC_API_ENDPOINT=https://$NETBIRD_DOMAIN:$NETBIRD_MGMT_GRPC_API_PORT
# Management Certficate file path. These are generated by the Dashboard container
NETBIRD_MGMT_API_CERT_FILE="/etc/letsencrypt/live/$NETBIRD_DOMAIN/fullchain.pem"
# Management Certficate key file path.

View File

@ -1,10 +1,32 @@
#!/bin/bash
source setup.env
source base.setup.env
if ! which envsubst > /dev/null 2>&1
then
echo "envsubst is needed to run this script"
if [[ $(uname) == "Darwin" ]]
then
echo "you can install it with homebrew (https://brew.sh):"
echo "brew install gettext"
else
if which apt-get > /dev/null 2>&1
then
echo "you can install it by running"
echo "apt-get update && apt-get install gettext-base"
else
echo "you can install it by installing the package gettext with your package manager"
fi
fi
exit 1
fi
if [[ "x-$NETBIRD_DOMAIN" == "x-" ]]
then
echo NETBIRD_DOMAIN is not set, please update your setup.env file
echo If you are migrating from old versions, you migh need to update your variables prefixes from
echo WIRETRUSTEE_.. TO NETBIRD_
exit 1
fi

View File

@ -42,8 +42,8 @@ services:
ports:
- $NETBIRD_MGMT_GRPC_API_PORT:33073 #gRPC port
- $NETBIRD_MGMT_API_PORT:33071 #API port
# # port and command for Let's Encrypt validation
# - 443:443
# # port and command for Let's Encrypt validation without dashboard container
# - 443:443
# command: ["--letsencrypt-domain", "$NETBIRD_DOMAIN", "--log-file", "console"]
# Coturn
coturn:

View File

@ -0,0 +1,16 @@
## example file, you can copy this file to setup.env and update its values
##
# Dashboard domain and auth0 configuration
# Dashboard domain. e.g. app.mydomain.com
NETBIRD_DOMAIN=""
# e.g. dev-24vkclam.us.auth0.com
NETBIRD_AUTH0_DOMAIN=""
# e.g. 61u3JMXRO0oOevc7gCkZLCwePQvT4lL0
NETBIRD_AUTH0_CLIENT_ID=""
# e.g. https://app.mydomain.com/ or https://app.mydomain.com,
# Make sure you used the exact same value for Identifier
# you used when creating your Auth0 API
NETBIRD_AUTH0_AUDIENCE=""
# e.g. hello@mydomain.com
NETBIRD_LETSENCRYPT_EMAIL=""

View File

@ -0,0 +1,16 @@
## example file, you can copy this file to setup.env and update its values
##
# Dashboard domain and auth0 configuration
# Dashboard domain. e.g. app.mydomain.com
NETBIRD_DOMAIN="localhost"
# e.g. dev-24vkclam.us.auth0.com
NETBIRD_AUTH0_DOMAIN=$CI_NETBIRD_AUTH0_DOMAIN
# e.g. 61u3JMXRO0oOevc7gCkZLCwePQvT4lL0
NETBIRD_AUTH0_CLIENT_ID=$CI_NETBIRD_AUTH0_CLIENT_ID
# e.g. https://app.mydomain.com/ or https://app.mydomain.com,
# Make sure you used the exact same value for Identifier
# you used when creating your Auth0 API
NETBIRD_AUTH0_AUDIENCE=$CI_NETBIRD_AUTH0_AUDIENCE
# e.g. hello@mydomain.com
NETBIRD_LETSENCRYPT_EMAIL=""