[relay] Feature/relay integration (#2244)

This update adds new relay integration for NetBird clients. The new relay is based on web sockets and listens on a single port.

- Adds new relay implementation with websocket with single port relaying mechanism
- refactor peer connection logic, allowing upgrade and downgrade from/to P2P connection
- peer connections are faster since it connects first to relay and then upgrades to P2P
- maintains compatibility with old clients by not using the new relay
- updates infrastructure scripts with new relay service

.github/workflows/test-infrastructure-files.yml

@@ -150,6 +150,13 @@ jobs:
           grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000"
           grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP
           grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN"
+          # check relay values
+          grep "NB_EXPOSED_ADDRESS=$CI_NETBIRD_DOMAIN:33445" docker-compose.yml
+          grep "NB_LISTEN_ADDRESS=:33445" docker-compose.yml
+          grep '33445:33445' docker-compose.yml
+          grep -A 10 'relay:' docker-compose.yml | egrep 'NB_AUTH_SECRET=.+$'
+          grep -A 7 Relay management.json | grep "rel://$CI_NETBIRD_DOMAIN:33445"
+          grep -A 7 Relay management.json | egrep '"Secret": ".+"'
 
       - name: Install modules
         run: go mod tidy
@@ -175,6 +182,15 @@ jobs:
         run: |
           docker build -t netbirdio/signal:latest .
 
+      - name: Build relay binary
+        working-directory: relay
+        run: CGO_ENABLED=0 go build -o netbird-relay main.go
+
+      - name: Build relay docker image
+        working-directory: relay
+        run: |
+          docker build -t netbirdio/relay:latest .
+
       - name: run docker compose up
         working-directory: infrastructure_files/artifacts
         run: |
@@ -186,7 +202,7 @@ jobs:
       - name: test running containers
         run: |
           count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running)
-          test $count -eq 4 || docker compose logs
+          test $count -eq 5 || docker compose logs
         working-directory: infrastructure_files/artifacts
 
       - name: test geolocation databases
@@ -205,6 +221,9 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: handle insisting image   # remove after release
+        run: docker pull netbirdio/relay:latest || docker pull netbirdio/signal:latest && docker tag netbirdio/signal:latest netbirdio/relay:latest
+
       - name: run script with Zitadel PostgreSQL
         run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh
 
@@ -229,6 +248,9 @@ jobs:
       - name: test dashboard.env file gen postgres
         run: test -f dashboard.env
 
+      - name: test relay.env file gen postgres
+        run: test -f relay.env
+
       - name: test zdb.env file gen postgres
         run: test -f zdb.env
 
@@ -237,6 +259,9 @@ jobs:
           docker compose down --volumes --rmi all
           rm -rf docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json zdb.env
 
+      - name: handle insisting image  gen CockroachDB # remove after release
+        run: docker pull netbirdio/relay:latest || docker pull netbirdio/signal:latest && docker tag netbirdio/signal:latest netbirdio/relay:latest
+
       - name: run script with Zitadel CockroachDB
         run: bash -x infrastructure_files/getting-started-with-zitadel.sh
         env:
@@ -264,6 +289,9 @@ jobs:
       - name: test dashboard.env file gen CockroachDB
         run: test -f dashboard.env
 
+      - name: test relay.env file gen CockroachDB
+        run: test -f relay.env
+
   test-download-geolite2-script:
     runs-on: ubuntu-latest
     steps:

.goreleaser.yaml

@@ -80,6 +80,20 @@ builds:
       - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
     mod_timestamp: '{{ .CommitTimestamp }}'
 
+  - id: netbird-relay
+    dir: relay
+    env: [CGO_ENABLED=0]
+    binary: netbird-relay
+    goos:
+      - linux
+    goarch:
+      - amd64
+      - arm64
+      - arm
+    ldflags:
+      - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
+    mod_timestamp: '{{ .CommitTimestamp }}'
+
 archives:
   - builds:
       - netbird
@@ -161,6 +175,52 @@ dockers:
       - "--label=org.opencontainers.image.revision={{.FullCommit}}"
       - "--label=org.opencontainers.image.version={{.Version}}"
       - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-amd64
+    ids:
+      - netbird-relay
+    goarch: amd64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/amd64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+    ids:
+      - netbird-relay
+    goarch: arm64
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm64"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
+  - image_templates:
+      - netbirdio/relay:{{ .Version }}-arm
+    ids:
+      - netbird-relay
+    goarch: arm
+    goarm: 6
+    use: buildx
+    dockerfile: relay/Dockerfile
+    build_flag_templates:
+      - "--platform=linux/arm"
+      - "--label=org.opencontainers.image.created={{.Date}}"
+      - "--label=org.opencontainers.image.title={{.ProjectName}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=org.opencontainers.image.revision={{.FullCommit}}"
+      - "--label=org.opencontainers.image.version={{.Version}}"
+      - "--label=maintainer=dev@netbird.io"
   - image_templates:
       - netbirdio/signal:{{ .Version }}-amd64
     ids:
@@ -313,6 +373,18 @@ docker_manifests:
       - netbirdio/netbird:{{ .Version }}-arm
       - netbirdio/netbird:{{ .Version }}-amd64
 
+  - name_template: netbirdio/relay:{{ .Version }}
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
+  - name_template: netbirdio/relay:latest
+    image_templates:
+      - netbirdio/relay:{{ .Version }}-arm64v8
+      - netbirdio/relay:{{ .Version }}-arm
+      - netbirdio/relay:{{ .Version }}-amd64
+
   - name_template: netbirdio/signal:{{ .Version }}
     image_templates:
       - netbirdio/signal:{{ .Version }}-arm64v8

client/cmd/status.go

@@ -31,9 +31,9 @@ type peerStateDetailOutput struct {
 	Status                 string           `json:"status" yaml:"status"`
 	LastStatusUpdate       time.Time        `json:"lastStatusUpdate" yaml:"lastStatusUpdate"`
 	ConnType               string           `json:"connectionType" yaml:"connectionType"`
-	Direct                 bool             `json:"direct" yaml:"direct"`
 	IceCandidateType       iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"`
 	IceCandidateEndpoint   iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"`
+	RelayAddress           string           `json:"relayAddress" yaml:"relayAddress"`
 	LastWireguardHandshake time.Time        `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"`
 	TransferReceived       int64            `json:"transferReceived" yaml:"transferReceived"`
 	TransferSent           int64            `json:"transferSent" yaml:"transferSent"`
@@ -335,16 +335,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput {
 
 func mapPeers(peers []*proto.PeerState) peersStateOutput {
 	var peersStateDetail []peerStateDetailOutput
-	localICE := ""
-	remoteICE := ""
-	localICEEndpoint := ""
-	remoteICEEndpoint := ""
-	connType := ""
 	peersConnected := 0
-	lastHandshake := time.Time{}
-	transferReceived := int64(0)
-	transferSent := int64(0)
 	for _, pbPeerState := range peers {
+		localICE := ""
+		remoteICE := ""
+		localICEEndpoint := ""
+		remoteICEEndpoint := ""
+		relayServerAddress := ""
+		connType := ""
+		lastHandshake := time.Time{}
+		transferReceived := int64(0)
+		transferSent := int64(0)
+
 		isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String()
 		if skipDetailByFilters(pbPeerState, isPeerConnected) {
 			continue
@@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			if pbPeerState.Relayed {
 				connType = "Relayed"
 			}
+			relayServerAddress = pbPeerState.GetRelayAddress()
 			lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local()
 			transferReceived = pbPeerState.GetBytesRx()
 			transferSent = pbPeerState.GetBytesTx()
@@ -372,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 			Status:           pbPeerState.GetConnStatus(),
 			LastStatusUpdate: timeLocal,
 			ConnType:         connType,
-			Direct:           pbPeerState.GetDirect(),
 			IceCandidateType: iceCandidateType{
 				Local:  localICE,
 				Remote: remoteICE,
@@ -381,6 +383,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput {
 				Local:  localICEEndpoint,
 				Remote: remoteICEEndpoint,
 			},
+			RelayAddress:           relayServerAddress,
 			FQDN:                   pbPeerState.GetFqdn(),
 			LastWireguardHandshake: lastHandshake,
 			TransferReceived:       transferReceived,
@@ -641,9 +644,9 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 				"  Status: %s\n"+
 				"  -- detail --\n"+
 				"  Connection type: %s\n"+
-				"  Direct: %t\n"+
 				"  ICE candidate (Local/Remote): %s/%s\n"+
 				"  ICE candidate endpoints (Local/Remote): %s/%s\n"+
+				"  Relay server address: %s\n"+
 				"  Last connection update: %s\n"+
 				"  Last WireGuard handshake: %s\n"+
 				"  Transfer status (received/sent) %s/%s\n"+
@@ -655,11 +658,11 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo
 			peerState.PubKey,
 			peerState.Status,
 			peerState.ConnType,
-			peerState.Direct,
 			localICE,
 			remoteICE,
 			localICEEndpoint,
 			remoteICEEndpoint,
+			peerState.RelayAddress,
 			timeAgo(peerState.LastStatusUpdate),
 			timeAgo(peerState.LastWireguardHandshake),
 			toIEC(peerState.TransferReceived),

client/cmd/status_test.go

@@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)),
 				Relayed:                    false,
-				Direct:                     true,
 				LocalIceCandidateType:      "",
 				RemoteIceCandidateType:     "",
 				LocalIceCandidateEndpoint:  "",
@@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{
 				ConnStatus:                 "Connected",
 				ConnStatusUpdate:           timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)),
 				Relayed:                    true,
-				Direct:                     false,
 				LocalIceCandidateType:      "relay",
 				RemoteIceCandidateType:     "prflx",
 				LocalIceCandidateEndpoint:  "10.0.0.1:10001",
@@ -137,7 +135,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC),
 				ConnType:         "P2P",
-				Direct:           true,
 				IceCandidateType: iceCandidateType{
 					Local:  "",
 					Remote: "",
@@ -161,7 +158,6 @@ var overview = statusOutputOverview{
 				Status:           "Connected",
 				LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC),
 				ConnType:         "Relayed",
-				Direct:           false,
 				IceCandidateType: iceCandidateType{
 					Local:  "relay",
 					Remote: "prflx",
@@ -283,7 +279,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2001-01-01T01:01:01Z",
                 "connectionType": "P2P",
-                "direct": true,
                 "iceCandidateType": {
                   "local": "",
                   "remote": ""
@@ -292,6 +287,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "",
                   "remote": ""
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2001-01-01T01:01:02Z",
                 "transferReceived": 200,
                 "transferSent": 100,
@@ -308,7 +304,6 @@ func TestParsingToJSON(t *testing.T) {
                 "status": "Connected",
                 "lastStatusUpdate": "2002-02-02T02:02:02Z",
                 "connectionType": "Relayed",
-                "direct": false,
                 "iceCandidateType": {
                   "local": "relay",
                   "remote": "prflx"
@@ -317,6 +312,7 @@ func TestParsingToJSON(t *testing.T) {
                   "local": "10.0.0.1:10001",
                   "remote": "10.0.10.1:10002"
                 },
+				"relayAddress": "",
                 "lastWireguardHandshake": "2002-02-02T02:02:03Z",
                 "transferReceived": 2000,
                 "transferSent": 1000,
@@ -408,13 +404,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2001-01-01T01:01:01Z
           connectionType: P2P
-          direct: true
           iceCandidateType:
             local: ""
             remote: ""
           iceCandidateEndpoint:
             local: ""
             remote: ""
+          relayAddress: ""
           lastWireguardHandshake: 2001-01-01T01:01:02Z
           transferReceived: 200
           transferSent: 100
@@ -428,13 +424,13 @@ func TestParsingToYAML(t *testing.T) {
           status: Connected
           lastStatusUpdate: 2002-02-02T02:02:02Z
           connectionType: Relayed
-          direct: false
           iceCandidateType:
             local: relay
             remote: prflx
           iceCandidateEndpoint:
             local: 10.0.0.1:10001
             remote: 10.0.10.1:10002
+          relayAddress: ""
           lastWireguardHandshake: 2002-02-02T02:02:03Z
           transferReceived: 2000
           transferSent: 1000
@@ -505,9 +501,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: P2P
-  Direct: true
   ICE candidate (Local/Remote): -/-
   ICE candidate endpoints (Local/Remote): -/-
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 200 B/100 B
@@ -521,9 +517,9 @@ func TestParsingToDetail(t *testing.T) {
   Status: Connected
   -- detail --
   Connection type: Relayed
-  Direct: false
   ICE candidate (Local/Remote): relay/prflx
   ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002
+  Relay server address: 
   Last connection update: %s
   Last WireGuard handshake: %s
   Transfer status (received/sent) 2.0 KiB/1000 B

client/cmd/testutil_test.go

@@ -98,8 +98,9 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/cmd/up.go

@@ -168,7 +168,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error {
 	ctx, cancel = context.WithCancel(ctx)
 	SetupCloseHandler(ctx, cancel)
 
-	connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String()))
+	r := peer.NewRecorder(config.ManagementURL.String())
+	r.GetFullStatus()
+
+	connectClient := internal.NewConnectClient(ctx, config, r)
 	return connectClient.Run()
 }
 

client/internal/connect.go

@@ -26,6 +26,8 @@ import (
 	"github.com/netbirdio/netbird/iface"
 	mgm "github.com/netbirdio/netbird/management/client"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/relay/auth/hmac"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	signal "github.com/netbirdio/netbird/signal/client"
 	"github.com/netbirdio/netbird/util"
 	"github.com/netbirdio/netbird/version"
@@ -158,10 +160,8 @@ func (c *ConnectClient) run(
 	defer c.statusRecorder.ClientStop()
 	operation := func() error {
 		// if context cancelled we not start new backoff cycle
-		select {
+		if c.isContextCancelled() {
-		case <-c.ctx.Done():
 			return nil
-		default:
 		}
 
 		state.Set(StatusConnecting)
@@ -183,8 +183,7 @@ func (c *ConnectClient) run(
 
 		log.Debugf("connected to the Management service %s", c.config.ManagementURL.Host)
 		defer func() {
-			err = mgmClient.Close()
+			if err = mgmClient.Close(); err != nil {
-			if err != nil {
 				log.Warnf("failed to close the Management service client %v", err)
 			}
 		}()
@@ -208,7 +207,6 @@ func (c *ConnectClient) run(
 			KernelInterface: iface.WireGuardModuleIsLoaded(),
 			FQDN:            loginResp.GetPeerConfig().GetFqdn(),
 		}
-
 		c.statusRecorder.UpdateLocalPeerState(localPeerState)
 
 		signalURL := fmt.Sprintf("%s://%s",
@@ -241,6 +239,23 @@ func (c *ConnectClient) run(
 
 		c.statusRecorder.MarkSignalConnected()
 
+		relayURLs, token := parseRelayInfo(loginResp)
+		relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String())
+		if len(relayURLs) > 0 {
+			if token != nil {
+				if err := relayManager.UpdateToken(token); err != nil {
+					log.Errorf("failed to update token: %s", err)
+					return wrapErr(err)
+				}
+			}
+			log.Infof("connecting to the Relay service(s): %s", strings.Join(relayURLs, ", "))
+			if err = relayManager.Serve(); err != nil {
+				log.Error(err)
+				return wrapErr(err)
+			}
+			c.statusRecorder.SetRelayMgr(relayManager)
+		}
+
 		peerConfig := loginResp.GetPeerConfig()
 
 		engineConfig, err := createEngineConfig(myPrivateKey, c.config, peerConfig)
@@ -252,11 +267,11 @@ func (c *ConnectClient) run(
 		checks := loginResp.GetChecks()
 
 		c.engineMutex.Lock()
-		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, engineConfig, mobileDependency, c.statusRecorder, probes, checks)
+		c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, probes, checks)
+
 		c.engineMutex.Unlock()
 
-		err = c.engine.Start()
+		if err := c.engine.Start(); err != nil {
-		if err != nil {
 			log.Errorf("error while starting Netbird Connection Engine: %s", err)
 			return wrapErr(err)
 		}
@@ -296,6 +311,20 @@ func (c *ConnectClient) run(
 	return nil
 }
 
+func parseRelayInfo(loginResp *mgmProto.LoginResponse) ([]string, *hmac.Token) {
+	relayCfg := loginResp.GetWiretrusteeConfig().GetRelay()
+	if relayCfg == nil {
+		return nil, nil
+	}
+
+	token := &hmac.Token{
+		Payload:   relayCfg.GetTokenPayload(),
+		Signature: relayCfg.GetTokenSignature(),
+	}
+
+	return relayCfg.GetUrls(), token
+}
+
 func (c *ConnectClient) Engine() *Engine {
 	if c == nil {
 		return nil
@@ -320,6 +349,15 @@ func (c *ConnectClient) Stop() error {
 	return c.engine.Stop()
 }
 
+func (c *ConnectClient) isContextCancelled() bool {
+	select {
+	case <-c.ctx.Done():
+		return true
+	default:
+		return false
+	}
+}
+
 // createEngineConfig converts configuration received from Management Service to EngineConfig
 func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) {
 	nm := false

client/internal/engine.go

@@ -25,6 +25,7 @@ import (
 	"github.com/netbirdio/netbird/client/firewall/manager"
 	"github.com/netbirdio/netbird/client/internal/acl"
 	"github.com/netbirdio/netbird/client/internal/dns"
+
 	"github.com/netbirdio/netbird/client/internal/networkmonitor"
 	"github.com/netbirdio/netbird/client/internal/peer"
 	"github.com/netbirdio/netbird/client/internal/relay"
@@ -40,6 +41,8 @@ import (
 	mgm "github.com/netbirdio/netbird/management/client"
 	"github.com/netbirdio/netbird/management/domain"
 	mgmProto "github.com/netbirdio/netbird/management/proto"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	signal "github.com/netbirdio/netbird/signal/client"
 	sProto "github.com/netbirdio/netbird/signal/proto"
@@ -102,7 +105,8 @@ type EngineConfig struct {
 // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers.
 type Engine struct {
 	// signal is a Signal Service client
-	signal signal.Client
+	signal   signal.Client
+	signaler *peer.Signaler
 	// mgmClient is a Management Service client
 	mgmClient mgm.Client
 	// peerConns is a map that holds all the peers that are known to this peer
@@ -159,10 +163,10 @@ type Engine struct {
 
 	probes *ProbeHolder
 
-	wgConnWorker sync.WaitGroup
-
 	// checks are the client-applied posture checks that need to be evaluated on the client
 	checks []*mgmProto.Checks
+
+	relayManager *relayClient.Manager
 }
 
 // Peer is an instance of the Connection Peer
@@ -177,6 +181,7 @@ func NewEngine(
 	clientCancel context.CancelFunc,
 	signalClient signal.Client,
 	mgmClient mgm.Client,
+	relayManager *relayClient.Manager,
 	config *EngineConfig,
 	mobileDep MobileDependency,
 	statusRecorder *peer.Status,
@@ -187,6 +192,7 @@ func NewEngine(
 		clientCancel,
 		signalClient,
 		mgmClient,
+		relayManager,
 		config,
 		mobileDep,
 		statusRecorder,
@@ -201,18 +207,20 @@ func NewEngineWithProbes(
 	clientCancel context.CancelFunc,
 	signalClient signal.Client,
 	mgmClient mgm.Client,
+	relayManager *relayClient.Manager,
 	config *EngineConfig,
 	mobileDep MobileDependency,
 	statusRecorder *peer.Status,
 	probes *ProbeHolder,
 	checks []*mgmProto.Checks,
 ) *Engine {
-
 	return &Engine{
 		clientCtx:      clientCtx,
 		clientCancel:   clientCancel,
 		signal:         signalClient,
+		signaler:       peer.NewSignaler(signalClient, config.WgPrivateKey),
 		mgmClient:      mgmClient,
+		relayManager:   relayManager,
 		peerConns:      make(map[string]*peer.Conn),
 		syncMsgMux:     &sync.Mutex{},
 		config:         config,
@@ -260,11 +268,7 @@ func (e *Engine) Stop() error {
 	time.Sleep(500 * time.Millisecond)
 
 	e.close()
-
+	log.Infof("stopped Netbird Engine")
-	e.wgConnWorker.Wait()
-
-	log.Infof("Engine stopped")
-
 	return nil
 }
 
@@ -314,7 +318,7 @@ func (e *Engine) Start() error {
 	}
 	e.dnsServer = dnsServer
 
-	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, initialRoutes)
+	e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, e.relayManager, initialRoutes)
 	beforePeerHook, afterPeerHook, err := e.routeManager.Init()
 	if err != nil {
 		log.Errorf("Failed to initialize route manager: %s", err)
@@ -463,78 +467,25 @@ func (e *Engine) removePeer(peerKey string) error {
 	conn, exists := e.peerConns[peerKey]
 	if exists {
 		delete(e.peerConns, peerKey)
-		err := conn.Close()
+		conn.Close()
-		if err != nil {
-			switch err.(type) {
-			case *peer.ConnectionAlreadyClosedError:
-				return nil
-			default:
-				return err
-			}
-		}
 	}
 	return nil
 }
 
-func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client) error {
-	err := s.Send(&sProto.Message{
-		Key:       myKey.PublicKey().String(),
-		RemoteKey: remoteKey.String(),
-		Body: &sProto.Body{
-			Type:    sProto.Body_CANDIDATE,
-			Payload: candidate.Marshal(),
-		},
-	})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func sendSignal(message *sProto.Message, s signal.Client) error {
-	return s.Send(message)
-}
-
-// SignalOfferAnswer signals either an offer or an answer to remote peer
-func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client,
-	isAnswer bool) error {
-	var t sProto.Body_Type
-	if isAnswer {
-		t = sProto.Body_ANSWER
-	} else {
-		t = sProto.Body_OFFER
-	}
-
-	msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{
-		UFrag: offerAnswer.IceCredentials.UFrag,
-		Pwd:   offerAnswer.IceCredentials.Pwd,
-	}, t, offerAnswer.RosenpassPubKey, offerAnswer.RosenpassAddr)
-	if err != nil {
-		return err
-	}
-
-	err = s.Send(msg)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (e *Engine) handleSync(update *mgmProto.SyncResponse) error {
 	e.syncMsgMux.Lock()
 	defer e.syncMsgMux.Unlock()
 
 	if update.GetWiretrusteeConfig() != nil {
-		err := e.updateTURNs(update.GetWiretrusteeConfig().GetTurns())
+		wCfg := update.GetWiretrusteeConfig()
+		err := e.updateTURNs(wCfg.GetTurns())
 		if err != nil {
-			return err
+			return fmt.Errorf("update TURNs: %w", err)
 		}
 
-		err = e.updateSTUNs(update.GetWiretrusteeConfig().GetStuns())
+		err = e.updateSTUNs(wCfg.GetStuns())
 		if err != nil {
-			return err
+			return fmt.Errorf("update STUNs: %w", err)
 		}
 
 		var stunTurn []*stun.URI
@@ -542,6 +493,19 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error {
 		stunTurn = append(stunTurn, e.TURNs...)
 		e.stunTurn.Store(stunTurn)
 
+		relayMsg := wCfg.GetRelay()
+		if relayMsg != nil {
+			c := &auth.Token{
+				Payload:   relayMsg.GetTokenPayload(),
+				Signature: relayMsg.GetTokenSignature(),
+			}
+			if err := e.relayManager.UpdateToken(c); err != nil {
+				log.Errorf("failed to update relay token: %v", err)
+				return fmt.Errorf("update relay token: %w", err)
+			}
+		}
+
+		// todo update relay address in the relay manager
 		// todo update signal
 	}
 
@@ -937,58 +901,11 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error {
 			log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err)
 		}
 
-		e.wgConnWorker.Add(1)
+		conn.Open()
-		go e.connWorker(conn, peerKey)
 	}
 	return nil
 }
 
-func (e *Engine) connWorker(conn *peer.Conn, peerKey string) {
-	defer e.wgConnWorker.Done()
-	for {
-
-		// randomize starting time a bit
-		minValue := 500
-		maxValue := 2000
-		duration := time.Duration(rand.Intn(maxValue-minValue)+minValue) * time.Millisecond
-		select {
-		case <-e.ctx.Done():
-			return
-		case <-time.After(duration):
-		}
-
-		// if peer has been removed -> give up
-		if !e.peerExists(peerKey) {
-			log.Debugf("peer %s doesn't exist anymore, won't retry connection", peerKey)
-			return
-		}
-
-		if !e.signal.Ready() {
-			log.Infof("signal client isn't ready, skipping connection attempt %s", peerKey)
-			continue
-		}
-
-		err := conn.Open(e.ctx)
-		if err != nil {
-			log.Debugf("connection to peer %s failed: %v", peerKey, err)
-			var connectionClosedError *peer.ConnectionClosedError
-			switch {
-			case errors.As(err, &connectionClosedError):
-				// conn has been forced to close, so we exit the loop
-				return
-			default:
-			}
-		}
-	}
-}
-
-func (e *Engine) peerExists(peerKey string) bool {
-	e.syncMsgMux.Lock()
-	defer e.syncMsgMux.Unlock()
-	_, ok := e.peerConns[peerKey]
-	return ok
-}
-
 func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) {
 	log.Debugf("creating peer connection %s", pubKey)
 
@@ -1040,37 +957,12 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e
 		},
 	}
 
-	peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover)
+	peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager)
 	if err != nil {
 		return nil, err
 	}
 
-	wgPubKey, err := wgtypes.ParseKey(pubKey)
-	if err != nil {
-		return nil, err
-	}
-
-	signalOffer := func(offerAnswer peer.OfferAnswer) error {
-		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false)
-	}
-
-	signalCandidate := func(candidate ice.Candidate) error {
-		return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal)
-	}
-
-	signalAnswer := func(offerAnswer peer.OfferAnswer) error {
-		return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true)
-	}
-
-	peerConn.SetSignalCandidate(signalCandidate)
-	peerConn.SetSignalOffer(signalOffer)
-	peerConn.SetSignalAnswer(signalAnswer)
-	peerConn.SetSendSignalMessage(func(message *sProto.Message) error {
-		return sendSignal(message, e.signal)
-	})
-
 	if e.rpManager != nil {
-
 		peerConn.SetOnConnected(e.rpManager.OnConnected)
 		peerConn.SetOnDisconnected(e.rpManager.OnDisconnected)
 	}
@@ -1113,6 +1005,7 @@ func (e *Engine) receiveSignalEvents() {
 					Version:         msg.GetBody().GetNetBirdVersion(),
 					RosenpassPubKey: rosenpassPubKey,
 					RosenpassAddr:   rosenpassAddr,
+					RelaySrvAddress: msg.GetBody().GetRelayServerAddress(),
 				})
 			case sProto.Body_ANSWER:
 				remoteCred, err := signal.UnMarshalCredential(msg)
@@ -1135,6 +1028,7 @@ func (e *Engine) receiveSignalEvents() {
 					Version:         msg.GetBody().GetNetBirdVersion(),
 					RosenpassPubKey: rosenpassPubKey,
 					RosenpassAddr:   rosenpassAddr,
+					RelaySrvAddress: msg.GetBody().GetRelayServerAddress(),
 				})
 			case sProto.Body_CANDIDATE:
 				candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload)
@@ -1143,7 +1037,7 @@ func (e *Engine) receiveSignalEvents() {
 					return err
 				}
 
-				conn.OnRemoteCandidate(candidate, e.GetClientRoutes())
+				go conn.OnRemoteCandidate(candidate, e.GetClientRoutes())
 			case sProto.Body_MODE:
 			}
 
@@ -1442,7 +1336,7 @@ func (e *Engine) receiveProbeEvents() {
 
 			for _, peer := range e.peerConns {
 				key := peer.GetKey()
-				wgStats, err := peer.GetConf().WgConfig.WgInterface.GetStats(key)
+				wgStats, err := peer.WgConfig().WgInterface.GetStats(key)
 				if err != nil {
 					log.Debugf("failed to get wg stats for peer %s: %s", key, err)
 				}

client/internal/engine_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/telemetry"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	signal "github.com/netbirdio/netbird/signal/client"
 	"github.com/netbirdio/netbird/signal/proto"
@@ -59,6 +60,12 @@ var (
 	}
 )
 
+func TestMain(m *testing.M) {
+	_ = util.InitLog("debug", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestEngine_SSH(t *testing.T) {
 	// todo resolve test execution on freebsd
 	if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" {
@@ -74,13 +81,23 @@ func TestEngine_SSH(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
-		WgIfaceName:      "utun101",
+	engine := NewEngine(
-		WgAddr:           "100.64.0.1/24",
+		ctx, cancel,
-		WgPrivateKey:     key,
+		&signal.MockClient{},
-		WgPort:           33100,
+		&mgmt.MockClient{},
-		ServerSSHAllowed: true,
+		relayMgr,
-	}, MobileDependency{}, peer.NewRecorder("https://mgm"), nil)
+		&EngineConfig{
+			WgIfaceName:      "utun101",
+			WgAddr:           "100.64.0.1/24",
+			WgPrivateKey:     key,
+			WgPort:           33100,
+			ServerSSHAllowed: true,
+		},
+		MobileDependency{},
+		peer.NewRecorder("https://mgm"),
+		nil,
+	)
 
 	engine.dnsServer = &dns.MockServer{
 		UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil },
@@ -209,12 +226,21 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
-		WgIfaceName:  "utun102",
+	engine := NewEngine(
-		WgAddr:       "100.64.0.1/24",
+		ctx, cancel,
-		WgPrivateKey: key,
+		&signal.MockClient{},
-		WgPort:       33100,
+		&mgmt.MockClient{},
-	}, MobileDependency{}, peer.NewRecorder("https://mgm"), nil)
+		relayMgr,
+		&EngineConfig{
+			WgIfaceName:  "utun102",
+			WgAddr:       "100.64.0.1/24",
+			WgPrivateKey: key,
+			WgPort:       33100,
+		},
+		MobileDependency{},
+		peer.NewRecorder("https://mgm"),
+		nil)
 
 	wgIface := &iface.MockWGIface{
 		RemovePeerFunc: func(peerKey string) error {
@@ -222,7 +248,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) {
 		},
 	}
 	engine.wgInterface = wgIface
-	engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, nil)
+	engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, relayMgr, nil)
 	engine.dnsServer = &dns.MockServer{
 		UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil },
 	}
@@ -404,8 +430,8 @@ func TestEngine_Sync(t *testing.T) {
 		}
 		return nil
 	}
-
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
-	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, &EngineConfig{
+	engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, relayMgr, &EngineConfig{
 		WgIfaceName:  "utun103",
 		WgAddr:       "100.64.0.1/24",
 		WgPrivateKey: key,
@@ -564,7 +590,8 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
 			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
 			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
 
-			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+			relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{
 				WgIfaceName:  wgIfaceName,
 				WgAddr:       wgAddr,
 				WgPrivateKey: key,
@@ -734,7 +761,8 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
 			wgIfaceName := fmt.Sprintf("utun%d", 104+n)
 			wgAddr := fmt.Sprintf("100.66.%d.1/24", n)
 
-			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{
+			relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+			engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{
 				WgIfaceName:  wgIfaceName,
 				WgAddr:       wgAddr,
 				WgPrivateKey: key,
@@ -1012,7 +1040,8 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin
 		WgPort:       wgPort,
 	}
 
-	e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil
+	relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String())
+	e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, relayMgr, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil
 	e.ctx = ctx
 	return e, err
 }
@@ -1046,6 +1075,11 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 	config := &server.Config{
 		Stuns:      []*server.Host{},
 		TURNConfig: &server.TURNConfig{},
+		Relay: &server.Relay{
+			Addresses:      []string{"127.0.0.1:1234"},
+			CredentialsTTL: util.Duration{Duration: time.Hour},
+			Secret:         "222222222222222222",
+		},
 		Signal: &server.Host{
 			Proto: "http",
 			URI:   "localhost:10000",
@@ -1080,8 +1114,9 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error)
 	if err != nil {
 		return nil, "", err
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		return nil, "", err
 	}

client/internal/peer/conn_test.go

@@ -2,6 +2,7 @@ package peer
 
 import (
 	"context"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -11,6 +12,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 	"github.com/netbirdio/netbird/client/internal/wgproxy"
 	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/util"
 )
 
 var connConf = ConnConfig{
@@ -23,6 +25,12 @@ var connConf = ConnConfig{
 	},
 }
 
+func TestMain(m *testing.M) {
+	_ = util.InitLog("trace", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
 func TestNewConn_interfaceFilter(t *testing.T) {
 	ignore := []string{iface.WgInterfaceDefault, "tun0", "zt", "ZeroTier", "utun", "wg", "ts",
 		"Tailscale", "tailscale"}
@@ -40,7 +48,7 @@ func TestConn_GetKey(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -55,7 +63,7 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -63,7 +71,7 @@ func TestConn_OnRemoteOffer(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
-		<-conn.remoteOffersCh
+		<-conn.handshaker.remoteOffersCh
 		wg.Done()
 	}()
 
@@ -92,7 +100,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
@@ -100,7 +108,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) {
 	wg := sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
-		<-conn.remoteAnswerCh
+		<-conn.handshaker.remoteAnswerCh
 		wg.Done()
 	}()
 
@@ -128,58 +136,33 @@ func TestConn_Status(t *testing.T) {
 	defer func() {
 		_ = wgProxyFactory.Free()
 	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
+	conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil)
 	if err != nil {
 		return
 	}
 
 	tables := []struct {
-		name   string
+		name        string
-		status ConnStatus
+		statusIce   ConnStatus
-		want   ConnStatus
+		statusRelay ConnStatus
+		want        ConnStatus
 	}{
-		{"StatusConnected", StatusConnected, StatusConnected},
+		{"StatusConnected", StatusConnected, StatusConnected, StatusConnected},
-		{"StatusDisconnected", StatusDisconnected, StatusDisconnected},
+		{"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected},
-		{"StatusConnecting", StatusConnecting, StatusConnecting},
+		{"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting},
+		{"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting},
+		{"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected},
+		{"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting},
+		{"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected},
 	}
 
 	for _, table := range tables {
 		t.Run(table.name, func(t *testing.T) {
-			conn.status = table.status
+			conn.statusICE = table.statusIce
+			conn.statusRelay = table.statusRelay
 
 			got := conn.Status()
 			assert.Equal(t, got, table.want, "they should be equal")
 		})
 	}
 }
-
-func TestConn_Close(t *testing.T) {
-	wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort)
-	defer func() {
-		_ = wgProxyFactory.Free()
-	}()
-	conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil)
-	if err != nil {
-		return
-	}
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go func() {
-		<-conn.closeCh
-		wg.Done()
-	}()
-
-	go func() {
-		for {
-			err := conn.Close()
-			if err != nil {
-				continue
-			} else {
-				return
-			}
-		}
-	}()
-
-	wg.Wait()
-}

client/internal/peer/handshaker.go

@@ -0,0 +1,192 @@
+package peer
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/version"
+)
+
+var (
+	ErrSignalIsNotReady = errors.New("signal is not ready")
+)
+
+// IceCredentials ICE protocol credentials struct
+type IceCredentials struct {
+	UFrag string
+	Pwd   string
+}
+
+// OfferAnswer represents a session establishment offer or answer
+type OfferAnswer struct {
+	IceCredentials IceCredentials
+	// WgListenPort is a remote WireGuard listen port.
+	// This field is used when establishing a direct WireGuard connection without any proxy.
+	// We can set the remote peer's endpoint with this port.
+	WgListenPort int
+
+	// Version of NetBird Agent
+	Version string
+	// RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message
+	// This value is the local Rosenpass server public key when sending the message
+	RosenpassPubKey []byte
+	// RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message
+	// This value is the local Rosenpass server address when sending the message
+	RosenpassAddr string
+
+	// relay server address
+	RelaySrvAddress string
+}
+
+type Handshaker struct {
+	mu                  sync.Mutex
+	ctx                 context.Context
+	log                 *log.Entry
+	config              ConnConfig
+	signaler            *Signaler
+	ice                 *WorkerICE
+	relay               *WorkerRelay
+	onNewOfferListeners []func(*OfferAnswer)
+
+	// remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection
+	remoteOffersCh chan OfferAnswer
+	// remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection
+	remoteAnswerCh chan OfferAnswer
+}
+
+func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker {
+	return &Handshaker{
+		ctx:            ctx,
+		log:            log,
+		config:         config,
+		signaler:       signaler,
+		ice:            ice,
+		relay:          relay,
+		remoteOffersCh: make(chan OfferAnswer),
+		remoteAnswerCh: make(chan OfferAnswer),
+	}
+}
+
+func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAnswer)) {
+	h.onNewOfferListeners = append(h.onNewOfferListeners, offer)
+}
+
+func (h *Handshaker) Listen() {
+	for {
+		h.log.Debugf("wait for remote offer confirmation")
+		remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation()
+		if err != nil {
+			var connectionClosedError *ConnectionClosedError
+			if errors.As(err, &connectionClosedError) {
+				h.log.Tracef("stop handshaker")
+				return
+			}
+			h.log.Errorf("failed to received remote offer confirmation: %s", err)
+			continue
+		}
+
+		h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort)
+		for _, listener := range h.onNewOfferListeners {
+			go listener(remoteOfferAnswer)
+		}
+	}
+}
+
+func (h *Handshaker) SendOffer() error {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.sendOffer()
+}
+
+// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
+// doesn't block, discards the message if connection wasn't ready
+func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool {
+	select {
+	case h.remoteOffersCh <- offer:
+		return true
+	default:
+		h.log.Debugf("OnRemoteOffer skipping message because is not ready")
+		// connection might not be ready yet to receive so we ignore the message
+		return false
+	}
+}
+
+// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
+// doesn't block, discards the message if connection wasn't ready
+func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool {
+	select {
+	case h.remoteAnswerCh <- answer:
+		return true
+	default:
+		// connection might not be ready yet to receive so we ignore the message
+		h.log.Debugf("OnRemoteAnswer skipping message because is not ready")
+		return false
+	}
+}
+
+func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) {
+	select {
+	case remoteOfferAnswer := <-h.remoteOffersCh:
+		// received confirmation from the remote peer -> ready to proceed
+		err := h.sendAnswer()
+		if err != nil {
+			return nil, err
+		}
+		return &remoteOfferAnswer, nil
+	case remoteOfferAnswer := <-h.remoteAnswerCh:
+		return &remoteOfferAnswer, nil
+	case <-h.ctx.Done():
+		// closed externally
+		return nil, NewConnectionClosedError(h.config.Key)
+	}
+}
+
+// sendOffer prepares local user credentials and signals them to the remote peer
+func (h *Handshaker) sendOffer() error {
+	if !h.signaler.Ready() {
+		return ErrSignalIsNotReady
+	}
+
+	iceUFrag, icePwd := h.ice.GetLocalUserCredentials()
+	offer := OfferAnswer{
+		IceCredentials:  IceCredentials{iceUFrag, icePwd},
+		WgListenPort:    h.config.LocalWgPort,
+		Version:         version.NetbirdVersion(),
+		RosenpassPubKey: h.config.RosenpassPubKey,
+		RosenpassAddr:   h.config.RosenpassAddr,
+	}
+
+	addr, err := h.relay.RelayInstanceAddress()
+	if err == nil {
+		offer.RelaySrvAddress = addr
+	}
+
+	return h.signaler.SignalOffer(offer, h.config.Key)
+}
+
+func (h *Handshaker) sendAnswer() error {
+	h.log.Debugf("sending answer")
+	uFrag, pwd := h.ice.GetLocalUserCredentials()
+
+	answer := OfferAnswer{
+		IceCredentials:  IceCredentials{uFrag, pwd},
+		WgListenPort:    h.config.LocalWgPort,
+		Version:         version.NetbirdVersion(),
+		RosenpassPubKey: h.config.RosenpassPubKey,
+		RosenpassAddr:   h.config.RosenpassAddr,
+	}
+	addr, err := h.relay.RelayInstanceAddress()
+	if err == nil {
+		answer.RelaySrvAddress = addr
+	}
+
+	err = h.signaler.SignalAnswer(answer, h.config.Key)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

client/internal/peer/signaler.go

@@ -0,0 +1,70 @@
+package peer
+
+import (
+	"github.com/pion/ice/v3"
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+
+	signal "github.com/netbirdio/netbird/signal/client"
+	sProto "github.com/netbirdio/netbird/signal/proto"
+)
+
+type Signaler struct {
+	signal       signal.Client
+	wgPrivateKey wgtypes.Key
+}
+
+func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler {
+	return &Signaler{
+		signal:       signal,
+		wgPrivateKey: wgPrivateKey,
+	}
+}
+
+func (s *Signaler) SignalOffer(offer OfferAnswer, remoteKey string) error {
+	return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER)
+}
+
+func (s *Signaler) SignalAnswer(offer OfferAnswer, remoteKey string) error {
+	return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER)
+}
+
+func (s *Signaler) SignalICECandidate(candidate ice.Candidate, remoteKey string) error {
+	return s.signal.Send(&sProto.Message{
+		Key:       s.wgPrivateKey.PublicKey().String(),
+		RemoteKey: remoteKey,
+		Body: &sProto.Body{
+			Type:    sProto.Body_CANDIDATE,
+			Payload: candidate.Marshal(),
+		},
+	})
+}
+
+func (s *Signaler) Ready() bool {
+	return s.signal.Ready()
+}
+
+// SignalOfferAnswer signals either an offer or an answer to remote peer
+func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error {
+	msg, err := signal.MarshalCredential(
+		s.wgPrivateKey,
+		offerAnswer.WgListenPort,
+		remoteKey,
+		&signal.Credential{
+			UFrag: offerAnswer.IceCredentials.UFrag,
+			Pwd:   offerAnswer.IceCredentials.Pwd,
+		},
+		bodyType,
+		offerAnswer.RosenpassPubKey,
+		offerAnswer.RosenpassAddr,
+		offerAnswer.RelaySrvAddress)
+	if err != nil {
+		return err
+	}
+
+	err = s.signal.Send(msg)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

client/internal/peer/status.go

@@ -3,6 +3,7 @@ package peer
 import (
 	"errors"
 	"net/netip"
+	"slices"
 	"sync"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/relay"
 	"github.com/netbirdio/netbird/iface"
 	"github.com/netbirdio/netbird/management/domain"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 )
 
 // State contains the latest state of a peer
@@ -24,11 +26,11 @@ type State struct {
 	ConnStatus                 ConnStatus
 	ConnStatusUpdate           time.Time
 	Relayed                    bool
-	Direct                     bool
 	LocalIceCandidateType      string
 	RemoteIceCandidateType     string
 	LocalIceCandidateEndpoint  string
 	RemoteIceCandidateEndpoint string
+	RelayServerAddress         string
 	LastWireguardHandshake     time.Time
 	BytesTx                    int64
 	BytesRx                    int64
@@ -142,6 +144,8 @@ type Status struct {
 	// Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events
 	// set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications()
 	peerListChangedForNotification bool
+
+	relayMgr *relayClient.Manager
 }
 
 // NewRecorder returns a new Status instance
@@ -156,6 +160,12 @@ func NewRecorder(mgmAddress string) *Status {
 	}
 }
 
+func (d *Status) SetRelayMgr(manager *relayClient.Manager) {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+	d.relayMgr = manager
+}
+
 // ReplaceOfflinePeers replaces
 func (d *Status) ReplaceOfflinePeers(replacement []State) {
 	d.mux.Lock()
@@ -231,17 +241,17 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 		peerState.SetRoutes(receivedState.GetRoutes())
 	}
 
-	skipNotification := shouldSkipNotify(receivedState, peerState)
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
 
 	if receivedState.ConnStatus != peerState.ConnStatus {
 		peerState.ConnStatus = receivedState.ConnStatus
 		peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
-		peerState.Direct = receivedState.Direct
 		peerState.Relayed = receivedState.Relayed
 		peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
 		peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
 		peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
 		peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+		peerState.RelayServerAddress = receivedState.RelayServerAddress
 		peerState.RosenpassEnabled = receivedState.RosenpassEnabled
 	}
 
@@ -261,6 +271,146 @@ func (d *Status) UpdatePeerState(receivedState State) error {
 	return nil
 }
 
+func (d *Status) UpdatePeerICEState(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	if receivedState.IP != "" {
+		peerState.IP = receivedState.IP
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.Relayed = receivedState.Relayed
+	peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
+	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
+	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
+	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+	peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerRelayedState(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.Relayed = receivedState.Relayed
+	peerState.RelayServerAddress = receivedState.RelayServerAddress
+	peerState.RosenpassEnabled = receivedState.RosenpassEnabled
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.Relayed = receivedState.Relayed
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.RelayServerAddress = ""
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
+func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error {
+	d.mux.Lock()
+	defer d.mux.Unlock()
+
+	peerState, ok := d.peers[receivedState.PubKey]
+	if !ok {
+		return errors.New("peer doesn't exist")
+	}
+
+	skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState)
+
+	peerState.ConnStatus = receivedState.ConnStatus
+	peerState.Relayed = receivedState.Relayed
+	peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate
+	peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType
+	peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType
+	peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint
+	peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint
+
+	d.peers[receivedState.PubKey] = peerState
+
+	if skipNotification {
+		return nil
+	}
+
+	ch, found := d.changeNotify[receivedState.PubKey]
+	if found && ch != nil {
+		close(ch)
+		d.changeNotify[receivedState.PubKey] = nil
+	}
+
+	d.notifyPeerListChanged()
+	return nil
+}
+
 // UpdateWireGuardPeerState updates the WireGuard bits of the peer state
 func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) error {
 	d.mux.Lock()
@@ -280,13 +430,13 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats)
 	return nil
 }
 
-func shouldSkipNotify(received, curr State) bool {
+func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool {
 	switch {
-	case received.ConnStatus == StatusConnecting:
+	case receivedConnStatus == StatusConnecting:
 		return true
-	case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
+	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting:
 		return true
-	case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
+	case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected:
 		return curr.IP != ""
 	default:
 		return false
@@ -502,8 +652,35 @@ func (d *Status) GetSignalState() SignalState {
 	}
 }
 
+// GetRelayStates returns the stun/turn/permanent relay states
 func (d *Status) GetRelayStates() []relay.ProbeResult {
-	return d.relayStates
+	if d.relayMgr == nil {
+		return d.relayStates
+	}
+
+	// extend the list of stun, turn servers with relay address
+	relayStates := slices.Clone(d.relayStates)
+
+	var relayState relay.ProbeResult
+
+	// if the server connection is not established then we will use the general address
+	// in case of connection we will use the instance specific address
+	instanceAddr, err := d.relayMgr.RelayInstanceAddress()
+	if err != nil {
+		// TODO add their status
+		if errors.Is(err, relayClient.ErrRelayClientNotConnected) {
+			for _, r := range d.relayMgr.ServerURLs() {
+				relayStates = append(relayStates, relay.ProbeResult{
+					URI: r,
+				})
+			}
+			return relayStates
+		}
+		relayState.Err = err
+	}
+
+	relayState.URI = instanceAddr
+	return append(relayStates, relayState)
 }
 
 func (d *Status) GetDNSStates() []NSGroupState {
@@ -535,7 +712,6 @@ func (d *Status) GetFullStatus() FullStatus {
 	}
 
 	fullStatus.Peers = append(fullStatus.Peers, d.offlinePeers...)
-
 	return fullStatus
 }
 

client/internal/peer/status_test.go

@@ -2,8 +2,8 @@ package peer
 
 import (
 	"errors"
-	"testing"
 	"sync"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -43,7 +43,7 @@ func TestUpdatePeerState(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -64,7 +64,7 @@ func TestStatus_UpdatePeerFQDN(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -83,7 +83,7 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState
@@ -108,7 +108,7 @@ func TestRemovePeer(t *testing.T) {
 	status := NewRecorder("https://mgm")
 	peerState := State{
 		PubKey: key,
-		Mux:	new(sync.RWMutex),
+		Mux:    new(sync.RWMutex),
 	}
 
 	status.peers[key] = peerState

client/internal/peer/stdnet.go

@@ -6,6 +6,6 @@ import (
 	"github.com/netbirdio/netbird/client/internal/stdnet"
 )
 
-func (conn *Conn) newStdNet() (*stdnet.Net, error) {
+func (w *WorkerICE) newStdNet() (*stdnet.Net, error) {
-	return stdnet.NewNet(conn.config.ICEConfig.InterfaceBlackList)
+	return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList)
 }

client/internal/peer/stdnet_android.go

@@ -2,6 +2,6 @@ package peer
 
 import "github.com/netbirdio/netbird/client/internal/stdnet"
 
-func (conn *Conn) newStdNet() (*stdnet.Net, error) {
+func (w *WorkerICE) newStdNet() (*stdnet.Net, error) {
-	return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.config.ICEConfig.InterfaceBlackList)
+	return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList)
 }

client/internal/peer/worker_ice.go

@@ -0,0 +1,470 @@
+package peer
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/pion/ice/v3"
+	"github.com/pion/randutil"
+	"github.com/pion/stun/v2"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/client/internal/stdnet"
+	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/iface/bind"
+	"github.com/netbirdio/netbird/route"
+)
+
+const (
+	iceKeepAliveDefault           = 4 * time.Second
+	iceDisconnectedTimeoutDefault = 6 * time.Second
+	// iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package
+	iceRelayAcceptanceMinWaitDefault = 2 * time.Second
+
+	lenUFrag   = 16
+	lenPwd     = 32
+	runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+)
+
+var (
+	failedTimeout = 6 * time.Second
+)
+
+type ICEConfig struct {
+	// StunTurn is a list of STUN and TURN URLs
+	StunTurn *atomic.Value // []*stun.URI
+
+	// InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering
+	// (e.g. if eth0 is in the list, host candidate of this interface won't be used)
+	InterfaceBlackList   []string
+	DisableIPv6Discovery bool
+
+	UDPMux      ice.UDPMux
+	UDPMuxSrflx ice.UniversalUDPMux
+
+	NATExternalIPs []string
+}
+
+type ICEConnInfo struct {
+	RemoteConn                 net.Conn
+	RosenpassPubKey            []byte
+	RosenpassAddr              string
+	LocalIceCandidateType      string
+	RemoteIceCandidateType     string
+	RemoteIceCandidateEndpoint string
+	LocalIceCandidateEndpoint  string
+	Relayed                    bool
+	RelayedOnLocal             bool
+}
+
+type WorkerICECallbacks struct {
+	OnConnReady     func(ConnPriority, ICEConnInfo)
+	OnStatusChanged func(ConnStatus)
+}
+
+type WorkerICE struct {
+	ctx               context.Context
+	log               *log.Entry
+	config            ConnConfig
+	signaler          *Signaler
+	iFaceDiscover     stdnet.ExternalIFaceDiscover
+	statusRecorder    *Status
+	hasRelayOnLocally bool
+	conn              WorkerICECallbacks
+
+	selectedPriority ConnPriority
+
+	agent    *ice.Agent
+	muxAgent sync.Mutex
+
+	StunTurn []*stun.URI
+
+	sentExtraSrflx bool
+
+	localUfrag string
+	localPwd   string
+}
+
+func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) {
+	w := &WorkerICE{
+		ctx:               ctx,
+		log:               log,
+		config:            config,
+		signaler:          signaler,
+		iFaceDiscover:     ifaceDiscover,
+		statusRecorder:    statusRecorder,
+		hasRelayOnLocally: hasRelayOnLocally,
+		conn:              callBacks,
+	}
+
+	localUfrag, localPwd, err := generateICECredentials()
+	if err != nil {
+		return nil, err
+	}
+	w.localUfrag = localUfrag
+	w.localPwd = localPwd
+	return w, nil
+}
+
+func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
+	w.log.Debugf("OnNewOffer for ICE")
+	w.muxAgent.Lock()
+
+	if w.agent != nil {
+		w.log.Debugf("agent already exists, skipping the offer")
+		w.muxAgent.Unlock()
+		return
+	}
+
+	var preferredCandidateTypes []ice.CandidateType
+	if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" {
+		w.selectedPriority = connPriorityICEP2P
+		preferredCandidateTypes = candidateTypesP2P()
+	} else {
+		w.selectedPriority = connPriorityICETurn
+		preferredCandidateTypes = candidateTypes()
+	}
+
+	w.log.Debugf("recreate ICE agent")
+	agentCtx, agentCancel := context.WithCancel(w.ctx)
+	agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes)
+	if err != nil {
+		w.log.Errorf("failed to recreate ICE Agent: %s", err)
+		w.muxAgent.Unlock()
+		return
+	}
+	w.agent = agent
+	w.muxAgent.Unlock()
+
+	w.log.Debugf("gather candidates")
+	err = w.agent.GatherCandidates()
+	if err != nil {
+		w.log.Debugf("failed to gather candidates: %s", err)
+		return
+	}
+
+	// will block until connection succeeded
+	// but it won't release if ICE Agent went into Disconnected or Failed state,
+	// so we have to cancel it with the provided context once agent detected a broken connection
+	w.log.Debugf("turn agent dial")
+	remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer)
+	if err != nil {
+		w.log.Debugf("failed to dial the remote peer: %s", err)
+		return
+	}
+	w.log.Debugf("agent dial succeeded")
+
+	pair, err := w.agent.GetSelectedCandidatePair()
+	if err != nil {
+		return
+	}
+
+	if !isRelayCandidate(pair.Local) {
+		// dynamically set remote WireGuard port if other side specified a different one from the default one
+		remoteWgPort := iface.DefaultWgPort
+		if remoteOfferAnswer.WgListenPort != 0 {
+			remoteWgPort = remoteOfferAnswer.WgListenPort
+		}
+
+		// To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port
+		go w.punchRemoteWGPort(pair, remoteWgPort)
+	}
+
+	ci := ICEConnInfo{
+		RemoteConn:                 remoteConn,
+		RosenpassPubKey:            remoteOfferAnswer.RosenpassPubKey,
+		RosenpassAddr:              remoteOfferAnswer.RosenpassAddr,
+		LocalIceCandidateType:      pair.Local.Type().String(),
+		RemoteIceCandidateType:     pair.Remote.Type().String(),
+		LocalIceCandidateEndpoint:  fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()),
+		RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()),
+		Relayed:                    isRelayed(pair),
+		RelayedOnLocal:             isRelayCandidate(pair.Local),
+	}
+	w.log.Debugf("on ICE conn read to use ready")
+	go w.conn.OnConnReady(w.selectedPriority, ci)
+}
+
+// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
+func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String())
+	if w.agent == nil {
+		w.log.Warnf("ICE Agent is not initialized yet")
+		return
+	}
+
+	if candidateViaRoutes(candidate, haRoutes) {
+		return
+	}
+
+	err := w.agent.AddRemoteCandidate(candidate)
+	if err != nil {
+		w.log.Errorf("error while handling remote candidate")
+		return
+	}
+}
+
+func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+	return w.localUfrag, w.localPwd
+}
+
+func (w *WorkerICE) Close() {
+	w.muxAgent.Lock()
+	defer w.muxAgent.Unlock()
+
+	if w.agent == nil {
+		return
+	}
+
+	err := w.agent.Close()
+	if err != nil {
+		w.log.Warnf("failed to close ICE agent: %s", err)
+	}
+}
+
+func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) {
+	transportNet, err := w.newStdNet()
+	if err != nil {
+		w.log.Errorf("failed to create pion's stdnet: %s", err)
+	}
+
+	iceKeepAlive := iceKeepAlive()
+	iceDisconnectedTimeout := iceDisconnectedTimeout()
+	iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
+
+	agentConfig := &ice.AgentConfig{
+		MulticastDNSMode:       ice.MulticastDNSModeDisabled,
+		NetworkTypes:           []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6},
+		Urls:                   w.config.ICEConfig.StunTurn.Load().([]*stun.URI),
+		CandidateTypes:         relaySupport,
+		InterfaceFilter:        stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList),
+		UDPMux:                 w.config.ICEConfig.UDPMux,
+		UDPMuxSrflx:            w.config.ICEConfig.UDPMuxSrflx,
+		NAT1To1IPs:             w.config.ICEConfig.NATExternalIPs,
+		Net:                    transportNet,
+		FailedTimeout:          &failedTimeout,
+		DisconnectedTimeout:    &iceDisconnectedTimeout,
+		KeepaliveInterval:      &iceKeepAlive,
+		RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait,
+		LocalUfrag:             w.localUfrag,
+		LocalPwd:               w.localPwd,
+	}
+
+	if w.config.ICEConfig.DisableIPv6Discovery {
+		agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4}
+	}
+
+	w.sentExtraSrflx = false
+	agent, err := ice.NewAgent(agentConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnCandidate(w.onICECandidate)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnConnectionStateChange(func(state ice.ConnectionState) {
+		w.log.Debugf("ICE ConnectionState has changed to %s", state.String())
+		if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected {
+			w.conn.OnStatusChanged(StatusDisconnected)
+
+			w.muxAgent.Lock()
+			agentCancel()
+			_ = agent.Close()
+			w.agent = nil
+
+			w.muxAgent.Unlock()
+		}
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair)
+	if err != nil {
+		return nil, err
+	}
+
+	err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) {
+		err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency())
+		if err != nil {
+			w.log.Debugf("failed to update latency for peer: %s", err)
+			return
+		}
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed setting binding response callback: %w", err)
+	}
+
+	return agent, nil
+}
+
+func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
+	// wait local endpoint configuration
+	time.Sleep(time.Second)
+	addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort))
+	if err != nil {
+		w.log.Warnf("got an error while resolving the udp address, err: %s", err)
+		return
+	}
+
+	mux, ok := w.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault)
+	if !ok {
+		w.log.Warn("invalid udp mux conversion")
+		return
+	}
+	_, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr)
+	if err != nil {
+		w.log.Warnf("got an error while sending the punch packet, err: %s", err)
+	}
+}
+
+// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates
+// and then signals them to the remote peer
+func (w *WorkerICE) onICECandidate(candidate ice.Candidate) {
+	// nil means candidate gathering has been ended
+	if candidate == nil {
+		return
+	}
+
+	// TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored
+	w.log.Debugf("discovered local candidate %s", candidate.String())
+	go func() {
+		err := w.signaler.SignalICECandidate(candidate, w.config.Key)
+		if err != nil {
+			w.log.Errorf("failed signaling candidate to the remote peer %s %s", w.config.Key, err)
+		}
+	}()
+
+	if !w.shouldSendExtraSrflxCandidate(candidate) {
+		return
+	}
+
+	// sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port)
+	// this is useful when network has an existing port forwarding rule for the wireguard port and this peer
+	extraSrflx, err := extraSrflxCandidate(candidate)
+	if err != nil {
+		w.log.Errorf("failed creating extra server reflexive candidate %s", err)
+		return
+	}
+	w.sentExtraSrflx = true
+
+	go func() {
+		err = w.signaler.SignalICECandidate(extraSrflx, w.config.Key)
+		if err != nil {
+			w.log.Errorf("failed signaling the extra server reflexive candidate: %s", err)
+		}
+	}()
+}
+
+func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) {
+	w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(),
+		w.config.Key)
+}
+
+func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool {
+	if !w.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port {
+		return true
+	}
+	return false
+}
+
+func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
+	isControlling := w.config.LocalKey > w.config.Key
+	if isControlling {
+		return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+	} else {
+		return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
+	}
+}
+
+func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) {
+	relatedAdd := candidate.RelatedAddress()
+	return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{
+		Network:   candidate.NetworkType().String(),
+		Address:   candidate.Address(),
+		Port:      relatedAdd.Port,
+		Component: candidate.Component(),
+		RelAddr:   relatedAdd.Address,
+		RelPort:   relatedAdd.Port,
+	})
+}
+
+func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool {
+	var routePrefixes []netip.Prefix
+	for _, routes := range clientRoutes {
+		if len(routes) > 0 && routes[0] != nil {
+			routePrefixes = append(routePrefixes, routes[0].Network)
+		}
+	}
+
+	addr, err := netip.ParseAddr(candidate.Address())
+	if err != nil {
+		log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err)
+		return false
+	}
+
+	for _, prefix := range routePrefixes {
+		// default route is
+		if prefix.Bits() == 0 {
+			continue
+		}
+
+		if prefix.Contains(addr) {
+			log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix)
+			return true
+		}
+	}
+	return false
+}
+
+func candidateTypes() []ice.CandidateType {
+	if hasICEForceRelayConn() {
+		return []ice.CandidateType{ice.CandidateTypeRelay}
+	}
+	// TODO: remove this once we have refactored userspace proxy into the bind package
+	if runtime.GOOS == "ios" {
+		return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+	}
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay}
+}
+
+func candidateTypesP2P() []ice.CandidateType {
+	return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive}
+}
+
+func isRelayCandidate(candidate ice.Candidate) bool {
+	return candidate.Type() == ice.CandidateTypeRelay
+}
+
+func isRelayed(pair *ice.CandidatePair) bool {
+	if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay {
+		return true
+	}
+	return false
+}
+
+func generateICECredentials() (string, string, error) {
+	ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+
+	pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha)
+	if err != nil {
+		return "", "", err
+	}
+	return ufrag, pwd, nil
+}

client/internal/peer/worker_relay.go

@@ -0,0 +1,223 @@
+package peer
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	relayClient "github.com/netbirdio/netbird/relay/client"
+)
+
+var (
+	wgHandshakePeriod   = 2 * time.Minute
+	wgHandshakeOvertime = 30 * time.Second
+)
+
+type RelayConnInfo struct {
+	relayedConn     net.Conn
+	rosenpassPubKey []byte
+	rosenpassAddr   string
+}
+
+type WorkerRelayCallbacks struct {
+	OnConnReady    func(RelayConnInfo)
+	OnDisconnected func()
+}
+
+type WorkerRelay struct {
+	log          *log.Entry
+	config       ConnConfig
+	relayManager relayClient.ManagerService
+	callBacks    WorkerRelayCallbacks
+
+	relayedConn      net.Conn
+	relayLock        sync.Mutex
+	ctxWgWatch       context.Context
+	ctxCancelWgWatch context.CancelFunc
+	ctxLock          sync.Mutex
+
+	relaySupportedOnRemotePeer atomic.Bool
+}
+
+func NewWorkerRelay(log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay {
+	r := &WorkerRelay{
+		log:          log,
+		config:       config,
+		relayManager: relayManager,
+		callBacks:    callbacks,
+	}
+	return r
+}
+
+func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
+	if !w.isRelaySupported(remoteOfferAnswer) {
+		w.log.Infof("Relay is not supported by remote peer")
+		w.relaySupportedOnRemotePeer.Store(false)
+		return
+	}
+	w.relaySupportedOnRemotePeer.Store(true)
+
+	// the relayManager will return with error in case if the connection has lost with relay server
+	currentRelayAddress, err := w.relayManager.RelayInstanceAddress()
+	if err != nil {
+		w.log.Errorf("failed to handle new offer: %s", err)
+		return
+	}
+
+	srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress)
+
+	relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key)
+	if err != nil {
+		if errors.Is(err, relayClient.ErrConnAlreadyExists) {
+			w.log.Infof("do not need to reopen relay connection")
+			return
+		}
+		w.log.Errorf("failed to open connection via Relay: %s", err)
+		return
+	}
+	w.relayLock.Lock()
+	w.relayedConn = relayedConn
+	w.relayLock.Unlock()
+
+	err = w.relayManager.AddCloseListener(srv, w.onRelayMGDisconnected)
+	if err != nil {
+		log.Errorf("failed to add close listener: %s", err)
+		_ = relayedConn.Close()
+		return
+	}
+
+	w.log.Debugf("peer conn opened via Relay: %s", srv)
+	go w.callBacks.OnConnReady(RelayConnInfo{
+		relayedConn:     relayedConn,
+		rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey,
+		rosenpassAddr:   remoteOfferAnswer.RosenpassAddr,
+	})
+}
+
+func (w *WorkerRelay) EnableWgWatcher(ctx context.Context) {
+	w.log.Debugf("enable WireGuard watcher")
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
+
+	if w.ctxWgWatch != nil && w.ctxWgWatch.Err() == nil {
+		return
+	}
+
+	ctx, ctxCancel := context.WithCancel(ctx)
+	go w.wgStateCheck(ctx)
+	w.ctxWgWatch = ctx
+	w.ctxCancelWgWatch = ctxCancel
+
+}
+
+func (w *WorkerRelay) DisableWgWatcher() {
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
+
+	if w.ctxCancelWgWatch == nil {
+		return
+	}
+
+	w.log.Debugf("disable WireGuard watcher")
+
+	w.ctxCancelWgWatch()
+}
+
+func (w *WorkerRelay) RelayInstanceAddress() (string, error) {
+	return w.relayManager.RelayInstanceAddress()
+}
+
+func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool {
+	return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally()
+}
+
+func (w *WorkerRelay) IsController() bool {
+	return w.config.LocalKey > w.config.Key
+}
+
+func (w *WorkerRelay) RelayIsSupportedLocally() bool {
+	return w.relayManager.HasRelayAddress()
+}
+
+func (w *WorkerRelay) CloseConn() {
+	w.relayLock.Lock()
+	defer w.relayLock.Unlock()
+	if w.relayedConn == nil {
+		return
+	}
+
+	err := w.relayedConn.Close()
+	if err != nil {
+		w.log.Warnf("failed to close relay connection: %v", err)
+	}
+}
+
+// wgStateCheck help to check the state of the wireguard handshake and relay connection
+func (w *WorkerRelay) wgStateCheck(ctx context.Context) {
+	timer := time.NewTimer(wgHandshakeOvertime)
+	defer timer.Stop()
+	expected := wgHandshakeOvertime
+	for {
+		select {
+		case <-timer.C:
+			lastHandshake, err := w.wgState()
+			if err != nil {
+				w.log.Errorf("failed to read wg stats: %v", err)
+				continue
+			}
+			w.log.Tracef("last handshake: %v", lastHandshake)
+
+			if time.Since(lastHandshake) > expected {
+				w.log.Infof("Wireguard handshake timed out, closing relay connection")
+				w.relayLock.Lock()
+				_ = w.relayedConn.Close()
+				w.relayLock.Unlock()
+				w.callBacks.OnDisconnected()
+				return
+			}
+			resetTime := time.Until(lastHandshake.Add(wgHandshakePeriod + wgHandshakeOvertime))
+			timer.Reset(resetTime)
+			expected = wgHandshakePeriod
+		case <-ctx.Done():
+			w.log.Debugf("WireGuard watcher stopped")
+			return
+		}
+	}
+}
+
+func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool {
+	if !w.relayManager.HasRelayAddress() {
+		return false
+	}
+	return answer.RelaySrvAddress != ""
+}
+
+func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string {
+	if w.IsController() {
+		return myRelayAddress
+	}
+	return remoteRelayAddress
+}
+
+func (w *WorkerRelay) wgState() (time.Time, error) {
+	wgState, err := w.config.WgConfig.WgInterface.GetStats(w.config.Key)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return wgState.LastHandshake, nil
+}
+
+func (w *WorkerRelay) onRelayMGDisconnected() {
+	w.ctxLock.Lock()
+	defer w.ctxLock.Unlock()
+
+	if w.ctxCancelWgWatch != nil {
+		w.ctxCancelWgWatch()
+	}
+	go w.callBacks.OnDisconnected()
+}

client/internal/relay/relay.go

@@ -17,7 +17,7 @@ import (
 
 // ProbeResult holds the info about the result of a relay probe request
 type ProbeResult struct {
-	URI  *stun.URI
+	URI  string
 	Err  error
 	Addr string
 }
@@ -176,7 +176,7 @@ func ProbeAll(
 		wg.Add(1)
 		go func(res *ProbeResult, stunURI *stun.URI) {
 			defer wg.Done()
-			res.URI = stunURI
+			res.URI = stunURI.String()
 			res.Addr, res.Err = fn(ctx, stunURI)
 		}(&results[i], uri)
 	}

client/internal/routemanager/client.go

@@ -22,7 +22,6 @@ import (
 type routerPeerStatus struct {
 	connected bool
 	relayed   bool
-	direct    bool
 	latency   time.Duration
 }
 
@@ -82,7 +81,6 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus {
 		routePeerStatuses[r.ID] = routerPeerStatus{
 			connected: peerStatus.ConnStatus == peer.StatusConnected,
 			relayed:   peerStatus.Relayed,
-			direct:    peerStatus.Direct,
 			latency:   peerStatus.Latency,
 		}
 	}
@@ -97,8 +95,8 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus {
 // * Connected peers: Only routes with connected peers are considered.
 // * Metric: Routes with lower metrics (better) are prioritized.
 // * Non-relayed: Routes without relays are preferred.
-// * Direct connections: Routes with direct peer connections are favored.
 // * Latency: Routes with lower latency are prioritized.
+// * we compare the current score + 10ms to the chosen score to avoid flapping between routes
 // * Stability: In case of equal scores, the currently active route (if any) is maintained.
 //
 // It returns the ID of the selected optimal route.
@@ -137,10 +135,6 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID]
 			tempScore++
 		}
 
-		if peerStatus.direct {
-			tempScore++
-		}
-
 		if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") {
 			chosen = r.ID
 			chosenScore = tempScore

client/internal/routemanager/client_test.go

@@ -24,7 +24,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -43,7 +42,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   true,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -62,7 +60,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   true,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -81,7 +78,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: false,
 					relayed:   false,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -100,12 +96,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -129,41 +123,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 				},
 				"route2": {
 					connected: true,
 					relayed:   true,
-					direct:    true,
-				},
-			},
-			existingRoutes: map[route.ID]*route.Route{
-				"route1": {
-					ID:     "route1",
-					Metric: route.MaxMetric,
-					Peer:   "peer1",
-				},
-				"route2": {
-					ID:     "route2",
-					Metric: route.MaxMetric,
-					Peer:   "peer2",
-				},
-			},
-			currentRoute:    "",
-			expectedRouteID: "route1",
-		},
-		{
-			name: "multiple connected peers with one direct",
-			statuses: map[route.ID]routerPeerStatus{
-				"route1": {
-					connected: true,
-					relayed:   false,
-					direct:    true,
-				},
-				"route2": {
-					connected: true,
-					relayed:   false,
-					direct:    false,
 				},
 			},
 			existingRoutes: map[route.ID]*route.Route{
@@ -241,13 +204,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   15 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},
@@ -272,13 +233,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   200 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},
@@ -303,13 +262,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) {
 				"route1": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   20 * time.Millisecond,
 				},
 				"route2": {
 					connected: true,
 					relayed:   false,
-					direct:    true,
 					latency:   10 * time.Millisecond,
 				},
 			},

client/internal/routemanager/manager.go

@@ -22,6 +22,7 @@ import (
 	"github.com/netbirdio/netbird/client/internal/routemanager/vars"
 	"github.com/netbirdio/netbird/client/internal/routeselector"
 	"github.com/netbirdio/netbird/iface"
+	relayClient "github.com/netbirdio/netbird/relay/client"
 	"github.com/netbirdio/netbird/route"
 	nbnet "github.com/netbirdio/netbird/util/net"
 	"github.com/netbirdio/netbird/version"
@@ -49,6 +50,7 @@ type DefaultManager struct {
 	serverRouter         serverRouter
 	sysOps               *systemops.SysOps
 	statusRecorder       *peer.Status
+	relayMgr             *relayClient.Manager
 	wgInterface          iface.IWGIface
 	pubKey               string
 	notifier             *notifier.Notifier
@@ -63,6 +65,7 @@ func NewManager(
 	dnsRouteInterval time.Duration,
 	wgInterface iface.IWGIface,
 	statusRecorder *peer.Status,
+	relayMgr *relayClient.Manager,
 	initialRoutes []*route.Route,
 ) *DefaultManager {
 	mCTX, cancel := context.WithCancel(ctx)
@@ -74,6 +77,7 @@ func NewManager(
 		stop:             cancel,
 		dnsRouteInterval: dnsRouteInterval,
 		clientNetworks:   make(map[route.HAUniqueID]*clientNetwork),
+		relayMgr:         relayMgr,
 		routeSelector:    routeselector.NewRouteSelector(),
 		sysOps:           sysOps,
 		statusRecorder:   statusRecorder,
@@ -124,9 +128,12 @@ func (m *DefaultManager) Init() (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error)
 		log.Warnf("Failed cleaning up routing: %v", err)
 	}
 
-	mgmtAddress := m.statusRecorder.GetManagementState().URL
+	initialAddresses := []string{m.statusRecorder.GetManagementState().URL, m.statusRecorder.GetSignalState().URL}
-	signalAddress := m.statusRecorder.GetSignalState().URL
+	if m.relayMgr != nil {
-	ips := resolveURLsToIPs([]string{mgmtAddress, signalAddress})
+		initialAddresses = append(initialAddresses, m.relayMgr.ServerURLs()...)
+	}
+
+	ips := resolveURLsToIPs(initialAddresses)
 
 	beforePeerHook, afterPeerHook, err := m.sysOps.SetupRouting(ips)
 	if err != nil {

client/internal/routemanager/manager_test.go

@@ -416,7 +416,7 @@ func TestManagerUpdateRoutes(t *testing.T) {
 
 			statusRecorder := peer.NewRecorder("https://mgm")
 			ctx := context.TODO()
-			routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil)
+			routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil, nil)
 
 			_, _, err = routeManager.Init()
 

client/internal/wgproxy/proxy_ebpf.go

@@ -181,7 +181,7 @@ func (p *WGEBPFProxy) proxyToRemote() {
 			conn, ok := p.turnConnStore[uint16(addr.Port)]
 			p.turnConnMutex.Unlock()
 			if !ok {
-				log.Infof("turn conn not found by port: %d", addr.Port)
+				log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port)
 				continue
 			}
 
@@ -206,7 +206,7 @@ func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) {
 }
 
 func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) {
-	log.Tracef("remove turn conn from store by port: %d", turnConnID)
+	log.Debugf("remove turn conn from store by port: %d", turnConnID)
 	p.turnConnMutex.Lock()
 	defer p.turnConnMutex.Unlock()
 	delete(p.turnConnStore, turnConnID)

client/internal/wgproxy/proxy_userspace.go

@@ -3,6 +3,7 @@ package wgproxy
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
 
 	log "github.com/sirupsen/logrus"
@@ -64,7 +65,6 @@ func (p *WGUserSpaceProxy) Free() error {
 // proxyToRemote proxies everything from Wireguard to the RemoteKey peer
 // blocks
 func (p *WGUserSpaceProxy) proxyToRemote() {
-
 	buf := make([]byte, 1500)
 	for {
 		select {
@@ -73,11 +73,17 @@ func (p *WGUserSpaceProxy) proxyToRemote() {
 		default:
 			n, err := p.localConn.Read(buf)
 			if err != nil {
+				log.Debugf("failed to read from wg interface conn: %s", err)
 				continue
 			}
 
 			_, err = p.remoteConn.Write(buf[:n])
 			if err != nil {
+				if err == io.EOF {
+					p.cancel()
+				} else {
+					log.Debugf("failed to write to remote conn: %s", err)
+				}
 				continue
 			}
 		}
@@ -96,11 +102,17 @@ func (p *WGUserSpaceProxy) proxyToLocal() {
 		default:
 			n, err := p.remoteConn.Read(buf)
 			if err != nil {
+				if err == io.EOF {
+					p.cancel()
+					return
+				}
+				log.Errorf("failed to read from remote conn: %s", err)
 				continue
 			}
 
 			_, err = p.localConn.Write(buf[:n])
 			if err != nil {
+				log.Debugf("failed to write to wg interface conn: %s", err)
 				continue
 			}
 		}

client/ios/NetBirdSDK/client.go

@@ -168,7 +168,6 @@ func (c *Client) GetStatusDetails() *StatusDetails {
 			BytesTx:                    p.BytesTx,
 			ConnStatus:                 p.ConnStatus.String(),
 			ConnStatusUpdate:           p.ConnStatusUpdate.Format("2006-01-02 15:04:05"),
-			Direct:                     p.Direct,
 			LastWireguardHandshake:     p.LastWireguardHandshake.String(),
 			Relayed:                    p.Relayed,
 			RosenpassEnabled:           p.RosenpassEnabled,

client/proto/daemon.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v4.23.4
+// 	protoc        v3.21.12
 // source: daemon.proto
 
 package proto
@@ -899,7 +899,6 @@ type PeerState struct {
 	ConnStatus                 string                 `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"`
 	ConnStatusUpdate           *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"`
 	Relayed                    bool                   `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"`
-	Direct                     bool                   `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"`
 	LocalIceCandidateType      string                 `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"`
 	RemoteIceCandidateType     string                 `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"`
 	Fqdn                       string                 `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"`
@@ -911,6 +910,7 @@ type PeerState struct {
 	RosenpassEnabled           bool                   `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"`
 	Routes                     []string               `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"`
 	Latency                    *durationpb.Duration   `protobuf:"bytes,17,opt,name=latency,proto3" json:"latency,omitempty"`
+	RelayAddress               string                 `protobuf:"bytes,18,opt,name=relayAddress,proto3" json:"relayAddress,omitempty"`
 }
 
 func (x *PeerState) Reset() {
@@ -980,13 +980,6 @@ func (x *PeerState) GetRelayed() bool {
 	return false
 }
 
-func (x *PeerState) GetDirect() bool {
-	if x != nil {
-		return x.Direct
-	}
-	return false
-}
-
 func (x *PeerState) GetLocalIceCandidateType() string {
 	if x != nil {
 		return x.LocalIceCandidateType
@@ -1064,6 +1057,13 @@ func (x *PeerState) GetLatency() *durationpb.Duration {
 	return nil
 }
 
+func (x *PeerState) GetRelayAddress() string {
+	if x != nil {
+		return x.RelayAddress
+	}
+	return ""
+}
+
 // LocalPeerState contains the latest state of the local peer
 type LocalPeerState struct {
 	state         protoimpl.MessageState
@@ -2243,7 +2243,7 @@ var file_daemon_proto_rawDesc = []byte{
 	0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e,
 	0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c,
 	0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50,
-	0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xce, 0x05, 0x0a, 0x09, 0x50, 0x65,
+	0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xda, 0x05, 0x0a, 0x09, 0x50, 0x65,
 	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20,
 	0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65,
 	0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12,
@@ -2255,209 +2255,210 @@ var file_daemon_proto_rawDesc = []byte{
 	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75,
 	0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79,
 	0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65,
-	0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28,
+	0x64, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e,
-	0x08, 0x52, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63,
+	0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09,
-	0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79,
+	0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64,
-	0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49,
+	0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74,
+	0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70,
+	0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49,
 	0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12,
-	0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64,
+	0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66,
-	0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43,
-	0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64,
+	0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
-	0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18,
+	0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65,
-	0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c,
+	0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
-	0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65,
+	0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61,
-	0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19,
+	0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18,
-	0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74,
+	0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65,
-	0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d,
+	0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
-	0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45,
+	0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61,
-	0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72,
+	0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28,
-	0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74,
+	0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73,
+	0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c,
-	0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68,
+	0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64,
-	0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67,
+	0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78,
-	0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65,
+	0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12,
-	0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67,
+	0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03,
-	0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a,
+	0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73,
-	0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07,
+	0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20,
-	0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73,
+	0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e,
-	0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54,
+	0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18,
-	0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e,
+	0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a,
-	0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73,
+	0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
-	0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a,
+	0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
-	0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72,
+	0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e,
-	0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79,
+	0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65,
-	0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+	0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f,
+	0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c,
-	0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c,
+	0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18,
-	0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62,
-	0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a,
+	0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65,
-	0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70,
+	0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72,
-	0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49,
+	0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e,
-	0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f,
+	0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66,
-	0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12,
+	0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12,
-	0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66,
+	0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62,
-	0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73,
+	0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e,
-	0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72,
+	0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72,
-	0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12,
+	0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69,
-	0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d,
+	0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70,
-	0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f,
+	0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a,
-	0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76,
+	0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72,
-	0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28,
+	0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53,
-	0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18,
+	0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f,
+	0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65,
-	0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63,
+	0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20,
-	0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f,
+	0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61,
-	0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a,
-	0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
+	0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12,
-	0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
+	0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01,
-	0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64,
+	0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65,
+	0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72,
-	0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
+	0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74,
-	0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79,
+	0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03,
-	0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01,
+	0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65,
-	0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c,
-	0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69,
+	0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09,
-	0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03,
+	0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e,
+	0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65,
-	0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73,
+	0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
-	0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65,
+	0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03,
-	0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73,
+	0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65,
-	0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12,
+	0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e,
-	0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08,
+	0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04,
-	0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a,
-	0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22,
+	0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61,
-	0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20,
-	0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
+	0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e,
-	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
+	0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61,
-	0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65,
+	0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a,
-	0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
+	0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01,
-	0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65,
+	0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
+	0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53,
-	0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65,
-	0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61,
+	0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64,
-	0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b,
+	0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53,
-	0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50,
+	0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53,
-	0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50,
+	0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20,
-	0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72,
+	0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65,
-	0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
+	0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a,
-	0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72,
+	0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e,
-	0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28,
+	0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74,
-	0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79,
+	0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73,
-	0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a,
+	0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14,
-	0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03,
+	0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53,
-	0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72,
+	0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
-	0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72,
+	0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65,
-	0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75,
-	0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73,
+	0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72,
-	0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
+	0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61,
-	0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74,
-	0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06,
+	0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74,
-	0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74,
+	0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75,
-	0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a,
+	0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75,
-	0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52,
+	0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18,
-	0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70,
+	0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a,
-	0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e,
+	0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22,
-	0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03,
+	0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52,
-	0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73,
-	0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49,
+	0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03,
-	0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03,
+	0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a,
-	0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74,
+	0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a,
-	0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49,
+	0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
-	0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01,
+	0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63,
-	0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73,
+	0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63,
-	0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73,
+	0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04,
-	0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a,
-	0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
+	0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03,
-	0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73,
+	0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74,
-	0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
+	0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74,
-	0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50,
+	0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a,
-	0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64,
+	0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e,
-	0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50,
-	0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
+	0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22,
-	0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
+	0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65,
-	0x02, 0x38, 0x01, 0x22, 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69,
-	0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f,
+	0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d,
-	0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e,
+	0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20,
-	0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75,
+	0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73,
-	0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12,
+	0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52,
-	0x1e, 0x0a, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20,
+	0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x29, 0x0a, 0x13, 0x44,
-	0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22,
+	0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65,
+	0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01,
+	0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65,
+	0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13,
-	0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52,
+	0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c,
+	0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c,
-	0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
+	0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53,
-	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22,
+	0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65,
+	0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01,
+	0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76,
-	0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f,
+	0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74,
-	0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a,
+	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70,
+	0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07,
-	0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c,
+	0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e,
-	0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a,
+	0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12,
-	0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41,
+	0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41,
-	0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08,
+	0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09,
-	0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f,
+	0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41,
-	0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a,
+	0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53,
-	0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65,
+	0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12,
-	0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f,
+	0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65,
-	0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c,
-	0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d,
+	0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b,
-	0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b,
-	0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67,
+	0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c,
-	0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74,
+	0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61,
-	0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69,
-	0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f,
+	0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55,
-	0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
+	0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71,
-	0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70,
-	0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74,
-	0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39,
+	0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74,
-	0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
+	0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61,
-	0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77,
+	0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52,
+	0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65,
-	0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42,
+	0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
-	0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61,
+	0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65,
+	0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47,
+	0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45,
-	0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64,
-	0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73,
+	0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73,
-	0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e,
-	0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61,
+	0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c,
+	0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53,
-	0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d,
+	0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52,
+	0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
+	0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70,
+	0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f,
-	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65,
+	0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65,
-	0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
+	0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65,
+	0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53,
+	0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65,
-	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75,
+	0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42,
-	0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65,
+	0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64,
-	0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c,
-	0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47,
-	0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
+	0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65,
-	0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a,
-	0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65,
-	0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65,
 	0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52,
-	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e,
-	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f,
+	0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65,
+	0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65,
-	0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70,
+	0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42,
+	0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
 }
 
 var (

client/proto/daemon.proto

@@ -168,7 +168,6 @@ message PeerState {
   string connStatus = 3;
   google.protobuf.Timestamp connStatusUpdate = 4;
   bool relayed = 5;
-  bool direct = 6;
   string localIceCandidateType = 7;
   string remoteIceCandidateType = 8;
   string fqdn = 9;
@@ -180,6 +179,7 @@ message PeerState {
   bool rosenpassEnabled = 15;
   repeated string routes = 16;
   google.protobuf.Duration latency = 17;
+  string relayAddress = 18;
 }
 
 // LocalPeerState contains the latest state of the local peer

client/server/debug.go

@@ -369,8 +369,8 @@ func seedFromStatus(a *anonymize.Anonymizer, status *peer.FullStatus) {
 	}
 
 	for _, relay := range status.Relays {
-		if relay.URI != nil {
+		if relay.URI != "" {
-			a.AnonymizeURI(relay.URI.String())
+			a.AnonymizeURI(relay.URI)
 		}
 	}
 }

client/server/server.go

@@ -758,11 +758,11 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus {
 			ConnStatus:                 peerState.ConnStatus.String(),
 			ConnStatusUpdate:           timestamppb.New(peerState.ConnStatusUpdate),
 			Relayed:                    peerState.Relayed,
-			Direct:                     peerState.Direct,
 			LocalIceCandidateType:      peerState.LocalIceCandidateType,
 			RemoteIceCandidateType:     peerState.RemoteIceCandidateType,
 			LocalIceCandidateEndpoint:  peerState.LocalIceCandidateEndpoint,
 			RemoteIceCandidateEndpoint: peerState.RemoteIceCandidateEndpoint,
+			RelayAddress:               peerState.RelayServerAddress,
 			Fqdn:                       peerState.FQDN,
 			LastWireguardHandshake:     timestamppb.New(peerState.LastWireguardHandshake),
 			BytesRx:                    peerState.BytesRx,
@@ -776,7 +776,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus {
 
 	for _, relayState := range fullStatus.Relays {
 		pbRelayState := &proto.RelayState{
-			URI:       relayState.URI.String(),
+			URI:       relayState.URI,
 			Available: relayState.Err == nil,
 		}
 		if err := relayState.Err; err != nil {

client/server/server_test.go

@@ -6,10 +6,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/netbirdio/management-integrations/integrations"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel"
 
+	"github.com/netbirdio/management-integrations/integrations"
+
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/keepalive"
@@ -129,8 +130,9 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve
 	if err != nil {
 		return nil, "", err
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		return nil, "", err
 	}

client/testdata/management.json

@@ -20,6 +20,13 @@
     "Secret": "c29tZV9wYXNzd29yZA==",
     "TimeBasedCredentials": true
   },
+  "Relay": {
+    "Addresses": [
+      "localhost:0"
+    ],
+    "CredentialsTTL": "1h",
+    "Secret": "b29tZV9wYXNzd29yZA=="
+  },
   "Signal": {
     "Proto": "http",
     "URI": "signal.wiretrustee.com:10000",

encryption/cert.go

@@ -0,0 +1,19 @@
+package encryption
+
+import "crypto/tls"
+
+func LoadTLSConfig(certFile, keyFile string) (*tls.Config, error) {
+	serverCert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	config := &tls.Config{
+		Certificates: []tls.Certificate{serverCert},
+		ClientAuth:   tls.NoClientCert,
+		NextProtos: []string{
+			"h2", "http/1.1", // enable HTTP/2
+		},
+	}
+	return config, nil
+}

encryption/letsencrypt.go

@@ -9,7 +9,7 @@ import (
 )
 
 // CreateCertManager wraps common logic of generating Let's encrypt certificate.
-func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Manager, error) {
+func CreateCertManager(datadir string, letsencryptDomain ...string) (*autocert.Manager, error) {
 	certDir := filepath.Join(datadir, "letsencrypt")
 
 	if _, err := os.Stat(certDir); os.IsNotExist(err) {
@@ -24,7 +24,7 @@ func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Mana
 	certManager := &autocert.Manager{
 		Prompt:     autocert.AcceptTOS,
 		Cache:      autocert.DirCache(certDir),
-		HostPolicy: autocert.HostWhitelist(letsencryptDomain),
+		HostPolicy: autocert.HostWhitelist(letsencryptDomain...),
 	}
 
 	return certManager, nil

encryption/route53.go

@@ -0,0 +1,87 @@
+package encryption
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/caddyserver/certmagic"
+	"github.com/libdns/route53"
+	log "github.com/sirupsen/logrus"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"golang.org/x/crypto/acme"
+)
+
+// Route53TLS by default, loads the AWS configuration from the environment.
+// env variables: AWS_REGION, AWS_PROFILE, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
+type Route53TLS struct {
+	DataDir string
+	Email   string
+	Domains []string
+	CA      string
+}
+
+func (r *Route53TLS) GetCertificate() (*tls.Config, error) {
+	if len(r.Domains) == 0 {
+		return nil, fmt.Errorf("no domains provided")
+	}
+
+	certmagic.Default.Logger = logger()
+	certmagic.Default.Storage = &certmagic.FileStorage{Path: r.DataDir}
+	certmagic.DefaultACME.Agreed = true
+	if r.Email != "" {
+		certmagic.DefaultACME.Email = r.Email
+	} else {
+		certmagic.DefaultACME.Email = emailFromDomain(r.Domains[0])
+	}
+
+	if r.CA == "" {
+		certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
+	} else {
+		certmagic.DefaultACME.CA = r.CA
+	}
+
+	certmagic.DefaultACME.DNS01Solver = &certmagic.DNS01Solver{
+		DNSManager: certmagic.DNSManager{
+			DNSProvider: &route53.Provider{},
+		},
+	}
+	cm := certmagic.NewDefault()
+	if err := cm.ManageSync(context.Background(), r.Domains); err != nil {
+		log.Errorf("failed to manage certificate: %v", err)
+		return nil, err
+	}
+
+	tlsConfig := &tls.Config{
+		GetCertificate: cm.GetCertificate,
+		NextProtos:     []string{"h2", "http/1.1", acme.ALPNProto},
+	}
+
+	return tlsConfig, nil
+}
+
+func emailFromDomain(domain string) string {
+	if domain == "" {
+		return ""
+	}
+
+	parts := strings.Split(domain, ".")
+	if len(parts) < 2 {
+		return ""
+	}
+	if parts[0] == "" {
+		return ""
+	}
+	return fmt.Sprintf("admin@%s.%s", parts[len(parts)-2], parts[len(parts)-1])
+}
+
+func logger() *zap.Logger {
+	return zap.New(zapcore.NewCore(
+		zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()),
+		os.Stderr,
+		zap.ErrorLevel,
+	))
+}

encryption/route53_test.go

@@ -0,0 +1,84 @@
+package encryption
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestRoute53TLSConfig(t *testing.T) {
+	t.SkipNow() // This test requires AWS credentials
+	exampleString := "Hello, world!"
+	rtls := &Route53TLS{
+		DataDir: t.TempDir(),
+		Email:   os.Getenv("LE_EMAIL_ROUTE53"),
+		Domains: []string{os.Getenv("DOMAIN")},
+	}
+	tlsConfig, err := rtls.GetCertificate()
+	if err != nil {
+		t.Errorf("Route53TLSConfig failed: %v", err)
+	}
+
+	server := &http.Server{
+		Addr: ":8443",
+		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write([]byte(exampleString))
+		}),
+		TLSConfig: tlsConfig,
+	}
+
+	go func() {
+		err := server.ListenAndServeTLS("", "")
+		if err != http.ErrServerClosed {
+			t.Errorf("Failed to start server: %v", err)
+		}
+	}()
+	defer func() {
+		if err := server.Shutdown(context.Background()); err != nil {
+			t.Errorf("Failed to shutdown server: %v", err)
+		}
+	}()
+
+	time.Sleep(1 * time.Second)
+	resp, err := http.Get("https://relay.godevltd.com:8443")
+	if err != nil {
+		t.Errorf("Failed to get response: %v", err)
+		return
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("Failed to read response body: %v", err)
+	}
+	if string(body) != exampleString {
+		t.Errorf("Unexpected response: %s", body)
+	}
+}
+
+func Test_emailFromDomain(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"example.com", "admin@example.com"},
+		{"x.example.com", "admin@example.com"},
+		{"x.x.example.com", "admin@example.com"},
+		{"*.example.com", "admin@example.com"},
+		{"example", ""},
+		{"", ""},
+		{".com", ""},
+	}
+	for _, tt := range tests {
+		t.Run("domain test", func(t *testing.T) {
+			if got := emailFromDomain(tt.input); got != tt.want {
+				t.Errorf("emailFromDomain() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/kardianos/service v1.2.3-0.20240613133416-becf2eb62b83
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.23.0
+	github.com/onsi/gomega v1.27.6
 	github.com/pion/ice/v3 v3.0.2
 	github.com/rs/cors v1.8.0
 	github.com/sirupsen/logrus v1.9.3
@@ -34,6 +34,7 @@ require (
 	fyne.io/systray v1.11.0
 	github.com/TheJumpCloud/jcapi-go v3.0.0+incompatible
 	github.com/c-robinson/iplib v1.0.3
+	github.com/caddyserver/certmagic v0.21.3
 	github.com/cilium/ebpf v0.15.0
 	github.com/coreos/go-iptables v0.7.0
 	github.com/creack/pty v1.1.18
@@ -50,11 +51,12 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-secure-stdlib/base62 v0.1.2
 	github.com/hashicorp/go-version v1.6.0
+	github.com/libdns/route53 v1.5.0
 	github.com/libp2p/go-netroute v0.2.1
 	github.com/magiconair/properties v1.8.7
 	github.com/mattn/go-sqlite3 v1.14.19
 	github.com/mdlayher/socket v0.4.1
-	github.com/miekg/dns v1.1.43
+	github.com/miekg/dns v1.1.59
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/nadoo/ipset v0.5.0
 	github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e
@@ -63,6 +65,7 @@ require (
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pion/logging v0.2.2
+	github.com/pion/randutil v0.1.0
 	github.com/pion/stun/v2 v2.0.0
 	github.com/pion/transport/v3 v3.0.1
 	github.com/pion/turn/v3 v3.0.1
@@ -70,6 +73,7 @@ require (
 	github.com/rs/xid v1.3.0
 	github.com/shirou/gopsutil/v3 v3.24.4
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
+	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
 	github.com/stretchr/testify v1.9.0
 	github.com/testcontainers/testcontainers-go v0.31.0
 	github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0
@@ -81,6 +85,7 @@ require (
 	go.opentelemetry.io/otel/exporters/prometheus v0.48.0
 	go.opentelemetry.io/otel/metric v1.26.0
 	go.opentelemetry.io/otel/sdk/metric v1.26.0
+	go.uber.org/zap v1.27.0
 	goauthentik.io/api/v3 v3.2023051.3
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a
@@ -93,6 +98,7 @@ require (
 	gorm.io/driver/postgres v1.5.7
 	gorm.io/driver/sqlite v1.5.3
 	gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde
+	nhooyr.io/websocket v1.8.11
 )
 
 require (
@@ -106,8 +112,23 @@ require (
 	github.com/Microsoft/hcsshim v0.12.3 // indirect
 	github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect
+	github.com/caddyserver/zerossl v0.1.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/containerd v1.7.16 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -140,7 +161,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-uuid v1.0.2 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
@@ -149,13 +170,17 @@ require (
 	github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/libdns/libdns v0.2.2 // indirect
 	github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
 	github.com/mdlayher/genetlink v1.3.2 // indirect
 	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mholt/acmez/v2 v2.0.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
@@ -164,12 +189,12 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pegasus-kv/thrift v0.13.0 // indirect
 	github.com/pion/dtls/v2 v2.2.10 // indirect
 	github.com/pion/mdns v0.0.12 // indirect
-	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/transport/v2 v2.2.4 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -186,10 +211,12 @@ require (
 	github.com/tklauser/numcpus v0.8.0 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	github.com/yuin/goldmark v1.7.1 // indirect
+	github.com/zeebo/blake3 v0.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.26.0 // indirect
 	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/image v0.18.0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/text v0.16.0 // indirect

go.sum

@@ -79,6 +79,34 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
+github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
+github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
+github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII=
+github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 h1:MmLCRqP4U4Cw9gJ4bNrCG0mWqEtBlmAVleyelcHARMU=
+github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3/go.mod h1:AMPjK2YnRh0YgOID3PqhJA1BRNfXDfGOnSsKHtAe8yA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ=
+github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
+github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -87,6 +115,10 @@ github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwel
 github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
 github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU=
 github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
+github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0=
+github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI=
+github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA=
+github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4=
 github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -207,6 +239,8 @@ github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZs
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-text/render v0.1.0 h1:osrmVDZNHuP1RSu3pNG7Z77Sd2xSbcb/xWytAj9kyVs=
 github.com/go-text/render v0.1.0/go.mod h1:jqEuNMenrmj6QRnkdpeaP0oKGFLDNhDkVKwGjsWWYU4=
 github.com/go-text/typesetting v0.1.0 h1:vioSaLPYcHwPEPLT7gsjCGDCoYSbljxoHJzMnKwVvHw=
@@ -350,8 +384,9 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
@@ -382,6 +417,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
 github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
 github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -401,6 +440,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
+github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -413,6 +455,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
+github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
+github.com/libdns/route53 v1.5.0 h1:2SKdpPFl/qgWsXQvsLNJJAoX7rSxlk7zgoL4jnWdXVA=
+github.com/libdns/route53 v1.5.0/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI=
 github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
@@ -431,9 +477,11 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/
 github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
 github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=
 github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
+github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k=
+github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
-github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -494,14 +542,14 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
-github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
-github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -592,6 +640,8 @@ github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrx
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
 github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
+github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
@@ -660,6 +710,12 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc=
 github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
+github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
@@ -695,8 +751,14 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4=
 goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -890,7 +952,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1187,6 +1248,8 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E=
+nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
+nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

infrastructure_files/base.setup.env

@@ -20,6 +20,12 @@ NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=${NETBIRD_MGMT_IDP_SIGNKEY_REFRESH:-false}
 NETBIRD_SIGNAL_PROTOCOL="http"
 NETBIRD_SIGNAL_PORT=${NETBIRD_SIGNAL_PORT:-10000}
 
+# Relay
+NETBIRD_RELAY_DOMAIN=${NETBIRD_RELAY_DOMAIN:-$NETBIRD_DOMAIN}
+NETBIRD_RELAY_PORT=${NETBIRD_RELAY_PORT:-33080}
+# Relay auth secret
+NETBIRD_RELAY_AUTH_SECRET=
+
 # Turn
 TURN_DOMAIN=${NETBIRD_TURN_DOMAIN:-$NETBIRD_DOMAIN}
 
@@ -69,7 +75,7 @@ NETBIRD_DASHBOARD_TAG=${NETBIRD_DASHBOARD_TAG:-"latest"}
 NETBIRD_SIGNAL_TAG=${NETBIRD_SIGNAL_TAG:-"latest"}
 NETBIRD_MANAGEMENT_TAG=${NETBIRD_MANAGEMENT_TAG:-"latest"}
 COTURN_TAG=${COTURN_TAG:-"latest"}
-
+NETBIRD_RELAY_TAG=${NETBIRD_RELAY_TAG:-"latest"}
 
 # exports
 export NETBIRD_DOMAIN
@@ -123,3 +129,7 @@ export NETBIRD_SIGNAL_TAG
 export NETBIRD_MANAGEMENT_TAG
 export COTURN_TAG
 export NETBIRD_TURN_EXTERNAL_IP
+export NETBIRD_RELAY_DOMAIN
+export NETBIRD_RELAY_PORT
+export NETBIRD_RELAY_AUTH_SECRET
+export NETBIRD_RELAY_TAG

infrastructure_files/configure.sh

@@ -89,6 +89,11 @@ fi
 
 export TURN_EXTERNAL_IP_CONFIG
 
+# if not provided, we generate a relay auth secret
+if [[ "x-$NETBIRD_RELAY_AUTH_SECRET" == "x-" ]]; then
+  export NETBIRD_RELAY_AUTH_SECRET=$(openssl rand -base64 32 | sed 's/=//g')
+fi
+
 artifacts_path="./artifacts"
 mkdir -p $artifacts_path
 

infrastructure_files/docker-compose.yml.tmpl

@@ -49,6 +49,23 @@ services:
       options:
         max-size: "500m"
         max-file: "2"
+  # Relay
+  relay:
+    image: netbirdio/relay:$NETBIRD_RELAY_TAG
+    restart: unless-stopped
+    environment:
+    - NB_LOG_LEVEL=info
+    - NB_LISTEN_ADDRESS=:$NETBIRD_RELAY_PORT
+    - NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT
+    # todo: change to a secure secret
+    - NB_AUTH_SECRET=$NETBIRD_RELAY_AUTH_SECRET
+    ports:
+      - $NETBIRD_RELAY_PORT:$NETBIRD_RELAY_PORT
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "500m"
+        max-file: "2"
 
   # Management
   management:

infrastructure_files/getting-started-with-zitadel.sh

@@ -103,13 +103,25 @@ wait_api() {
     INSTANCE_URL=$1
     PAT=$2
     set +e
+    counter=1
     while true; do
-      curl -s --fail -o /dev/null "$INSTANCE_URL/auth/v1/users/me" -H "Authorization: Bearer $PAT"
+      FLAGS="-s"
+      if [[ $counter -eq 45 ]]; then
+        FLAGS="-v"
+        echo ""
+      fi
+
+      curl $FLAGS --fail --connect-timeout 1 -o /dev/null "$INSTANCE_URL/auth/v1/users/me" -H "Authorization: Bearer $PAT"
       if [[ $? -eq 0 ]]; then
         break
       fi
+      if [[ $counter -eq 45 ]]; then
+        echo ""
+        echo "Unable to connect to Zitadel for more than 45s, please check the output above, your firewall rules and the caddy container logs to confirm if there are any issues provisioning TLS certificates"
+      fi
       echo -n " ."
       sleep 1
+      counter=$((counter + 1))
     done
     echo " done"
     set -e
@@ -424,8 +436,10 @@ initEnvironment() {
   ZITADEL_MASTERKEY="$(openssl rand -base64 32 | head -c 32)"
   NETBIRD_PORT=80
   NETBIRD_HTTP_PROTOCOL="http"
+  NETBIRD_RELAY_PROTO="rel"
   TURN_USER="self"
   TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g')
+  NETBIRD_RELAY_AUTH_SECRET=$(openssl rand -base64 32 | sed 's/=//g')
   TURN_MIN_PORT=49152
   TURN_MAX_PORT=65535
   TURN_EXTERNAL_IP_CONFIG=$(get_turn_external_ip)
@@ -442,6 +456,7 @@ initEnvironment() {
     NETBIRD_PORT=443
     CADDY_SECURE_DOMAIN=", $NETBIRD_DOMAIN:$NETBIRD_PORT"
     NETBIRD_HTTP_PROTOCOL="https"
+    NETBIRD_RELAY_PROTO="rels"
   fi
 
   if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -458,7 +473,7 @@ initEnvironment() {
     echo "Generated files already exist, if you want to reinitialize the environment, please remove them first."
     echo "You can use the following commands:"
     echo "  $DOCKER_COMPOSE_COMMAND down --volumes # to remove all containers and volumes"
-    echo "  rm -f docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json"
+    echo "  rm -f docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json relay.env"
     echo "Be aware that this will remove all data from the database, and you will have to reconfigure the dashboard."
     exit 1
   fi
@@ -484,6 +499,7 @@ initEnvironment() {
   echo "" > dashboard.env
   echo "" > turnserver.conf
   echo "" > management.json
+  echo "" > relay.env
 
   mkdir -p machinekey
   chmod 777 machinekey
@@ -498,6 +514,7 @@ initEnvironment() {
   renderTurnServerConf > turnserver.conf
   renderManagementJson > management.json
   renderDashboardEnv > dashboard.env
+  renderRelayEnv > relay.env
 
   echo -e "\nStarting NetBird services\n"
   $DOCKER_COMPOSE_COMMAND up -d
@@ -559,6 +576,8 @@ renderCaddyfile() {
 
 :80${CADDY_SECURE_DOMAIN} {
     import security_headers
+    # relay
+    reverse_proxy /relay* relay:80
     # Signal
     reverse_proxy /signalexchange.SignalExchange/* h2c://signal:10000
     # Management
@@ -629,6 +648,11 @@ renderManagementJson() {
         ],
         "TimeBasedCredentials": false
     },
+    "Relay": {
+        "Addresses": ["$NETBIRD_RELAY_PROTO://$NETBIRD_DOMAIN:$NETBIRD_PORT"],
+        "CredentialsTTL": "24h",
+        "Secret": "$NETBIRD_RELAY_AUTH_SECRET"
+    },
     "Signal": {
         "Proto": "$NETBIRD_HTTP_PROTOCOL",
         "URI": "$NETBIRD_DOMAIN:$NETBIRD_PORT"
@@ -744,6 +768,15 @@ POSTGRES_PASSWORD=$POSTGRES_ROOT_PASSWORD
 EOF
 }
 
+renderRelayEnv() {
+  cat <<EOF
+NB_LOG_LEVEL=info
+NB_LISTEN_ADDRESS=:80
+NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_PROTO://$NETBIRD_DOMAIN:$NETBIRD_PORT
+NB_AUTH_SECRET=$NETBIRD_RELAY_AUTH_SECRET
+EOF
+}
+
 renderDockerCompose() {
   cat <<EOF
 version: "3.4"
@@ -782,6 +815,18 @@ services:
       options:
         max-size: "500m"
         max-file: "2"
+  # Relay
+  relay:
+    image: netbirdio/relay:latest
+    restart: unless-stopped
+    networks: [netbird]
+    env_file:
+      - ./relay.env
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "500m"
+        max-file: "2"
   # Management
   management:
     image: netbirdio/management:latest

infrastructure_files/management.json.tmpl

@@ -20,6 +20,11 @@
         "Secret": "secret",
         "TimeBasedCredentials": false
     },
+    "Relay": {
+        "Addresses": ["rel://$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT"],
+        "CredentialsTTL": "24h",
+        "Secret": "$NETBIRD_RELAY_AUTH_SECRET"
+    },
     "Signal": {
         "Proto": "$NETBIRD_SIGNAL_PROTOCOL",
         "URI": "$NETBIRD_DOMAIN:$NETBIRD_SIGNAL_PORT",

infrastructure_files/setup.env.example

@@ -7,6 +7,7 @@ NETBIRD_DASHBOARD_TAG=""
 NETBIRD_SIGNAL_TAG=""
 NETBIRD_MANAGEMENT_TAG=""
 COTURN_TAG=""
+NETBIRD_RELAY_TAG=""
 
 # Dashboard domain. e.g. app.mydomain.com
 NETBIRD_DOMAIN=""
@@ -91,3 +92,14 @@ NETBIRD_LETSENCRYPT_EMAIL=""
 NETBIRD_DISABLE_ANONYMOUS_METRICS=false
 # DNS DOMAIN configures the domain name used for peer resolution. By default it is netbird.selfhosted
 NETBIRD_MGMT_DNS_DOMAIN=netbird.selfhosted
+
+# -------------------------------------------
+# Relay settings
+# -------------------------------------------
+# Relay server domain. e.g. relay.mydomain.com
+# if not specified it will assume NETBIRD_DOMAIN
+NETBIRD_RELAY_DOMAIN=""
+
+# Relay server connection port. If none is supplied
+# it will default to 33080
+NETBIRD_RELAY_PORT=""

infrastructure_files/tests/setup.env

@@ -26,3 +26,4 @@ NETBIRD_SIGNAL_PORT=12345
 NETBIRD_STORE_CONFIG_ENGINE=$CI_NETBIRD_STORE_CONFIG_ENGINE
 NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=$CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH
 NETBIRD_TURN_EXTERNAL_IP=1.2.3.4
+NETBIRD_RELAY_PORT=33445

management/client/client_test.go

@@ -82,8 +82,9 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

management/cmd/management.go

@@ -195,7 +195,7 @@ var (
 				return fmt.Errorf("failed to build default manager: %v", err)
 			}
 
-			turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+			secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
 
 			trustedPeers := config.ReverseProxy.TrustedPeers
 			defaultTrustedPeers := []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0")}
@@ -271,7 +271,7 @@ var (
 			ephemeralManager.LoadInitialPeers(ctx)
 
 			gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
-			srv, err := server.NewServer(ctx, config, accountManager, peersUpdateManager, turnManager, appMetrics, ephemeralManager)
+			srv, err := server.NewServer(ctx, config, accountManager, peersUpdateManager, secretsManager, appMetrics, ephemeralManager)
 			if err != nil {
 				return fmt.Errorf("failed creating gRPC API handler: %v", err)
 			}
@@ -538,6 +538,10 @@ func loadMgmtConfig(ctx context.Context, mgmtConfigPath string) (*server.Config,
 		}
 	}
 
+	if loadedConfig.Relay != nil {
+		log.Infof("Relay addresses: %v", loadedConfig.Relay.Addresses)
+	}
+
 	return loadedConfig, err
 }
 

management/cmd/management_test.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"testing"
+)
+
+const (
+	exampleConfig = `{
+	  "Relay": {
+		"Addresses": [
+		  "rel://192.168.100.1:8085",
+		  "rel://192.168.100.1:8086"
+		],
+		"CredentialsTTL": "12h0m0s",
+		"Secret": "8f7e9d6c5b4a3f2e1d0c9b8a7f6e5d4c3b2a1f0e9d8c7b6a5f4e3d2c1b0a9f8"
+	  },
+	  "HttpConfig": {
+		"AuthAudience": "https://stageapp/",
+		"AuthIssuer": "https://something.eu.auth0.com/",
+		"OIDCConfigEndpoint": "https://something.eu.auth0.com/.well-known/openid-configuration"
+	  }
+	}`
+)
+
+func Test_loadMgmtConfig(t *testing.T) {
+	tmpFile, err := createConfig()
+	if err != nil {
+		t.Fatalf("failed to create config: %s", err)
+	}
+
+	cfg, err := loadMgmtConfig(context.Background(), tmpFile)
+	if err != nil {
+		t.Fatalf("failed to load management config: %s", err)
+	}
+	if cfg.Relay == nil {
+		t.Fatalf("config is nil")
+	}
+	if len(cfg.Relay.Addresses) == 0 {
+		t.Fatalf("relay address is empty")
+	}
+}
+
+func createConfig() (string, error) {
+	tmpfile, err := os.CreateTemp("", "config.json")
+	if err != nil {
+		return "", err
+	}
+	_, err = tmpfile.Write([]byte(exampleConfig))
+	if err != nil {
+		return "", err
+	}
+
+	if err := tmpfile.Close(); err != nil {
+		return "", err
+	}
+	return tmpfile.Name(), nil
+}

management/proto/management.proto

@@ -177,6 +177,8 @@ message WiretrusteeConfig {
 
   // a Signal server config
   HostConfig signal = 3;
+
+  RelayConfig relay = 4;
 }
 
 // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management)
@@ -193,6 +195,13 @@ message HostConfig {
     DTLS = 4;
   }
 }
+
+message RelayConfig {
+  repeated string urls = 1;
+  string tokenPayload = 2;
+  string tokenSignature = 3;
+}
+
 // ProtectedHostConfig is similar to HostConfig but has additional user and password
 // Mostly used for TURN servers
 message ProtectedHostConfig {

management/server/config.go

@@ -34,6 +34,7 @@ const (
 type Config struct {
 	Stuns      []*Host
 	TURNConfig *TURNConfig
+	Relay      *Relay
 	Signal     *Host
 
 	Datadir                string
@@ -75,6 +76,12 @@ type TURNConfig struct {
 	Turns                []*Host
 }
 
+type Relay struct {
+	Addresses      []string
+	CredentialsTTL util.Duration
+	Secret         string
+}
+
 // HttpServerConfig is a config of the HTTP Management service server
 type HttpServerConfig struct {
 	LetsEncryptDomain string

management/server/grpcserver.go

@@ -16,13 +16,12 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	nbContext "github.com/netbirdio/netbird/management/server/context"
-	"github.com/netbirdio/netbird/management/server/posture"
-
 	"github.com/netbirdio/netbird/encryption"
 	"github.com/netbirdio/netbird/management/proto"
+	nbContext "github.com/netbirdio/netbird/management/server/context"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
 	nbpeer "github.com/netbirdio/netbird/management/server/peer"
+	"github.com/netbirdio/netbird/management/server/posture"
 	internalStatus "github.com/netbirdio/netbird/management/server/status"
 	"github.com/netbirdio/netbird/management/server/telemetry"
 )
@@ -32,17 +31,25 @@ type GRPCServer struct {
 	accountManager AccountManager
 	wgKey          wgtypes.Key
 	proto.UnimplementedManagementServiceServer
-	peersUpdateManager     *PeersUpdateManager
+	peersUpdateManager *PeersUpdateManager
-	config                 *Config
+	config             *Config
-	turnCredentialsManager TURNCredentialsManager
+	secretsManager     SecretsManager
-	jwtValidator           *jwtclaims.JWTValidator
+	jwtValidator       *jwtclaims.JWTValidator
-	jwtClaimsExtractor     *jwtclaims.ClaimsExtractor
+	jwtClaimsExtractor *jwtclaims.ClaimsExtractor
-	appMetrics             telemetry.AppMetrics
+	appMetrics         telemetry.AppMetrics
-	ephemeralManager       *EphemeralManager
+	ephemeralManager   *EphemeralManager
 }
 
 // NewServer creates a new Management server
-func NewServer(ctx context.Context, config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnCredentialsManager TURNCredentialsManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) {
+func NewServer(
+	ctx context.Context,
+	config *Config,
+	accountManager AccountManager,
+	peersUpdateManager *PeersUpdateManager,
+	secretsManager SecretsManager,
+	appMetrics telemetry.AppMetrics,
+	ephemeralManager *EphemeralManager,
+) (*GRPCServer, error) {
 	key, err := wgtypes.GeneratePrivateKey()
 	if err != nil {
 		return nil, err
@@ -88,14 +95,14 @@ func NewServer(ctx context.Context, config *Config, accountManager AccountManage
 	return &GRPCServer{
 		wgKey: key,
 		// peerKey -> event channel
-		peersUpdateManager:     peersUpdateManager,
+		peersUpdateManager: peersUpdateManager,
-		accountManager:         accountManager,
+		accountManager:     accountManager,
-		config:                 config,
+		config:             config,
-		turnCredentialsManager: turnCredentialsManager,
+		secretsManager:     secretsManager,
-		jwtValidator:           jwtValidator,
+		jwtValidator:       jwtValidator,
-		jwtClaimsExtractor:     jwtClaimsExtractor,
+		jwtClaimsExtractor: jwtClaimsExtractor,
-		appMetrics:             appMetrics,
+		appMetrics:         appMetrics,
-		ephemeralManager:       ephemeralManager,
+		ephemeralManager:   ephemeralManager,
 	}, nil
 }
 
@@ -177,9 +184,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi
 
 	s.ephemeralManager.OnPeerConnected(ctx, peer)
 
-	if s.config.TURNConfig.TimeBasedCredentials {
+	s.secretsManager.SetupRefresh(ctx, peer.ID)
-		s.turnCredentialsManager.SetupRefresh(ctx, peer.ID)
-	}
 
 	if s.appMetrics != nil {
 		s.appMetrics.GRPCMetrics().CountSyncRequestDuration(time.Since(reqStart))
@@ -241,7 +246,7 @@ func (s *GRPCServer) sendUpdate(ctx context.Context, accountID string, peerKey w
 
 func (s *GRPCServer) cancelPeerRoutines(ctx context.Context, accountID string, peer *nbpeer.Peer) {
 	s.peersUpdateManager.CloseChannel(ctx, peer.ID)
-	s.turnCredentialsManager.CancelRefresh(peer.ID)
+	s.secretsManager.CancelRefresh(peer.ID)
 	_ = s.accountManager.OnPeerDisconnected(ctx, accountID, peer.Key)
 	s.ephemeralManager.OnPeerDisconnected(ctx, peer)
 }
@@ -427,9 +432,17 @@ func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*p
 		s.ephemeralManager.OnPeerDisconnected(ctx, peer)
 	}
 
+	var relayToken *Token
+	if s.config.Relay != nil && len(s.config.Relay.Addresses) > 0 {
+		relayToken, err = s.secretsManager.GenerateRelayToken()
+		if err != nil {
+			log.Errorf("failed generating Relay token: %v", err)
+		}
+	}
+
 	// if peer has reached this point then it has logged in
 	loginResp := &proto.LoginResponse{
-		WiretrusteeConfig: toWiretrusteeConfig(s.config, nil),
+		WiretrusteeConfig: toWiretrusteeConfig(s.config, nil, relayToken),
 		PeerConfig:        toPeerConfig(peer, netMap.Network, s.accountManager.GetDNSDomain()),
 		Checks:            toProtocolChecks(ctx, postureChecks),
 	}
@@ -487,10 +500,11 @@ func ToResponseProto(configProto Protocol) proto.HostConfig_Protocol {
 	}
 }
 
-func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *proto.WiretrusteeConfig {
+func toWiretrusteeConfig(config *Config, turnCredentials *Token, relayToken *Token) *proto.WiretrusteeConfig {
 	if config == nil {
 		return nil
 	}
+
 	var stuns []*proto.HostConfig
 	for _, stun := range config.Stuns {
 		stuns = append(stuns, &proto.HostConfig{
@@ -498,25 +512,40 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot
 			Protocol: ToResponseProto(stun.Proto),
 		})
 	}
+
 	var turns []*proto.ProtectedHostConfig
-	for _, turn := range config.TURNConfig.Turns {
+	if config.TURNConfig != nil {
-		var username string
+		for _, turn := range config.TURNConfig.Turns {
-		var password string
+			var username string
-		if turnCredentials != nil {
+			var password string
-			username = turnCredentials.Username
+			if turnCredentials != nil {
-			password = turnCredentials.Password
+				username = turnCredentials.Payload
-		} else {
+				password = turnCredentials.Signature
-			username = turn.Username
+			} else {
-			password = turn.Password
+				username = turn.Username
+				password = turn.Password
+			}
+			turns = append(turns, &proto.ProtectedHostConfig{
+				HostConfig: &proto.HostConfig{
+					Uri:      turn.URI,
+					Protocol: ToResponseProto(turn.Proto),
+				},
+				User:     username,
+				Password: password,
+			})
+		}
+	}
+
+	var relayCfg *proto.RelayConfig
+	if config.Relay != nil && len(config.Relay.Addresses) > 0 {
+		relayCfg = &proto.RelayConfig{
+			Urls: config.Relay.Addresses,
+		}
+
+		if relayToken != nil {
+			relayCfg.TokenPayload = relayToken.Payload
+			relayCfg.TokenSignature = relayToken.Signature
 		}
-		turns = append(turns, &proto.ProtectedHostConfig{
-			HostConfig: &proto.HostConfig{
-				Uri:      turn.URI,
-				Protocol: ToResponseProto(turn.Proto),
-			},
-			User:     username,
-			Password: password,
-		})
 	}
 
 	return &proto.WiretrusteeConfig{
@@ -526,6 +555,7 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot
 			Uri:      config.Signal.URI,
 			Protocol: ToResponseProto(config.Signal.Proto),
 		},
+		Relay: relayCfg,
 	}
 }
 
@@ -539,9 +569,9 @@ func toPeerConfig(peer *nbpeer.Peer, network *Network, dnsName string) *proto.Pe
 	}
 }
 
-func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *TURNCredentials, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse {
+func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *Token, relayCredentials *Token, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse {
 	response := &proto.SyncResponse{
-		WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials),
+		WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials, relayCredentials),
 		PeerConfig:        toPeerConfig(peer, networkMap.Network, dnsName),
 		NetworkMap: &proto.NetworkMap{
 			Serial:    networkMap.Network.CurrentSerial(),
@@ -588,15 +618,25 @@ func (s *GRPCServer) IsHealthy(ctx context.Context, req *proto.Empty) (*proto.Em
 
 // sendInitialSync sends initial proto.SyncResponse to the peer requesting synchronization
 func (s *GRPCServer) sendInitialSync(ctx context.Context, peerKey wgtypes.Key, peer *nbpeer.Peer, networkMap *NetworkMap, postureChecks []*posture.Checks, srv proto.ManagementService_SyncServer) error {
-	// make secret time based TURN credentials optional
+	var err error
-	var turnCredentials *TURNCredentials
+
-	if s.config.TURNConfig.TimeBasedCredentials {
+	var turnToken *Token
-		creds := s.turnCredentialsManager.GenerateCredentials()
+	if s.config.TURNConfig != nil && s.config.TURNConfig.TimeBasedCredentials {
-		turnCredentials = &creds
+		turnToken, err = s.secretsManager.GenerateTurnToken()
-	} else {
+		if err != nil {
-		turnCredentials = nil
+			log.Errorf("failed generating TURN token: %v", err)
+		}
 	}
-	plainResp := toSyncResponse(ctx, s.config, peer, turnCredentials, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil)
+
+	var relayToken *Token
+	if s.config.Relay != nil && len(s.config.Relay.Addresses) > 0 {
+		relayToken, err = s.secretsManager.GenerateRelayToken()
+		if err != nil {
+			log.Errorf("failed generating Relay token: %v", err)
+		}
+	}
+
+	plainResp := toSyncResponse(ctx, s.config, peer, turnToken, relayToken, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil)
 
 	encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, plainResp)
 	if err != nil {

management/server/management_proto_test.go

@@ -439,10 +439,11 @@ func startManagementForTest(t TestingT, config *Config) (*grpc.Server, *DefaultA
 	if err != nil {
 		return nil, nil, "", err
 	}
-	turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
+	secretsManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
 
 	ephemeralMgr := NewEphemeralManager(store, accountManager)
-	mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, ephemeralMgr)
+	mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, ephemeralMgr)
 	if err != nil {
 		return nil, nil, "", err
 	}

management/server/management_test.go

@@ -552,8 +552,9 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) {
 	if err != nil {
 		log.Fatalf("failed creating a manager: %v", err)
 	}
-	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
+
-	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil)
+	secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay)
+	mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil)
 	Expect(err).NotTo(HaveOccurred())
 	mgmtProto.RegisterManagementServiceServer(s, mgmtServer)
 	go func() {

management/server/peer.go

@@ -964,7 +964,7 @@ func (am *DefaultAccountManager) updateAccountPeers(ctx context.Context, account
 
 			postureChecks := am.getPeerPostureChecks(account, p)
 			remotePeerNetworkMap := account.GetPeerNetworkMap(ctx, p.ID, customZone, approvedPeersMap, am.metrics.AccountManagerMetrics())
-			update := toSyncResponse(ctx, nil, p, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache)
+			update := toSyncResponse(ctx, nil, p, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache)
 			am.peersUpdateManager.SendUpdate(ctx, p.ID, &UpdateMessage{Update: update})
 		}(peer)
 	}

management/server/peer_test.go

@@ -848,9 +848,9 @@ func TestToSyncResponse(t *testing.T) {
 		DNSLabel:   "peer1",
 		SSHKey:     "peer1-ssh-key",
 	}
-	turnCredentials := &TURNCredentials{
+	turnRelayToken := &Token{
-		Username: "turn-user",
+		Payload:   "turn-user",
-		Password: "turn-pass",
+		Signature: "turn-pass",
 	}
 	networkMap := &NetworkMap{
 		Network:      &Network{Net: *ipnet, Serial: 1000},
@@ -916,7 +916,7 @@ func TestToSyncResponse(t *testing.T) {
 	}
 	dnsCache := &DNSConfigCache{}
 
-	response := toSyncResponse(context.Background(), config, peer, turnCredentials, networkMap, dnsName, checks, dnsCache)
+	response := toSyncResponse(context.Background(), config, peer, turnRelayToken, turnRelayToken, networkMap, dnsName, checks, dnsCache)
 
 	assert.NotNil(t, response)
 	// assert peer config

management/server/token_mgr.go

@@ -0,0 +1,222 @@
+package server
+
+import (
+	"context"
+	"crypto/sha1"
+	"crypto/sha256"
+	"fmt"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/management/proto"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+)
+
+const defaultDuration = 12 * time.Hour
+
+// SecretsManager used to manage TURN and relay secrets
+type SecretsManager interface {
+	GenerateTurnToken() (*Token, error)
+	GenerateRelayToken() (*Token, error)
+	SetupRefresh(ctx context.Context, peerKey string)
+	CancelRefresh(peerKey string)
+}
+
+// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server
+type TimeBasedAuthSecretsManager struct {
+	mux            sync.Mutex
+	turnCfg        *TURNConfig
+	relayCfg       *Relay
+	turnHmacToken  *auth.TimedHMAC
+	relayHmacToken *auth.TimedHMAC
+	updateManager  *PeersUpdateManager
+	turnCancelMap  map[string]chan struct{}
+	relayCancelMap map[string]chan struct{}
+}
+
+type Token auth.Token
+
+func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayCfg *Relay) *TimeBasedAuthSecretsManager {
+	mgr := &TimeBasedAuthSecretsManager{
+		updateManager:  updateManager,
+		turnCfg:        turnCfg,
+		relayCfg:       relayCfg,
+		turnCancelMap:  make(map[string]chan struct{}),
+		relayCancelMap: make(map[string]chan struct{}),
+	}
+
+	if turnCfg != nil {
+		duration := turnCfg.CredentialsTTL.Duration
+		if turnCfg.CredentialsTTL.Duration <= 0 {
+			log.Warnf("TURN credentials TTL is not set or invalid, using default value %s", defaultDuration)
+			duration = defaultDuration
+		}
+		mgr.turnHmacToken = auth.NewTimedHMAC(turnCfg.Secret, duration)
+	}
+
+	if relayCfg != nil {
+		duration := relayCfg.CredentialsTTL.Duration
+		if relayCfg.CredentialsTTL.Duration <= 0 {
+			log.Warnf("Relay credentials TTL is not set or invalid, using default value %s", defaultDuration)
+			duration = defaultDuration
+		}
+
+		mgr.relayHmacToken = auth.NewTimedHMAC(relayCfg.Secret, duration)
+	}
+
+	return mgr
+}
+
+// GenerateTurnToken generates new time-based secret credentials for TURN
+func (m *TimeBasedAuthSecretsManager) GenerateTurnToken() (*Token, error) {
+	if m.turnHmacToken == nil {
+		return nil, fmt.Errorf("TURN configuration is not set")
+	}
+	turnToken, err := m.turnHmacToken.GenerateToken(sha1.New)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate TURN token: %s", err)
+	}
+	return (*Token)(turnToken), nil
+}
+
+// GenerateRelayToken generates new time-based secret credentials for relay
+func (m *TimeBasedAuthSecretsManager) GenerateRelayToken() (*Token, error) {
+	if m.relayHmacToken == nil {
+		return nil, fmt.Errorf("relay configuration is not set")
+	}
+	relayToken, err := m.relayHmacToken.GenerateToken(sha256.New)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate relay token: %s", err)
+	}
+	return (*Token)(relayToken), nil
+}
+
+func (m *TimeBasedAuthSecretsManager) cancelTURN(peerID string) {
+	if channel, ok := m.turnCancelMap[peerID]; ok {
+		close(channel)
+		delete(m.turnCancelMap, peerID)
+	}
+}
+
+func (m *TimeBasedAuthSecretsManager) cancelRelay(peerID string) {
+	if channel, ok := m.relayCancelMap[peerID]; ok {
+		close(channel)
+		delete(m.relayCancelMap, peerID)
+	}
+}
+
+// CancelRefresh cancels scheduled peer credentials refresh
+func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) {
+	m.mux.Lock()
+	defer m.mux.Unlock()
+	m.cancelTURN(peerID)
+	m.cancelRelay(peerID)
+}
+
+// SetupRefresh starts peer credentials refresh
+func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) {
+	m.mux.Lock()
+	defer m.mux.Unlock()
+
+	m.cancelTURN(peerID)
+	m.cancelRelay(peerID)
+
+	if m.turnCfg != nil && m.turnCfg.TimeBasedCredentials {
+		turnCancel := make(chan struct{}, 1)
+		m.turnCancelMap[peerID] = turnCancel
+		go m.refreshTURNTokens(ctx, peerID, turnCancel)
+		log.WithContext(ctx).Debugf("starting TURN refresh for %s", peerID)
+	}
+
+	if m.relayCfg != nil {
+		relayCancel := make(chan struct{}, 1)
+		m.relayCancelMap[peerID] = relayCancel
+		go m.refreshRelayTokens(ctx, peerID, relayCancel)
+		log.WithContext(ctx).Debugf("starting relay refresh for %s", peerID)
+	}
+}
+
+func (m *TimeBasedAuthSecretsManager) refreshTURNTokens(ctx context.Context, peerID string, cancel chan struct{}) {
+	ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-cancel:
+			log.WithContext(ctx).Debugf("stopping TURN refresh for %s", peerID)
+			return
+		case <-ticker.C:
+			m.pushNewTURNTokens(ctx, peerID)
+		}
+	}
+}
+
+func (m *TimeBasedAuthSecretsManager) refreshRelayTokens(ctx context.Context, peerID string, cancel chan struct{}) {
+	ticker := time.NewTicker(m.relayCfg.CredentialsTTL.Duration / 4 * 3)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-cancel:
+			log.WithContext(ctx).Debugf("stopping relay refresh for %s", peerID)
+			return
+		case <-ticker.C:
+			m.pushNewRelayTokens(ctx, peerID)
+		}
+	}
+}
+
+func (m *TimeBasedAuthSecretsManager) pushNewTURNTokens(ctx context.Context, peerID string) {
+	turnToken, err := m.turnHmacToken.GenerateToken(sha1.New)
+	if err != nil {
+		log.Errorf("failed to generate token for peer '%s': %s", peerID, err)
+		return
+	}
+
+	var turns []*proto.ProtectedHostConfig
+	for _, host := range m.turnCfg.Turns {
+		turn := &proto.ProtectedHostConfig{
+			HostConfig: &proto.HostConfig{
+				Uri:      host.URI,
+				Protocol: ToResponseProto(host.Proto),
+			},
+			User:     turnToken.Payload,
+			Password: turnToken.Signature,
+		}
+		turns = append(turns, turn)
+	}
+
+	update := &proto.SyncResponse{
+		WiretrusteeConfig: &proto.WiretrusteeConfig{
+			Turns: turns,
+			// omit Relay to avoid updates there
+		},
+	}
+
+	log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID)
+	m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update})
+}
+
+func (m *TimeBasedAuthSecretsManager) pushNewRelayTokens(ctx context.Context, peerID string) {
+	relayToken, err := m.relayHmacToken.GenerateToken(sha256.New)
+	if err != nil {
+		log.Errorf("failed to generate relay token for peer '%s': %s", peerID, err)
+		return
+	}
+
+	update := &proto.SyncResponse{
+		WiretrusteeConfig: &proto.WiretrusteeConfig{
+			Relay: &proto.RelayConfig{
+				Urls:           m.relayCfg.Addresses,
+				TokenPayload:   relayToken.Payload,
+				TokenSignature: relayToken.Signature,
+			},
+			// omit Turns to avoid updates there
+		},
+	}
+
+	log.WithContext(ctx).Debugf("sending new relay credentials to peer %s", peerID)
+	m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update})
+}

management/server/token_mgr_test.go

@@ -0,0 +1,218 @@
+package server
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"hash"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/management/proto"
+	"github.com/netbirdio/netbird/util"
+)
+
+var TurnTestHost = &Host{
+	Proto:    UDP,
+	URI:      "turn:turn.wiretrustee.com:77777",
+	Username: "username",
+	Password: "",
+}
+
+func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) {
+	ttl := util.Duration{Duration: time.Hour}
+	secret := "some_secret"
+	peersManager := NewPeersUpdateManager(nil)
+
+	rc := &Relay{
+		Addresses:      []string{"localhost:0"},
+		CredentialsTTL: ttl,
+		Secret:         secret,
+	}
+
+	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
+		CredentialsTTL:       ttl,
+		Secret:               secret,
+		Turns:                []*Host{TurnTestHost},
+		TimeBasedCredentials: true,
+	}, rc)
+
+	turnCredentials, err := tested.GenerateTurnToken()
+	require.NoError(t, err)
+
+	if turnCredentials.Payload == "" {
+		t.Errorf("expected generated TURN username not to be empty, got empty")
+	}
+	if turnCredentials.Signature == "" {
+		t.Errorf("expected generated TURN password not to be empty, got empty")
+	}
+
+	validateMAC(t, sha1.New, turnCredentials.Payload, turnCredentials.Signature, []byte(secret))
+
+	relayCredentials, err := tested.GenerateRelayToken()
+	require.NoError(t, err)
+
+	if relayCredentials.Payload == "" {
+		t.Errorf("expected generated relay payload not to be empty, got empty")
+	}
+	if relayCredentials.Signature == "" {
+		t.Errorf("expected generated relay signature not to be empty, got empty")
+	}
+
+	validateMAC(t, sha256.New, relayCredentials.Payload, relayCredentials.Signature, []byte(secret))
+}
+
+func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) {
+	ttl := util.Duration{Duration: 2 * time.Second}
+	secret := "some_secret"
+	peersManager := NewPeersUpdateManager(nil)
+	peer := "some_peer"
+	updateChannel := peersManager.CreateChannel(context.Background(), peer)
+
+	rc := &Relay{
+		Addresses:      []string{"localhost:0"},
+		CredentialsTTL: ttl,
+		Secret:         secret,
+	}
+	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
+		CredentialsTTL:       ttl,
+		Secret:               secret,
+		Turns:                []*Host{TurnTestHost},
+		TimeBasedCredentials: true,
+	}, rc)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	tested.SetupRefresh(ctx, peer)
+
+	if _, ok := tested.turnCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in the turn cancel map, got not present")
+	}
+
+	if _, ok := tested.relayCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in the relay cancel map, got not present")
+	}
+
+	var updates []*UpdateMessage
+
+loop:
+	for timeout := time.After(5 * time.Second); ; {
+		select {
+		case update := <-updateChannel:
+			updates = append(updates, update)
+		case <-timeout:
+			break loop
+		}
+
+		if len(updates) >= 2 {
+			break loop
+		}
+	}
+
+	if len(updates) < 2 {
+		t.Errorf("expecting at least 2 peer credentials updates, got %v", len(updates))
+	}
+
+	var turnUpdates, relayUpdates int
+	var firstTurnUpdate, secondTurnUpdate *proto.ProtectedHostConfig
+	var firstRelayUpdate, secondRelayUpdate *proto.RelayConfig
+
+	for _, update := range updates {
+		if turns := update.Update.GetWiretrusteeConfig().GetTurns(); len(turns) > 0 {
+			turnUpdates++
+			if turnUpdates == 1 {
+				firstTurnUpdate = turns[0]
+			} else {
+				secondTurnUpdate = turns[0]
+			}
+		}
+		if relay := update.Update.GetWiretrusteeConfig().GetRelay(); relay != nil {
+			relayUpdates++
+			if relayUpdates == 1 {
+				firstRelayUpdate = relay
+			} else {
+				secondRelayUpdate = relay
+			}
+		}
+	}
+
+	if turnUpdates < 1 {
+		t.Errorf("expecting at least 1 TURN credential update, got %v", turnUpdates)
+	}
+	if relayUpdates < 1 {
+		t.Errorf("expecting at least 1 relay credential update, got %v", relayUpdates)
+	}
+
+	if firstTurnUpdate != nil && secondTurnUpdate != nil {
+		if firstTurnUpdate.Password == secondTurnUpdate.Password {
+			t.Errorf("expecting first TURN credential update password %v to be different from second, got equal", firstTurnUpdate.Password)
+		}
+	}
+
+	if firstRelayUpdate != nil && secondRelayUpdate != nil {
+		if firstRelayUpdate.TokenSignature == secondRelayUpdate.TokenSignature {
+			t.Errorf("expecting first relay credential update signature %v to be different from second, got equal", firstRelayUpdate.TokenSignature)
+		}
+	}
+}
+
+func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) {
+	ttl := util.Duration{Duration: time.Hour}
+	secret := "some_secret"
+	peersManager := NewPeersUpdateManager(nil)
+	peer := "some_peer"
+
+	rc := &Relay{
+		Addresses:      []string{"localhost:0"},
+		CredentialsTTL: ttl,
+		Secret:         secret,
+	}
+	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
+		CredentialsTTL:       ttl,
+		Secret:               secret,
+		Turns:                []*Host{TurnTestHost},
+		TimeBasedCredentials: true,
+	}, rc)
+
+	tested.SetupRefresh(context.Background(), peer)
+	if _, ok := tested.turnCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in turn cancel map, got not present")
+	}
+	if _, ok := tested.relayCancelMap[peer]; !ok {
+		t.Errorf("expecting peer to be present in relay cancel map, got not present")
+	}
+
+	tested.CancelRefresh(peer)
+	if _, ok := tested.turnCancelMap[peer]; ok {
+		t.Errorf("expecting peer to be not present in turn cancel map, got present")
+	}
+	if _, ok := tested.relayCancelMap[peer]; ok {
+		t.Errorf("expecting peer to be not present in relay cancel map, got present")
+	}
+}
+
+func validateMAC(t *testing.T, algo func() hash.Hash, username string, actualMAC string, key []byte) {
+	t.Helper()
+	mac := hmac.New(algo, key)
+
+	_, err := mac.Write([]byte(username))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedMAC := mac.Sum(nil)
+	decodedMAC, err := base64.StdEncoding.DecodeString(actualMAC)
+	if err != nil {
+		t.Fatal(err)
+	}
+	equal := hmac.Equal(decodedMAC, expectedMAC)
+
+	if !equal {
+		t.Errorf("expected password MAC to be %s. got %s", expectedMAC, decodedMAC)
+	}
+}

management/server/turncredentials.go

@@ -1,126 +0,0 @@
-package server
-
-import (
-	"context"
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/base64"
-	"fmt"
-	"sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/netbirdio/netbird/management/proto"
-)
-
-// TURNCredentialsManager used to manage TURN credentials
-type TURNCredentialsManager interface {
-	GenerateCredentials() TURNCredentials
-	SetupRefresh(ctx context.Context, peerKey string)
-	CancelRefresh(peerKey string)
-}
-
-// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server
-type TimeBasedAuthSecretsManager struct {
-	mux           sync.Mutex
-	config        *TURNConfig
-	updateManager *PeersUpdateManager
-	cancelMap     map[string]chan struct{}
-}
-
-type TURNCredentials struct {
-	Username string
-	Password string
-}
-
-func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, config *TURNConfig) *TimeBasedAuthSecretsManager {
-	return &TimeBasedAuthSecretsManager{
-		mux:           sync.Mutex{},
-		config:        config,
-		updateManager: updateManager,
-		cancelMap:     make(map[string]chan struct{}),
-	}
-}
-
-// GenerateCredentials generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret
-func (m *TimeBasedAuthSecretsManager) GenerateCredentials() TURNCredentials {
-	mac := hmac.New(sha1.New, []byte(m.config.Secret))
-
-	timeAuth := time.Now().Add(m.config.CredentialsTTL.Duration).Unix()
-
-	username := fmt.Sprint(timeAuth)
-
-	_, err := mac.Write([]byte(username))
-	if err != nil {
-		log.Errorln("Generating turn password failed with error: ", err)
-	}
-
-	bytePassword := mac.Sum(nil)
-	password := base64.StdEncoding.EncodeToString(bytePassword)
-
-	return TURNCredentials{
-		Username: username,
-		Password: password,
-	}
-
-}
-
-func (m *TimeBasedAuthSecretsManager) cancel(peerID string) {
-	if channel, ok := m.cancelMap[peerID]; ok {
-		close(channel)
-		delete(m.cancelMap, peerID)
-	}
-}
-
-// CancelRefresh cancels scheduled peer credentials refresh
-func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) {
-	m.mux.Lock()
-	defer m.mux.Unlock()
-	m.cancel(peerID)
-}
-
-// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer.
-// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone.
-func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) {
-	m.mux.Lock()
-	defer m.mux.Unlock()
-	m.cancel(peerID)
-	cancel := make(chan struct{}, 1)
-	m.cancelMap[peerID] = cancel
-	log.WithContext(ctx).Debugf("starting turn refresh for %s", peerID)
-
-	go func() {
-		// we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL)
-		ticker := time.NewTicker(m.config.CredentialsTTL.Duration / 4 * 3)
-
-		for {
-			select {
-			case <-cancel:
-				log.WithContext(ctx).Debugf("stopping turn refresh for %s", peerID)
-				return
-			case <-ticker.C:
-				c := m.GenerateCredentials()
-				var turns []*proto.ProtectedHostConfig
-				for _, host := range m.config.Turns {
-					turns = append(turns, &proto.ProtectedHostConfig{
-						HostConfig: &proto.HostConfig{
-							Uri:      host.URI,
-							Protocol: ToResponseProto(host.Proto),
-						},
-						User:     c.Username,
-						Password: c.Password,
-					})
-				}
-
-				update := &proto.SyncResponse{
-					WiretrusteeConfig: &proto.WiretrusteeConfig{
-						Turns: turns,
-					},
-				}
-				log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID)
-				m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update})
-			}
-		}
-	}()
-}

management/server/turncredentials_test.go

@@ -1,136 +0,0 @@
-package server
-
-import (
-	"context"
-	"crypto/hmac"
-	"crypto/sha1"
-	"encoding/base64"
-	"testing"
-	"time"
-
-	"github.com/netbirdio/netbird/util"
-)
-
-var TurnTestHost = &Host{
-	Proto:    UDP,
-	URI:      "turn:turn.wiretrustee.com:77777",
-	Username: "username",
-	Password: "",
-}
-
-func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) {
-	ttl := util.Duration{Duration: time.Hour}
-	secret := "some_secret"
-	peersManager := NewPeersUpdateManager(nil)
-
-	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
-		CredentialsTTL: ttl,
-		Secret:         secret,
-		Turns:          []*Host{TurnTestHost},
-	})
-
-	credentials := tested.GenerateCredentials()
-
-	if credentials.Username == "" {
-		t.Errorf("expected generated TURN username not to be empty, got empty")
-	}
-	if credentials.Password == "" {
-		t.Errorf("expected generated TURN password not to be empty, got empty")
-	}
-
-	validateMAC(t, credentials.Username, credentials.Password, []byte(secret))
-
-}
-
-func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) {
-	ttl := util.Duration{Duration: 2 * time.Second}
-	secret := "some_secret"
-	peersManager := NewPeersUpdateManager(nil)
-	peer := "some_peer"
-	updateChannel := peersManager.CreateChannel(context.Background(), peer)
-
-	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
-		CredentialsTTL: ttl,
-		Secret:         secret,
-		Turns:          []*Host{TurnTestHost},
-	})
-
-	tested.SetupRefresh(context.Background(), peer)
-
-	if _, ok := tested.cancelMap[peer]; !ok {
-		t.Errorf("expecting peer to be present in a cancel map, got not present")
-	}
-
-	var updates []*UpdateMessage
-
-loop:
-	for timeout := time.After(5 * time.Second); ; {
-
-		select {
-		case update := <-updateChannel:
-			updates = append(updates, update)
-		case <-timeout:
-			break loop
-		}
-
-		if len(updates) >= 2 {
-			break loop
-		}
-	}
-
-	if len(updates) < 2 {
-		t.Errorf("expecting 2 peer credentials updates, got %v", len(updates))
-	}
-
-	firstUpdate := updates[0].Update.GetWiretrusteeConfig().Turns[0]
-	secondUpdate := updates[1].Update.GetWiretrusteeConfig().Turns[0]
-
-	if firstUpdate.Password == secondUpdate.Password {
-		t.Errorf("expecting first credential update password %v to be diffeerent from second, got equal", firstUpdate.Password)
-	}
-
-}
-
-func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) {
-	ttl := util.Duration{Duration: time.Hour}
-	secret := "some_secret"
-	peersManager := NewPeersUpdateManager(nil)
-	peer := "some_peer"
-
-	tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{
-		CredentialsTTL: ttl,
-		Secret:         secret,
-		Turns:          []*Host{TurnTestHost},
-	})
-
-	tested.SetupRefresh(context.Background(), peer)
-	if _, ok := tested.cancelMap[peer]; !ok {
-		t.Errorf("expecting peer to be present in a cancel map, got not present")
-	}
-
-	tested.CancelRefresh(peer)
-	if _, ok := tested.cancelMap[peer]; ok {
-		t.Errorf("expecting peer to be not present in a cancel map, got present")
-	}
-}
-
-func validateMAC(t *testing.T, username string, actualMAC string, key []byte) {
-	t.Helper()
-	mac := hmac.New(sha1.New, key)
-
-	_, err := mac.Write([]byte(username))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	expectedMAC := mac.Sum(nil)
-	decodedMAC, err := base64.StdEncoding.DecodeString(actualMAC)
-	if err != nil {
-		t.Fatal(err)
-	}
-	equal := hmac.Equal(decodedMAC, expectedMAC)
-
-	if !equal {
-		t.Errorf("expected password MAC to be %s. got %s", expectedMAC, decodedMAC)
-	}
-}

relay/Dockerfile

@@ -0,0 +1,4 @@
+FROM gcr.io/distroless/base:debug
+ENTRYPOINT [ "/go/bin/netbird-relay" ]
+ENV NB_LOG_FILE=console
+COPY netbird-relay /go/bin/netbird-relay

relay/auth/allow/allow_all.go

@@ -0,0 +1,12 @@
+package allow
+
+import "hash"
+
+// Auth is a Validator that allows all connections.
+// Used this for testing purposes only.
+type Auth struct {
+}
+
+func (a *Auth) Validate(func() hash.Hash, any) error {
+	return nil
+}

relay/auth/doc.go

@@ -0,0 +1,26 @@
+/*
+Package auth manages the authentication process with the relay server.
+
+Key Components:
+
+Validator: The Validator interface defines the Validate method. Any type that provides this method can be used as a
+Validator.
+
+Methods:
+
+Validate(func() hash.Hash, any): This method is defined in the Validator interface and is used to validate the authentication.
+
+Usage:
+
+To create a new AllowAllAuth validator, simply instantiate it:
+
+	validator := &allow.Auth{}
+
+To validate the authentication, use the Validate method:
+
+	err := validator.Validate(sha256.New, any)
+
+This package provides a simple and effective way to manage authentication with the relay server, ensuring that the
+peers are authenticated properly.
+*/
+package auth

relay/auth/hmac/doc.go

@@ -0,0 +1,8 @@
+/*
+This package uses a similar HMAC method for authentication with the TURN server. The Management server provides the
+tokens for the peers. The peers manage these tokens in the token store. The token store is a simple thread safe store
+that keeps the tokens in memory. These tokens are used to authenticate the peers with the Relay server in the hello
+message.
+*/
+
+package hmac

relay/auth/hmac/store.go

@@ -0,0 +1,36 @@
+package hmac
+
+import (
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// TokenStore is a simple in-memory store for token
+// With this can update the token in thread safe way
+type TokenStore struct {
+	mu    sync.Mutex
+	token []byte
+}
+
+func (a *TokenStore) UpdateToken(token *Token) error {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if token == nil {
+		return nil
+	}
+
+	t, err := marshalToken(*token)
+	if err != nil {
+		log.Debugf("failed to marshal token: %s", err)
+		return err
+	}
+	a.token = t
+	return nil
+}
+
+func (a *TokenStore) TokenBinary() []byte {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.token
+}

relay/auth/hmac/token.go

@@ -0,0 +1,105 @@
+package hmac
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"encoding/base64"
+	"encoding/gob"
+	"fmt"
+	"hash"
+	"strconv"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type Token struct {
+	Payload   string
+	Signature string
+}
+
+func marshalToken(token Token) ([]byte, error) {
+	var buffer bytes.Buffer
+	encoder := gob.NewEncoder(&buffer)
+	err := encoder.Encode(token)
+	if err != nil {
+		log.Debugf("failed to marshal token: %s", err)
+		return nil, fmt.Errorf("failed to marshal token: %w", err)
+	}
+	return buffer.Bytes(), nil
+}
+
+func unmarshalToken(payload []byte) (Token, error) {
+	var creds Token
+	buffer := bytes.NewBuffer(payload)
+	decoder := gob.NewDecoder(buffer)
+	err := decoder.Decode(&creds)
+	return creds, err
+}
+
+// TimedHMAC generates a token with TTL and uses a pre-shared secret known to the relay server
+type TimedHMAC struct {
+	secret     string
+	timeToLive time.Duration
+}
+
+// NewTimedHMAC creates a new TimedHMAC instance
+func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC {
+	return &TimedHMAC{
+		secret:     secret,
+		timeToLive: timeToLive,
+	}
+}
+
+// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC
+// hash of a timestamp with a preshared TURN secret
+func (m *TimedHMAC) GenerateToken(algo func() hash.Hash) (*Token, error) {
+	timeAuth := time.Now().Add(m.timeToLive).Unix()
+	timeStamp := strconv.FormatInt(timeAuth, 10)
+
+	checksum, err := m.generate(algo, timeStamp)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Token{
+		Payload:   timeStamp,
+		Signature: base64.StdEncoding.EncodeToString(checksum),
+	}, nil
+}
+
+// Validate checks if the token is valid
+func (m *TimedHMAC) Validate(algo func() hash.Hash, token Token) error {
+	expectedMAC, err := m.generate(algo, token.Payload)
+	if err != nil {
+		return err
+	}
+
+	expectedSignature := base64.StdEncoding.EncodeToString(expectedMAC)
+
+	if !hmac.Equal([]byte(expectedSignature), []byte(token.Signature)) {
+		return fmt.Errorf("signature mismatch")
+	}
+
+	timeAuthInt, err := strconv.ParseInt(token.Payload, 10, 64)
+	if err != nil {
+		return fmt.Errorf("invalid payload: %w", err)
+	}
+
+	if time.Now().Unix() > timeAuthInt {
+		return fmt.Errorf("expired token")
+	}
+
+	return nil
+}
+
+func (m *TimedHMAC) generate(algo func() hash.Hash, payload string) ([]byte, error) {
+	mac := hmac.New(algo, []byte(m.secret))
+	_, err := mac.Write([]byte(payload))
+	if err != nil {
+		log.Debugf("failed to generate token: %s", err)
+		return nil, fmt.Errorf("failed to generate token: %w", err)
+	}
+
+	return mac.Sum(nil), nil
+}

relay/auth/hmac/token_test.go

@@ -0,0 +1,105 @@
+package hmac
+
+import (
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestGenerateCredentials(t *testing.T) {
+	secret := "secret"
+	timeToLive := 1 * time.Hour
+	v := NewTimedHMAC(secret, timeToLive)
+
+	creds, err := v.GenerateToken(sha1.New)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if creds.Payload == "" {
+		t.Fatalf("expected non-empty payload")
+	}
+
+	_, err = strconv.ParseInt(creds.Payload, 10, 64)
+	if err != nil {
+		t.Fatalf("expected payload to be a valid unix timestamp, got %v", err)
+	}
+
+	_, err = base64.StdEncoding.DecodeString(creds.Signature)
+	if err != nil {
+		t.Fatalf("expected signature to be base64 encoded, got %v", err)
+	}
+}
+
+func TestValidateCredentials(t *testing.T) {
+	secret := "supersecret"
+	timeToLive := 1 * time.Hour
+	manager := NewTimedHMAC(secret, timeToLive)
+
+	// Test valid token
+	creds, err := manager.GenerateToken(sha1.New)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if err := manager.Validate(sha1.New, *creds); err != nil {
+		t.Fatalf("expected valid token: %s", err)
+	}
+}
+
+func TestInvalidSignature(t *testing.T) {
+	secret := "supersecret"
+	timeToLive := 1 * time.Hour
+	manager := NewTimedHMAC(secret, timeToLive)
+
+	creds, err := manager.GenerateToken(sha256.New)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	invalidCreds := &Token{
+		Payload:   creds.Payload,
+		Signature: "invalidsignature",
+	}
+
+	if err = manager.Validate(sha1.New, *invalidCreds); err == nil {
+		t.Fatalf("expected invalid token due to signature mismatch")
+	}
+}
+
+func TestExpired(t *testing.T) {
+	secret := "supersecret"
+	v := NewTimedHMAC(secret, -1*time.Hour)
+	expiredCreds, err := v.GenerateToken(sha256.New)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	if err = v.Validate(sha1.New, *expiredCreds); err == nil {
+		t.Fatalf("expected invalid token due to expiration")
+	}
+}
+
+func TestInvalidPayload(t *testing.T) {
+	secret := "supersecret"
+	timeToLive := 1 * time.Hour
+	v := NewTimedHMAC(secret, timeToLive)
+
+	creds, err := v.GenerateToken(sha256.New)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	// Test invalid payload
+	invalidPayloadCreds := &Token{
+		Payload:   "invalidtimestamp",
+		Signature: creds.Signature,
+	}
+
+	if err = v.Validate(sha1.New, *invalidPayloadCreds); err == nil {
+		t.Fatalf("expected invalid token due to invalid payload")
+	}
+}

relay/auth/hmac/validator.go

@@ -0,0 +1,33 @@
+package hmac
+
+import (
+	"fmt"
+	"hash"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type TimedHMACValidator struct {
+	*TimedHMAC
+}
+
+func NewTimedHMACValidator(secret string, duration time.Duration) *TimedHMACValidator {
+	ta := NewTimedHMAC(secret, duration)
+	return &TimedHMACValidator{
+		ta,
+	}
+}
+
+func (a *TimedHMACValidator) Validate(algo func() hash.Hash, credentials any) error {
+	b, ok := credentials.([]byte)
+	if !ok {
+		return fmt.Errorf("invalid credentials type")
+	}
+	c, err := unmarshalToken(b)
+	if err != nil {
+		log.Debugf("failed to unmarshal token: %s", err)
+		return err
+	}
+	return a.TimedHMAC.Validate(algo, c)
+}

relay/auth/validator.go

@@ -0,0 +1,8 @@
+package auth
+
+import "hash"
+
+// Validator is an interface that defines the Validate method.
+type Validator interface {
+	Validate(func() hash.Hash, any) error
+}

relay/client/addr.go

@@ -0,0 +1,13 @@
+package client
+
+type RelayAddr struct {
+	addr string
+}
+
+func (a RelayAddr) Network() string {
+	return "relay"
+}
+
+func (a RelayAddr) String() string {
+	return a.addr
+}

relay/client/client.go

@@ -0,0 +1,553 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/relay/client/dialer/ws"
+	"github.com/netbirdio/netbird/relay/healthcheck"
+	"github.com/netbirdio/netbird/relay/messages"
+	"github.com/netbirdio/netbird/relay/messages/address"
+	auth2 "github.com/netbirdio/netbird/relay/messages/auth"
+)
+
+const (
+	bufferSize            = 8820
+	serverResponseTimeout = 8 * time.Second
+)
+
+var (
+	ErrConnAlreadyExists = fmt.Errorf("connection already exists")
+)
+
+type internalStopFlag struct {
+	sync.Mutex
+	stop bool
+}
+
+func newInternalStopFlag() *internalStopFlag {
+	return &internalStopFlag{}
+}
+
+func (isf *internalStopFlag) set() {
+	isf.Lock()
+	defer isf.Unlock()
+	isf.stop = true
+}
+
+func (isf *internalStopFlag) isSet() bool {
+	isf.Lock()
+	defer isf.Unlock()
+	return isf.stop
+}
+
+// Msg carry the payload from the server to the client. With this struct, the net.Conn can free the buffer.
+type Msg struct {
+	Payload []byte
+
+	bufPool *sync.Pool
+	bufPtr  *[]byte
+}
+
+func (m *Msg) Free() {
+	m.bufPool.Put(m.bufPtr)
+}
+
+type connContainer struct {
+	conn        *Conn
+	messages    chan Msg
+	msgChanLock sync.Mutex
+	closed      bool // flag to check if channel is closed
+}
+
+func newConnContainer(conn *Conn, messages chan Msg) *connContainer {
+	return &connContainer{
+		conn:     conn,
+		messages: messages,
+	}
+}
+
+func (cc *connContainer) writeMsg(msg Msg) {
+	cc.msgChanLock.Lock()
+	defer cc.msgChanLock.Unlock()
+	if cc.closed {
+		return
+	}
+	cc.messages <- msg
+}
+
+func (cc *connContainer) close() {
+	cc.msgChanLock.Lock()
+	defer cc.msgChanLock.Unlock()
+	if cc.closed {
+		return
+	}
+	close(cc.messages)
+	cc.closed = true
+}
+
+// Client is a client for the relay server. It is responsible for establishing a connection to the relay server and
+// managing connections to other peers. All exported functions are safe to call concurrently. After close the connection,
+// the client can be reused by calling Connect again. When the client is closed, all connections are closed too.
+// While the Connect is in progress, the OpenConn function will block until the connection is established with relay server.
+type Client struct {
+	log            *log.Entry
+	parentCtx      context.Context
+	connectionURL  string
+	authTokenStore *auth.TokenStore
+	hashedID       []byte
+
+	bufPool *sync.Pool
+
+	relayConn        net.Conn
+	conns            map[string]*connContainer
+	serviceIsRunning bool
+	mu               sync.Mutex // protect serviceIsRunning and conns
+	readLoopMutex    sync.Mutex
+	wgReadLoop       sync.WaitGroup
+	instanceURL      *RelayAddr
+	muInstanceURL    sync.Mutex
+
+	onDisconnectListener func()
+	listenerMutex        sync.Mutex
+}
+
+// NewClient creates a new client for the relay server. The client is not connected to the server until the Connect
+func NewClient(ctx context.Context, serverURL string, authTokenStore *auth.TokenStore, peerID string) *Client {
+	hashedID, hashedStringId := messages.HashID(peerID)
+	return &Client{
+		log:            log.WithField("client_id", hashedStringId),
+		parentCtx:      ctx,
+		connectionURL:  serverURL,
+		authTokenStore: authTokenStore,
+		hashedID:       hashedID,
+		bufPool: &sync.Pool{
+			New: func() any {
+				buf := make([]byte, bufferSize)
+				return &buf
+			},
+		},
+		conns: make(map[string]*connContainer),
+	}
+}
+
+// Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs.
+func (c *Client) Connect() error {
+	c.log.Infof("connecting to relay server: %s", c.connectionURL)
+	c.readLoopMutex.Lock()
+	defer c.readLoopMutex.Unlock()
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.serviceIsRunning {
+		return nil
+	}
+
+	err := c.connect()
+	if err != nil {
+		return err
+	}
+
+	c.serviceIsRunning = true
+
+	c.wgReadLoop.Add(1)
+	go c.readLoop(c.relayConn)
+
+	c.log.Infof("relay connection established with: %s", c.connectionURL)
+	return nil
+}
+
+// OpenConn create a new net.Conn for the destination peer ID. In case if the connection is in progress
+// to the relay server, the function will block until the connection is established or timed out. Otherwise,
+// it will return immediately.
+// todo: what should happen if call with the same peerID with multiple times?
+func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if !c.serviceIsRunning {
+		return nil, fmt.Errorf("relay connection is not established")
+	}
+
+	hashedID, hashedStringID := messages.HashID(dstPeerID)
+	_, ok := c.conns[hashedStringID]
+	if ok {
+		return nil, ErrConnAlreadyExists
+	}
+
+	log.Infof("open connection to peer: %s", hashedStringID)
+	msgChannel := make(chan Msg, 2)
+	conn := NewConn(c, hashedID, hashedStringID, msgChannel, c.instanceURL)
+
+	c.conns[hashedStringID] = newConnContainer(conn, msgChannel)
+	return conn, nil
+}
+
+// ServerInstanceURL returns the address of the relay server. It could change after the close and reopen the connection.
+func (c *Client) ServerInstanceURL() (string, error) {
+	c.muInstanceURL.Lock()
+	defer c.muInstanceURL.Unlock()
+	if c.instanceURL == nil {
+		return "", fmt.Errorf("relay connection is not established")
+	}
+	return c.instanceURL.String(), nil
+}
+
+// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed.
+func (c *Client) SetOnDisconnectListener(fn func()) {
+	c.listenerMutex.Lock()
+	defer c.listenerMutex.Unlock()
+	c.onDisconnectListener = fn
+}
+
+// HasConns returns true if there are connections.
+func (c *Client) HasConns() bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return len(c.conns) > 0
+}
+
+// Close closes the connection to the relay server and all connections to other peers.
+func (c *Client) Close() error {
+	return c.close(true)
+}
+
+func (c *Client) connect() error {
+	conn, err := ws.Dial(c.connectionURL)
+	if err != nil {
+		return err
+	}
+	c.relayConn = conn
+
+	err = c.handShake()
+	if err != nil {
+		cErr := conn.Close()
+		if cErr != nil {
+			log.Errorf("failed to close connection: %s", cErr)
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) handShake() error {
+	authMsg := &auth2.Msg{
+		AuthAlgorithm:  auth2.AlgoHMACSHA256,
+		AdditionalData: c.authTokenStore.TokenBinary(),
+	}
+
+	authData, err := authMsg.Marshal()
+	if err != nil {
+		return fmt.Errorf("marshal auth message: %w", err)
+	}
+
+	msg, err := messages.MarshalHelloMsg(c.hashedID, authData)
+	if err != nil {
+		log.Errorf("failed to marshal hello message: %s", err)
+		return err
+	}
+
+	_, err = c.relayConn.Write(msg)
+	if err != nil {
+		log.Errorf("failed to send hello message: %s", err)
+		return err
+	}
+	buf := make([]byte, messages.MaxHandshakeSize)
+	n, err := c.readWithTimeout(buf)
+	if err != nil {
+		log.Errorf("failed to read hello response: %s", err)
+		return err
+	}
+
+	_, err = messages.ValidateVersion(buf[:n])
+	if err != nil {
+		return fmt.Errorf("validate version: %w", err)
+	}
+
+	msgType, err := messages.DetermineServerMessageType(buf[messages.SizeOfVersionByte:n])
+	if err != nil {
+		log.Errorf("failed to determine message type: %s", err)
+		return err
+	}
+
+	if msgType != messages.MsgTypeHelloResponse {
+		log.Errorf("unexpected message type: %s", msgType)
+		return fmt.Errorf("unexpected message type")
+	}
+
+	additionalData, err := messages.UnmarshalHelloResponse(buf[messages.SizeOfProtoHeader:n])
+	if err != nil {
+		return err
+	}
+
+	addr, err := address.Unmarshal(additionalData)
+	if err != nil {
+		return fmt.Errorf("unmarshal address: %w", err)
+	}
+
+	c.muInstanceURL.Lock()
+	c.instanceURL = &RelayAddr{addr: addr.URL}
+	c.muInstanceURL.Unlock()
+	return nil
+}
+
+func (c *Client) readLoop(relayConn net.Conn) {
+	internallyStoppedFlag := newInternalStopFlag()
+	hc := healthcheck.NewReceiver()
+	go c.listenForStopEvents(hc, relayConn, internallyStoppedFlag)
+
+	var (
+		errExit error
+		n       int
+	)
+	for {
+		bufPtr := c.bufPool.Get().(*[]byte)
+		buf := *bufPtr
+		n, errExit = relayConn.Read(buf)
+		if errExit != nil {
+			c.mu.Lock()
+			if c.serviceIsRunning && !internallyStoppedFlag.isSet() {
+				c.log.Debugf("failed to read message from relay server: %s", errExit)
+			}
+			c.mu.Unlock()
+			break
+		}
+
+		_, err := messages.ValidateVersion(buf[:n])
+		if err != nil {
+			c.log.Errorf("failed to validate protocol version: %s", err)
+			c.bufPool.Put(bufPtr)
+			continue
+		}
+
+		msgType, err := messages.DetermineServerMessageType(buf[messages.SizeOfVersionByte:n])
+		if err != nil {
+			c.log.Errorf("failed to determine message type: %s", err)
+			c.bufPool.Put(bufPtr)
+			continue
+		}
+
+		if !c.handleMsg(msgType, buf[messages.SizeOfProtoHeader:n], bufPtr, hc, internallyStoppedFlag) {
+			break
+		}
+	}
+
+	hc.Stop()
+
+	c.muInstanceURL.Lock()
+	c.instanceURL = nil
+	c.muInstanceURL.Unlock()
+
+	c.notifyDisconnected()
+	c.wgReadLoop.Done()
+	_ = c.close(false)
+}
+
+func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) {
+	switch msgType {
+	case messages.MsgTypeHealthCheck:
+		c.handleHealthCheck(hc, internallyStoppedFlag)
+		c.bufPool.Put(bufPtr)
+	case messages.MsgTypeTransport:
+		return c.handleTransportMsg(buf, bufPtr, internallyStoppedFlag)
+	case messages.MsgTypeClose:
+		log.Debugf("relay connection close by server")
+		c.bufPool.Put(bufPtr)
+		return false
+	}
+
+	return true
+}
+
+func (c *Client) handleHealthCheck(hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) {
+	msg := messages.MarshalHealthcheck()
+	_, wErr := c.relayConn.Write(msg)
+	if wErr != nil {
+		if c.serviceIsRunning && !internallyStoppedFlag.isSet() {
+			c.log.Errorf("failed to send heartbeat: %s", wErr)
+		}
+	}
+	hc.Heartbeat()
+}
+
+func (c *Client) handleTransportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool {
+	peerID, payload, err := messages.UnmarshalTransportMsg(buf)
+	if err != nil {
+		if c.serviceIsRunning && !internallyStoppedFlag.isSet() {
+			c.log.Errorf("failed to parse transport message: %v", err)
+		}
+
+		c.bufPool.Put(bufPtr)
+		return true
+	}
+
+	stringID := messages.HashIDToString(peerID)
+
+	c.mu.Lock()
+	if !c.serviceIsRunning {
+		c.mu.Unlock()
+		c.bufPool.Put(bufPtr)
+		return false
+	}
+	container, ok := c.conns[stringID]
+	c.mu.Unlock()
+	if !ok {
+		c.log.Errorf("peer not found: %s", stringID)
+		c.bufPool.Put(bufPtr)
+		return true
+	}
+	msg := Msg{
+		bufPool: c.bufPool,
+		bufPtr:  bufPtr,
+		Payload: payload,
+	}
+	container.writeMsg(msg)
+	return true
+}
+
+func (c *Client) writeTo(connReference *Conn, id string, dstID []byte, payload []byte) (int, error) {
+	c.mu.Lock()
+	conn, ok := c.conns[id]
+	c.mu.Unlock()
+	if !ok {
+		return 0, io.EOF
+	}
+
+	if conn.conn != connReference {
+		return 0, io.EOF
+	}
+
+	// todo: use buffer pool instead of create new transport msg.
+	msg, err := messages.MarshalTransportMsg(dstID, payload)
+	if err != nil {
+		log.Errorf("failed to marshal transport message: %s", err)
+		return 0, err
+	}
+
+	// the write always return with 0 length because the underling does not support the size feedback.
+	_, err = c.relayConn.Write(msg)
+	if err != nil {
+		log.Errorf("failed to write transport message: %s", err)
+	}
+	return len(payload), err
+}
+
+func (c *Client) listenForStopEvents(hc *healthcheck.Receiver, conn net.Conn, internalStopFlag *internalStopFlag) {
+	for {
+		select {
+		case _, ok := <-hc.OnTimeout:
+			if !ok {
+				return
+			}
+			c.log.Errorf("health check timeout")
+			internalStopFlag.set()
+			_ = conn.Close() // ignore the err because the readLoop will handle it
+			return
+		case <-c.parentCtx.Done():
+			err := c.close(true)
+			if err != nil {
+				log.Errorf("failed to teardown connection: %s", err)
+			}
+			return
+		}
+	}
+}
+
+func (c *Client) closeAllConns() {
+	for _, container := range c.conns {
+		container.close()
+	}
+	c.conns = make(map[string]*connContainer)
+}
+
+func (c *Client) closeConn(connReference *Conn, id string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	container, ok := c.conns[id]
+	if !ok {
+		return fmt.Errorf("connection already closed")
+	}
+
+	if container.conn != connReference {
+		return fmt.Errorf("conn reference mismatch")
+	}
+	container.close()
+	delete(c.conns, id)
+
+	return nil
+}
+
+func (c *Client) close(gracefullyExit bool) error {
+	c.readLoopMutex.Lock()
+	defer c.readLoopMutex.Unlock()
+
+	c.mu.Lock()
+	var err error
+	if !c.serviceIsRunning {
+		c.mu.Unlock()
+		return nil
+	}
+
+	c.serviceIsRunning = false
+	c.closeAllConns()
+	if gracefullyExit {
+		c.writeCloseMsg()
+	}
+	err = c.relayConn.Close()
+	c.mu.Unlock()
+
+	c.wgReadLoop.Wait()
+	c.log.Infof("relay connection closed with: %s", c.connectionURL)
+	return err
+}
+
+func (c *Client) notifyDisconnected() {
+	c.listenerMutex.Lock()
+	defer c.listenerMutex.Unlock()
+
+	if c.onDisconnectListener == nil {
+		return
+	}
+	go c.onDisconnectListener()
+}
+
+func (c *Client) writeCloseMsg() {
+	msg := messages.MarshalCloseMsg()
+	_, err := c.relayConn.Write(msg)
+	if err != nil {
+		c.log.Errorf("failed to send close message: %s", err)
+	}
+}
+
+func (c *Client) readWithTimeout(buf []byte) (int, error) {
+	ctx, cancel := context.WithTimeout(c.parentCtx, serverResponseTimeout)
+	defer cancel()
+
+	readDone := make(chan struct{})
+	var (
+		n   int
+		err error
+	)
+
+	go func() {
+		n, err = c.relayConn.Read(buf)
+		close(readDone)
+	}()
+
+	select {
+	case <-ctx.Done():
+		return 0, fmt.Errorf("read operation timed out")
+	case <-readDone:
+		return n, err
+	}
+}

relay/client/client_test.go

@@ -0,0 +1,631 @@
+package client
+
+import (
+	"context"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/relay/auth/allow"
+	"github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/util"
+
+	"github.com/netbirdio/netbird/relay/server"
+)
+
+var (
+	av               = &allow.Auth{}
+	hmacTokenStore   = &hmac.TokenStore{}
+	serverListenAddr = "127.0.0.1:1234"
+	serverURL        = "rel://127.0.0.1:1234"
+)
+
+func TestMain(m *testing.M) {
+	_ = util.InitLog("error", "console")
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestClient(t *testing.T) {
+	ctx := context.Background()
+
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		listenCfg := server.ListenerConfig{Address: serverListenAddr}
+		err := srv.Listen(listenCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for server to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+	t.Log("alice connecting to server")
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer clientAlice.Close()
+
+	t.Log("placeholder connecting to server")
+	clientPlaceHolder := NewClient(ctx, serverURL, hmacTokenStore, "clientPlaceHolder")
+	err = clientPlaceHolder.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer clientPlaceHolder.Close()
+
+	t.Log("Bob connecting to server")
+	clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob")
+	err = clientBob.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer clientBob.Close()
+
+	t.Log("Alice open connection to Bob")
+	connAliceToBob, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	t.Log("Bob open connection to Alice")
+	connBobToAlice, err := clientBob.OpenConn("alice")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	payload := "hello bob, I am alice"
+	_, err = connAliceToBob.Write([]byte(payload))
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+	log.Debugf("alice sent message to bob")
+
+	buf := make([]byte, 65535)
+	n, err := connBobToAlice.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+	log.Debugf("on new message from alice to bob")
+
+	if payload != string(buf[:n]) {
+		t.Fatalf("expected %s, got %s", payload, string(buf[:n]))
+	}
+}
+
+func TestRegistration(t *testing.T) {
+	ctx := context.Background()
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	// wait for server to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		_ = srv.Shutdown(ctx)
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close conn: %s", err)
+	}
+	err = srv.Shutdown(ctx)
+	if err != nil {
+		t.Errorf("failed to close server: %s", err)
+	}
+}
+
+func TestRegistrationTimeout(t *testing.T) {
+	ctx := context.Background()
+	fakeUDPListener, err := net.ListenUDP("udp", &net.UDPAddr{
+		Port: 1234,
+		IP:   net.ParseIP("0.0.0.0"),
+	})
+	if err != nil {
+		t.Fatalf("failed to bind UDP server: %s", err)
+	}
+	defer func(fakeUDPListener *net.UDPConn) {
+		_ = fakeUDPListener.Close()
+	}(fakeUDPListener)
+
+	fakeTCPListener, err := net.ListenTCP("tcp", &net.TCPAddr{
+		Port: 1234,
+		IP:   net.ParseIP("0.0.0.0"),
+	})
+	if err != nil {
+		t.Fatalf("failed to bind TCP server: %s", err)
+	}
+	defer func(fakeTCPListener *net.TCPListener) {
+		_ = fakeTCPListener.Close()
+	}(fakeTCPListener)
+
+	clientAlice := NewClient(ctx, "127.0.0.1:1234", hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err == nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+	log.Debugf("%s", err)
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close conn: %s", err)
+	}
+}
+
+func TestEcho(t *testing.T) {
+	ctx := context.Background()
+	idAlice := "alice"
+	idBob := "bob"
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice)
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer func() {
+		err := clientAlice.Close()
+		if err != nil {
+			t.Errorf("failed to close Alice client: %s", err)
+		}
+	}()
+
+	clientBob := NewClient(ctx, serverURL, hmacTokenStore, idBob)
+	err = clientBob.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+	defer func() {
+		err := clientBob.Close()
+		if err != nil {
+			t.Errorf("failed to close Bob client: %s", err)
+		}
+	}()
+
+	connAliceToBob, err := clientAlice.OpenConn(idBob)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	connBobToAlice, err := clientBob.OpenConn(idAlice)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	payload := "hello bob, I am alice"
+	_, err = connAliceToBob.Write([]byte(payload))
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	buf := make([]byte, 65535)
+	n, err := connBobToAlice.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	_, err = connBobToAlice.Write(buf[:n])
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	n, err = connAliceToBob.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	if payload != string(buf[:n]) {
+		t.Fatalf("expected %s, got %s", payload, string(buf[:n]))
+	}
+}
+
+func TestBindToUnavailabePeer(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		log.Infof("closing server")
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+	_, err = clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	log.Infof("closing client")
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+}
+
+func TestBindReconnect(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		log.Infof("closing server")
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	_, err = clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob")
+	err = clientBob.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	chBob, err := clientBob.OpenConn("alice")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	log.Infof("closing client Alice")
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+
+	clientAlice = NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	chAlice, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	testString := "hello alice, I am bob"
+	_, err = chBob.Write([]byte(testString))
+	if err != nil {
+		t.Errorf("failed to write to channel: %s", err)
+	}
+
+	buf := make([]byte, 65535)
+	n, err := chAlice.Read(buf)
+	if err != nil {
+		t.Errorf("failed to read from channel: %s", err)
+	}
+
+	if testString != string(buf[:n]) {
+		t.Errorf("expected %s, got %s", testString, string(buf[:n]))
+	}
+
+	log.Infof("closing client")
+	err = clientAlice.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+}
+
+func TestCloseConn(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		log.Infof("closing server")
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Errorf("failed to connect to server: %s", err)
+	}
+
+	conn, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	log.Infof("closing connection")
+	err = conn.Close()
+	if err != nil {
+		t.Errorf("failed to close connection: %s", err)
+	}
+
+	_, err = conn.Read(make([]byte, 1))
+	if err == nil {
+		t.Errorf("unexpected reading from closed connection")
+	}
+
+	_, err = conn.Write([]byte("hello"))
+	if err == nil {
+		t.Errorf("unexpected writing from closed connection")
+	}
+}
+
+func TestCloseRelayConn(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			log.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice")
+	err = clientAlice.Connect()
+	if err != nil {
+		t.Fatalf("failed to connect to server: %s", err)
+	}
+
+	conn, err := clientAlice.OpenConn("bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	_ = clientAlice.relayConn.Close()
+
+	_, err = conn.Read(make([]byte, 1))
+	if err == nil {
+		t.Errorf("unexpected reading from closed connection")
+	}
+
+	_, err = clientAlice.OpenConn("bob")
+	if err == nil {
+		t.Errorf("unexpected opening connection to closed server")
+	}
+}
+
+func TestCloseByServer(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv1, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+
+	go func() {
+		err := srv1.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice)
+	err = relayClient.Connect()
+	if err != nil {
+		log.Fatalf("failed to connect to server: %s", err)
+	}
+
+	disconnected := make(chan struct{})
+	relayClient.SetOnDisconnectListener(func() {
+		log.Infof("client disconnected")
+		close(disconnected)
+	})
+
+	err = srv1.Shutdown(ctx)
+	if err != nil {
+		t.Fatalf("failed to close server: %s", err)
+	}
+
+	select {
+	case <-disconnected:
+	case <-time.After(3 * time.Second):
+		log.Fatalf("timeout waiting for client to disconnect")
+	}
+
+	_, err = relayClient.OpenConn("bob")
+	if err == nil {
+		t.Errorf("unexpected opening connection to closed server")
+	}
+}
+
+func TestCloseByClient(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg := server.ListenerConfig{Address: serverListenAddr}
+	srv, err := server.NewServer(otel.Meter(""), serverURL, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	// wait for servers to start
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice)
+	err = relayClient.Connect()
+	if err != nil {
+		log.Fatalf("failed to connect to server: %s", err)
+	}
+
+	err = relayClient.Close()
+	if err != nil {
+		t.Errorf("failed to close client: %s", err)
+	}
+
+	_, err = relayClient.OpenConn("bob")
+	if err == nil {
+		t.Errorf("unexpected opening connection to closed server")
+	}
+
+	err = srv.Shutdown(ctx)
+	if err != nil {
+		t.Fatalf("failed to close server: %s", err)
+	}
+}
+
+func waitForServerToStart(errChan chan error) error {
+	select {
+	case err := <-errChan:
+		if err != nil {
+			return err
+		}
+	case <-time.After(300 * time.Millisecond):
+		return nil
+	}
+	return nil
+}

relay/client/conn.go

@@ -0,0 +1,76 @@
+package client
+
+import (
+	"io"
+	"net"
+	"time"
+)
+
+// Conn represent a connection to a relayed remote peer.
+type Conn struct {
+	client      *Client
+	dstID       []byte
+	dstStringID string
+	messageChan chan Msg
+	instanceURL *RelayAddr
+}
+
+// NewConn creates a new connection to a relayed remote peer.
+// client: the client instance, it used to send messages to the destination peer
+// dstID: the destination peer ID
+// dstStringID: the destination peer ID in string format
+// messageChan: the channel where the messages will be received
+// instanceURL: the relay instance URL, it used to get the proper server instance address for the remote peer
+func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg, instanceURL *RelayAddr) *Conn {
+	c := &Conn{
+		client:      client,
+		dstID:       dstID,
+		dstStringID: dstStringID,
+		messageChan: messageChan,
+		instanceURL: instanceURL,
+	}
+
+	return c
+}
+
+func (c *Conn) Write(p []byte) (n int, err error) {
+	return c.client.writeTo(c, c.dstStringID, c.dstID, p)
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	msg, ok := <-c.messageChan
+	if !ok {
+		return 0, io.EOF
+	}
+
+	n = copy(b, msg.Payload)
+	msg.Free()
+	return n, nil
+}
+
+func (c *Conn) Close() error {
+	return c.client.closeConn(c, c.dstStringID)
+}
+
+func (c *Conn) LocalAddr() net.Addr {
+	return c.client.relayConn.LocalAddr()
+}
+
+func (c *Conn) RemoteAddr() net.Addr {
+	return c.instanceURL
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	//TODO implement me
+	panic("SetDeadline is not implemented")
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	//TODO implement me
+	panic("SetReadDeadline is not implemented")
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	//TODO implement me
+	panic("SetReadDeadline is not implemented")
+}

relay/client/dialer/ws/addr.go

@@ -0,0 +1,13 @@
+package ws
+
+type WebsocketAddr struct {
+	addr string
+}
+
+func (a WebsocketAddr) Network() string {
+	return "websocket"
+}
+
+func (a WebsocketAddr) String() string {
+	return a.addr
+}

relay/client/dialer/ws/conn.go

@@ -0,0 +1,66 @@
+package ws
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"nhooyr.io/websocket"
+)
+
+type Conn struct {
+	ctx context.Context
+	*websocket.Conn
+	remoteAddr WebsocketAddr
+}
+
+func NewConn(wsConn *websocket.Conn, serverAddress string) net.Conn {
+	return &Conn{
+		ctx:        context.Background(),
+		Conn:       wsConn,
+		remoteAddr: WebsocketAddr{serverAddress},
+	}
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	t, ioReader, err := c.Conn.Reader(c.ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	if t != websocket.MessageBinary {
+		return 0, fmt.Errorf("unexpected message type")
+	}
+
+	return ioReader.Read(b)
+}
+
+func (c *Conn) Write(b []byte) (n int, err error) {
+	err = c.Conn.Write(c.ctx, websocket.MessageBinary, b)
+	return 0, err
+}
+
+func (c *Conn) RemoteAddr() net.Addr {
+	return c.remoteAddr
+}
+
+func (c *Conn) LocalAddr() net.Addr {
+	return WebsocketAddr{addr: "unknown"}
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	return fmt.Errorf("SetReadDeadline is not implemented")
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	return fmt.Errorf("SetWriteDeadline is not implemented")
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	return fmt.Errorf("SetDeadline is not implemented")
+}
+
+func (c *Conn) Close() error {
+	return c.Conn.CloseNow()
+}

relay/client/dialer/ws/ws.go

@@ -0,0 +1,67 @@
+package ws
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"nhooyr.io/websocket"
+
+	"github.com/netbirdio/netbird/relay/server/listener/ws"
+	nbnet "github.com/netbirdio/netbird/util/net"
+)
+
+func Dial(address string) (net.Conn, error) {
+	wsURL, err := prepareURL(address)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := &websocket.DialOptions{
+		HTTPClient: httpClientNbDialer(),
+	}
+
+	parsedURL, err := url.Parse(wsURL)
+	if err != nil {
+		return nil, err
+	}
+	parsedURL.Path = ws.URLPath
+
+	wsConn, resp, err := websocket.Dial(context.Background(), parsedURL.String(), opts)
+	if err != nil {
+		log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err)
+		return nil, err
+	}
+	if resp.Body != nil {
+		_ = resp.Body.Close()
+	}
+
+	conn := NewConn(wsConn, address)
+	return conn, nil
+}
+
+func prepareURL(address string) (string, error) {
+	if !strings.HasPrefix(address, "rel:") && !strings.HasPrefix(address, "rels:") {
+		return "", fmt.Errorf("unsupported scheme: %s", address)
+	}
+
+	return strings.Replace(address, "rel", "ws", 1), nil
+}
+
+func httpClientNbDialer() *http.Client {
+	customDialer := nbnet.NewDialer()
+
+	customTransport := &http.Transport{
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			return customDialer.DialContext(ctx, network, addr)
+		},
+	}
+
+	return &http.Client{
+		Transport: customTransport,
+	}
+}

relay/client/doc.go

@@ -0,0 +1,12 @@
+/*
+Package client contains the implementation of the Relay client.
+
+The Relay client is responsible for establishing a connection with the Relay server and sending and receiving messages,
+Keep persistent connection with the Relay server and handle the connection issues.
+It uses the WebSocket protocol for communication and optionally supports TLS (Transport Layer Security).
+
+If a peer wants to communicate with a peer on a different relay server, the manager will establish a new connection to
+the relay server. The connection with these relay servers will be closed if there is no active connection. The peers
+negotiate the common relay instance via signaling service.
+*/
+package client

relay/client/guard.go

@@ -0,0 +1,48 @@
+package client
+
+import (
+	"context"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+var (
+	reconnectingTimeout = 5 * time.Second
+)
+
+// Guard manage the reconnection tries to the Relay server in case of disconnection event.
+type Guard struct {
+	ctx         context.Context
+	relayClient *Client
+}
+
+// NewGuard creates a new guard for the relay client.
+func NewGuard(context context.Context, relayClient *Client) *Guard {
+	g := &Guard{
+		ctx:         context,
+		relayClient: relayClient,
+	}
+	return g
+}
+
+// OnDisconnected is called when the relay client is disconnected from the relay server. It will trigger the reconnection
+// todo prevent multiple reconnection instances. In the current usage it should not happen, but it is better to prevent
+func (g *Guard) OnDisconnected() {
+	ticker := time.NewTicker(reconnectingTimeout)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			err := g.relayClient.Connect()
+			if err != nil {
+				log.Errorf("failed to reconnect to relay server: %s", err)
+				continue
+			}
+			return
+		case <-g.ctx.Done():
+			return
+		}
+	}
+}

relay/client/manager.go

@@ -0,0 +1,365 @@
+package client
+
+import (
+	"container/list"
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"reflect"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+
+	relayAuth "github.com/netbirdio/netbird/relay/auth/hmac"
+)
+
+var (
+	relayCleanupInterval = 60 * time.Second
+	connectionTimeout    = 30 * time.Second
+	maxConcurrentServers = 7
+
+	ErrRelayClientNotConnected = fmt.Errorf("relay client not connected")
+)
+
+// RelayTrack hold the relay clients for the foreign relay servers.
+// With the mutex can ensure we can open new connection in case the relay connection has been established with
+// the relay server.
+type RelayTrack struct {
+	sync.RWMutex
+	relayClient *Client
+}
+
+func NewRelayTrack() *RelayTrack {
+	return &RelayTrack{}
+}
+
+type OnServerCloseListener func()
+
+// ManagerService is the interface for the relay manager.
+type ManagerService interface {
+	Serve() error
+	OpenConn(serverAddress, peerKey string) (net.Conn, error)
+	AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error
+	RelayInstanceAddress() (string, error)
+	ServerURLs() []string
+	HasRelayAddress() bool
+	UpdateToken(token *relayAuth.Token) error
+}
+
+// Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL
+// and automatically reconnect to them in case disconnection.
+// The manager also manage temporary relay connection. If a client wants to communicate with a client on a
+// different relay server, the manager will establish a new connection to the relay server. The connection with these
+// relay servers will be closed if there is no active connection. Periodically the manager will check if there is any
+// unused relay connection and close it.
+type Manager struct {
+	ctx        context.Context
+	serverURLs []string
+	peerID     string
+	tokenStore *relayAuth.TokenStore
+
+	relayClient    *Client
+	reconnectGuard *Guard
+
+	relayClients      map[string]*RelayTrack
+	relayClientsMutex sync.RWMutex
+
+	onDisconnectedListeners map[string]*list.List
+	listenerLock            sync.Mutex
+}
+
+// NewManager creates a new manager instance.
+// The serverURL address can be empty. In this case, the manager will not serve.
+func NewManager(ctx context.Context, serverURLs []string, peerID string) *Manager {
+	return &Manager{
+		ctx:                     ctx,
+		serverURLs:              serverURLs,
+		peerID:                  peerID,
+		tokenStore:              &relayAuth.TokenStore{},
+		relayClients:            make(map[string]*RelayTrack),
+		onDisconnectedListeners: make(map[string]*list.List),
+	}
+}
+
+// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop for
+// the unused relay connections. The manager will automatically reconnect to the relay server in case of disconnection.
+func (m *Manager) Serve() error {
+	if m.relayClient != nil {
+		return fmt.Errorf("manager already serving")
+	}
+	log.Debugf("starting relay client manager with %v relay servers", m.serverURLs)
+
+	totalServers := len(m.serverURLs)
+
+	successChan := make(chan *Client, 1)
+	errChan := make(chan error, len(m.serverURLs))
+
+	ctx, cancel := context.WithTimeout(m.ctx, connectionTimeout)
+	defer cancel()
+
+	sem := make(chan struct{}, maxConcurrentServers)
+
+	for _, url := range m.serverURLs {
+		sem <- struct{}{}
+		go func(url string) {
+			defer func() { <-sem }()
+			m.connect(m.ctx, url, successChan, errChan)
+		}(url)
+	}
+
+	var errCount int
+
+	for {
+		select {
+		case client := <-successChan:
+			log.Infof("Successfully connected to relay server: %s", client.connectionURL)
+
+			m.relayClient = client
+
+			m.reconnectGuard = NewGuard(m.ctx, m.relayClient)
+			m.relayClient.SetOnDisconnectListener(func() {
+				m.onServerDisconnected(client.connectionURL)
+			})
+			m.startCleanupLoop()
+			return nil
+		case err := <-errChan:
+			errCount++
+			log.Warnf("Connection attempt failed: %v", err)
+			if errCount == totalServers {
+				return errors.New("failed to connect to any relay server: all attempts failed")
+			}
+		case <-ctx.Done():
+			return fmt.Errorf("failed to connect to any relay server: %w", ctx.Err())
+		}
+	}
+}
+
+func (m *Manager) connect(ctx context.Context, serverURL string, successChan chan<- *Client, errChan chan<- error) {
+	// TODO: abort the connection if another connection was successful
+	relayClient := NewClient(ctx, serverURL, m.tokenStore, m.peerID)
+	if err := relayClient.Connect(); err != nil {
+		errChan <- fmt.Errorf("failed to connect to %s: %w", serverURL, err)
+		return
+	}
+
+	select {
+	case successChan <- relayClient:
+		// This client was the first to connect successfully
+	default:
+		if err := relayClient.Close(); err != nil {
+			log.Debugf("failed to close relay client: %s", err)
+		}
+	}
+}
+
+// OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be
+// established via the relay server. If the peer is on a different relay server, the manager will establish a new
+// connection to the relay server. It returns back with a net.Conn what represent the remote peer connection.
+func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) {
+	if m.relayClient == nil {
+		return nil, ErrRelayClientNotConnected
+	}
+
+	foreign, err := m.isForeignServer(serverAddress)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		netConn net.Conn
+	)
+	if !foreign {
+		log.Debugf("open peer connection via permanent server: %s", peerKey)
+		netConn, err = m.relayClient.OpenConn(peerKey)
+	} else {
+		log.Debugf("open peer connection via foreign server: %s", serverAddress)
+		netConn, err = m.openConnVia(serverAddress, peerKey)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return netConn, err
+}
+
+// AddCloseListener adds a listener to the given server instance address. The listener will be called if the connection
+// closed.
+func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error {
+	foreign, err := m.isForeignServer(serverAddress)
+	if err != nil {
+		return err
+	}
+
+	var listenerAddr string
+	if foreign {
+		listenerAddr = serverAddress
+	} else {
+		listenerAddr = m.relayClient.connectionURL
+	}
+	m.addListener(listenerAddr, onClosedListener)
+	return nil
+}
+
+// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is
+// lost. This address will be sent to the target peer to choose the common relay server for the communication.
+func (m *Manager) RelayInstanceAddress() (string, error) {
+	if m.relayClient == nil {
+		return "", ErrRelayClientNotConnected
+	}
+	return m.relayClient.ServerInstanceURL()
+}
+
+// ServerURLs returns the addresses of the relay servers.
+func (m *Manager) ServerURLs() []string {
+	return m.serverURLs
+}
+
+// HasRelayAddress returns true if the manager is serving. With this method can check if the peer can communicate with
+// Relay service.
+func (m *Manager) HasRelayAddress() bool {
+	return len(m.serverURLs) > 0
+}
+
+// UpdateToken updates the token in the token store.
+func (m *Manager) UpdateToken(token *relayAuth.Token) error {
+	return m.tokenStore.UpdateToken(token)
+}
+
+func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) {
+	// check if already has a connection to the desired relay server
+	m.relayClientsMutex.RLock()
+	rt, ok := m.relayClients[serverAddress]
+	if ok {
+		rt.RLock()
+		m.relayClientsMutex.RUnlock()
+		defer rt.RUnlock()
+		return rt.relayClient.OpenConn(peerKey)
+	}
+	m.relayClientsMutex.RUnlock()
+
+	// if not, establish a new connection but check it again (because changed the lock type) before starting the
+	// connection
+	m.relayClientsMutex.Lock()
+	rt, ok = m.relayClients[serverAddress]
+	if ok {
+		rt.RLock()
+		m.relayClientsMutex.Unlock()
+		defer rt.RUnlock()
+		return rt.relayClient.OpenConn(peerKey)
+	}
+
+	// create a new relay client and store it in the relayClients map
+	rt = NewRelayTrack()
+	rt.Lock()
+	m.relayClients[serverAddress] = rt
+	m.relayClientsMutex.Unlock()
+
+	relayClient := NewClient(m.ctx, serverAddress, m.tokenStore, m.peerID)
+	err := relayClient.Connect()
+	if err != nil {
+		rt.Unlock()
+		m.relayClientsMutex.Lock()
+		delete(m.relayClients, serverAddress)
+		m.relayClientsMutex.Unlock()
+		return nil, err
+	}
+	// if connection closed then delete the relay client from the list
+	relayClient.SetOnDisconnectListener(func() {
+		m.onServerDisconnected(serverAddress)
+	})
+	rt.relayClient = relayClient
+	rt.Unlock()
+
+	conn, err := relayClient.OpenConn(peerKey)
+	if err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+func (m *Manager) onServerDisconnected(serverAddress string) {
+	if serverAddress == m.relayClient.connectionURL {
+		go m.reconnectGuard.OnDisconnected()
+	}
+
+	m.notifyOnDisconnectListeners(serverAddress)
+}
+
+func (m *Manager) isForeignServer(address string) (bool, error) {
+	rAddr, err := m.relayClient.ServerInstanceURL()
+	if err != nil {
+		return false, fmt.Errorf("relay client not connected")
+	}
+	return rAddr != address, nil
+}
+
+func (m *Manager) startCleanupLoop() {
+	if m.ctx.Err() != nil {
+		return
+	}
+
+	ticker := time.NewTicker(relayCleanupInterval)
+	go func() {
+		defer ticker.Stop()
+		for {
+			select {
+			case <-m.ctx.Done():
+				return
+			case <-ticker.C:
+				m.cleanUpUnusedRelays()
+			}
+		}
+	}()
+}
+
+func (m *Manager) cleanUpUnusedRelays() {
+	m.relayClientsMutex.Lock()
+	defer m.relayClientsMutex.Unlock()
+
+	for addr, rt := range m.relayClients {
+		rt.Lock()
+		if rt.relayClient.HasConns() {
+			rt.Unlock()
+			continue
+		}
+		rt.relayClient.SetOnDisconnectListener(nil)
+		go func() {
+			_ = rt.relayClient.Close()
+		}()
+		log.Debugf("clean up unused relay server connection: %s", addr)
+		delete(m.relayClients, addr)
+		rt.Unlock()
+	}
+}
+
+func (m *Manager) addListener(serverAddress string, onClosedListener OnServerCloseListener) {
+	m.listenerLock.Lock()
+	defer m.listenerLock.Unlock()
+	l, ok := m.onDisconnectedListeners[serverAddress]
+	if !ok {
+		l = list.New()
+	}
+	for e := l.Front(); e != nil; e = e.Next() {
+		if reflect.ValueOf(e.Value).Pointer() == reflect.ValueOf(onClosedListener).Pointer() {
+			return
+		}
+	}
+	l.PushBack(onClosedListener)
+	m.onDisconnectedListeners[serverAddress] = l
+}
+
+func (m *Manager) notifyOnDisconnectListeners(serverAddress string) {
+	m.listenerLock.Lock()
+	defer m.listenerLock.Unlock()
+
+	l, ok := m.onDisconnectedListeners[serverAddress]
+	if !ok {
+		return
+	}
+	for e := l.Front(); e != nil; e = e.Next() {
+		go e.Value.(OnServerCloseListener)()
+	}
+	delete(m.onDisconnectedListeners, serverAddress)
+}

relay/client/manager_test.go

@@ -0,0 +1,432 @@
+package client
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel"
+
+	"github.com/netbirdio/netbird/relay/server"
+)
+
+func TestEmptyURL(t *testing.T) {
+	mgr := NewManager(context.Background(), nil, "alice")
+	err := mgr.Serve()
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+}
+
+func TestForeignConn(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	srvCfg2 := server.ListenerConfig{
+		Address: "localhost:2234",
+	}
+	srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan2 := make(chan error, 1)
+	go func() {
+		err := srv2.Listen(srvCfg2)
+		if err != nil {
+			errChan2 <- err
+		}
+	}()
+
+	defer func() {
+		err := srv2.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan2); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = clientAlice.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+
+	idBob := "bob"
+	log.Debugf("connect by bob")
+	clientBob := NewManager(mCtx, toURL(srvCfg2), idBob)
+	err = clientBob.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+	bobsSrvAddr, err := clientBob.RelayInstanceAddress()
+	if err != nil {
+		t.Fatalf("failed to get relay address: %s", err)
+	}
+	connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+	connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice)
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	payload := "hello bob, I am alice"
+	_, err = connAliceToBob.Write([]byte(payload))
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	buf := make([]byte, 65535)
+	n, err := connBobToAlice.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	_, err = connBobToAlice.Write(buf[:n])
+	if err != nil {
+		t.Fatalf("failed to write to channel: %s", err)
+	}
+
+	n, err = connAliceToBob.Read(buf)
+	if err != nil {
+		t.Fatalf("failed to read from channel: %s", err)
+	}
+
+	if payload != string(buf[:n]) {
+		t.Fatalf("expected %s, got %s", payload, string(buf[:n]))
+	}
+}
+
+func TestForeginConnClose(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	srvCfg2 := server.ListenerConfig{
+		Address: "localhost:2234",
+	}
+	srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan2 := make(chan error, 1)
+	go func() {
+		err := srv2.Listen(srvCfg2)
+		if err != nil {
+			errChan2 <- err
+		}
+	}()
+
+	defer func() {
+		err := srv2.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan2); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	mgr := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = mgr.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+	conn, err := mgr.OpenConn(toURL(srvCfg2)[0], "anotherpeer")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	err = conn.Close()
+	if err != nil {
+		t.Fatalf("failed to close connection: %s", err)
+	}
+}
+
+func TestForeginAutoClose(t *testing.T) {
+	ctx := context.Background()
+	relayCleanupInterval = 1 * time.Second
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		t.Log("binding server 1.")
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		t.Logf("closing server 1.")
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+		t.Logf("server 1. closed")
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	srvCfg2 := server.ListenerConfig{
+		Address: "localhost:2234",
+	}
+	srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan2 := make(chan error, 1)
+	go func() {
+		t.Log("binding server 2.")
+		err := srv2.Listen(srvCfg2)
+		if err != nil {
+			errChan2 <- err
+		}
+	}()
+	defer func() {
+		t.Logf("closing server 2.")
+		err := srv2.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+		t.Logf("server 2 closed.")
+	}()
+
+	if err := waitForServerToStart(errChan2); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	t.Log("connect to server 1.")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	mgr := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = mgr.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+
+	t.Log("open connection to another peer")
+	conn, err := mgr.OpenConn(toURL(srvCfg2)[0], "anotherpeer")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	t.Log("close conn")
+	err = conn.Close()
+	if err != nil {
+		t.Fatalf("failed to close connection: %s", err)
+	}
+
+	t.Logf("waiting for relay cleanup: %s", relayCleanupInterval+1*time.Second)
+	time.Sleep(relayCleanupInterval + 1*time.Second)
+	if len(mgr.relayClients) != 0 {
+		t.Errorf("expected 0, got %d", len(mgr.relayClients))
+	}
+
+	t.Logf("closing manager")
+}
+
+func TestAutoReconnect(t *testing.T) {
+	ctx := context.Background()
+	reconnectingTimeout = 2 * time.Second
+
+	srvCfg := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv, err := server.NewServer(otel.Meter(""), srvCfg.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv.Listen(srvCfg)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv.Shutdown(ctx)
+		if err != nil {
+			log.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	clientAlice := NewManager(mCtx, toURL(srvCfg), "alice")
+	err = clientAlice.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+	ra, err := clientAlice.RelayInstanceAddress()
+	if err != nil {
+		t.Errorf("failed to get relay address: %s", err)
+	}
+	conn, err := clientAlice.OpenConn(ra, "bob")
+	if err != nil {
+		t.Errorf("failed to bind channel: %s", err)
+	}
+
+	t.Log("closing client relay connection")
+	// todo figure out moc server
+	_ = clientAlice.relayClient.relayConn.Close()
+	t.Log("start test reading")
+	_, err = conn.Read(make([]byte, 1))
+	if err == nil {
+		t.Errorf("unexpected reading from closed connection")
+	}
+
+	log.Infof("waiting for reconnection")
+	time.Sleep(reconnectingTimeout + 1*time.Second)
+
+	log.Infof("reopent the connection")
+	_, err = clientAlice.OpenConn(ra, "bob")
+	if err != nil {
+		t.Errorf("failed to open channel: %s", err)
+	}
+}
+
+func TestNotifierDoubleAdd(t *testing.T) {
+	ctx := context.Background()
+
+	srvCfg1 := server.ListenerConfig{
+		Address: "localhost:1234",
+	}
+	srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av)
+	if err != nil {
+		t.Fatalf("failed to create server: %s", err)
+	}
+	errChan := make(chan error, 1)
+	go func() {
+		err := srv1.Listen(srvCfg1)
+		if err != nil {
+			errChan <- err
+		}
+	}()
+
+	defer func() {
+		err := srv1.Shutdown(ctx)
+		if err != nil {
+			t.Errorf("failed to close server: %s", err)
+		}
+	}()
+
+	if err := waitForServerToStart(errChan); err != nil {
+		t.Fatalf("failed to start server: %s", err)
+	}
+
+	idAlice := "alice"
+	log.Debugf("connect by alice")
+	mCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice)
+	err = clientAlice.Serve()
+	if err != nil {
+		t.Fatalf("failed to serve manager: %s", err)
+	}
+
+	conn1, err := clientAlice.OpenConn(clientAlice.ServerURLs()[0], "idBob")
+	if err != nil {
+		t.Fatalf("failed to bind channel: %s", err)
+	}
+
+	fnCloseListener := OnServerCloseListener(func() {
+		log.Infof("close listener")
+	})
+
+	err = clientAlice.AddCloseListener(clientAlice.ServerURLs()[0], fnCloseListener)
+	if err != nil {
+		t.Fatalf("failed to add close listener: %s", err)
+	}
+
+	err = clientAlice.AddCloseListener(clientAlice.ServerURLs()[0], fnCloseListener)
+	if err != nil {
+		t.Fatalf("failed to add close listener: %s", err)
+	}
+
+	err = conn1.Close()
+	if err != nil {
+		t.Errorf("failed to close connection: %s", err)
+	}
+
+}
+
+func toURL(address server.ListenerConfig) []string {
+	return []string{"rel://" + address.Address}
+}

relay/cmd/env.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"os"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+// setFlagsFromEnvVars reads and updates flag values from environment variables with prefix NB_
+func setFlagsFromEnvVars(cmd *cobra.Command) {
+	flags := cmd.PersistentFlags()
+	flags.VisitAll(func(f *pflag.Flag) {
+		newEnvVar := flagNameToEnvVar(f.Name, "NB_")
+		value, present := os.LookupEnv(newEnvVar)
+		if !present {
+			return
+		}
+
+		err := flags.Set(f.Name, value)
+		if err != nil {
+			log.Infof("unable to configure flag %s using variable %s, err: %v", f.Name, newEnvVar, err)
+		}
+	})
+}
+
+// flagNameToEnvVar converts flag name to environment var name adding a prefix,
+// replacing dashes and making all uppercase (e.g. setup-keys is converted to NB_SETUP_KEYS according to the input prefix)
+func flagNameToEnvVar(cmdFlag string, prefix string) string {
+	parsed := strings.ReplaceAll(cmdFlag, "-", "_")
+	upper := strings.ToUpper(parsed)
+	return prefix + upper
+}

relay/cmd/root.go

@@ -0,0 +1,214 @@
+package cmd
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/go-multierror"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/netbirdio/netbird/encryption"
+	auth "github.com/netbirdio/netbird/relay/auth/hmac"
+	"github.com/netbirdio/netbird/relay/server"
+	"github.com/netbirdio/netbird/signal/metrics"
+	"github.com/netbirdio/netbird/util"
+)
+
+const (
+	metricsPort = 9090
+)
+
+type Config struct {
+	ListenAddress string
+	// in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection
+	// it is a domain:port or ip:port
+	ExposedAddress     string
+	LetsencryptEmail   string
+	LetsencryptDataDir string
+	LetsencryptDomains []string
+	// in case of using Route 53 for DNS challenge the credentials should be provided in the environment variables or
+	// in the AWS credentials file
+	LetsencryptAWSRoute53 bool
+	TlsCertFile           string
+	TlsKeyFile            string
+	AuthSecret            string
+	LogLevel              string
+	LogFile               string
+}
+
+func (c Config) Validate() error {
+	if c.ExposedAddress == "" {
+		return fmt.Errorf("exposed address is required")
+	}
+	if c.AuthSecret == "" {
+		return fmt.Errorf("auth secret is required")
+	}
+	return nil
+}
+
+func (c Config) HasCertConfig() bool {
+	return c.TlsCertFile != "" && c.TlsKeyFile != ""
+}
+
+func (c Config) HasLetsEncrypt() bool {
+	return c.LetsencryptDataDir != "" && c.LetsencryptDomains != nil && len(c.LetsencryptDomains) > 0
+}
+
+var (
+	cobraConfig *Config
+	rootCmd     = &cobra.Command{
+		Use:           "relay",
+		Short:         "Relay service",
+		Long:          "Relay service for Netbird agents",
+		SilenceUsage:  true,
+		SilenceErrors: true,
+		RunE:          execute,
+	}
+)
+
+func init() {
+	_ = util.InitLog("trace", "console")
+	cobraConfig = &Config{}
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.ListenAddress, "listen-address", "l", ":443", "listen address")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.ExposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.LetsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.")
+	rootCmd.PersistentFlags().StringSliceVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS")
+	rootCmd.PersistentFlags().StringVar(&cobraConfig.LetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration")
+	rootCmd.PersistentFlags().BoolVar(&cobraConfig.LetsencryptAWSRoute53, "letsencrypt-aws-route53", false, "use AWS Route 53 for Let's Encrypt DNS challenge")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsCertFile, "tls-cert-file", "c", "", "")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsKeyFile, "tls-key-file", "k", "", "")
+	rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "auth secret")
+	rootCmd.PersistentFlags().StringVar(&cobraConfig.LogLevel, "log-level", "info", "log level")
+	rootCmd.PersistentFlags().StringVar(&cobraConfig.LogFile, "log-file", "console", "log file")
+
+	setFlagsFromEnvVars(rootCmd)
+}
+
+func Execute() error {
+	return rootCmd.Execute()
+}
+
+func waitForExitSignal() {
+	osSigs := make(chan os.Signal, 1)
+	signal.Notify(osSigs, syscall.SIGINT, syscall.SIGTERM)
+	<-osSigs
+}
+
+func execute(cmd *cobra.Command, args []string) error {
+	err := cobraConfig.Validate()
+	if err != nil {
+		log.Debugf("invalid config: %s", err)
+		return fmt.Errorf("invalid config: %s", err)
+	}
+
+	err = util.InitLog(cobraConfig.LogLevel, cobraConfig.LogFile)
+	if err != nil {
+		log.Debugf("failed to initialize log: %s", err)
+		return fmt.Errorf("failed to initialize log: %s", err)
+	}
+
+	metricsServer, err := metrics.NewServer(metricsPort, "")
+	if err != nil {
+		log.Debugf("setup metrics: %v", err)
+		return fmt.Errorf("setup metrics: %v", err)
+	}
+
+	go func() {
+		log.Infof("running metrics server: %s%s", metricsServer.Addr, metricsServer.Endpoint)
+		if err := metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+			log.Fatalf("Failed to start metrics server: %v", err)
+		}
+	}()
+
+	srvListenerCfg := server.ListenerConfig{
+		Address: cobraConfig.ListenAddress,
+	}
+
+	tlsConfig, tlsSupport, err := handleTLSConfig(cobraConfig)
+	if err != nil {
+		log.Debugf("failed to setup TLS config: %s", err)
+		return fmt.Errorf("failed to setup TLS config: %s", err)
+	}
+	srvListenerCfg.TLSConfig = tlsConfig
+
+	authenticator := auth.NewTimedHMACValidator(cobraConfig.AuthSecret, 24*time.Hour)
+	srv, err := server.NewServer(metricsServer.Meter, cobraConfig.ExposedAddress, tlsSupport, authenticator)
+	if err != nil {
+		log.Debugf("failed to create relay server: %v", err)
+		return fmt.Errorf("failed to create relay server: %v", err)
+	}
+	log.Infof("server will be available on: %s", srv.InstanceURL())
+	go func() {
+		if err := srv.Listen(srvListenerCfg); err != nil {
+			log.Fatalf("failed to bind server: %s", err)
+		}
+	}()
+
+	// it will block until exit signal
+	waitForExitSignal()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	var shutDownErrors error
+	if err := srv.Shutdown(ctx); err != nil {
+		shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close server: %s", err))
+	}
+
+	log.Infof("shutting down metrics server")
+	if err := metricsServer.Shutdown(ctx); err != nil {
+		shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err))
+	}
+	return shutDownErrors
+}
+
+func handleTLSConfig(cfg *Config) (*tls.Config, bool, error) {
+	if cfg.LetsencryptAWSRoute53 {
+		log.Debugf("using Let's Encrypt DNS resolver with Route 53 support")
+		r53 := encryption.Route53TLS{
+			DataDir: cfg.LetsencryptDataDir,
+			Email:   cfg.LetsencryptEmail,
+			Domains: cfg.LetsencryptDomains,
+		}
+		tlsCfg, err := r53.GetCertificate()
+		if err != nil {
+			return nil, false, fmt.Errorf("%s", err)
+		}
+		return tlsCfg, true, nil
+	}
+
+	if cfg.HasLetsEncrypt() {
+		log.Infof("setting up TLS with Let's Encrypt.")
+		tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...)
+		if err != nil {
+			return nil, false, fmt.Errorf("%s", err)
+		}
+		return tlsCfg, true, nil
+	}
+
+	if cfg.HasCertConfig() {
+		log.Debugf("using file based TLS config")
+		tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile)
+		if err != nil {
+			return nil, false, fmt.Errorf("%s", err)
+		}
+		return tlsCfg, true, nil
+	}
+	return nil, false, nil
+}
+
+func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) {
+	certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...)
+	if err != nil {
+		return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err)
+	}
+	return certManager.TLSConfig(), nil
+}

relay/doc.go

@@ -0,0 +1,14 @@
+//Package main
+/*
+The `relay` package contains the implementation of the Relay server and client. The Relay server can be used to relay
+messages between peers on a single network channel. In this implementation the transport layer is the WebSocket
+protocol.
+
+Between the server and client communication has been design a custom protocol and message format. These messages are
+transported over the WebSocket connection. Optionally the server can use TLS to secure the communication.
+
+The service can support multiple Relay server instances. For this purpose the peers must know the server instance URL.
+This URL will be sent to the target peer to choose the common Relay server for the communication via Signal service.
+
+*/
+package main

relay/healthcheck/doc.go

@@ -0,0 +1,17 @@
+/*
+The `healthcheck` package is responsible for managing the health checks between the client and the relay server. It
+ensures that the connection between the client and the server are alive and functioning properly.
+
+The `Sender` struct is responsible for sending health check signals to the receiver. The receiver listens for these
+signals and sends a new signal back to the sender to acknowledge that the signal has been received. If the sender does
+not receive an acknowledgment signal within a certain time frame, it will send a timeout signal via timeout channel
+and stop working.
+
+The `Receiver` struct is responsible for receiving the health check signals from the sender. If the receiver does not
+receive a signal within a certain time frame, it will send a timeout signal via the OnTimeout channel and stop working.
+
+In the Relay usage the signal is sent to the peer in message type Healthcheck. In case of timeout the connection is
+closed and the peer is removed from the relay.
+*/
+
+package healthcheck

relay/healthcheck/receiver.go

@@ -0,0 +1,82 @@
+package healthcheck
+
+import (
+	"context"
+	"time"
+)
+
+var (
+	heartbeatTimeout = healthCheckInterval + 3*time.Second
+)
+
+// Receiver is a healthcheck receiver
+// It will listen for heartbeat and check if the heartbeat is not received in a certain time
+// If the heartbeat is not received in a certain time, it will send a timeout signal and stop to work
+// The heartbeat timeout is a bit longer than the sender's healthcheck interval
+type Receiver struct {
+	OnTimeout chan struct{}
+
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+	heartbeat chan struct{}
+	alive     bool
+}
+
+// NewReceiver creates a new healthcheck receiver and start the timer in the background
+func NewReceiver() *Receiver {
+	ctx, ctxCancel := context.WithCancel(context.Background())
+
+	r := &Receiver{
+		OnTimeout: make(chan struct{}, 1),
+		ctx:       ctx,
+		ctxCancel: ctxCancel,
+		heartbeat: make(chan struct{}, 1),
+	}
+
+	go r.waitForHealthcheck()
+	return r
+}
+
+// Heartbeat acknowledge the heartbeat has been received
+func (r *Receiver) Heartbeat() {
+	select {
+	case r.heartbeat <- struct{}{}:
+	default:
+	}
+}
+
+// Stop check the timeout and do not send new notifications
+func (r *Receiver) Stop() {
+	r.ctxCancel()
+}
+
+func (r *Receiver) waitForHealthcheck() {
+	ticker := time.NewTicker(heartbeatTimeout)
+	defer ticker.Stop()
+	defer r.ctxCancel()
+	defer close(r.OnTimeout)
+
+	for {
+		select {
+		case <-r.heartbeat:
+			r.alive = true
+		case <-ticker.C:
+			if r.alive {
+				r.alive = false
+				continue
+			}
+
+			r.notifyTimeout()
+			return
+		case <-r.ctx.Done():
+			return
+		}
+	}
+}
+
+func (r *Receiver) notifyTimeout() {
+	select {
+	case r.OnTimeout <- struct{}{}:
+	default:
+	}
+}

relay/healthcheck/receiver_test.go

@@ -0,0 +1,42 @@
+package healthcheck
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNewReceiver(t *testing.T) {
+	heartbeatTimeout = 5 * time.Second
+	r := NewReceiver()
+
+	select {
+	case <-r.OnTimeout:
+		t.Error("unexpected timeout")
+	case <-time.After(1 * time.Second):
+
+	}
+}
+
+func TestNewReceiverNotReceive(t *testing.T) {
+	heartbeatTimeout = 1 * time.Second
+	r := NewReceiver()
+
+	select {
+	case <-r.OnTimeout:
+	case <-time.After(2 * time.Second):
+		t.Error("timeout not received")
+	}
+}
+
+func TestNewReceiverAck(t *testing.T) {
+	heartbeatTimeout = 2 * time.Second
+	r := NewReceiver()
+
+	r.Heartbeat()
+
+	select {
+	case <-r.OnTimeout:
+		t.Error("unexpected timeout")
+	case <-time.After(3 * time.Second):
+	}
+}

relay/healthcheck/sender.go

@@ -0,0 +1,68 @@
+package healthcheck
+
+import (
+	"context"
+	"time"
+)
+
+var (
+	healthCheckInterval = 25 * time.Second
+	healthCheckTimeout  = 5 * time.Second
+)
+
+// Sender is a healthcheck sender
+// It will send healthcheck signal to the receiver
+// If the receiver does not receive the signal in a certain time, it will send a timeout signal and stop to work
+// It will also stop if the context is canceled
+type Sender struct {
+	// HealthCheck is a channel to send health check signal to the peer
+	HealthCheck chan struct{}
+	// Timeout is a channel to the health check signal is not received in a certain time
+	Timeout chan struct{}
+
+	ack chan struct{}
+}
+
+// NewSender creates a new healthcheck sender
+func NewSender() *Sender {
+	hc := &Sender{
+		HealthCheck: make(chan struct{}, 1),
+		Timeout:     make(chan struct{}, 1),
+		ack:         make(chan struct{}, 1),
+	}
+
+	return hc
+}
+
+// OnHCResponse sends an acknowledgment signal to the sender
+func (hc *Sender) OnHCResponse() {
+	select {
+	case hc.ack <- struct{}{}:
+	default:
+	}
+}
+
+func (hc *Sender) StartHealthCheck(ctx context.Context) {
+	ticker := time.NewTicker(healthCheckInterval)
+	defer ticker.Stop()
+
+	timeoutTimer := time.NewTimer(healthCheckInterval + healthCheckTimeout)
+	defer timeoutTimer.Stop()
+
+	defer close(hc.HealthCheck)
+	defer close(hc.Timeout)
+
+	for {
+		select {
+		case <-ticker.C:
+			hc.HealthCheck <- struct{}{}
+		case <-timeoutTimer.C:
+			hc.Timeout <- struct{}{}
+			return
+		case <-hc.ack:
+			timeoutTimer.Reset(healthCheckInterval + healthCheckTimeout)
+		case <-ctx.Done():
+			return
+		}
+	}
+}

relay/healthcheck/sender_test.go

@@ -0,0 +1,103 @@
+package healthcheck
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestMain(m *testing.M) {
+	// override the health check interval to speed up the test
+	healthCheckInterval = 2 * time.Second
+	healthCheckTimeout = 100 * time.Millisecond
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestNewHealthPeriod(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	hc := NewSender()
+	go hc.StartHealthCheck(ctx)
+
+	iterations := 0
+	for i := 0; i < 3; i++ {
+		select {
+		case <-hc.HealthCheck:
+			iterations++
+			hc.OnHCResponse()
+		case <-hc.Timeout:
+			t.Fatalf("health check is timed out")
+		case <-time.After(healthCheckInterval + 100*time.Millisecond):
+			t.Fatalf("health check not received")
+		}
+	}
+}
+
+func TestNewHealthFailed(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	hc := NewSender()
+	go hc.StartHealthCheck(ctx)
+
+	select {
+	case <-hc.Timeout:
+	case <-time.After(healthCheckInterval + healthCheckTimeout + 100*time.Millisecond):
+		t.Fatalf("health check is not timed out")
+	}
+}
+
+func TestNewHealthcheckStop(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	hc := NewSender()
+	go hc.StartHealthCheck(ctx)
+
+	time.Sleep(100 * time.Millisecond)
+	cancel()
+
+	select {
+	case _, ok := <-hc.HealthCheck:
+		if ok {
+			t.Fatalf("health check on received")
+		}
+	case _, ok := <-hc.Timeout:
+		if ok {
+			t.Fatalf("health check on received")
+		}
+	case <-ctx.Done():
+		// expected
+	case <-time.After(10 * time.Second):
+		t.Fatalf("is not exited")
+	}
+}
+
+func TestTimeoutReset(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	hc := NewSender()
+	go hc.StartHealthCheck(ctx)
+
+	iterations := 0
+	for i := 0; i < 3; i++ {
+		select {
+		case <-hc.HealthCheck:
+			iterations++
+			hc.OnHCResponse()
+		case <-hc.Timeout:
+			t.Fatalf("health check is timed out")
+		case <-time.After(healthCheckInterval + 100*time.Millisecond):
+			t.Fatalf("health check not received")
+		}
+	}
+
+	select {
+	case <-hc.HealthCheck:
+	case <-hc.Timeout:
+		// expected
+	case <-ctx.Done():
+		t.Fatalf("context is done")
+	case <-time.After(10 * time.Second):
+		t.Fatalf("is not exited")
+	}
+}

relay/main.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	log "github.com/sirupsen/logrus"
+
+	"github.com/netbirdio/netbird/relay/cmd"
+)
+
+func main() {
+	if err := cmd.Execute(); err != nil {
+		log.Fatalf("failed to execute command: %v", err)
+	}
+}

relay/messages/address/address.go

@@ -0,0 +1,30 @@
+package address
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+)
+
+type Address struct {
+	URL string
+}
+
+func (addr *Address) Marshal() ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	if err := enc.Encode(addr); err != nil {
+		return nil, fmt.Errorf("encode Address: %w", err)
+	}
+	return buf.Bytes(), nil
+}
+
+func Unmarshal(data []byte) (*Address, error) {
+	var addr Address
+	buf := bytes.NewBuffer(data)
+	dec := gob.NewDecoder(buf)
+	if err := dec.Decode(&addr); err != nil {
+		return nil, fmt.Errorf("decode Address: %w", err)
+	}
+	return &addr, nil
+}

relay/messages/auth/auth.go

@@ -0,0 +1,51 @@
+package auth
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+)
+
+type Algorithm int
+
+const (
+	AlgoUnknown Algorithm = iota
+	AlgoHMACSHA256
+	AlgoHMACSHA512
+)
+
+func (a Algorithm) String() string {
+	switch a {
+	case AlgoHMACSHA256:
+		return "HMAC-SHA256"
+	case AlgoHMACSHA512:
+		return "HMAC-SHA512"
+	default:
+		return "Unknown"
+	}
+}
+
+type Msg struct {
+	AuthAlgorithm  Algorithm
+	AdditionalData []byte
+}
+
+func (msg *Msg) Marshal() ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	if err := enc.Encode(msg); err != nil {
+		return nil, fmt.Errorf("encode Msg: %w", err)
+	}
+	return buf.Bytes(), nil
+}
+
+func UnmarshalMsg(data []byte) (*Msg, error) {
+	var msg *Msg
+
+	buf := bytes.NewBuffer(data)
+	dec := gob.NewDecoder(buf)
+	if err := dec.Decode(&msg); err != nil {
+		return nil, fmt.Errorf("decode Msg: %w", err)
+	}
+	return msg, nil
+}

relay/messages/doc.go

@@ -0,0 +1,5 @@
+/*
+Package messages provides the message types that are used to communicate between the relay and the client.
+This package is used to determine the type of message that is being sent and received between the relay and the client.
+*/
+package messages

relay/messages/id.go

@@ -0,0 +1,31 @@
+package messages
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+)
+
+const (
+	prefixLength = 4
+	IDSize       = prefixLength + sha256.Size
+)
+
+var (
+	prefix = []byte("sha-") // 4 bytes
+)
+
+// HashID generates a sha256 hash from the peerID and returns the hash and the human-readable string
+func HashID(peerID string) ([]byte, string) {
+	idHash := sha256.Sum256([]byte(peerID))
+	idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:])
+	var prefixedHash []byte
+	prefixedHash = append(prefixedHash, prefix...)
+	prefixedHash = append(prefixedHash, idHash[:]...)
+	return prefixedHash, idHashString
+}
+
+// HashIDToString converts a hash to a human-readable string
+func HashIDToString(idHash []byte) string {
+	return fmt.Sprintf("%s%s", idHash[:prefixLength], base64.StdEncoding.EncodeToString(idHash[prefixLength:]))
+}

relay/messages/id_test.go

@@ -0,0 +1,13 @@
+package messages
+
+import (
+	"testing"
+)
+
+func TestHashID(t *testing.T) {
+	hashedID, hashedStringId := HashID("alice")
+	enc := HashIDToString(hashedID)
+	if enc != hashedStringId {
+		t.Errorf("expected %s, got %s", hashedStringId, enc)
+	}
+}

relay/messages/message.go

@@ -0,0 +1,239 @@
+package messages
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+)
+
+const (
+	MsgTypeUnknown       MsgType = 0
+	MsgTypeHello         MsgType = 1
+	MsgTypeHelloResponse MsgType = 2
+	MsgTypeTransport     MsgType = 3
+	MsgTypeClose         MsgType = 4
+	MsgTypeHealthCheck   MsgType = 5
+
+	SizeOfVersionByte = 1
+	SizeOfMsgType     = 1
+
+	SizeOfProtoHeader = SizeOfVersionByte + SizeOfMsgType
+
+	sizeOfMagicByte = 4
+
+	headerSizeTransport = IDSize
+	headerSizeHello     = sizeOfMagicByte + IDSize
+	headerSizeHelloResp = 0
+
+	MaxHandshakeSize = 8192
+
+	CurrentProtocolVersion = 1
+)
+
+var (
+	ErrInvalidMessageLength = errors.New("invalid message length")
+	ErrUnsupportedVersion   = errors.New("unsupported version")
+
+	magicHeader = []byte{0x21, 0x12, 0xA4, 0x42}
+
+	healthCheckMsg = []byte{byte(CurrentProtocolVersion), byte(MsgTypeHealthCheck)}
+)
+
+type MsgType byte
+
+func (m MsgType) String() string {
+	switch m {
+	case MsgTypeHello:
+		return "hello"
+	case MsgTypeHelloResponse:
+		return "hello response"
+	case MsgTypeTransport:
+		return "transport"
+	case MsgTypeClose:
+		return "close"
+	case MsgTypeHealthCheck:
+		return "health check"
+	default:
+		return "unknown"
+	}
+}
+
+type HelloResponse struct {
+	InstanceAddress string
+}
+
+// ValidateVersion checks if the given version is supported by the protocol
+func ValidateVersion(msg []byte) (int, error) {
+	if len(msg) < SizeOfVersionByte {
+		return 0, ErrInvalidMessageLength
+	}
+	version := int(msg[0])
+	if version != CurrentProtocolVersion {
+		return 0, fmt.Errorf("%d: %w", version, ErrUnsupportedVersion)
+	}
+	return version, nil
+}
+
+// DetermineClientMessageType determines the message type from the first the message
+func DetermineClientMessageType(msg []byte) (MsgType, error) {
+	if len(msg) < SizeOfMsgType {
+		return 0, ErrInvalidMessageLength
+	}
+
+	msgType := MsgType(msg[0])
+	switch msgType {
+	case
+		MsgTypeHello,
+		MsgTypeTransport,
+		MsgTypeClose,
+		MsgTypeHealthCheck:
+		return msgType, nil
+	default:
+		return MsgTypeUnknown, fmt.Errorf("invalid msg type %d", msgType)
+	}
+}
+
+// DetermineServerMessageType determines the message type from the first the message
+func DetermineServerMessageType(msg []byte) (MsgType, error) {
+	if len(msg) < SizeOfMsgType {
+		return 0, ErrInvalidMessageLength
+	}
+
+	msgType := MsgType(msg[0])
+	switch msgType {
+	case
+		MsgTypeHelloResponse,
+		MsgTypeTransport,
+		MsgTypeClose,
+		MsgTypeHealthCheck:
+		return msgType, nil
+	default:
+		return MsgTypeUnknown, fmt.Errorf("invalid msg type %d", msgType)
+	}
+}
+
+// MarshalHelloMsg initial hello message
+// The Hello message is the first message sent by a client after establishing a connection with the Relay server. This
+// message is used to authenticate the client with the server. The authentication is done using an HMAC method.
+// The protocol does not limit to use HMAC, it can be any other method. If the authentication failed the server will
+// close the network connection without any response.
+func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) {
+	if len(peerID) != IDSize {
+		return nil, fmt.Errorf("invalid peerID length: %d", len(peerID))
+	}
+
+	msg := make([]byte, SizeOfProtoHeader+sizeOfMagicByte, SizeOfProtoHeader+headerSizeHello+len(additions))
+
+	msg[0] = byte(CurrentProtocolVersion)
+	msg[1] = byte(MsgTypeHello)
+
+	copy(msg[SizeOfProtoHeader:SizeOfProtoHeader+sizeOfMagicByte], magicHeader)
+
+	msg = append(msg, peerID...)
+	msg = append(msg, additions...)
+
+	return msg, nil
+}
+
+// UnmarshalHelloMsg extracts peerID and the additional data from the hello message. The Additional data is used to
+// authenticate the client with the server.
+func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) {
+	if len(msg) < headerSizeHello {
+		return nil, nil, ErrInvalidMessageLength
+	}
+	if !bytes.Equal(msg[:sizeOfMagicByte], magicHeader) {
+		return nil, nil, errors.New("invalid magic header")
+	}
+
+	return msg[sizeOfMagicByte:headerSizeHello], msg[headerSizeHello:], nil
+}
+
+// MarshalHelloResponse creates a response message to the hello message.
+// In case of success connection the server response with a Hello Response message. This message contains the server's
+// instance URL. This URL will be used by choose the common Relay server in case if the peers are in different Relay
+// servers.
+func MarshalHelloResponse(additionalData []byte) ([]byte, error) {
+	msg := make([]byte, SizeOfProtoHeader, SizeOfProtoHeader+headerSizeHelloResp+len(additionalData))
+
+	msg[0] = byte(CurrentProtocolVersion)
+	msg[1] = byte(MsgTypeHelloResponse)
+
+	msg = append(msg, additionalData...)
+
+	return msg, nil
+}
+
+// UnmarshalHelloResponse extracts the additional data from the hello response message.
+func UnmarshalHelloResponse(msg []byte) ([]byte, error) {
+	if len(msg) < headerSizeHelloResp {
+		return nil, ErrInvalidMessageLength
+	}
+	return msg, nil
+}
+
+// MarshalCloseMsg creates a close message.
+// The close message is used to close the connection gracefully between the client and the server. The server and the
+// client can send this message. After receiving this message, the server or client will close the connection.
+func MarshalCloseMsg() []byte {
+	msg := make([]byte, SizeOfProtoHeader)
+
+	msg[0] = byte(CurrentProtocolVersion)
+	msg[1] = byte(MsgTypeClose)
+
+	return msg
+}
+
+// MarshalTransportMsg creates a transport message.
+// The transport message is used to exchange data between peers. The message contains the data to be exchanged and the
+// destination peer hashed ID.
+func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) {
+	if len(peerID) != IDSize {
+		return nil, fmt.Errorf("invalid peerID length: %d", len(peerID))
+	}
+
+	msg := make([]byte, SizeOfProtoHeader+headerSizeTransport, SizeOfProtoHeader+headerSizeTransport+len(payload))
+
+	msg[0] = byte(CurrentProtocolVersion)
+	msg[1] = byte(MsgTypeTransport)
+
+	copy(msg[SizeOfProtoHeader:], peerID)
+
+	msg = append(msg, payload...)
+
+	return msg, nil
+}
+
+// UnmarshalTransportMsg extracts the peerID and the payload from the transport message.
+func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) {
+	if len(buf) < headerSizeTransport {
+		return nil, nil, ErrInvalidMessageLength
+	}
+
+	return buf[:headerSizeTransport], buf[headerSizeTransport:], nil
+}
+
+// UnmarshalTransportID extracts the peerID from the transport message.
+func UnmarshalTransportID(buf []byte) ([]byte, error) {
+	if len(buf) < headerSizeTransport {
+		return nil, ErrInvalidMessageLength
+	}
+	return buf[:headerSizeTransport], nil
+}
+
+// UpdateTransportMsg updates the peerID in the transport message.
+// With this function the server can reuse the given byte slice to update the peerID in the transport message. So do
+// need to allocate a new byte slice.
+func UpdateTransportMsg(msg []byte, peerID []byte) error {
+	if len(msg) < len(peerID) {
+		return ErrInvalidMessageLength
+	}
+	copy(msg, peerID)
+	return nil
+}
+
+// MarshalHealthcheck creates a health check message.
+// Health check message is sent by the server periodically. The client will respond with a health check response
+// message. If the client does not respond to the health check message, the server will close the connection.
+func MarshalHealthcheck() []byte {
+	return healthCheckMsg
+}

relay/messages/message_test.go

@@ -0,0 +1,43 @@
+package messages
+
+import (
+	"testing"
+)
+
+func TestMarshalHelloMsg(t *testing.T) {
+	peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+")
+	bHello, err := MarshalHelloMsg(peerID, nil)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	receivedPeerID, _, err := UnmarshalHelloMsg(bHello[SizeOfProtoHeader:])
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+	if string(receivedPeerID) != string(peerID) {
+		t.Errorf("expected %s, got %s", peerID, receivedPeerID)
+	}
+}
+
+func TestMarshalTransportMsg(t *testing.T) {
+	peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+")
+	payload := []byte("payload")
+	msg, err := MarshalTransportMsg(peerID, payload)
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	id, respPayload, err := UnmarshalTransportMsg(msg[SizeOfProtoHeader:])
+	if err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	if string(id) != string(peerID) {
+		t.Errorf("expected %s, got %s", peerID, id)
+	}
+
+	if string(respPayload) != string(payload) {
+		t.Errorf("expected %s, got %s", payload, respPayload)
+	}
+}

relay/metrics/realy.go

@@ -0,0 +1,136 @@
+package metrics
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const (
+	idleTimeout = 30 * time.Second
+)
+
+type Metrics struct {
+	metric.Meter
+
+	TransferBytesSent metric.Int64Counter
+	TransferBytesRecv metric.Int64Counter
+
+	peers            metric.Int64UpDownCounter
+	peerActivityChan chan string
+	peerLastActive   map[string]time.Time
+	mutexActivity    sync.Mutex
+	ctx              context.Context
+}
+
+func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) {
+	bytesSent, err := meter.Int64Counter("relay_transfer_sent_bytes_total")
+	if err != nil {
+		return nil, err
+	}
+
+	bytesRecv, err := meter.Int64Counter("relay_transfer_received_bytes_total")
+	if err != nil {
+		return nil, err
+	}
+
+	peers, err := meter.Int64UpDownCounter("relay_peers")
+	if err != nil {
+		return nil, err
+	}
+
+	peersActive, err := meter.Int64ObservableGauge("relay_peers_active")
+	if err != nil {
+		return nil, err
+	}
+
+	peersIdle, err := meter.Int64ObservableGauge("relay_peers_idle")
+	if err != nil {
+		return nil, err
+	}
+
+	m := &Metrics{
+		Meter:             meter,
+		TransferBytesSent: bytesSent,
+		TransferBytesRecv: bytesRecv,
+		peers:             peers,
+
+		ctx:              ctx,
+		peerActivityChan: make(chan string, 10),
+		peerLastActive:   make(map[string]time.Time),
+	}
+
+	_, err = meter.RegisterCallback(
+		func(ctx context.Context, o metric.Observer) error {
+			active, idle := m.calculateActiveIdleConnections()
+			o.ObserveInt64(peersActive, active)
+			o.ObserveInt64(peersIdle, idle)
+			return nil
+		},
+		peersActive, peersIdle,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	go m.readPeerActivity()
+	return m, nil
+}
+
+// PeerConnected increments the number of connected peers and increments number of idle connections
+func (m *Metrics) PeerConnected(id string) {
+	m.peers.Add(m.ctx, 1)
+	m.mutexActivity.Lock()
+	defer m.mutexActivity.Unlock()
+
+	m.peerLastActive[id] = time.Time{}
+}
+
+// PeerDisconnected decrements the number of connected peers and decrements number of idle or active connections
+func (m *Metrics) PeerDisconnected(id string) {
+	m.peers.Add(m.ctx, -1)
+	m.mutexActivity.Lock()
+	defer m.mutexActivity.Unlock()
+
+	delete(m.peerLastActive, id)
+}
+
+// PeerActivity increases the active connections
+func (m *Metrics) PeerActivity(peerID string) {
+	select {
+	case m.peerActivityChan <- peerID:
+	default:
+		log.Errorf("peer activity channel is full, dropping activity metrics for peer %s", peerID)
+	}
+}
+
+func (m *Metrics) calculateActiveIdleConnections() (int64, int64) {
+	active, idle := int64(0), int64(0)
+	m.mutexActivity.Lock()
+	defer m.mutexActivity.Unlock()
+
+	for _, lastActive := range m.peerLastActive {
+		if time.Since(lastActive) > idleTimeout {
+			idle++
+		} else {
+			active++
+		}
+	}
+	return active, idle
+}
+
+func (m *Metrics) readPeerActivity() {
+	for {
+		select {
+		case peerID := <-m.peerActivityChan:
+			m.mutexActivity.Lock()
+			m.peerLastActive[peerID] = time.Now()
+			m.mutexActivity.Unlock()
+		case <-m.ctx.Done():
+			return
+		}
+	}
+}

relay/server/listener/listener.go

@@ -0,0 +1,11 @@
+package listener
+
+import (
+	"context"
+	"net"
+)
+
+type Listener interface {
+	Listen(func(conn net.Conn)) error
+	Shutdown(ctx context.Context) error
+}

relay/server/listener/ws/conn.go

@@ -0,0 +1,114 @@
+package ws
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"nhooyr.io/websocket"
+)
+
+const (
+	writeTimeout = 10 * time.Second
+)
+
+type Conn struct {
+	*websocket.Conn
+	lAddr *net.TCPAddr
+	rAddr *net.TCPAddr
+
+	closed   bool
+	closedMu sync.Mutex
+	ctx      context.Context
+}
+
+func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn {
+	return &Conn{
+		Conn:  wsConn,
+		lAddr: lAddr,
+		rAddr: rAddr,
+		ctx:   context.Background(),
+	}
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	t, r, err := c.Reader(c.ctx)
+	if err != nil {
+		return 0, c.ioErrHandling(err)
+	}
+
+	if t != websocket.MessageBinary {
+		log.Errorf("unexpected message type: %d", t)
+		return 0, fmt.Errorf("unexpected message type")
+	}
+
+	n, err = r.Read(b)
+	if err != nil {
+		return 0, c.ioErrHandling(err)
+	}
+	return n, err
+}
+
+// Write writes a binary message with the given payload.
+// It does not block until fill the internal buffer.
+// If the buffer filled up, wait until the buffer is drained or timeout.
+func (c *Conn) Write(b []byte) (int, error) {
+	ctx, ctxCancel := context.WithTimeout(c.ctx, writeTimeout)
+	defer ctxCancel()
+
+	err := c.Conn.Write(ctx, websocket.MessageBinary, b)
+	return len(b), err
+}
+
+func (c *Conn) LocalAddr() net.Addr {
+	return c.lAddr
+}
+
+func (c *Conn) RemoteAddr() net.Addr {
+	return c.rAddr
+}
+
+func (c *Conn) SetReadDeadline(t time.Time) error {
+	return fmt.Errorf("SetReadDeadline is not implemented")
+}
+
+func (c *Conn) SetWriteDeadline(t time.Time) error {
+	return fmt.Errorf("SetWriteDeadline is not implemented")
+}
+
+func (c *Conn) SetDeadline(t time.Time) error {
+	return fmt.Errorf("SetDeadline is not implemented")
+}
+
+func (c *Conn) Close() error {
+	c.closedMu.Lock()
+	c.closed = true
+	c.closedMu.Unlock()
+	return c.Conn.CloseNow()
+}
+
+func (c *Conn) isClosed() bool {
+	c.closedMu.Lock()
+	defer c.closedMu.Unlock()
+	return c.closed
+}
+
+func (c *Conn) ioErrHandling(err error) error {
+	if c.isClosed() {
+		return io.EOF
+	}
+
+	var wErr *websocket.CloseError
+	if !errors.As(err, &wErr) {
+		return err
+	}
+	if wErr.Code == websocket.StatusNormalClosure {
+		return io.EOF
+	}
+	return err
+}