From 0c039274a41ea0d2284545f49527ce0c2a38386b Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 8 Sep 2024 12:06:14 +0200 Subject: [PATCH] [relay] Feature/relay integration (#2244) This update adds new relay integration for NetBird clients. The new relay is based on web sockets and listens on a single port. - Adds new relay implementation with websocket with single port relaying mechanism - refactor peer connection logic, allowing upgrade and downgrade from/to P2P connection - peer connections are faster since it connects first to relay and then upgrades to P2P - maintains compatibility with old clients by not using the new relay - updates infrastructure scripts with new relay service --- .../workflows/test-infrastructure-files.yml | 30 +- .goreleaser.yaml | 74 +- client/cmd/status.go | 27 +- client/cmd/status_test.go | 16 +- client/cmd/testutil_test.go | 5 +- client/cmd/up.go | 5 +- client/internal/connect.go | 56 +- client/internal/engine.go | 184 +-- client/internal/engine_test.go | 77 +- client/internal/peer/conn.go | 1326 +++++++++-------- client/internal/peer/conn_test.go | 71 +- client/internal/peer/handshaker.go | 192 +++ client/internal/peer/signaler.go | 70 + client/internal/peer/status.go | 194 ++- client/internal/peer/status_test.go | 10 +- client/internal/peer/stdnet.go | 4 +- client/internal/peer/stdnet_android.go | 4 +- client/internal/peer/worker_ice.go | 470 ++++++ client/internal/peer/worker_relay.go | 223 +++ client/internal/relay/relay.go | 4 +- client/internal/routemanager/client.go | 8 +- client/internal/routemanager/client_test.go | 43 - client/internal/routemanager/manager.go | 13 +- client/internal/routemanager/manager_test.go | 2 +- client/internal/wgproxy/proxy_ebpf.go | 4 +- client/internal/wgproxy/proxy_userspace.go | 14 +- client/ios/NetBirdSDK/client.go | 1 - client/proto/daemon.pb.go | 423 +++--- client/proto/daemon.proto | 2 +- client/server/debug.go | 4 +- client/server/server.go | 4 +- client/server/server_test.go | 8 +- client/testdata/management.json | 9 +- encryption/cert.go | 19 + encryption/letsencrypt.go | 4 +- encryption/route53.go | 87 ++ encryption/route53_test.go | 84 ++ go.mod | 35 +- go.sum | 79 +- infrastructure_files/base.setup.env | 12 +- infrastructure_files/configure.sh | 5 + infrastructure_files/docker-compose.yml.tmpl | 17 + .../getting-started-with-zitadel.sh | 49 +- infrastructure_files/management.json.tmpl | 5 + infrastructure_files/setup.env.example | 12 + infrastructure_files/tests/setup.env | 3 +- management/client/client_test.go | 5 +- management/cmd/management.go | 8 +- management/cmd/management_test.go | 59 + management/proto/management.pb.go | 859 ++++++----- management/proto/management.proto | 9 + management/server/config.go | 7 + management/server/grpcserver.go | 144 +- management/server/management_proto_test.go | 5 +- management/server/management_test.go | 5 +- management/server/peer.go | 2 +- management/server/peer_test.go | 8 +- management/server/token_mgr.go | 222 +++ management/server/token_mgr_test.go | 218 +++ management/server/turncredentials.go | 126 -- management/server/turncredentials_test.go | 136 -- relay/Dockerfile | 4 + relay/auth/allow/allow_all.go | 12 + relay/auth/doc.go | 26 + relay/auth/hmac/doc.go | 8 + relay/auth/hmac/store.go | 36 + relay/auth/hmac/token.go | 105 ++ relay/auth/hmac/token_test.go | 105 ++ relay/auth/hmac/validator.go | 33 + relay/auth/validator.go | 8 + relay/client/addr.go | 13 + relay/client/client.go | 553 +++++++ relay/client/client_test.go | 631 ++++++++ relay/client/conn.go | 76 + relay/client/dialer/ws/addr.go | 13 + relay/client/dialer/ws/conn.go | 66 + relay/client/dialer/ws/ws.go | 67 + relay/client/doc.go | 12 + relay/client/guard.go | 48 + relay/client/manager.go | 365 +++++ relay/client/manager_test.go | 432 ++++++ relay/cmd/env.go | 35 + relay/cmd/root.go | 214 +++ relay/doc.go | 14 + relay/healthcheck/doc.go | 17 + relay/healthcheck/receiver.go | 82 + relay/healthcheck/receiver_test.go | 42 + relay/healthcheck/sender.go | 68 + relay/healthcheck/sender_test.go | 103 ++ relay/main.go | 13 + relay/messages/address/address.go | 30 + relay/messages/auth/auth.go | 51 + relay/messages/doc.go | 5 + relay/messages/id.go | 31 + relay/messages/id_test.go | 13 + relay/messages/message.go | 239 +++ relay/messages/message_test.go | 43 + relay/metrics/realy.go | 136 ++ relay/server/listener/listener.go | 11 + relay/server/listener/ws/conn.go | 114 ++ relay/server/listener/ws/listener.go | 92 ++ relay/server/peer.go | 203 +++ relay/server/relay.go | 206 +++ relay/server/relay_test.go | 36 + relay/server/server.go | 76 + relay/server/store.go | 64 + relay/server/store_test.go | 40 + relay/test/benchmark_test.go | 386 +++++ relay/testec2/main.go | 258 ++++ relay/testec2/relay.go | 176 +++ relay/testec2/signal.go | 91 ++ relay/testec2/start_msg.go | 39 + relay/testec2/tun/proxy.go | 72 + relay/testec2/tun/tun.go | 110 ++ relay/testec2/turn.go | 181 +++ relay/testec2/turn_allocator.go | 83 ++ signal/client/client.go | 6 +- signal/proto/signalexchange.pb.go | 20 +- signal/proto/signalexchange.proto | 3 + util/net/dialer_nonios.go | 2 + 120 files changed, 9879 insertions(+), 1940 deletions(-) create mode 100644 client/internal/peer/handshaker.go create mode 100644 client/internal/peer/signaler.go create mode 100644 client/internal/peer/worker_ice.go create mode 100644 client/internal/peer/worker_relay.go create mode 100644 encryption/cert.go create mode 100644 encryption/route53.go create mode 100644 encryption/route53_test.go create mode 100644 management/cmd/management_test.go create mode 100644 management/server/token_mgr.go create mode 100644 management/server/token_mgr_test.go delete mode 100644 management/server/turncredentials.go delete mode 100644 management/server/turncredentials_test.go create mode 100644 relay/Dockerfile create mode 100644 relay/auth/allow/allow_all.go create mode 100644 relay/auth/doc.go create mode 100644 relay/auth/hmac/doc.go create mode 100644 relay/auth/hmac/store.go create mode 100644 relay/auth/hmac/token.go create mode 100644 relay/auth/hmac/token_test.go create mode 100644 relay/auth/hmac/validator.go create mode 100644 relay/auth/validator.go create mode 100644 relay/client/addr.go create mode 100644 relay/client/client.go create mode 100644 relay/client/client_test.go create mode 100644 relay/client/conn.go create mode 100644 relay/client/dialer/ws/addr.go create mode 100644 relay/client/dialer/ws/conn.go create mode 100644 relay/client/dialer/ws/ws.go create mode 100644 relay/client/doc.go create mode 100644 relay/client/guard.go create mode 100644 relay/client/manager.go create mode 100644 relay/client/manager_test.go create mode 100644 relay/cmd/env.go create mode 100644 relay/cmd/root.go create mode 100644 relay/doc.go create mode 100644 relay/healthcheck/doc.go create mode 100644 relay/healthcheck/receiver.go create mode 100644 relay/healthcheck/receiver_test.go create mode 100644 relay/healthcheck/sender.go create mode 100644 relay/healthcheck/sender_test.go create mode 100644 relay/main.go create mode 100644 relay/messages/address/address.go create mode 100644 relay/messages/auth/auth.go create mode 100644 relay/messages/doc.go create mode 100644 relay/messages/id.go create mode 100644 relay/messages/id_test.go create mode 100644 relay/messages/message.go create mode 100644 relay/messages/message_test.go create mode 100644 relay/metrics/realy.go create mode 100644 relay/server/listener/listener.go create mode 100644 relay/server/listener/ws/conn.go create mode 100644 relay/server/listener/ws/listener.go create mode 100644 relay/server/peer.go create mode 100644 relay/server/relay.go create mode 100644 relay/server/relay_test.go create mode 100644 relay/server/server.go create mode 100644 relay/server/store.go create mode 100644 relay/server/store_test.go create mode 100644 relay/test/benchmark_test.go create mode 100644 relay/testec2/main.go create mode 100644 relay/testec2/relay.go create mode 100644 relay/testec2/signal.go create mode 100644 relay/testec2/start_msg.go create mode 100644 relay/testec2/tun/proxy.go create mode 100644 relay/testec2/tun/tun.go create mode 100644 relay/testec2/turn.go create mode 100644 relay/testec2/turn_allocator.go diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index f758e74bd..03ecbd445 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -150,6 +150,13 @@ jobs: grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000" grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN" + # check relay values + grep "NB_EXPOSED_ADDRESS=$CI_NETBIRD_DOMAIN:33445" docker-compose.yml + grep "NB_LISTEN_ADDRESS=:33445" docker-compose.yml + grep '33445:33445' docker-compose.yml + grep -A 10 'relay:' docker-compose.yml | egrep 'NB_AUTH_SECRET=.+$' + grep -A 7 Relay management.json | grep "rel://$CI_NETBIRD_DOMAIN:33445" + grep -A 7 Relay management.json | egrep '"Secret": ".+"' - name: Install modules run: go mod tidy @@ -175,6 +182,15 @@ jobs: run: | docker build -t netbirdio/signal:latest . + - name: Build relay binary + working-directory: relay + run: CGO_ENABLED=0 go build -o netbird-relay main.go + + - name: Build relay docker image + working-directory: relay + run: | + docker build -t netbirdio/relay:latest . + - name: run docker compose up working-directory: infrastructure_files/artifacts run: | @@ -186,7 +202,7 @@ jobs: - name: test running containers run: | count=$(docker compose ps --format json | jq '. | select(.Name | contains("artifacts")) | .State' | grep -c running) - test $count -eq 4 || docker compose logs + test $count -eq 5 || docker compose logs working-directory: infrastructure_files/artifacts - name: test geolocation databases @@ -205,6 +221,9 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + - name: handle insisting image # remove after release + run: docker pull netbirdio/relay:latest || docker pull netbirdio/signal:latest && docker tag netbirdio/signal:latest netbirdio/relay:latest + - name: run script with Zitadel PostgreSQL run: NETBIRD_DOMAIN=use-ip bash -x infrastructure_files/getting-started-with-zitadel.sh @@ -229,6 +248,9 @@ jobs: - name: test dashboard.env file gen postgres run: test -f dashboard.env + - name: test relay.env file gen postgres + run: test -f relay.env + - name: test zdb.env file gen postgres run: test -f zdb.env @@ -237,6 +259,9 @@ jobs: docker compose down --volumes --rmi all rm -rf docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json zdb.env + - name: handle insisting image gen CockroachDB # remove after release + run: docker pull netbirdio/relay:latest || docker pull netbirdio/signal:latest && docker tag netbirdio/signal:latest netbirdio/relay:latest + - name: run script with Zitadel CockroachDB run: bash -x infrastructure_files/getting-started-with-zitadel.sh env: @@ -264,6 +289,9 @@ jobs: - name: test dashboard.env file gen CockroachDB run: test -f dashboard.env + - name: test relay.env file gen CockroachDB + run: test -f relay.env + test-download-geolite2-script: runs-on: ubuntu-latest steps: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7a219110a..068864d6e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -80,6 +80,20 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: '{{ .CommitTimestamp }}' + - id: netbird-relay + dir: relay + env: [CGO_ENABLED=0] + binary: netbird-relay + goos: + - linux + goarch: + - amd64 + - arm64 + - arm + ldflags: + - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: '{{ .CommitTimestamp }}' + archives: - builds: - netbird @@ -161,6 +175,52 @@ dockers: - "--label=org.opencontainers.image.revision={{.FullCommit}}" - "--label=org.opencontainers.image.version={{.Version}}" - "--label=maintainer=dev@netbird.io" + - image_templates: + - netbirdio/relay:{{ .Version }}-amd64 + ids: + - netbird-relay + goarch: amd64 + use: buildx + dockerfile: relay/Dockerfile + build_flag_templates: + - "--platform=linux/amd64" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=maintainer=dev@netbird.io" + - image_templates: + - netbirdio/relay:{{ .Version }}-arm64v8 + ids: + - netbird-relay + goarch: arm64 + use: buildx + dockerfile: relay/Dockerfile + build_flag_templates: + - "--platform=linux/arm64" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=maintainer=dev@netbird.io" + - image_templates: + - netbirdio/relay:{{ .Version }}-arm + ids: + - netbird-relay + goarch: arm + goarm: 6 + use: buildx + dockerfile: relay/Dockerfile + build_flag_templates: + - "--platform=linux/arm" + - "--label=org.opencontainers.image.created={{.Date}}" + - "--label=org.opencontainers.image.title={{.ProjectName}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=org.opencontainers.image.revision={{.FullCommit}}" + - "--label=org.opencontainers.image.version={{.Version}}" + - "--label=maintainer=dev@netbird.io" - image_templates: - netbirdio/signal:{{ .Version }}-amd64 ids: @@ -313,6 +373,18 @@ docker_manifests: - netbirdio/netbird:{{ .Version }}-arm - netbirdio/netbird:{{ .Version }}-amd64 + - name_template: netbirdio/relay:{{ .Version }} + image_templates: + - netbirdio/relay:{{ .Version }}-arm64v8 + - netbirdio/relay:{{ .Version }}-arm + - netbirdio/relay:{{ .Version }}-amd64 + + - name_template: netbirdio/relay:latest + image_templates: + - netbirdio/relay:{{ .Version }}-arm64v8 + - netbirdio/relay:{{ .Version }}-arm + - netbirdio/relay:{{ .Version }}-amd64 + - name_template: netbirdio/signal:{{ .Version }} image_templates: - netbirdio/signal:{{ .Version }}-arm64v8 @@ -386,4 +458,4 @@ checksum: release: extra_files: - glob: ./infrastructure_files/getting-started-with-zitadel.sh - - glob: ./release_files/install.sh \ No newline at end of file + - glob: ./release_files/install.sh diff --git a/client/cmd/status.go b/client/cmd/status.go index d9b7a9c91..1ef8b4913 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -31,9 +31,9 @@ type peerStateDetailOutput struct { Status string `json:"status" yaml:"status"` LastStatusUpdate time.Time `json:"lastStatusUpdate" yaml:"lastStatusUpdate"` ConnType string `json:"connectionType" yaml:"connectionType"` - Direct bool `json:"direct" yaml:"direct"` IceCandidateType iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"` IceCandidateEndpoint iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"` + RelayAddress string `json:"relayAddress" yaml:"relayAddress"` LastWireguardHandshake time.Time `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"` TransferReceived int64 `json:"transferReceived" yaml:"transferReceived"` TransferSent int64 `json:"transferSent" yaml:"transferSent"` @@ -335,16 +335,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput { func mapPeers(peers []*proto.PeerState) peersStateOutput { var peersStateDetail []peerStateDetailOutput - localICE := "" - remoteICE := "" - localICEEndpoint := "" - remoteICEEndpoint := "" - connType := "" peersConnected := 0 - lastHandshake := time.Time{} - transferReceived := int64(0) - transferSent := int64(0) for _, pbPeerState := range peers { + localICE := "" + remoteICE := "" + localICEEndpoint := "" + remoteICEEndpoint := "" + relayServerAddress := "" + connType := "" + lastHandshake := time.Time{} + transferReceived := int64(0) + transferSent := int64(0) + isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String() if skipDetailByFilters(pbPeerState, isPeerConnected) { continue @@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { if pbPeerState.Relayed { connType = "Relayed" } + relayServerAddress = pbPeerState.GetRelayAddress() lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local() transferReceived = pbPeerState.GetBytesRx() transferSent = pbPeerState.GetBytesTx() @@ -372,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { Status: pbPeerState.GetConnStatus(), LastStatusUpdate: timeLocal, ConnType: connType, - Direct: pbPeerState.GetDirect(), IceCandidateType: iceCandidateType{ Local: localICE, Remote: remoteICE, @@ -381,6 +383,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { Local: localICEEndpoint, Remote: remoteICEEndpoint, }, + RelayAddress: relayServerAddress, FQDN: pbPeerState.GetFqdn(), LastWireguardHandshake: lastHandshake, TransferReceived: transferReceived, @@ -641,9 +644,9 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo " Status: %s\n"+ " -- detail --\n"+ " Connection type: %s\n"+ - " Direct: %t\n"+ " ICE candidate (Local/Remote): %s/%s\n"+ " ICE candidate endpoints (Local/Remote): %s/%s\n"+ + " Relay server address: %s\n"+ " Last connection update: %s\n"+ " Last WireGuard handshake: %s\n"+ " Transfer status (received/sent) %s/%s\n"+ @@ -655,11 +658,11 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo peerState.PubKey, peerState.Status, peerState.ConnType, - peerState.Direct, localICE, remoteICE, localICEEndpoint, remoteICEEndpoint, + peerState.RelayAddress, timeAgo(peerState.LastStatusUpdate), timeAgo(peerState.LastWireguardHandshake), toIEC(peerState.TransferReceived), diff --git a/client/cmd/status_test.go b/client/cmd/status_test.go index 46620a956..ca43df8a5 100644 --- a/client/cmd/status_test.go +++ b/client/cmd/status_test.go @@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{ ConnStatus: "Connected", ConnStatusUpdate: timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)), Relayed: false, - Direct: true, LocalIceCandidateType: "", RemoteIceCandidateType: "", LocalIceCandidateEndpoint: "", @@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{ ConnStatus: "Connected", ConnStatusUpdate: timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)), Relayed: true, - Direct: false, LocalIceCandidateType: "relay", RemoteIceCandidateType: "prflx", LocalIceCandidateEndpoint: "10.0.0.1:10001", @@ -137,7 +135,6 @@ var overview = statusOutputOverview{ Status: "Connected", LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC), ConnType: "P2P", - Direct: true, IceCandidateType: iceCandidateType{ Local: "", Remote: "", @@ -161,7 +158,6 @@ var overview = statusOutputOverview{ Status: "Connected", LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC), ConnType: "Relayed", - Direct: false, IceCandidateType: iceCandidateType{ Local: "relay", Remote: "prflx", @@ -283,7 +279,6 @@ func TestParsingToJSON(t *testing.T) { "status": "Connected", "lastStatusUpdate": "2001-01-01T01:01:01Z", "connectionType": "P2P", - "direct": true, "iceCandidateType": { "local": "", "remote": "" @@ -292,6 +287,7 @@ func TestParsingToJSON(t *testing.T) { "local": "", "remote": "" }, + "relayAddress": "", "lastWireguardHandshake": "2001-01-01T01:01:02Z", "transferReceived": 200, "transferSent": 100, @@ -308,7 +304,6 @@ func TestParsingToJSON(t *testing.T) { "status": "Connected", "lastStatusUpdate": "2002-02-02T02:02:02Z", "connectionType": "Relayed", - "direct": false, "iceCandidateType": { "local": "relay", "remote": "prflx" @@ -317,6 +312,7 @@ func TestParsingToJSON(t *testing.T) { "local": "10.0.0.1:10001", "remote": "10.0.10.1:10002" }, + "relayAddress": "", "lastWireguardHandshake": "2002-02-02T02:02:03Z", "transferReceived": 2000, "transferSent": 1000, @@ -408,13 +404,13 @@ func TestParsingToYAML(t *testing.T) { status: Connected lastStatusUpdate: 2001-01-01T01:01:01Z connectionType: P2P - direct: true iceCandidateType: local: "" remote: "" iceCandidateEndpoint: local: "" remote: "" + relayAddress: "" lastWireguardHandshake: 2001-01-01T01:01:02Z transferReceived: 200 transferSent: 100 @@ -428,13 +424,13 @@ func TestParsingToYAML(t *testing.T) { status: Connected lastStatusUpdate: 2002-02-02T02:02:02Z connectionType: Relayed - direct: false iceCandidateType: local: relay remote: prflx iceCandidateEndpoint: local: 10.0.0.1:10001 remote: 10.0.10.1:10002 + relayAddress: "" lastWireguardHandshake: 2002-02-02T02:02:03Z transferReceived: 2000 transferSent: 1000 @@ -505,9 +501,9 @@ func TestParsingToDetail(t *testing.T) { Status: Connected -- detail -- Connection type: P2P - Direct: true ICE candidate (Local/Remote): -/- ICE candidate endpoints (Local/Remote): -/- + Relay server address: Last connection update: %s Last WireGuard handshake: %s Transfer status (received/sent) 200 B/100 B @@ -521,9 +517,9 @@ func TestParsingToDetail(t *testing.T) { Status: Connected -- detail -- Connection type: Relayed - Direct: false ICE candidate (Local/Remote): relay/prflx ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002 + Relay server address: Last connection update: %s Last WireGuard handshake: %s Transfer status (received/sent) 2.0 KiB/1000 B diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index 984aa6df7..780cc8b04 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -98,8 +98,9 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) - mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) + + secretsManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay) + mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil) if err != nil { t.Fatal(err) } diff --git a/client/cmd/up.go b/client/cmd/up.go index 2ed6e41d2..b447f7141 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -168,7 +168,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error { ctx, cancel = context.WithCancel(ctx) SetupCloseHandler(ctx, cancel) - connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String())) + r := peer.NewRecorder(config.ManagementURL.String()) + r.GetFullStatus() + + connectClient := internal.NewConnectClient(ctx, config, r) return connectClient.Run() } diff --git a/client/internal/connect.go b/client/internal/connect.go index 5dacde746..515321f7f 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -26,6 +26,8 @@ import ( "github.com/netbirdio/netbird/iface" mgm "github.com/netbirdio/netbird/management/client" mgmProto "github.com/netbirdio/netbird/management/proto" + "github.com/netbirdio/netbird/relay/auth/hmac" + relayClient "github.com/netbirdio/netbird/relay/client" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/util" "github.com/netbirdio/netbird/version" @@ -158,10 +160,8 @@ func (c *ConnectClient) run( defer c.statusRecorder.ClientStop() operation := func() error { // if context cancelled we not start new backoff cycle - select { - case <-c.ctx.Done(): + if c.isContextCancelled() { return nil - default: } state.Set(StatusConnecting) @@ -183,8 +183,7 @@ func (c *ConnectClient) run( log.Debugf("connected to the Management service %s", c.config.ManagementURL.Host) defer func() { - err = mgmClient.Close() - if err != nil { + if err = mgmClient.Close(); err != nil { log.Warnf("failed to close the Management service client %v", err) } }() @@ -208,7 +207,6 @@ func (c *ConnectClient) run( KernelInterface: iface.WireGuardModuleIsLoaded(), FQDN: loginResp.GetPeerConfig().GetFqdn(), } - c.statusRecorder.UpdateLocalPeerState(localPeerState) signalURL := fmt.Sprintf("%s://%s", @@ -241,6 +239,23 @@ func (c *ConnectClient) run( c.statusRecorder.MarkSignalConnected() + relayURLs, token := parseRelayInfo(loginResp) + relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String()) + if len(relayURLs) > 0 { + if token != nil { + if err := relayManager.UpdateToken(token); err != nil { + log.Errorf("failed to update token: %s", err) + return wrapErr(err) + } + } + log.Infof("connecting to the Relay service(s): %s", strings.Join(relayURLs, ", ")) + if err = relayManager.Serve(); err != nil { + log.Error(err) + return wrapErr(err) + } + c.statusRecorder.SetRelayMgr(relayManager) + } + peerConfig := loginResp.GetPeerConfig() engineConfig, err := createEngineConfig(myPrivateKey, c.config, peerConfig) @@ -252,11 +267,11 @@ func (c *ConnectClient) run( checks := loginResp.GetChecks() c.engineMutex.Lock() - c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, engineConfig, mobileDependency, c.statusRecorder, probes, checks) + c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, probes, checks) + c.engineMutex.Unlock() - err = c.engine.Start() - if err != nil { + if err := c.engine.Start(); err != nil { log.Errorf("error while starting Netbird Connection Engine: %s", err) return wrapErr(err) } @@ -296,6 +311,20 @@ func (c *ConnectClient) run( return nil } +func parseRelayInfo(loginResp *mgmProto.LoginResponse) ([]string, *hmac.Token) { + relayCfg := loginResp.GetWiretrusteeConfig().GetRelay() + if relayCfg == nil { + return nil, nil + } + + token := &hmac.Token{ + Payload: relayCfg.GetTokenPayload(), + Signature: relayCfg.GetTokenSignature(), + } + + return relayCfg.GetUrls(), token +} + func (c *ConnectClient) Engine() *Engine { if c == nil { return nil @@ -320,6 +349,15 @@ func (c *ConnectClient) Stop() error { return c.engine.Stop() } +func (c *ConnectClient) isContextCancelled() bool { + select { + case <-c.ctx.Done(): + return true + default: + return false + } +} + // createEngineConfig converts configuration received from Management Service to EngineConfig func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) { nm := false diff --git a/client/internal/engine.go b/client/internal/engine.go index 0d80806a4..47a36c4bf 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -25,6 +25,7 @@ import ( "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/internal/acl" "github.com/netbirdio/netbird/client/internal/dns" + "github.com/netbirdio/netbird/client/internal/networkmonitor" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/relay" @@ -40,6 +41,8 @@ import ( mgm "github.com/netbirdio/netbird/management/client" "github.com/netbirdio/netbird/management/domain" mgmProto "github.com/netbirdio/netbird/management/proto" + auth "github.com/netbirdio/netbird/relay/auth/hmac" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" sProto "github.com/netbirdio/netbird/signal/proto" @@ -102,7 +105,8 @@ type EngineConfig struct { // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers. type Engine struct { // signal is a Signal Service client - signal signal.Client + signal signal.Client + signaler *peer.Signaler // mgmClient is a Management Service client mgmClient mgm.Client // peerConns is a map that holds all the peers that are known to this peer @@ -159,10 +163,10 @@ type Engine struct { probes *ProbeHolder - wgConnWorker sync.WaitGroup - // checks are the client-applied posture checks that need to be evaluated on the client checks []*mgmProto.Checks + + relayManager *relayClient.Manager } // Peer is an instance of the Connection Peer @@ -177,6 +181,7 @@ func NewEngine( clientCancel context.CancelFunc, signalClient signal.Client, mgmClient mgm.Client, + relayManager *relayClient.Manager, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, @@ -187,6 +192,7 @@ func NewEngine( clientCancel, signalClient, mgmClient, + relayManager, config, mobileDep, statusRecorder, @@ -201,18 +207,20 @@ func NewEngineWithProbes( clientCancel context.CancelFunc, signalClient signal.Client, mgmClient mgm.Client, + relayManager *relayClient.Manager, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, probes *ProbeHolder, checks []*mgmProto.Checks, ) *Engine { - return &Engine{ clientCtx: clientCtx, clientCancel: clientCancel, signal: signalClient, + signaler: peer.NewSignaler(signalClient, config.WgPrivateKey), mgmClient: mgmClient, + relayManager: relayManager, peerConns: make(map[string]*peer.Conn), syncMsgMux: &sync.Mutex{}, config: config, @@ -260,11 +268,7 @@ func (e *Engine) Stop() error { time.Sleep(500 * time.Millisecond) e.close() - - e.wgConnWorker.Wait() - - log.Infof("Engine stopped") - + log.Infof("stopped Netbird Engine") return nil } @@ -314,7 +318,7 @@ func (e *Engine) Start() error { } e.dnsServer = dnsServer - e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, initialRoutes) + e.routeManager = routemanager.NewManager(e.ctx, e.config.WgPrivateKey.PublicKey().String(), e.config.DNSRouteInterval, e.wgInterface, e.statusRecorder, e.relayManager, initialRoutes) beforePeerHook, afterPeerHook, err := e.routeManager.Init() if err != nil { log.Errorf("Failed to initialize route manager: %s", err) @@ -463,78 +467,25 @@ func (e *Engine) removePeer(peerKey string) error { conn, exists := e.peerConns[peerKey] if exists { delete(e.peerConns, peerKey) - err := conn.Close() - if err != nil { - switch err.(type) { - case *peer.ConnectionAlreadyClosedError: - return nil - default: - return err - } - } + conn.Close() } return nil } -func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client) error { - err := s.Send(&sProto.Message{ - Key: myKey.PublicKey().String(), - RemoteKey: remoteKey.String(), - Body: &sProto.Body{ - Type: sProto.Body_CANDIDATE, - Payload: candidate.Marshal(), - }, - }) - if err != nil { - return err - } - - return nil -} - -func sendSignal(message *sProto.Message, s signal.Client) error { - return s.Send(message) -} - -// SignalOfferAnswer signals either an offer or an answer to remote peer -func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client, - isAnswer bool) error { - var t sProto.Body_Type - if isAnswer { - t = sProto.Body_ANSWER - } else { - t = sProto.Body_OFFER - } - - msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{ - UFrag: offerAnswer.IceCredentials.UFrag, - Pwd: offerAnswer.IceCredentials.Pwd, - }, t, offerAnswer.RosenpassPubKey, offerAnswer.RosenpassAddr) - if err != nil { - return err - } - - err = s.Send(msg) - if err != nil { - return err - } - - return nil -} - func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { e.syncMsgMux.Lock() defer e.syncMsgMux.Unlock() if update.GetWiretrusteeConfig() != nil { - err := e.updateTURNs(update.GetWiretrusteeConfig().GetTurns()) + wCfg := update.GetWiretrusteeConfig() + err := e.updateTURNs(wCfg.GetTurns()) if err != nil { - return err + return fmt.Errorf("update TURNs: %w", err) } - err = e.updateSTUNs(update.GetWiretrusteeConfig().GetStuns()) + err = e.updateSTUNs(wCfg.GetStuns()) if err != nil { - return err + return fmt.Errorf("update STUNs: %w", err) } var stunTurn []*stun.URI @@ -542,6 +493,19 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { stunTurn = append(stunTurn, e.TURNs...) e.stunTurn.Store(stunTurn) + relayMsg := wCfg.GetRelay() + if relayMsg != nil { + c := &auth.Token{ + Payload: relayMsg.GetTokenPayload(), + Signature: relayMsg.GetTokenSignature(), + } + if err := e.relayManager.UpdateToken(c); err != nil { + log.Errorf("failed to update relay token: %v", err) + return fmt.Errorf("update relay token: %w", err) + } + } + + // todo update relay address in the relay manager // todo update signal } @@ -937,58 +901,11 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err) } - e.wgConnWorker.Add(1) - go e.connWorker(conn, peerKey) + conn.Open() } return nil } -func (e *Engine) connWorker(conn *peer.Conn, peerKey string) { - defer e.wgConnWorker.Done() - for { - - // randomize starting time a bit - minValue := 500 - maxValue := 2000 - duration := time.Duration(rand.Intn(maxValue-minValue)+minValue) * time.Millisecond - select { - case <-e.ctx.Done(): - return - case <-time.After(duration): - } - - // if peer has been removed -> give up - if !e.peerExists(peerKey) { - log.Debugf("peer %s doesn't exist anymore, won't retry connection", peerKey) - return - } - - if !e.signal.Ready() { - log.Infof("signal client isn't ready, skipping connection attempt %s", peerKey) - continue - } - - err := conn.Open(e.ctx) - if err != nil { - log.Debugf("connection to peer %s failed: %v", peerKey, err) - var connectionClosedError *peer.ConnectionClosedError - switch { - case errors.As(err, &connectionClosedError): - // conn has been forced to close, so we exit the loop - return - default: - } - } - } -} - -func (e *Engine) peerExists(peerKey string) bool { - e.syncMsgMux.Lock() - defer e.syncMsgMux.Unlock() - _, ok := e.peerConns[peerKey] - return ok -} - func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) { log.Debugf("creating peer connection %s", pubKey) @@ -1040,37 +957,12 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e }, } - peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover) + peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager) if err != nil { return nil, err } - wgPubKey, err := wgtypes.ParseKey(pubKey) - if err != nil { - return nil, err - } - - signalOffer := func(offerAnswer peer.OfferAnswer) error { - return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false) - } - - signalCandidate := func(candidate ice.Candidate) error { - return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal) - } - - signalAnswer := func(offerAnswer peer.OfferAnswer) error { - return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true) - } - - peerConn.SetSignalCandidate(signalCandidate) - peerConn.SetSignalOffer(signalOffer) - peerConn.SetSignalAnswer(signalAnswer) - peerConn.SetSendSignalMessage(func(message *sProto.Message) error { - return sendSignal(message, e.signal) - }) - if e.rpManager != nil { - peerConn.SetOnConnected(e.rpManager.OnConnected) peerConn.SetOnDisconnected(e.rpManager.OnDisconnected) } @@ -1113,6 +1005,7 @@ func (e *Engine) receiveSignalEvents() { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), }) case sProto.Body_ANSWER: remoteCred, err := signal.UnMarshalCredential(msg) @@ -1135,6 +1028,7 @@ func (e *Engine) receiveSignalEvents() { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), }) case sProto.Body_CANDIDATE: candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload) @@ -1143,7 +1037,7 @@ func (e *Engine) receiveSignalEvents() { return err } - conn.OnRemoteCandidate(candidate, e.GetClientRoutes()) + go conn.OnRemoteCandidate(candidate, e.GetClientRoutes()) case sProto.Body_MODE: } @@ -1442,7 +1336,7 @@ func (e *Engine) receiveProbeEvents() { for _, peer := range e.peerConns { key := peer.GetKey() - wgStats, err := peer.GetConf().WgConfig.WgInterface.GetStats(key) + wgStats, err := peer.WgConfig().WgInterface.GetStats(key) if err != nil { log.Debugf("failed to get wg stats for peer %s: %s", key, err) } diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index e024dd323..f30566380 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -38,6 +38,7 @@ import ( "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/activity" "github.com/netbirdio/netbird/management/server/telemetry" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/signal/proto" @@ -59,6 +60,12 @@ var ( } ) +func TestMain(m *testing.M) { + _ = util.InitLog("debug", "console") + code := m.Run() + os.Exit(code) +} + func TestEngine_SSH(t *testing.T) { // todo resolve test execution on freebsd if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" { @@ -74,13 +81,23 @@ func TestEngine_SSH(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ - WgIfaceName: "utun101", - WgAddr: "100.64.0.1/24", - WgPrivateKey: key, - WgPort: 33100, - ServerSSHAllowed: true, - }, MobileDependency{}, peer.NewRecorder("https://mgm"), nil) + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String()) + engine := NewEngine( + ctx, cancel, + &signal.MockClient{}, + &mgmt.MockClient{}, + relayMgr, + &EngineConfig{ + WgIfaceName: "utun101", + WgAddr: "100.64.0.1/24", + WgPrivateKey: key, + WgPort: 33100, + ServerSSHAllowed: true, + }, + MobileDependency{}, + peer.NewRecorder("https://mgm"), + nil, + ) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, @@ -209,12 +226,21 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ - WgIfaceName: "utun102", - WgAddr: "100.64.0.1/24", - WgPrivateKey: key, - WgPort: 33100, - }, MobileDependency{}, peer.NewRecorder("https://mgm"), nil) + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String()) + engine := NewEngine( + ctx, cancel, + &signal.MockClient{}, + &mgmt.MockClient{}, + relayMgr, + &EngineConfig{ + WgIfaceName: "utun102", + WgAddr: "100.64.0.1/24", + WgPrivateKey: key, + WgPort: 33100, + }, + MobileDependency{}, + peer.NewRecorder("https://mgm"), + nil) wgIface := &iface.MockWGIface{ RemovePeerFunc: func(peerKey string) error { @@ -222,7 +248,7 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { }, } engine.wgInterface = wgIface - engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, nil) + engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, relayMgr, nil) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, } @@ -404,8 +430,8 @@ func TestEngine_Sync(t *testing.T) { } return nil } - - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, relayMgr, &EngineConfig{ WgIfaceName: "utun103", WgAddr: "100.64.0.1/24", WgPrivateKey: key, @@ -564,7 +590,8 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) { wgIfaceName := fmt.Sprintf("utun%d", 104+n) wgAddr := fmt.Sprintf("100.66.%d.1/24", n) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, WgPrivateKey: key, @@ -734,7 +761,8 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) { wgIfaceName := fmt.Sprintf("utun%d", 104+n) wgAddr := fmt.Sprintf("100.66.%d.1/24", n) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, WgPrivateKey: key, @@ -1012,7 +1040,8 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin WgPort: wgPort, } - e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil + relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String()) + e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, relayMgr, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil e.ctx = ctx return e, err } @@ -1046,6 +1075,11 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error) config := &server.Config{ Stuns: []*server.Host{}, TURNConfig: &server.TURNConfig{}, + Relay: &server.Relay{ + Addresses: []string{"127.0.0.1:1234"}, + CredentialsTTL: util.Duration{Duration: time.Hour}, + Secret: "222222222222222222", + }, Signal: &server.Host{ Proto: "http", URI: "localhost:10000", @@ -1080,8 +1114,9 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error) if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) - mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) + + secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay) + mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil) if err != nil { return nil, "", err } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 7c3b60011..8b8b3c5c0 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -2,36 +2,35 @@ package peer import ( "context" - "fmt" + "math/rand" "net" + "os" "runtime" "strings" "sync" - "sync/atomic" "time" + "github.com/cenkalti/backoff/v4" "github.com/pion/ice/v3" - "github.com/pion/stun/v2" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" - "github.com/netbirdio/netbird/iface/bind" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" - sProto "github.com/netbirdio/netbird/signal/proto" nbnet "github.com/netbirdio/netbird/util/net" - "github.com/netbirdio/netbird/version" ) -const ( - iceKeepAliveDefault = 4 * time.Second - iceDisconnectedTimeoutDefault = 6 * time.Second - // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package - iceRelayAcceptanceMinWaitDefault = 2 * time.Second +type ConnPriority int +const ( defaultWgKeepAlive = 25 * time.Second + + connPriorityRelay ConnPriority = 1 + connPriorityICETurn ConnPriority = 1 + connPriorityICEP2P ConnPriority = 2 ) type WgConfig struct { @@ -42,21 +41,6 @@ type WgConfig struct { PreSharedKey *wgtypes.Key } -type ICEConfig struct { - // StunTurn is a list of STUN and TURN URLs - StunTurn *atomic.Value // []*stun.URI - - // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering - // (e.g. if eth0 is in the list, host candidate of this interface won't be used) - InterfaceBlackList []string - DisableIPv6Discovery bool - - UDPMux ice.UDPMux - UDPMuxSrflx ice.UniversalUDPMux - - NATExternalIPs []string -} - // ConnConfig is a peer Connection configuration type ConnConfig struct { // Key is a public key of a remote peer @@ -79,493 +63,215 @@ type ConnConfig struct { ICEConfig ICEConfig } -// OfferAnswer represents a session establishment offer or answer -type OfferAnswer struct { - IceCredentials IceCredentials - // WgListenPort is a remote WireGuard listen port. - // This field is used when establishing a direct WireGuard connection without any proxy. - // We can set the remote peer's endpoint with this port. - WgListenPort int +type WorkerCallbacks struct { + OnRelayReadyCallback func(info RelayConnInfo) + OnRelayStatusChanged func(ConnStatus) - // Version of NetBird Agent - Version string - // RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message - // This value is the local Rosenpass server public key when sending the message - RosenpassPubKey []byte - // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message - // This value is the local Rosenpass server address when sending the message - RosenpassAddr string -} - -// IceCredentials ICE protocol credentials struct -type IceCredentials struct { - UFrag string - Pwd string + OnICEConnReadyCallback func(ConnPriority, ICEConnInfo) + OnICEStatusChanged func(ConnStatus) } type Conn struct { - config ConnConfig - mu sync.Mutex - - // signalCandidate is a handler function to signal remote peer about local connection candidate - signalCandidate func(candidate ice.Candidate) error - // signalOffer is a handler function to signal remote peer our connection offer (credentials) - signalOffer func(OfferAnswer) error - signalAnswer func(OfferAnswer) error - sendSignalMessage func(message *sProto.Message) error - onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) - onDisconnected func(remotePeer string, wgIP string) - - // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection - remoteOffersCh chan OfferAnswer - // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection - remoteAnswerCh chan OfferAnswer - closeCh chan struct{} - ctx context.Context - notifyDisconnected context.CancelFunc - - agent *ice.Agent - status ConnStatus - + log *log.Entry + mu sync.Mutex + ctx context.Context + ctxCancel context.CancelFunc + config ConnConfig statusRecorder *Status - wgProxyFactory *wgproxy.Factory - wgProxy wgproxy.Proxy + wgProxyICE wgproxy.Proxy + wgProxyRelay wgproxy.Proxy + signaler *Signaler + relayManager *relayClient.Manager + allowedIPsIP string + handshaker *Handshaker - adapter iface.TunAdapter - iFaceDiscover stdnet.ExternalIFaceDiscover - sentExtraSrflx bool + onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) + onDisconnected func(remotePeer string, wgIP string) - connID nbnet.ConnectionID + statusRelay ConnStatus + statusICE ConnStatus + currentConnPriority ConnPriority + opened bool // this flag is used to prevent close in case of not opened connection + + workerICE *WorkerICE + workerRelay *WorkerRelay + + connIDRelay nbnet.ConnectionID + connIDICE nbnet.ConnectionID beforeAddPeerHooks []nbnet.AddHookFunc afterRemovePeerHooks []nbnet.RemoveHookFunc -} -// GetConf returns the connection config -func (conn *Conn) GetConf() ConnConfig { - return conn.config -} + endpointRelay *net.UDPAddr -// WgConfig returns the WireGuard config -func (conn *Conn) WgConfig() WgConfig { - return conn.config.WgConfig + // for reconnection operations + iCEDisconnected chan bool + relayDisconnected chan bool } // NewConn creates a new not opened Conn to the remote peer. // To establish a connection run Conn.Open -func NewConn(config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover) (*Conn, error) { - return &Conn{ - config: config, - mu: sync.Mutex{}, - status: StatusDisconnected, - closeCh: make(chan struct{}), - remoteOffersCh: make(chan OfferAnswer), - remoteAnswerCh: make(chan OfferAnswer), - statusRecorder: statusRecorder, - wgProxyFactory: wgProxyFactory, - adapter: adapter, - iFaceDiscover: iFaceDiscover, - }, nil +func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { + _, allowedIPsIP, err := net.ParseCIDR(config.WgConfig.AllowedIps) + if err != nil { + log.Errorf("failed to parse allowedIPS: %v", err) + return nil, err + } + + ctx, ctxCancel := context.WithCancel(engineCtx) + connLog := log.WithField("peer", config.Key) + + var conn = &Conn{ + log: connLog, + ctx: ctx, + ctxCancel: ctxCancel, + config: config, + statusRecorder: statusRecorder, + wgProxyFactory: wgProxyFactory, + signaler: signaler, + relayManager: relayManager, + allowedIPsIP: allowedIPsIP.String(), + statusRelay: StatusDisconnected, + statusICE: StatusDisconnected, + iCEDisconnected: make(chan bool, 1), + relayDisconnected: make(chan bool, 1), + } + + rFns := WorkerRelayCallbacks{ + OnConnReady: conn.relayConnectionIsReady, + OnDisconnected: conn.onWorkerRelayStateDisconnected, + } + + wFns := WorkerICECallbacks{ + OnConnReady: conn.iCEConnectionIsReady, + OnStatusChanged: conn.onWorkerICEStateDisconnected, + } + + conn.workerRelay = NewWorkerRelay(connLog, config, relayManager, rFns) + + relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() + conn.workerICE, err = NewWorkerICE(ctx, connLog, config, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns) + if err != nil { + return nil, err + } + + conn.handshaker = NewHandshaker(ctx, connLog, config, signaler, conn.workerICE, conn.workerRelay) + + conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer) + if os.Getenv("NB_FORCE_RELAY") != "true" { + conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer) + } + + go conn.handshaker.Listen() + + return conn, nil } -func (conn *Conn) reCreateAgent() error { +// Open opens connection to the remote peer +// It will try to establish a connection using ICE and in parallel with relay. The higher priority connection type will +// be used. +func (conn *Conn) Open() { + conn.log.Debugf("open connection to peer") conn.mu.Lock() defer conn.mu.Unlock() - - failedTimeout := 6 * time.Second - - var err error - transportNet, err := conn.newStdNet() - if err != nil { - log.Errorf("failed to create pion's stdnet: %s", err) - } - - iceKeepAlive := iceKeepAlive() - iceDisconnectedTimeout := iceDisconnectedTimeout() - iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() - - agentConfig := &ice.AgentConfig{ - MulticastDNSMode: ice.MulticastDNSModeDisabled, - NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: conn.config.ICEConfig.StunTurn.Load().([]*stun.URI), - CandidateTypes: conn.candidateTypes(), - FailedTimeout: &failedTimeout, - InterfaceFilter: stdnet.InterfaceFilter(conn.config.ICEConfig.InterfaceBlackList), - UDPMux: conn.config.ICEConfig.UDPMux, - UDPMuxSrflx: conn.config.ICEConfig.UDPMuxSrflx, - NAT1To1IPs: conn.config.ICEConfig.NATExternalIPs, - Net: transportNet, - DisconnectedTimeout: &iceDisconnectedTimeout, - KeepaliveInterval: &iceKeepAlive, - RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, - } - - if conn.config.ICEConfig.DisableIPv6Discovery { - agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} - } - - conn.agent, err = ice.NewAgent(agentConfig) - if err != nil { - return err - } - - err = conn.agent.OnCandidate(conn.onICECandidate) - if err != nil { - return err - } - - err = conn.agent.OnConnectionStateChange(conn.onICEConnectionStateChange) - if err != nil { - return err - } - - err = conn.agent.OnSelectedCandidatePairChange(conn.onICESelectedCandidatePair) - if err != nil { - return err - } - - err = conn.agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { - err := conn.statusRecorder.UpdateLatency(conn.config.Key, p.Latency()) - if err != nil { - log.Debugf("failed to update latency for peer %s: %s", conn.config.Key, err) - return - } - }) - if err != nil { - return fmt.Errorf("failed setting binding response callback: %w", err) - } - - return nil -} - -func (conn *Conn) candidateTypes() []ice.CandidateType { - if hasICEForceRelayConn() { - return []ice.CandidateType{ice.CandidateTypeRelay} - } - // TODO: remove this once we have refactored userspace proxy into the bind package - if runtime.GOOS == "ios" { - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} - } - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} -} - -// Open opens connection to the remote peer starting ICE candidate gathering process. -// Blocks until connection has been closed or connection timeout. -// ConnStatus will be set accordingly -func (conn *Conn) Open(ctx context.Context) error { - log.Debugf("trying to connect to peer %s", conn.config.Key) + conn.opened = true peerState := State{ PubKey: conn.config.Key, IP: strings.Split(conn.config.WgConfig.AllowedIps, "/")[0], ConnStatusUpdate: time.Now(), - ConnStatus: conn.status, + ConnStatus: StatusDisconnected, Mux: new(sync.RWMutex), } err := conn.statusRecorder.UpdatePeerState(peerState) if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + conn.log.Warnf("error while updating the state err: %v", err) } - defer func() { - err := conn.cleanup() - if err != nil { - log.Warnf("error while cleaning up peer connection %s: %v", conn.config.Key, err) - return - } - }() + go conn.startHandshakeAndReconnect() +} - err = conn.reCreateAgent() +func (conn *Conn) startHandshakeAndReconnect() { + conn.waitInitialRandomSleepTime() + + err := conn.handshaker.sendOffer() if err != nil { - return err + conn.log.Errorf("failed to send initial offer: %v", err) } - err = conn.sendOffer() - if err != nil { - return err - } - - log.Debugf("connection offer sent to peer %s, waiting for the confirmation", conn.config.Key) - - // Only continue once we got a connection confirmation from the remote peer. - // The connection timeout could have happened before a confirmation received from the remote. - // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) - var remoteOfferAnswer OfferAnswer - select { - case remoteOfferAnswer = <-conn.remoteOffersCh: - // received confirmation from the remote peer -> ready to proceed - err = conn.sendAnswer() - if err != nil { - return err - } - case remoteOfferAnswer = <-conn.remoteAnswerCh: - case <-time.After(conn.config.Timeout): - return NewConnectionTimeoutError(conn.config.Key, conn.config.Timeout) - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - } - - log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", - conn.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) - - // at this point we received offer/answer and we are ready to gather candidates - conn.mu.Lock() - conn.status = StatusConnecting - conn.ctx, conn.notifyDisconnected = context.WithCancel(ctx) - defer conn.notifyDisconnected() - conn.mu.Unlock() - - peerState = State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) - } - - err = conn.agent.GatherCandidates() - if err != nil { - return fmt.Errorf("gather candidates: %v", err) - } - - // will block until connection succeeded - // but it won't release if ICE Agent went into Disconnected or Failed state, - // so we have to cancel it with the provided context once agent detected a broken connection - isControlling := conn.config.LocalKey > conn.config.Key - var remoteConn *ice.Conn - if isControlling { - remoteConn, err = conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + if conn.workerRelay.IsController() { + conn.reconnectLoopWithRetry() } else { - remoteConn, err = conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) - } - if err != nil { - return err - } - - // dynamically set remote WireGuard port if other side specified a different one from the default one - remoteWgPort := iface.DefaultWgPort - if remoteOfferAnswer.WgListenPort != 0 { - remoteWgPort = remoteOfferAnswer.WgListenPort - } - - // the ice connection has been established successfully so we are ready to start the proxy - remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort, remoteOfferAnswer.RosenpassPubKey, - remoteOfferAnswer.RosenpassAddr) - if err != nil { - return err - } - - log.Infof("connected to peer %s, endpoint address: %s", conn.config.Key, remoteAddr.String()) - - // wait until connection disconnected or has been closed externally (upper layer, e.g. engine) - select { - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - case <-conn.ctx.Done(): - // disconnected from the remote peer - return NewConnectionDisconnectedError(conn.config.Key) + conn.reconnectLoopForOnDisconnectedEvent() } } -func isRelayCandidate(candidate ice.Candidate) bool { - return candidate.Type() == ice.CandidateTypeRelay +// Close closes this peer Conn issuing a close event to the Conn closeCh +func (conn *Conn) Close() { + conn.mu.Lock() + defer conn.mu.Unlock() + + conn.log.Infof("close peer connection") + conn.ctxCancel() + + if !conn.opened { + conn.log.Debugf("ignore close connection to peer") + return + } + + conn.workerRelay.DisableWgWatcher() + conn.workerRelay.CloseConn() + conn.workerICE.Close() + + if conn.wgProxyRelay != nil { + err := conn.wgProxyRelay.CloseConn() + if err != nil { + conn.log.Errorf("failed to close wg proxy for relay: %v", err) + } + conn.wgProxyRelay = nil + } + + if conn.wgProxyICE != nil { + err := conn.wgProxyICE.CloseConn() + if err != nil { + conn.log.Errorf("failed to close wg proxy for ice: %v", err) + } + conn.wgProxyICE = nil + } + + err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if err != nil { + conn.log.Errorf("failed to remove wg endpoint: %v", err) + } + + conn.freeUpConnID() + + if conn.evalStatus() == StatusConnected && conn.onDisconnected != nil { + conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) + } + + conn.setStatusToDisconnected() +} + +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { + conn.log.Debugf("OnRemoteAnswer, status ICE: %s, status relay: %s", conn.statusICE, conn.statusRelay) + return conn.handshaker.OnRemoteAnswer(answer) +} + +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + conn.workerICE.OnRemoteCandidate(candidate, haRoutes) } func (conn *Conn) AddBeforeAddPeerHook(hook nbnet.AddHookFunc) { conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook) } - func (conn *Conn) AddAfterRemovePeerHook(hook nbnet.RemoveHookFunc) { conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook) } -// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected -func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) (net.Addr, error) { - conn.mu.Lock() - defer conn.mu.Unlock() - - pair, err := conn.agent.GetSelectedCandidatePair() - if err != nil { - return nil, err - } - - var endpoint net.Addr - if isRelayCandidate(pair.Local) { - log.Debugf("setup relay connection") - conn.wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - endpoint, err = conn.wgProxy.AddTurnConn(remoteConn) - if err != nil { - return nil, err - } - } else { - // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port - go conn.punchRemoteWGPort(pair, remoteWgPort) - endpoint = remoteConn.RemoteAddr() - } - - endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) - log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) - - conn.connID = nbnet.GenerateConnID() - for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { - log.Errorf("Before add peer hook failed: %v", err) - } - } - - err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) - if err != nil { - if conn.wgProxy != nil { - if err := conn.wgProxy.CloseConn(); err != nil { - log.Warnf("Failed to close turn connection: %v", err) - } - } - return nil, fmt.Errorf("update peer: %w", err) - } - - conn.status = StatusConnected - rosenpassEnabled := false - if remoteRosenpassPubKey != nil { - rosenpassEnabled = true - } - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - LocalIceCandidateType: pair.Local.Type().String(), - RemoteIceCandidateType: pair.Remote.Type().String(), - LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), - RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), - Direct: !isRelayCandidate(pair.Local), - RosenpassEnabled: rosenpassEnabled, - Mux: new(sync.RWMutex), - } - if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { - peerState.Relayed = true - } - - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("unable to save peer's state, got error: %v", err) - } - - _, ipNet, err := net.ParseCIDR(conn.config.WgConfig.AllowedIps) - if err != nil { - return nil, err - } - - if runtime.GOOS == "ios" { - runtime.GC() - } - - if conn.onConnected != nil { - conn.onConnected(conn.config.Key, remoteRosenpassPubKey, ipNet.IP.String(), remoteRosenpassAddr) - } - - return endpoint, nil -} - -func (conn *Conn) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { - // wait local endpoint configuration - time.Sleep(time.Second) - addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) - if err != nil { - log.Warnf("got an error while resolving the udp address, err: %s", err) - return - } - - mux, ok := conn.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) - if !ok { - log.Warn("invalid udp mux conversion") - return - } - _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) - if err != nil { - log.Warnf("got an error while sending the punch packet, err: %s", err) - } -} - -// cleanup closes all open resources and sets status to StatusDisconnected -func (conn *Conn) cleanup() error { - log.Debugf("trying to cleanup %s", conn.config.Key) - conn.mu.Lock() - defer conn.mu.Unlock() - - conn.sentExtraSrflx = false - - var err1, err2, err3 error - if conn.agent != nil { - err1 = conn.agent.Close() - if err1 == nil { - conn.agent = nil - } - } - - if conn.wgProxy != nil { - err2 = conn.wgProxy.CloseConn() - conn.wgProxy = nil - } - - // todo: is it problem if we try to remove a peer what is never existed? - err3 = conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) - - if conn.connID != "" { - for _, hook := range conn.afterRemovePeerHooks { - if err := hook(conn.connID); err != nil { - log.Errorf("After remove peer hook failed: %v", err) - } - } - } - conn.connID = "" - - if conn.notifyDisconnected != nil { - conn.notifyDisconnected() - conn.notifyDisconnected = nil - } - - if conn.status == StatusConnected && conn.onDisconnected != nil { - conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) - } - - conn.status = StatusDisconnected - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err := conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - // pretty common error because by that time Engine can already remove the peer and status won't be available. - // todo rethink status updates - log.Debugf("error while updating peer's %s state, err: %v", conn.config.Key, err) - } - if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { - log.Debugf("failed to reset wireguard stats for peer %s: %s", conn.config.Key, err) - } - - log.Debugf("cleaned up connection to peer %s", conn.config.Key) - if err1 != nil { - return err1 - } - if err2 != nil { - return err2 - } - return err3 -} - -// SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer -func (conn *Conn) SetSignalOffer(handler func(offer OfferAnswer) error) { - conn.signalOffer = handler -} - // SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) { conn.onConnected = handler @@ -576,218 +282,514 @@ func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string) conn.onDisconnected = handler } -// SetSignalAnswer sets a handler function to be triggered by Conn when a new connection answer has to be signalled to the remote peer -func (conn *Conn) SetSignalAnswer(handler func(answer OfferAnswer) error) { - conn.signalAnswer = handler +func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { + conn.log.Debugf("OnRemoteOffer, on status ICE: %s, status Relay: %s", conn.statusICE, conn.statusRelay) + return conn.handshaker.OnRemoteOffer(offer) } -// SetSignalCandidate sets a handler function to be triggered by Conn when a new ICE local connection candidate has to be signalled to the remote peer -func (conn *Conn) SetSignalCandidate(handler func(candidate ice.Candidate) error) { - conn.signalCandidate = handler -} - -// SetSendSignalMessage sets a handler function to be triggered by Conn when there is new message to send via signal -func (conn *Conn) SetSendSignalMessage(handler func(message *sProto.Message) error) { - conn.sendSignalMessage = handler -} - -// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates -// and then signals them to the remote peer -func (conn *Conn) onICECandidate(candidate ice.Candidate) { - // nil means candidate gathering has been ended - if candidate == nil { - return - } - - // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored - log.Debugf("discovered local candidate %s", candidate.String()) - go func() { - err := conn.signalCandidate(candidate) - if err != nil { - log.Errorf("failed signaling candidate to the remote peer %s %s", conn.config.Key, err) - } - }() - - if !conn.shouldSendExtraSrflxCandidate(candidate) { - return - } - - // sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port) - // this is useful when network has an existing port forwarding rule for the wireguard port and this peer - extraSrflx, err := extraSrflxCandidate(candidate) - if err != nil { - log.Errorf("failed creating extra server reflexive candidate %s", err) - return - } - conn.sentExtraSrflx = true - - go func() { - err = conn.signalCandidate(extraSrflx) - if err != nil { - log.Errorf("failed signaling the extra server reflexive candidate to the remote peer %s: %s", conn.config.Key, err) - } - }() -} - -func (conn *Conn) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { - log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), - conn.config.Key) -} - -// onICEConnectionStateChange registers callback of an ICE Agent to track connection state -func (conn *Conn) onICEConnectionStateChange(state ice.ConnectionState) { - log.Debugf("peer %s ICE ConnectionState has changed to %s", conn.config.Key, state.String()) - if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { - conn.notifyDisconnected() - } -} - -func (conn *Conn) sendAnswer() error { - conn.mu.Lock() - defer conn.mu.Unlock() - - localUFrag, localPwd, err := conn.agent.GetLocalUserCredentials() - if err != nil { - return err - } - - log.Debugf("sending answer to %s", conn.config.Key) - err = conn.signalAnswer(OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, - WgListenPort: conn.config.LocalWgPort, - Version: version.NetbirdVersion(), - RosenpassPubKey: conn.config.RosenpassPubKey, - RosenpassAddr: conn.config.RosenpassAddr, - }) - if err != nil { - return err - } - - return nil -} - -// sendOffer prepares local user credentials and signals them to the remote peer -func (conn *Conn) sendOffer() error { - conn.mu.Lock() - defer conn.mu.Unlock() - - localUFrag, localPwd, err := conn.agent.GetLocalUserCredentials() - if err != nil { - return err - } - err = conn.signalOffer(OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, - WgListenPort: conn.config.LocalWgPort, - Version: version.NetbirdVersion(), - RosenpassPubKey: conn.config.RosenpassPubKey, - RosenpassAddr: conn.config.RosenpassAddr, - }) - if err != nil { - return err - } - return nil -} - -// Close closes this peer Conn issuing a close event to the Conn closeCh -func (conn *Conn) Close() error { - conn.mu.Lock() - defer conn.mu.Unlock() - select { - case conn.closeCh <- struct{}{}: - return nil - default: - // probably could happen when peer has been added and removed right after not even starting to connect - // todo further investigate - // this really happens due to unordered messages coming from management - // more importantly it causes inconsistency -> 2 Conn objects for the same peer - // e.g. this flow: - // update from management has peers: [1,2,3,4] - // engine creates a Conn for peers: [1,2,3,4] and schedules Open in ~1sec - // before conn.Open() another update from management arrives with peers: [1,2,3] - // engine removes peer 4 and calls conn.Close() which does nothing (this default clause) - // before conn.Open() another update from management arrives with peers: [1,2,3,4,5] - // engine adds a new Conn for 4 and 5 - // therefore peer 4 has 2 Conn objects - log.Warnf("Connection has been already closed or attempted closing not started connection %s", conn.config.Key) - return NewConnectionAlreadyClosed(conn.config.Key) - } +// WgConfig returns the WireGuard config +func (conn *Conn) WgConfig() WgConfig { + return conn.config.WgConfig } // Status returns current status of the Conn func (conn *Conn) Status() ConnStatus { conn.mu.Lock() defer conn.mu.Unlock() - return conn.status -} - -// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { - log.Debugf("OnRemoteOffer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteOffersCh <- offer: - return true - default: - log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - // connection might not be ready yet to receive so we ignore the message - return false - } -} - -// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { - log.Debugf("OnRemoteAnswer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteAnswerCh <- answer: - return true - default: - // connection might not be ready yet to receive so we ignore the message - log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - return false - } -} - -// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. -func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) - go func() { - conn.mu.Lock() - defer conn.mu.Unlock() - - if conn.agent == nil { - return - } - - err := conn.agent.AddRemoteCandidate(candidate) - if err != nil { - log.Errorf("error while handling remote candidate from peer %s", conn.config.Key) - return - } - }() + return conn.evalStatus() } func (conn *Conn) GetKey() string { return conn.config.Key } -func (conn *Conn) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { - if !conn.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { - return true +func (conn *Conn) reconnectLoopWithRetry() { + // Give chance to the peer to establish the initial connection. + // With it, we can decrease to send necessary offer + select { + case <-conn.ctx.Done(): + case <-time.After(3 * time.Second): + } + + ticker := conn.prepareExponentTicker() + defer ticker.Stop() + time.Sleep(1 * time.Second) + for { + select { + case t := <-ticker.C: + if t.IsZero() { + // in case if the ticker has been canceled by context then avoid the temporary loop + return + } + + if conn.workerRelay.IsRelayConnectionSupportedWithPeer() { + if conn.statusRelay == StatusDisconnected || conn.statusICE == StatusDisconnected { + conn.log.Tracef("connectivity guard timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + } + } else { + if conn.statusICE == StatusDisconnected { + conn.log.Tracef("connectivity guard timedout, ice state: %s", conn.statusICE) + } + } + + // checks if there is peer connection is established via relay or ice + if conn.isConnected() { + continue + } + + err := conn.handshaker.sendOffer() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, reset reconnect timer") + ticker.Stop() + ticker = conn.prepareExponentTicker() + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, reset reconnect timer") + ticker.Stop() + ticker = conn.prepareExponentTicker() + case <-conn.ctx.Done(): + conn.log.Debugf("context is done, stop reconnect loop") + return + } } - return false } -func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { - relatedAdd := candidate.RelatedAddress() - return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ - Network: candidate.NetworkType().String(), - Address: candidate.Address(), - Port: relatedAdd.Port, - Component: candidate.Component(), - RelAddr: relatedAdd.Address, - RelPort: relatedAdd.Port, - }) +func (conn *Conn) prepareExponentTicker() *backoff.Ticker { + bo := backoff.WithContext(&backoff.ExponentialBackOff{ + InitialInterval: 800 * time.Millisecond, + RandomizationFactor: 0.01, + Multiplier: 2, + MaxInterval: conn.config.Timeout, + MaxElapsedTime: 0, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + }, conn.ctx) + + ticker := backoff.NewTicker(bo) + <-ticker.C // consume the initial tick what is happening right after the ticker has been created + + return ticker +} + +// reconnectLoopForOnDisconnectedEvent is used when the peer is not a controller and it should reconnect to the peer +// when the connection is lost. It will try to establish a connection only once time if before the connection was established +// It track separately the ice and relay connection status. Just because a lover priority connection reestablished it does not +// mean that to switch to it. We always force to use the higher priority connection. +func (conn *Conn) reconnectLoopForOnDisconnectedEvent() { + for { + select { + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, try to send new offer") + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, try to send new offer") + case <-conn.ctx.Done(): + conn.log.Debugf("context is done, stop reconnect loop") + return + } + + err := conn.handshaker.SendOffer() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + } +} + +// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected +func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICEConnInfo) { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + conn.log.Debugf("ICE connection is ready") + + conn.statusICE = StatusConnected + + defer conn.updateIceState(iceConnInfo) + + if conn.currentConnPriority > priority { + return + } + + conn.log.Infof("set ICE to active connection") + + endpoint, wgProxy, err := conn.getEndpointForICEConnInfo(iceConnInfo) + if err != nil { + return + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.log.Debugf("Conn resolved IP is %s for endopint %s", endpoint, endpointUdpAddr.IP) + + conn.connIDICE = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connIDICE, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) + } + } + + conn.workerRelay.DisableWgWatcher() + + err = conn.configureWGEndpoint(endpointUdpAddr) + if err != nil { + if wgProxy != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close turn connection: %v", err) + } + } + conn.log.Warnf("Failed to update wg peer configuration: %v", err) + return + } + wgConfigWorkaround() + + if conn.wgProxyICE != nil { + if err := conn.wgProxyICE.CloseConn(); err != nil { + conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err) + } + } + conn.wgProxyICE = wgProxy + + conn.currentConnPriority = priority + + conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) +} + +// todo review to make sense to handle connecting and disconnected status also? +func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + conn.log.Tracef("ICE connection state changed to %s", newState) + + // switch back to relay connection + if conn.endpointRelay != nil && conn.currentConnPriority != connPriorityRelay { + conn.log.Debugf("ICE disconnected, set Relay to active connection") + err := conn.configureWGEndpoint(conn.endpointRelay) + if err != nil { + conn.log.Errorf("failed to switch to relay conn: %v", err) + } + conn.workerRelay.EnableWgWatcher(conn.ctx) + conn.currentConnPriority = connPriorityRelay + } + + changed := conn.statusICE != newState && newState != StatusConnecting + conn.statusICE = newState + + select { + case conn.iCEDisconnected <- changed: + default: + } + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), + ConnStatusUpdate: time.Now(), + } + + err := conn.statusRecorder.UpdatePeerICEStateToDisconnected(peerState) + if err != nil { + conn.log.Warnf("unable to set peer's state to disconnected ice, got error: %v", err) + } +} + +func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + conn.log.Debugf("Relay connection is ready to use") + conn.statusRelay = StatusConnected + + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + endpoint, err := wgProxy.AddTurnConn(rci.relayedConn) + if err != nil { + conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err) + return + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.endpointRelay = endpointUdpAddr + conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + + defer conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey) + + if conn.currentConnPriority > connPriorityRelay { + if conn.statusICE == StatusConnected { + log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnPriority) + return + } + } + + conn.connIDRelay = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connIDRelay, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) + } + } + + err = conn.configureWGEndpoint(endpointUdpAddr) + if err != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close relay connection: %v", err) + } + conn.log.Errorf("Failed to update wg peer configuration: %v", err) + return + } + wgConfigWorkaround() + conn.workerRelay.EnableWgWatcher(conn.ctx) + + if conn.wgProxyRelay != nil { + if err := conn.wgProxyRelay.CloseConn(); err != nil { + conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err) + } + } + conn.wgProxyRelay = wgProxy + conn.currentConnPriority = connPriorityRelay + + conn.log.Infof("start to communicate with peer via relay") + conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr) +} + +func (conn *Conn) onWorkerRelayStateDisconnected() { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + if conn.wgProxyRelay != nil { + log.Debugf("relayed connection is closed, clean up WireGuard config") + err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if err != nil { + conn.log.Errorf("failed to remove wg endpoint: %v", err) + } + + conn.endpointRelay = nil + _ = conn.wgProxyRelay.CloseConn() + conn.wgProxyRelay = nil + } + + changed := conn.statusRelay != StatusDisconnected + conn.statusRelay = StatusDisconnected + + select { + case conn.relayDisconnected <- changed: + default: + } + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), + ConnStatusUpdate: time.Now(), + } + + err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's state to Relay disconnected, got error: %v", err) + } +} + +func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error { + return conn.config.WgConfig.WgInterface.UpdatePeer( + conn.config.WgConfig.RemoteKey, + conn.config.WgConfig.AllowedIps, + defaultWgKeepAlive, + addr, + conn.config.WgConfig.PreSharedKey, + ) +} + +func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []byte) { + peerState := State{ + PubKey: conn.config.Key, + ConnStatusUpdate: time.Now(), + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), + RelayServerAddress: relayServerAddr, + RosenpassEnabled: isRosenpassEnabled(rosenpassPubKey), + } + + err := conn.statusRecorder.UpdatePeerRelayedState(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's Relay state, got error: %v", err) + } +} + +func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) { + peerState := State{ + PubKey: conn.config.Key, + ConnStatusUpdate: time.Now(), + ConnStatus: conn.evalStatus(), + Relayed: iceConnInfo.Relayed, + LocalIceCandidateType: iceConnInfo.LocalIceCandidateType, + RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, + LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, + RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, + RosenpassEnabled: isRosenpassEnabled(iceConnInfo.RosenpassPubKey), + } + + err := conn.statusRecorder.UpdatePeerICEState(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's ICE state, got error: %v", err) + } +} + +func (conn *Conn) setStatusToDisconnected() { + conn.statusRelay = StatusDisconnected + conn.statusICE = StatusDisconnected + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: StatusDisconnected, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + err := conn.statusRecorder.UpdatePeerState(peerState) + if err != nil { + // pretty common error because by that time Engine can already remove the peer and status won't be available. + // todo rethink status updates + conn.log.Debugf("error while updating peer's state, err: %v", err) + } + if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { + conn.log.Debugf("failed to reset wireguard stats for peer: %s", err) + } +} + +func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { + if runtime.GOOS == "ios" { + runtime.GC() + } + + if conn.onConnected != nil { + conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIPsIP, remoteRosenpassAddr) + } +} + +func (conn *Conn) waitInitialRandomSleepTime() { + minWait := 100 + maxWait := 800 + duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + + timeout := time.NewTimer(duration) + defer timeout.Stop() + + select { + case <-conn.ctx.Done(): + case <-timeout.C: + } +} + +func (conn *Conn) isRelayed() bool { + if conn.statusRelay == StatusDisconnected && (conn.statusICE == StatusDisconnected || conn.statusICE == StatusConnecting) { + return false + } + + if conn.currentConnPriority == connPriorityICEP2P { + return false + } + + return true +} + +func (conn *Conn) evalStatus() ConnStatus { + if conn.statusRelay == StatusConnected || conn.statusICE == StatusConnected { + return StatusConnected + } + + if conn.statusRelay == StatusConnecting || conn.statusICE == StatusConnecting { + return StatusConnecting + } + + return StatusDisconnected +} + +func (conn *Conn) isConnected() bool { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.statusICE != StatusConnected && conn.statusICE != StatusConnecting { + return false + } + + if conn.workerRelay.IsRelayConnectionSupportedWithPeer() { + if conn.statusRelay != StatusConnected { + return false + } + } + + return true +} + +func (conn *Conn) freeUpConnID() { + if conn.connIDRelay != "" { + for _, hook := range conn.afterRemovePeerHooks { + if err := hook(conn.connIDRelay); err != nil { + conn.log.Errorf("After remove peer hook failed: %v", err) + } + } + conn.connIDRelay = "" + } + + if conn.connIDICE != "" { + for _, hook := range conn.afterRemovePeerHooks { + if err := hook(conn.connIDICE); err != nil { + conn.log.Errorf("After remove peer hook failed: %v", err) + } + } + conn.connIDICE = "" + } +} + +func (conn *Conn) getEndpointForICEConnInfo(iceConnInfo ICEConnInfo) (net.Addr, wgproxy.Proxy, error) { + if !iceConnInfo.RelayedOnLocal { + return iceConnInfo.RemoteConn.RemoteAddr(), nil, nil + } + conn.log.Debugf("setup ice turn connection") + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + ep, err := wgProxy.AddTurnConn(iceConnInfo.RemoteConn) + if err != nil { + conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) + err = wgProxy.CloseConn() + if err != nil { + conn.log.Warnf("failed to close turn proxy connection: %v", err) + } + return nil, nil, err + } + return ep, wgProxy, nil +} + +func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { + return remoteRosenpassPubKey != nil +} + +// wgConfigWorkaround is a workaround for the issue with WireGuard configuration update +// When update a peer configuration in near to each other time, the second update can be ignored by WireGuard +func wgConfigWorkaround() { + time.Sleep(100 * time.Millisecond) } diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index c124208d1..59f249b82 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -2,6 +2,7 @@ package peer import ( "context" + "os" "sync" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" + "github.com/netbirdio/netbird/util" ) var connConf = ConnConfig{ @@ -23,6 +25,12 @@ var connConf = ConnConfig{ }, } +func TestMain(m *testing.M) { + _ = util.InitLog("trace", "console") + code := m.Run() + os.Exit(code) +} + func TestNewConn_interfaceFilter(t *testing.T) { ignore := []string{iface.WgInterfaceDefault, "tun0", "zt", "ZeroTier", "utun", "wg", "ts", "Tailscale", "tailscale"} @@ -40,7 +48,7 @@ func TestConn_GetKey(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -55,7 +63,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -63,7 +71,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) go func() { - <-conn.remoteOffersCh + <-conn.handshaker.remoteOffersCh wg.Done() }() @@ -92,7 +100,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -100,7 +108,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) go func() { - <-conn.remoteAnswerCh + <-conn.handshaker.remoteAnswerCh wg.Done() }() @@ -128,58 +136,33 @@ func TestConn_Status(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } tables := []struct { - name string - status ConnStatus - want ConnStatus + name string + statusIce ConnStatus + statusRelay ConnStatus + want ConnStatus }{ - {"StatusConnected", StatusConnected, StatusConnected}, - {"StatusDisconnected", StatusDisconnected, StatusDisconnected}, - {"StatusConnecting", StatusConnecting, StatusConnecting}, + {"StatusConnected", StatusConnected, StatusConnected, StatusConnected}, + {"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected}, + {"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting}, + {"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting}, + {"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected}, + {"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting}, + {"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected}, } for _, table := range tables { t.Run(table.name, func(t *testing.T) { - conn.status = table.status + conn.statusICE = table.statusIce + conn.statusRelay = table.statusRelay got := conn.Status() assert.Equal(t, got, table.want, "they should be equal") }) } } - -func TestConn_Close(t *testing.T) { - wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort) - defer func() { - _ = wgProxyFactory.Free() - }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) - if err != nil { - return - } - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - <-conn.closeCh - wg.Done() - }() - - go func() { - for { - err := conn.Close() - if err != nil { - continue - } else { - return - } - } - }() - - wg.Wait() -} diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go new file mode 100644 index 000000000..545f81966 --- /dev/null +++ b/client/internal/peer/handshaker.go @@ -0,0 +1,192 @@ +package peer + +import ( + "context" + "errors" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/version" +) + +var ( + ErrSignalIsNotReady = errors.New("signal is not ready") +) + +// IceCredentials ICE protocol credentials struct +type IceCredentials struct { + UFrag string + Pwd string +} + +// OfferAnswer represents a session establishment offer or answer +type OfferAnswer struct { + IceCredentials IceCredentials + // WgListenPort is a remote WireGuard listen port. + // This field is used when establishing a direct WireGuard connection without any proxy. + // We can set the remote peer's endpoint with this port. + WgListenPort int + + // Version of NetBird Agent + Version string + // RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message + // This value is the local Rosenpass server public key when sending the message + RosenpassPubKey []byte + // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message + // This value is the local Rosenpass server address when sending the message + RosenpassAddr string + + // relay server address + RelaySrvAddress string +} + +type Handshaker struct { + mu sync.Mutex + ctx context.Context + log *log.Entry + config ConnConfig + signaler *Signaler + ice *WorkerICE + relay *WorkerRelay + onNewOfferListeners []func(*OfferAnswer) + + // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection + remoteOffersCh chan OfferAnswer + // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection + remoteAnswerCh chan OfferAnswer +} + +func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker { + return &Handshaker{ + ctx: ctx, + log: log, + config: config, + signaler: signaler, + ice: ice, + relay: relay, + remoteOffersCh: make(chan OfferAnswer), + remoteAnswerCh: make(chan OfferAnswer), + } +} + +func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAnswer)) { + h.onNewOfferListeners = append(h.onNewOfferListeners, offer) +} + +func (h *Handshaker) Listen() { + for { + h.log.Debugf("wait for remote offer confirmation") + remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() + if err != nil { + var connectionClosedError *ConnectionClosedError + if errors.As(err, &connectionClosedError) { + h.log.Tracef("stop handshaker") + return + } + h.log.Errorf("failed to received remote offer confirmation: %s", err) + continue + } + + h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) + for _, listener := range h.onNewOfferListeners { + go listener(remoteOfferAnswer) + } + } +} + +func (h *Handshaker) SendOffer() error { + h.mu.Lock() + defer h.mu.Unlock() + return h.sendOffer() +} + +// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { + select { + case h.remoteOffersCh <- offer: + return true + default: + h.log.Debugf("OnRemoteOffer skipping message because is not ready") + // connection might not be ready yet to receive so we ignore the message + return false + } +} + +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { + select { + case h.remoteAnswerCh <- answer: + return true + default: + // connection might not be ready yet to receive so we ignore the message + h.log.Debugf("OnRemoteAnswer skipping message because is not ready") + return false + } +} + +func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { + select { + case remoteOfferAnswer := <-h.remoteOffersCh: + // received confirmation from the remote peer -> ready to proceed + err := h.sendAnswer() + if err != nil { + return nil, err + } + return &remoteOfferAnswer, nil + case remoteOfferAnswer := <-h.remoteAnswerCh: + return &remoteOfferAnswer, nil + case <-h.ctx.Done(): + // closed externally + return nil, NewConnectionClosedError(h.config.Key) + } +} + +// sendOffer prepares local user credentials and signals them to the remote peer +func (h *Handshaker) sendOffer() error { + if !h.signaler.Ready() { + return ErrSignalIsNotReady + } + + iceUFrag, icePwd := h.ice.GetLocalUserCredentials() + offer := OfferAnswer{ + IceCredentials: IceCredentials{iceUFrag, icePwd}, + WgListenPort: h.config.LocalWgPort, + Version: version.NetbirdVersion(), + RosenpassPubKey: h.config.RosenpassPubKey, + RosenpassAddr: h.config.RosenpassAddr, + } + + addr, err := h.relay.RelayInstanceAddress() + if err == nil { + offer.RelaySrvAddress = addr + } + + return h.signaler.SignalOffer(offer, h.config.Key) +} + +func (h *Handshaker) sendAnswer() error { + h.log.Debugf("sending answer") + uFrag, pwd := h.ice.GetLocalUserCredentials() + + answer := OfferAnswer{ + IceCredentials: IceCredentials{uFrag, pwd}, + WgListenPort: h.config.LocalWgPort, + Version: version.NetbirdVersion(), + RosenpassPubKey: h.config.RosenpassPubKey, + RosenpassAddr: h.config.RosenpassAddr, + } + addr, err := h.relay.RelayInstanceAddress() + if err == nil { + answer.RelaySrvAddress = addr + } + + err = h.signaler.SignalAnswer(answer, h.config.Key) + if err != nil { + return err + } + + return nil +} diff --git a/client/internal/peer/signaler.go b/client/internal/peer/signaler.go new file mode 100644 index 000000000..713123e5d --- /dev/null +++ b/client/internal/peer/signaler.go @@ -0,0 +1,70 @@ +package peer + +import ( + "github.com/pion/ice/v3" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + signal "github.com/netbirdio/netbird/signal/client" + sProto "github.com/netbirdio/netbird/signal/proto" +) + +type Signaler struct { + signal signal.Client + wgPrivateKey wgtypes.Key +} + +func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler { + return &Signaler{ + signal: signal, + wgPrivateKey: wgPrivateKey, + } +} + +func (s *Signaler) SignalOffer(offer OfferAnswer, remoteKey string) error { + return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER) +} + +func (s *Signaler) SignalAnswer(offer OfferAnswer, remoteKey string) error { + return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER) +} + +func (s *Signaler) SignalICECandidate(candidate ice.Candidate, remoteKey string) error { + return s.signal.Send(&sProto.Message{ + Key: s.wgPrivateKey.PublicKey().String(), + RemoteKey: remoteKey, + Body: &sProto.Body{ + Type: sProto.Body_CANDIDATE, + Payload: candidate.Marshal(), + }, + }) +} + +func (s *Signaler) Ready() bool { + return s.signal.Ready() +} + +// SignalOfferAnswer signals either an offer or an answer to remote peer +func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error { + msg, err := signal.MarshalCredential( + s.wgPrivateKey, + offerAnswer.WgListenPort, + remoteKey, + &signal.Credential{ + UFrag: offerAnswer.IceCredentials.UFrag, + Pwd: offerAnswer.IceCredentials.Pwd, + }, + bodyType, + offerAnswer.RosenpassPubKey, + offerAnswer.RosenpassAddr, + offerAnswer.RelaySrvAddress) + if err != nil { + return err + } + + err = s.signal.Send(msg) + if err != nil { + return err + } + + return nil +} diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index a7cfb95c4..f116f3fef 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -3,6 +3,7 @@ package peer import ( "errors" "net/netip" + "slices" "sync" "time" @@ -13,6 +14,7 @@ import ( "github.com/netbirdio/netbird/client/internal/relay" "github.com/netbirdio/netbird/iface" "github.com/netbirdio/netbird/management/domain" + relayClient "github.com/netbirdio/netbird/relay/client" ) // State contains the latest state of a peer @@ -24,11 +26,11 @@ type State struct { ConnStatus ConnStatus ConnStatusUpdate time.Time Relayed bool - Direct bool LocalIceCandidateType string RemoteIceCandidateType string LocalIceCandidateEndpoint string RemoteIceCandidateEndpoint string + RelayServerAddress string LastWireguardHandshake time.Time BytesTx int64 BytesRx int64 @@ -142,6 +144,8 @@ type Status struct { // Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events // set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications() peerListChangedForNotification bool + + relayMgr *relayClient.Manager } // NewRecorder returns a new Status instance @@ -156,6 +160,12 @@ func NewRecorder(mgmAddress string) *Status { } } +func (d *Status) SetRelayMgr(manager *relayClient.Manager) { + d.mux.Lock() + defer d.mux.Unlock() + d.relayMgr = manager +} + // ReplaceOfflinePeers replaces func (d *Status) ReplaceOfflinePeers(replacement []State) { d.mux.Lock() @@ -231,17 +241,17 @@ func (d *Status) UpdatePeerState(receivedState State) error { peerState.SetRoutes(receivedState.GetRoutes()) } - skipNotification := shouldSkipNotify(receivedState, peerState) + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) if receivedState.ConnStatus != peerState.ConnStatus { peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + peerState.RelayServerAddress = receivedState.RelayServerAddress peerState.RosenpassEnabled = receivedState.RosenpassEnabled } @@ -261,6 +271,146 @@ func (d *Status) UpdatePeerState(receivedState State) error { return nil } +func (d *Status) UpdatePeerICEState(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + if receivedState.IP != "" { + peerState.IP = receivedState.IP + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.Relayed = receivedState.Relayed + peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType + peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType + peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint + peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + peerState.RosenpassEnabled = receivedState.RosenpassEnabled + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerRelayedState(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.Relayed = receivedState.Relayed + peerState.RelayServerAddress = receivedState.RelayServerAddress + peerState.RosenpassEnabled = receivedState.RosenpassEnabled + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.Relayed = receivedState.Relayed + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.RelayServerAddress = "" + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.Relayed = receivedState.Relayed + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType + peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType + peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint + peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + // UpdateWireGuardPeerState updates the WireGuard bits of the peer state func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) error { d.mux.Lock() @@ -280,13 +430,13 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) return nil } -func shouldSkipNotify(received, curr State) bool { +func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool { switch { - case received.ConnStatus == StatusConnecting: + case receivedConnStatus == StatusConnecting: return true - case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting: + case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting: return true - case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected: + case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected: return curr.IP != "" default: return false @@ -502,8 +652,35 @@ func (d *Status) GetSignalState() SignalState { } } +// GetRelayStates returns the stun/turn/permanent relay states func (d *Status) GetRelayStates() []relay.ProbeResult { - return d.relayStates + if d.relayMgr == nil { + return d.relayStates + } + + // extend the list of stun, turn servers with relay address + relayStates := slices.Clone(d.relayStates) + + var relayState relay.ProbeResult + + // if the server connection is not established then we will use the general address + // in case of connection we will use the instance specific address + instanceAddr, err := d.relayMgr.RelayInstanceAddress() + if err != nil { + // TODO add their status + if errors.Is(err, relayClient.ErrRelayClientNotConnected) { + for _, r := range d.relayMgr.ServerURLs() { + relayStates = append(relayStates, relay.ProbeResult{ + URI: r, + }) + } + return relayStates + } + relayState.Err = err + } + + relayState.URI = instanceAddr + return append(relayStates, relayState) } func (d *Status) GetDNSStates() []NSGroupState { @@ -535,7 +712,6 @@ func (d *Status) GetFullStatus() FullStatus { } fullStatus.Peers = append(fullStatus.Peers, d.offlinePeers...) - return fullStatus } diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index a4a6e6081..1d283433b 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -2,8 +2,8 @@ package peer import ( "errors" - "testing" "sync" + "testing" "github.com/stretchr/testify/assert" ) @@ -43,7 +43,7 @@ func TestUpdatePeerState(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -64,7 +64,7 @@ func TestStatus_UpdatePeerFQDN(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -83,7 +83,7 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -108,7 +108,7 @@ func TestRemovePeer(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState diff --git a/client/internal/peer/stdnet.go b/client/internal/peer/stdnet.go index 1faa30ce3..ae31ebbf0 100644 --- a/client/internal/peer/stdnet.go +++ b/client/internal/peer/stdnet.go @@ -6,6 +6,6 @@ import ( "github.com/netbirdio/netbird/client/internal/stdnet" ) -func (conn *Conn) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNet(conn.config.ICEConfig.InterfaceBlackList) +func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList) } diff --git a/client/internal/peer/stdnet_android.go b/client/internal/peer/stdnet_android.go index 90865242b..b411405bb 100644 --- a/client/internal/peer/stdnet_android.go +++ b/client/internal/peer/stdnet_android.go @@ -2,6 +2,6 @@ package peer import "github.com/netbirdio/netbird/client/internal/stdnet" -func (conn *Conn) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.config.ICEConfig.InterfaceBlackList) +func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList) } diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go new file mode 100644 index 000000000..8bf1b7568 --- /dev/null +++ b/client/internal/peer/worker_ice.go @@ -0,0 +1,470 @@ +package peer + +import ( + "context" + "fmt" + "net" + "net/netip" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/pion/ice/v3" + "github.com/pion/randutil" + "github.com/pion/stun/v2" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/iface" + "github.com/netbirdio/netbird/iface/bind" + "github.com/netbirdio/netbird/route" +) + +const ( + iceKeepAliveDefault = 4 * time.Second + iceDisconnectedTimeoutDefault = 6 * time.Second + // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package + iceRelayAcceptanceMinWaitDefault = 2 * time.Second + + lenUFrag = 16 + lenPwd = 32 + runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +var ( + failedTimeout = 6 * time.Second +) + +type ICEConfig struct { + // StunTurn is a list of STUN and TURN URLs + StunTurn *atomic.Value // []*stun.URI + + // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering + // (e.g. if eth0 is in the list, host candidate of this interface won't be used) + InterfaceBlackList []string + DisableIPv6Discovery bool + + UDPMux ice.UDPMux + UDPMuxSrflx ice.UniversalUDPMux + + NATExternalIPs []string +} + +type ICEConnInfo struct { + RemoteConn net.Conn + RosenpassPubKey []byte + RosenpassAddr string + LocalIceCandidateType string + RemoteIceCandidateType string + RemoteIceCandidateEndpoint string + LocalIceCandidateEndpoint string + Relayed bool + RelayedOnLocal bool +} + +type WorkerICECallbacks struct { + OnConnReady func(ConnPriority, ICEConnInfo) + OnStatusChanged func(ConnStatus) +} + +type WorkerICE struct { + ctx context.Context + log *log.Entry + config ConnConfig + signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + statusRecorder *Status + hasRelayOnLocally bool + conn WorkerICECallbacks + + selectedPriority ConnPriority + + agent *ice.Agent + muxAgent sync.Mutex + + StunTurn []*stun.URI + + sentExtraSrflx bool + + localUfrag string + localPwd string +} + +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) { + w := &WorkerICE{ + ctx: ctx, + log: log, + config: config, + signaler: signaler, + iFaceDiscover: ifaceDiscover, + statusRecorder: statusRecorder, + hasRelayOnLocally: hasRelayOnLocally, + conn: callBacks, + } + + localUfrag, localPwd, err := generateICECredentials() + if err != nil { + return nil, err + } + w.localUfrag = localUfrag + w.localPwd = localPwd + return w, nil +} + +func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { + w.log.Debugf("OnNewOffer for ICE") + w.muxAgent.Lock() + + if w.agent != nil { + w.log.Debugf("agent already exists, skipping the offer") + w.muxAgent.Unlock() + return + } + + var preferredCandidateTypes []ice.CandidateType + if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { + w.selectedPriority = connPriorityICEP2P + preferredCandidateTypes = candidateTypesP2P() + } else { + w.selectedPriority = connPriorityICETurn + preferredCandidateTypes = candidateTypes() + } + + w.log.Debugf("recreate ICE agent") + agentCtx, agentCancel := context.WithCancel(w.ctx) + agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes) + if err != nil { + w.log.Errorf("failed to recreate ICE Agent: %s", err) + w.muxAgent.Unlock() + return + } + w.agent = agent + w.muxAgent.Unlock() + + w.log.Debugf("gather candidates") + err = w.agent.GatherCandidates() + if err != nil { + w.log.Debugf("failed to gather candidates: %s", err) + return + } + + // will block until connection succeeded + // but it won't release if ICE Agent went into Disconnected or Failed state, + // so we have to cancel it with the provided context once agent detected a broken connection + w.log.Debugf("turn agent dial") + remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer) + if err != nil { + w.log.Debugf("failed to dial the remote peer: %s", err) + return + } + w.log.Debugf("agent dial succeeded") + + pair, err := w.agent.GetSelectedCandidatePair() + if err != nil { + return + } + + if !isRelayCandidate(pair.Local) { + // dynamically set remote WireGuard port if other side specified a different one from the default one + remoteWgPort := iface.DefaultWgPort + if remoteOfferAnswer.WgListenPort != 0 { + remoteWgPort = remoteOfferAnswer.WgListenPort + } + + // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port + go w.punchRemoteWGPort(pair, remoteWgPort) + } + + ci := ICEConnInfo{ + RemoteConn: remoteConn, + RosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + RosenpassAddr: remoteOfferAnswer.RosenpassAddr, + LocalIceCandidateType: pair.Local.Type().String(), + RemoteIceCandidateType: pair.Remote.Type().String(), + LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), + RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), + Relayed: isRelayed(pair), + RelayedOnLocal: isRelayCandidate(pair.Local), + } + w.log.Debugf("on ICE conn read to use ready") + go w.conn.OnConnReady(w.selectedPriority, ci) +} + +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String()) + if w.agent == nil { + w.log.Warnf("ICE Agent is not initialized yet") + return + } + + if candidateViaRoutes(candidate, haRoutes) { + return + } + + err := w.agent.AddRemoteCandidate(candidate) + if err != nil { + w.log.Errorf("error while handling remote candidate") + return + } +} + +func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) { + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + return w.localUfrag, w.localPwd +} + +func (w *WorkerICE) Close() { + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + + if w.agent == nil { + return + } + + err := w.agent.Close() + if err != nil { + w.log.Warnf("failed to close ICE agent: %s", err) + } +} + +func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { + transportNet, err := w.newStdNet() + if err != nil { + w.log.Errorf("failed to create pion's stdnet: %s", err) + } + + iceKeepAlive := iceKeepAlive() + iceDisconnectedTimeout := iceDisconnectedTimeout() + iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() + + agentConfig := &ice.AgentConfig{ + MulticastDNSMode: ice.MulticastDNSModeDisabled, + NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, + Urls: w.config.ICEConfig.StunTurn.Load().([]*stun.URI), + CandidateTypes: relaySupport, + InterfaceFilter: stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList), + UDPMux: w.config.ICEConfig.UDPMux, + UDPMuxSrflx: w.config.ICEConfig.UDPMuxSrflx, + NAT1To1IPs: w.config.ICEConfig.NATExternalIPs, + Net: transportNet, + FailedTimeout: &failedTimeout, + DisconnectedTimeout: &iceDisconnectedTimeout, + KeepaliveInterval: &iceKeepAlive, + RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, + LocalUfrag: w.localUfrag, + LocalPwd: w.localPwd, + } + + if w.config.ICEConfig.DisableIPv6Discovery { + agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} + } + + w.sentExtraSrflx = false + agent, err := ice.NewAgent(agentConfig) + if err != nil { + return nil, err + } + + err = agent.OnCandidate(w.onICECandidate) + if err != nil { + return nil, err + } + + err = agent.OnConnectionStateChange(func(state ice.ConnectionState) { + w.log.Debugf("ICE ConnectionState has changed to %s", state.String()) + if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { + w.conn.OnStatusChanged(StatusDisconnected) + + w.muxAgent.Lock() + agentCancel() + _ = agent.Close() + w.agent = nil + + w.muxAgent.Unlock() + } + }) + if err != nil { + return nil, err + } + + err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair) + if err != nil { + return nil, err + } + + err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { + err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency()) + if err != nil { + w.log.Debugf("failed to update latency for peer: %s", err) + return + } + }) + if err != nil { + return nil, fmt.Errorf("failed setting binding response callback: %w", err) + } + + return agent, nil +} + +func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { + // wait local endpoint configuration + time.Sleep(time.Second) + addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) + if err != nil { + w.log.Warnf("got an error while resolving the udp address, err: %s", err) + return + } + + mux, ok := w.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) + if !ok { + w.log.Warn("invalid udp mux conversion") + return + } + _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) + if err != nil { + w.log.Warnf("got an error while sending the punch packet, err: %s", err) + } +} + +// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates +// and then signals them to the remote peer +func (w *WorkerICE) onICECandidate(candidate ice.Candidate) { + // nil means candidate gathering has been ended + if candidate == nil { + return + } + + // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored + w.log.Debugf("discovered local candidate %s", candidate.String()) + go func() { + err := w.signaler.SignalICECandidate(candidate, w.config.Key) + if err != nil { + w.log.Errorf("failed signaling candidate to the remote peer %s %s", w.config.Key, err) + } + }() + + if !w.shouldSendExtraSrflxCandidate(candidate) { + return + } + + // sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port) + // this is useful when network has an existing port forwarding rule for the wireguard port and this peer + extraSrflx, err := extraSrflxCandidate(candidate) + if err != nil { + w.log.Errorf("failed creating extra server reflexive candidate %s", err) + return + } + w.sentExtraSrflx = true + + go func() { + err = w.signaler.SignalICECandidate(extraSrflx, w.config.Key) + if err != nil { + w.log.Errorf("failed signaling the extra server reflexive candidate: %s", err) + } + }() +} + +func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { + w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), + w.config.Key) +} + +func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { + if !w.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { + return true + } + return false +} + +func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { + isControlling := w.config.LocalKey > w.config.Key + if isControlling { + return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } else { + return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } +} + +func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { + relatedAdd := candidate.RelatedAddress() + return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ + Network: candidate.NetworkType().String(), + Address: candidate.Address(), + Port: relatedAdd.Port, + Component: candidate.Component(), + RelAddr: relatedAdd.Address, + RelPort: relatedAdd.Port, + }) +} + +func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool { + var routePrefixes []netip.Prefix + for _, routes := range clientRoutes { + if len(routes) > 0 && routes[0] != nil { + routePrefixes = append(routePrefixes, routes[0].Network) + } + } + + addr, err := netip.ParseAddr(candidate.Address()) + if err != nil { + log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err) + return false + } + + for _, prefix := range routePrefixes { + // default route is + if prefix.Bits() == 0 { + continue + } + + if prefix.Contains(addr) { + log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix) + return true + } + } + return false +} + +func candidateTypes() []ice.CandidateType { + if hasICEForceRelayConn() { + return []ice.CandidateType{ice.CandidateTypeRelay} + } + // TODO: remove this once we have refactored userspace proxy into the bind package + if runtime.GOOS == "ios" { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} + } + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} +} + +func candidateTypesP2P() []ice.CandidateType { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} +} + +func isRelayCandidate(candidate ice.Candidate) bool { + return candidate.Type() == ice.CandidateTypeRelay +} + +func isRelayed(pair *ice.CandidatePair) bool { + if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { + return true + } + return false +} + +func generateICECredentials() (string, string, error) { + ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha) + if err != nil { + return "", "", err + } + + pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha) + if err != nil { + return "", "", err + } + return ufrag, pwd, nil +} diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go new file mode 100644 index 000000000..930a8f5b6 --- /dev/null +++ b/client/internal/peer/worker_relay.go @@ -0,0 +1,223 @@ +package peer + +import ( + "context" + "errors" + "net" + "sync" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + + relayClient "github.com/netbirdio/netbird/relay/client" +) + +var ( + wgHandshakePeriod = 2 * time.Minute + wgHandshakeOvertime = 30 * time.Second +) + +type RelayConnInfo struct { + relayedConn net.Conn + rosenpassPubKey []byte + rosenpassAddr string +} + +type WorkerRelayCallbacks struct { + OnConnReady func(RelayConnInfo) + OnDisconnected func() +} + +type WorkerRelay struct { + log *log.Entry + config ConnConfig + relayManager relayClient.ManagerService + callBacks WorkerRelayCallbacks + + relayedConn net.Conn + relayLock sync.Mutex + ctxWgWatch context.Context + ctxCancelWgWatch context.CancelFunc + ctxLock sync.Mutex + + relaySupportedOnRemotePeer atomic.Bool +} + +func NewWorkerRelay(log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay { + r := &WorkerRelay{ + log: log, + config: config, + relayManager: relayManager, + callBacks: callbacks, + } + return r +} + +func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { + if !w.isRelaySupported(remoteOfferAnswer) { + w.log.Infof("Relay is not supported by remote peer") + w.relaySupportedOnRemotePeer.Store(false) + return + } + w.relaySupportedOnRemotePeer.Store(true) + + // the relayManager will return with error in case if the connection has lost with relay server + currentRelayAddress, err := w.relayManager.RelayInstanceAddress() + if err != nil { + w.log.Errorf("failed to handle new offer: %s", err) + return + } + + srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress) + + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) + if err != nil { + if errors.Is(err, relayClient.ErrConnAlreadyExists) { + w.log.Infof("do not need to reopen relay connection") + return + } + w.log.Errorf("failed to open connection via Relay: %s", err) + return + } + w.relayLock.Lock() + w.relayedConn = relayedConn + w.relayLock.Unlock() + + err = w.relayManager.AddCloseListener(srv, w.onRelayMGDisconnected) + if err != nil { + log.Errorf("failed to add close listener: %s", err) + _ = relayedConn.Close() + return + } + + w.log.Debugf("peer conn opened via Relay: %s", srv) + go w.callBacks.OnConnReady(RelayConnInfo{ + relayedConn: relayedConn, + rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + rosenpassAddr: remoteOfferAnswer.RosenpassAddr, + }) +} + +func (w *WorkerRelay) EnableWgWatcher(ctx context.Context) { + w.log.Debugf("enable WireGuard watcher") + w.ctxLock.Lock() + defer w.ctxLock.Unlock() + + if w.ctxWgWatch != nil && w.ctxWgWatch.Err() == nil { + return + } + + ctx, ctxCancel := context.WithCancel(ctx) + go w.wgStateCheck(ctx) + w.ctxWgWatch = ctx + w.ctxCancelWgWatch = ctxCancel + +} + +func (w *WorkerRelay) DisableWgWatcher() { + w.ctxLock.Lock() + defer w.ctxLock.Unlock() + + if w.ctxCancelWgWatch == nil { + return + } + + w.log.Debugf("disable WireGuard watcher") + + w.ctxCancelWgWatch() +} + +func (w *WorkerRelay) RelayInstanceAddress() (string, error) { + return w.relayManager.RelayInstanceAddress() +} + +func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool { + return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally() +} + +func (w *WorkerRelay) IsController() bool { + return w.config.LocalKey > w.config.Key +} + +func (w *WorkerRelay) RelayIsSupportedLocally() bool { + return w.relayManager.HasRelayAddress() +} + +func (w *WorkerRelay) CloseConn() { + w.relayLock.Lock() + defer w.relayLock.Unlock() + if w.relayedConn == nil { + return + } + + err := w.relayedConn.Close() + if err != nil { + w.log.Warnf("failed to close relay connection: %v", err) + } +} + +// wgStateCheck help to check the state of the wireguard handshake and relay connection +func (w *WorkerRelay) wgStateCheck(ctx context.Context) { + timer := time.NewTimer(wgHandshakeOvertime) + defer timer.Stop() + expected := wgHandshakeOvertime + for { + select { + case <-timer.C: + lastHandshake, err := w.wgState() + if err != nil { + w.log.Errorf("failed to read wg stats: %v", err) + continue + } + w.log.Tracef("last handshake: %v", lastHandshake) + + if time.Since(lastHandshake) > expected { + w.log.Infof("Wireguard handshake timed out, closing relay connection") + w.relayLock.Lock() + _ = w.relayedConn.Close() + w.relayLock.Unlock() + w.callBacks.OnDisconnected() + return + } + resetTime := time.Until(lastHandshake.Add(wgHandshakePeriod + wgHandshakeOvertime)) + timer.Reset(resetTime) + expected = wgHandshakePeriod + case <-ctx.Done(): + w.log.Debugf("WireGuard watcher stopped") + return + } + } +} + +func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { + if !w.relayManager.HasRelayAddress() { + return false + } + return answer.RelaySrvAddress != "" +} + +func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { + if w.IsController() { + return myRelayAddress + } + return remoteRelayAddress +} + +func (w *WorkerRelay) wgState() (time.Time, error) { + wgState, err := w.config.WgConfig.WgInterface.GetStats(w.config.Key) + if err != nil { + return time.Time{}, err + } + return wgState.LastHandshake, nil +} + +func (w *WorkerRelay) onRelayMGDisconnected() { + w.ctxLock.Lock() + defer w.ctxLock.Unlock() + + if w.ctxCancelWgWatch != nil { + w.ctxCancelWgWatch() + } + go w.callBacks.OnDisconnected() +} diff --git a/client/internal/relay/relay.go b/client/internal/relay/relay.go index 4542a37fe..7d98a6060 100644 --- a/client/internal/relay/relay.go +++ b/client/internal/relay/relay.go @@ -17,7 +17,7 @@ import ( // ProbeResult holds the info about the result of a relay probe request type ProbeResult struct { - URI *stun.URI + URI string Err error Addr string } @@ -176,7 +176,7 @@ func ProbeAll( wg.Add(1) go func(res *ProbeResult, stunURI *stun.URI) { defer wg.Done() - res.URI = stunURI + res.URI = stunURI.String() res.Addr, res.Err = fn(ctx, stunURI) }(&results[i], uri) } diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index cebdd2b0f..db2caea7f 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -22,7 +22,6 @@ import ( type routerPeerStatus struct { connected bool relayed bool - direct bool latency time.Duration } @@ -82,7 +81,6 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { routePeerStatuses[r.ID] = routerPeerStatus{ connected: peerStatus.ConnStatus == peer.StatusConnected, relayed: peerStatus.Relayed, - direct: peerStatus.Direct, latency: peerStatus.Latency, } } @@ -97,8 +95,8 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { // * Connected peers: Only routes with connected peers are considered. // * Metric: Routes with lower metrics (better) are prioritized. // * Non-relayed: Routes without relays are preferred. -// * Direct connections: Routes with direct peer connections are favored. // * Latency: Routes with lower latency are prioritized. +// * we compare the current score + 10ms to the chosen score to avoid flapping between routes // * Stability: In case of equal scores, the currently active route (if any) is maintained. // // It returns the ID of the selected optimal route. @@ -137,10 +135,6 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID] tempScore++ } - if peerStatus.direct { - tempScore++ - } - if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") { chosen = r.ID chosenScore = tempScore diff --git a/client/internal/routemanager/client_test.go b/client/internal/routemanager/client_test.go index 0ae10e568..583156e4d 100644 --- a/client/internal/routemanager/client_test.go +++ b/client/internal/routemanager/client_test.go @@ -24,7 +24,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -43,7 +42,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: true, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -62,7 +60,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: true, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -81,7 +78,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: false, relayed: false, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -100,12 +96,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: false, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -129,41 +123,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: true, - direct: true, - }, - }, - existingRoutes: map[route.ID]*route.Route{ - "route1": { - ID: "route1", - Metric: route.MaxMetric, - Peer: "peer1", - }, - "route2": { - ID: "route2", - Metric: route.MaxMetric, - Peer: "peer2", - }, - }, - currentRoute: "", - expectedRouteID: "route1", - }, - { - name: "multiple connected peers with one direct", - statuses: map[route.ID]routerPeerStatus{ - "route1": { - connected: true, - relayed: false, - direct: true, - }, - "route2": { - connected: true, - relayed: false, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -241,13 +204,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 15 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, @@ -272,13 +233,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 200 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, @@ -303,13 +262,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 20 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 597eddd51..cdfd322bd 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -22,6 +22,7 @@ import ( "github.com/netbirdio/netbird/client/internal/routemanager/vars" "github.com/netbirdio/netbird/client/internal/routeselector" "github.com/netbirdio/netbird/iface" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" nbnet "github.com/netbirdio/netbird/util/net" "github.com/netbirdio/netbird/version" @@ -49,6 +50,7 @@ type DefaultManager struct { serverRouter serverRouter sysOps *systemops.SysOps statusRecorder *peer.Status + relayMgr *relayClient.Manager wgInterface iface.IWGIface pubKey string notifier *notifier.Notifier @@ -63,6 +65,7 @@ func NewManager( dnsRouteInterval time.Duration, wgInterface iface.IWGIface, statusRecorder *peer.Status, + relayMgr *relayClient.Manager, initialRoutes []*route.Route, ) *DefaultManager { mCTX, cancel := context.WithCancel(ctx) @@ -74,6 +77,7 @@ func NewManager( stop: cancel, dnsRouteInterval: dnsRouteInterval, clientNetworks: make(map[route.HAUniqueID]*clientNetwork), + relayMgr: relayMgr, routeSelector: routeselector.NewRouteSelector(), sysOps: sysOps, statusRecorder: statusRecorder, @@ -124,9 +128,12 @@ func (m *DefaultManager) Init() (nbnet.AddHookFunc, nbnet.RemoveHookFunc, error) log.Warnf("Failed cleaning up routing: %v", err) } - mgmtAddress := m.statusRecorder.GetManagementState().URL - signalAddress := m.statusRecorder.GetSignalState().URL - ips := resolveURLsToIPs([]string{mgmtAddress, signalAddress}) + initialAddresses := []string{m.statusRecorder.GetManagementState().URL, m.statusRecorder.GetSignalState().URL} + if m.relayMgr != nil { + initialAddresses = append(initialAddresses, m.relayMgr.ServerURLs()...) + } + + ips := resolveURLsToIPs(initialAddresses) beforePeerHook, afterPeerHook, err := m.sysOps.SetupRouting(ips) if err != nil { diff --git a/client/internal/routemanager/manager_test.go b/client/internal/routemanager/manager_test.go index 455c7ac0b..2995e2740 100644 --- a/client/internal/routemanager/manager_test.go +++ b/client/internal/routemanager/manager_test.go @@ -416,7 +416,7 @@ func TestManagerUpdateRoutes(t *testing.T) { statusRecorder := peer.NewRecorder("https://mgm") ctx := context.TODO() - routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil) + routeManager := NewManager(ctx, localPeerKey, 0, wgInterface, statusRecorder, nil, nil) _, _, err = routeManager.Init() diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index bbd00d6e2..d385cc4ca 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -181,7 +181,7 @@ func (p *WGEBPFProxy) proxyToRemote() { conn, ok := p.turnConnStore[uint16(addr.Port)] p.turnConnMutex.Unlock() if !ok { - log.Infof("turn conn not found by port: %d", addr.Port) + log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port) continue } @@ -206,7 +206,7 @@ func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) { } func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) { - log.Tracef("remove turn conn from store by port: %d", turnConnID) + log.Debugf("remove turn conn from store by port: %d", turnConnID) p.turnConnMutex.Lock() defer p.turnConnMutex.Unlock() delete(p.turnConnStore, turnConnID) diff --git a/client/internal/wgproxy/proxy_userspace.go b/client/internal/wgproxy/proxy_userspace.go index 234ea2a42..c2c8a9b51 100644 --- a/client/internal/wgproxy/proxy_userspace.go +++ b/client/internal/wgproxy/proxy_userspace.go @@ -3,6 +3,7 @@ package wgproxy import ( "context" "fmt" + "io" "net" log "github.com/sirupsen/logrus" @@ -64,7 +65,6 @@ func (p *WGUserSpaceProxy) Free() error { // proxyToRemote proxies everything from Wireguard to the RemoteKey peer // blocks func (p *WGUserSpaceProxy) proxyToRemote() { - buf := make([]byte, 1500) for { select { @@ -73,11 +73,17 @@ func (p *WGUserSpaceProxy) proxyToRemote() { default: n, err := p.localConn.Read(buf) if err != nil { + log.Debugf("failed to read from wg interface conn: %s", err) continue } _, err = p.remoteConn.Write(buf[:n]) if err != nil { + if err == io.EOF { + p.cancel() + } else { + log.Debugf("failed to write to remote conn: %s", err) + } continue } } @@ -96,11 +102,17 @@ func (p *WGUserSpaceProxy) proxyToLocal() { default: n, err := p.remoteConn.Read(buf) if err != nil { + if err == io.EOF { + p.cancel() + return + } + log.Errorf("failed to read from remote conn: %s", err) continue } _, err = p.localConn.Write(buf[:n]) if err != nil { + log.Debugf("failed to write to wg interface conn: %s", err) continue } } diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 779c27a4d..dc13706bf 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -168,7 +168,6 @@ func (c *Client) GetStatusDetails() *StatusDetails { BytesTx: p.BytesTx, ConnStatus: p.ConnStatus.String(), ConnStatusUpdate: p.ConnStatusUpdate.Format("2006-01-02 15:04:05"), - Direct: p.Direct, LastWireguardHandshake: p.LastWireguardHandshake.String(), Relayed: p.Relayed, RosenpassEnabled: p.RosenpassEnabled, diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index fb10a38d3..b942d8b6e 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v4.23.4 +// protoc v3.21.12 // source: daemon.proto package proto @@ -899,7 +899,6 @@ type PeerState struct { ConnStatus string `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"` ConnStatusUpdate *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"` Relayed bool `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"` - Direct bool `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"` LocalIceCandidateType string `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"` RemoteIceCandidateType string `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"` Fqdn string `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"` @@ -911,6 +910,7 @@ type PeerState struct { RosenpassEnabled bool `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` Routes []string `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"` Latency *durationpb.Duration `protobuf:"bytes,17,opt,name=latency,proto3" json:"latency,omitempty"` + RelayAddress string `protobuf:"bytes,18,opt,name=relayAddress,proto3" json:"relayAddress,omitempty"` } func (x *PeerState) Reset() { @@ -980,13 +980,6 @@ func (x *PeerState) GetRelayed() bool { return false } -func (x *PeerState) GetDirect() bool { - if x != nil { - return x.Direct - } - return false -} - func (x *PeerState) GetLocalIceCandidateType() string { if x != nil { return x.LocalIceCandidateType @@ -1064,6 +1057,13 @@ func (x *PeerState) GetLatency() *durationpb.Duration { return nil } +func (x *PeerState) GetRelayAddress() string { + if x != nil { + return x.RelayAddress + } + return "" +} + // LocalPeerState contains the latest state of the local peer type LocalPeerState struct { state protoimpl.MessageState @@ -2243,7 +2243,7 @@ var file_daemon_proto_rawDesc = []byte{ 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xce, 0x05, 0x0a, 0x09, 0x50, 0x65, + 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xda, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, @@ -2255,209 +2255,210 @@ var file_daemon_proto_rawDesc = []byte{ 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, - 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, + 0x64, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, + 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, - 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, - 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, - 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, - 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, - 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, - 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, - 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, - 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, - 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, - 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, - 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, - 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, - 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, - 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, - 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, - 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, - 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, - 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, - 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, - 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, - 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, - 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, - 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, - 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, - 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, - 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, - 0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, - 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, - 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, - 0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, - 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, - 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, - 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x1e, 0x0a, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, - 0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, - 0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, - 0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, - 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, - 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, - 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, - 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, - 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, - 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, - 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, - 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, - 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, - 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, - 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, - 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, - 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, + 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, + 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, + 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, + 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, + 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, + 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, + 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, + 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, + 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, + 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, + 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, + 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, + 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, + 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, + 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, + 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, + 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, + 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, + 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, + 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, + 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, + 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, + 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, + 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, + 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a, + 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a, + 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, + 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, + 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, + 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x29, 0x0a, 0x13, 0x44, + 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, + 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e, + 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12, + 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, + 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09, + 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, + 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, + 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, + 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, + 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, + 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 43c379fb5..384bc0e62 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -168,7 +168,6 @@ message PeerState { string connStatus = 3; google.protobuf.Timestamp connStatusUpdate = 4; bool relayed = 5; - bool direct = 6; string localIceCandidateType = 7; string remoteIceCandidateType = 8; string fqdn = 9; @@ -180,6 +179,7 @@ message PeerState { bool rosenpassEnabled = 15; repeated string routes = 16; google.protobuf.Duration latency = 17; + string relayAddress = 18; } // LocalPeerState contains the latest state of the local peer diff --git a/client/server/debug.go b/client/server/debug.go index 1187f3187..5ed43293b 100644 --- a/client/server/debug.go +++ b/client/server/debug.go @@ -369,8 +369,8 @@ func seedFromStatus(a *anonymize.Anonymizer, status *peer.FullStatus) { } for _, relay := range status.Relays { - if relay.URI != nil { - a.AnonymizeURI(relay.URI.String()) + if relay.URI != "" { + a.AnonymizeURI(relay.URI) } } } diff --git a/client/server/server.go b/client/server/server.go index d8d32e1ce..0a4c18131 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -758,11 +758,11 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { ConnStatus: peerState.ConnStatus.String(), ConnStatusUpdate: timestamppb.New(peerState.ConnStatusUpdate), Relayed: peerState.Relayed, - Direct: peerState.Direct, LocalIceCandidateType: peerState.LocalIceCandidateType, RemoteIceCandidateType: peerState.RemoteIceCandidateType, LocalIceCandidateEndpoint: peerState.LocalIceCandidateEndpoint, RemoteIceCandidateEndpoint: peerState.RemoteIceCandidateEndpoint, + RelayAddress: peerState.RelayServerAddress, Fqdn: peerState.FQDN, LastWireguardHandshake: timestamppb.New(peerState.LastWireguardHandshake), BytesRx: peerState.BytesRx, @@ -776,7 +776,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { for _, relayState := range fullStatus.Relays { pbRelayState := &proto.RelayState{ - URI: relayState.URI.String(), + URI: relayState.URI, Available: relayState.Err == nil, } if err := relayState.Err; err != nil { diff --git a/client/server/server_test.go b/client/server/server_test.go index 242d399ec..795060fab 100644 --- a/client/server/server_test.go +++ b/client/server/server_test.go @@ -6,10 +6,11 @@ import ( "testing" "time" - "github.com/netbirdio/management-integrations/integrations" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" + "github.com/netbirdio/management-integrations/integrations" + log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" @@ -129,8 +130,9 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) - mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) + + secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay) + mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil) if err != nil { return nil, "", err } diff --git a/client/testdata/management.json b/client/testdata/management.json index 4745f2e8c..674c66e06 100644 --- a/client/testdata/management.json +++ b/client/testdata/management.json @@ -20,6 +20,13 @@ "Secret": "c29tZV9wYXNzd29yZA==", "TimeBasedCredentials": true }, + "Relay": { + "Addresses": [ + "localhost:0" + ], + "CredentialsTTL": "1h", + "Secret": "b29tZV9wYXNzd29yZA==" + }, "Signal": { "Proto": "http", "URI": "signal.wiretrustee.com:10000", @@ -34,4 +41,4 @@ "AuthAudience": "", "AuthKeysLocation": "" } -} \ No newline at end of file +} diff --git a/encryption/cert.go b/encryption/cert.go new file mode 100644 index 000000000..3f6d5c679 --- /dev/null +++ b/encryption/cert.go @@ -0,0 +1,19 @@ +package encryption + +import "crypto/tls" + +func LoadTLSConfig(certFile, keyFile string) (*tls.Config, error) { + serverCert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.NoClientCert, + NextProtos: []string{ + "h2", "http/1.1", // enable HTTP/2 + }, + } + return config, nil +} diff --git a/encryption/letsencrypt.go b/encryption/letsencrypt.go index cfe54ec5a..27a5e3110 100644 --- a/encryption/letsencrypt.go +++ b/encryption/letsencrypt.go @@ -9,7 +9,7 @@ import ( ) // CreateCertManager wraps common logic of generating Let's encrypt certificate. -func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Manager, error) { +func CreateCertManager(datadir string, letsencryptDomain ...string) (*autocert.Manager, error) { certDir := filepath.Join(datadir, "letsencrypt") if _, err := os.Stat(certDir); os.IsNotExist(err) { @@ -24,7 +24,7 @@ func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Mana certManager := &autocert.Manager{ Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(certDir), - HostPolicy: autocert.HostWhitelist(letsencryptDomain), + HostPolicy: autocert.HostWhitelist(letsencryptDomain...), } return certManager, nil diff --git a/encryption/route53.go b/encryption/route53.go new file mode 100644 index 000000000..3c81ab103 --- /dev/null +++ b/encryption/route53.go @@ -0,0 +1,87 @@ +package encryption + +import ( + "context" + "crypto/tls" + "fmt" + "os" + "strings" + + "github.com/caddyserver/certmagic" + "github.com/libdns/route53" + log "github.com/sirupsen/logrus" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/crypto/acme" +) + +// Route53TLS by default, loads the AWS configuration from the environment. +// env variables: AWS_REGION, AWS_PROFILE, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN +type Route53TLS struct { + DataDir string + Email string + Domains []string + CA string +} + +func (r *Route53TLS) GetCertificate() (*tls.Config, error) { + if len(r.Domains) == 0 { + return nil, fmt.Errorf("no domains provided") + } + + certmagic.Default.Logger = logger() + certmagic.Default.Storage = &certmagic.FileStorage{Path: r.DataDir} + certmagic.DefaultACME.Agreed = true + if r.Email != "" { + certmagic.DefaultACME.Email = r.Email + } else { + certmagic.DefaultACME.Email = emailFromDomain(r.Domains[0]) + } + + if r.CA == "" { + certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA + } else { + certmagic.DefaultACME.CA = r.CA + } + + certmagic.DefaultACME.DNS01Solver = &certmagic.DNS01Solver{ + DNSManager: certmagic.DNSManager{ + DNSProvider: &route53.Provider{}, + }, + } + cm := certmagic.NewDefault() + if err := cm.ManageSync(context.Background(), r.Domains); err != nil { + log.Errorf("failed to manage certificate: %v", err) + return nil, err + } + + tlsConfig := &tls.Config{ + GetCertificate: cm.GetCertificate, + NextProtos: []string{"h2", "http/1.1", acme.ALPNProto}, + } + + return tlsConfig, nil +} + +func emailFromDomain(domain string) string { + if domain == "" { + return "" + } + + parts := strings.Split(domain, ".") + if len(parts) < 2 { + return "" + } + if parts[0] == "" { + return "" + } + return fmt.Sprintf("admin@%s.%s", parts[len(parts)-2], parts[len(parts)-1]) +} + +func logger() *zap.Logger { + return zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()), + os.Stderr, + zap.ErrorLevel, + )) +} diff --git a/encryption/route53_test.go b/encryption/route53_test.go new file mode 100644 index 000000000..765b60f84 --- /dev/null +++ b/encryption/route53_test.go @@ -0,0 +1,84 @@ +package encryption + +import ( + "context" + "io" + "net/http" + "os" + "testing" + "time" +) + +func TestRoute53TLSConfig(t *testing.T) { + t.SkipNow() // This test requires AWS credentials + exampleString := "Hello, world!" + rtls := &Route53TLS{ + DataDir: t.TempDir(), + Email: os.Getenv("LE_EMAIL_ROUTE53"), + Domains: []string{os.Getenv("DOMAIN")}, + } + tlsConfig, err := rtls.GetCertificate() + if err != nil { + t.Errorf("Route53TLSConfig failed: %v", err) + } + + server := &http.Server{ + Addr: ":8443", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(exampleString)) + }), + TLSConfig: tlsConfig, + } + + go func() { + err := server.ListenAndServeTLS("", "") + if err != http.ErrServerClosed { + t.Errorf("Failed to start server: %v", err) + } + }() + defer func() { + if err := server.Shutdown(context.Background()); err != nil { + t.Errorf("Failed to shutdown server: %v", err) + } + }() + + time.Sleep(1 * time.Second) + resp, err := http.Get("https://relay.godevltd.com:8443") + if err != nil { + t.Errorf("Failed to get response: %v", err) + return + } + defer func() { + _ = resp.Body.Close() + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("Failed to read response body: %v", err) + } + if string(body) != exampleString { + t.Errorf("Unexpected response: %s", body) + } +} + +func Test_emailFromDomain(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"example.com", "admin@example.com"}, + {"x.example.com", "admin@example.com"}, + {"x.x.example.com", "admin@example.com"}, + {"*.example.com", "admin@example.com"}, + {"example", ""}, + {"", ""}, + {".com", ""}, + } + for _, tt := range tests { + t.Run("domain test", func(t *testing.T) { + if got := emailFromDomain(tt.input); got != tt.want { + t.Errorf("emailFromDomain() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/go.mod b/go.mod index 9e440e342..7d5817769 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/kardianos/service v1.2.3-0.20240613133416-becf2eb62b83 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.23.0 + github.com/onsi/gomega v1.27.6 github.com/pion/ice/v3 v3.0.2 github.com/rs/cors v1.8.0 github.com/sirupsen/logrus v1.9.3 @@ -34,6 +34,7 @@ require ( fyne.io/systray v1.11.0 github.com/TheJumpCloud/jcapi-go v3.0.0+incompatible github.com/c-robinson/iplib v1.0.3 + github.com/caddyserver/certmagic v0.21.3 github.com/cilium/ebpf v0.15.0 github.com/coreos/go-iptables v0.7.0 github.com/creack/pty v1.1.18 @@ -50,11 +51,12 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 github.com/hashicorp/go-version v1.6.0 + github.com/libdns/route53 v1.5.0 github.com/libp2p/go-netroute v0.2.1 github.com/magiconair/properties v1.8.7 github.com/mattn/go-sqlite3 v1.14.19 github.com/mdlayher/socket v0.4.1 - github.com/miekg/dns v1.1.43 + github.com/miekg/dns v1.1.59 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/nadoo/ipset v0.5.0 github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e @@ -63,6 +65,7 @@ require ( github.com/oschwald/maxminddb-golang v1.12.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pion/logging v0.2.2 + github.com/pion/randutil v0.1.0 github.com/pion/stun/v2 v2.0.0 github.com/pion/transport/v3 v3.0.1 github.com/pion/turn/v3 v3.0.1 @@ -70,6 +73,7 @@ require ( github.com/rs/xid v1.3.0 github.com/shirou/gopsutil/v3 v3.24.4 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.31.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0 @@ -81,6 +85,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.26.0 go.opentelemetry.io/otel/sdk/metric v1.26.0 + go.uber.org/zap v1.27.0 goauthentik.io/api/v3 v3.2023051.3 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a @@ -93,6 +98,7 @@ require ( gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.3 gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde + nhooyr.io/websocket v1.8.11 ) require ( @@ -106,8 +112,23 @@ require ( github.com/Microsoft/hcsshim v0.12.3 // indirect github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/containerd v1.7.16 // indirect github.com/containerd/log v0.1.0 // indirect @@ -140,7 +161,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -149,13 +170,17 @@ require ( github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/mdlayher/genetlink v1.3.2 // indirect github.com/mdlayher/netlink v1.7.2 // indirect + github.com/mholt/acmez/v2 v2.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect @@ -164,12 +189,12 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pegasus-kv/thrift v0.13.0 // indirect github.com/pion/dtls/v2 v2.2.10 // indirect github.com/pion/mdns v0.0.12 // indirect - github.com/pion/randutil v0.1.0 // indirect github.com/pion/transport/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -186,10 +211,12 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/yuin/goldmark v1.7.1 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index 916f1f0c8..7a587c0d1 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,34 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= +github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 h1:MmLCRqP4U4Cw9gJ4bNrCG0mWqEtBlmAVleyelcHARMU= +github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3/go.mod h1:AMPjK2YnRh0YgOID3PqhJA1BRNfXDfGOnSsKHtAe8yA= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -87,6 +115,10 @@ github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwel github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= +github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0= +github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -207,6 +239,8 @@ github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZs github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-text/render v0.1.0 h1:osrmVDZNHuP1RSu3pNG7Z77Sd2xSbcb/xWytAj9kyVs= github.com/go-text/render v0.1.0/go.mod h1:jqEuNMenrmj6QRnkdpeaP0oKGFLDNhDkVKwGjsWWYU4= github.com/go-text/typesetting v0.1.0 h1:vioSaLPYcHwPEPLT7gsjCGDCoYSbljxoHJzMnKwVvHw= @@ -350,8 +384,9 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -382,6 +417,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -401,6 +440,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -413,6 +455,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= +github.com/libdns/route53 v1.5.0 h1:2SKdpPFl/qgWsXQvsLNJJAoX7rSxlk7zgoL4jnWdXVA= +github.com/libdns/route53 v1.5.0/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -431,9 +477,11 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k= +github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -494,14 +542,14 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -592,6 +640,8 @@ github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrx github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -660,6 +710,12 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -695,8 +751,14 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4= goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -890,7 +952,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1187,6 +1248,8 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/infrastructure_files/base.setup.env b/infrastructure_files/base.setup.env index 296e165f0..45dce8d88 100644 --- a/infrastructure_files/base.setup.env +++ b/infrastructure_files/base.setup.env @@ -20,6 +20,12 @@ NETBIRD_MGMT_IDP_SIGNKEY_REFRESH=${NETBIRD_MGMT_IDP_SIGNKEY_REFRESH:-false} NETBIRD_SIGNAL_PROTOCOL="http" NETBIRD_SIGNAL_PORT=${NETBIRD_SIGNAL_PORT:-10000} +# Relay +NETBIRD_RELAY_DOMAIN=${NETBIRD_RELAY_DOMAIN:-$NETBIRD_DOMAIN} +NETBIRD_RELAY_PORT=${NETBIRD_RELAY_PORT:-33080} +# Relay auth secret +NETBIRD_RELAY_AUTH_SECRET= + # Turn TURN_DOMAIN=${NETBIRD_TURN_DOMAIN:-$NETBIRD_DOMAIN} @@ -69,7 +75,7 @@ NETBIRD_DASHBOARD_TAG=${NETBIRD_DASHBOARD_TAG:-"latest"} NETBIRD_SIGNAL_TAG=${NETBIRD_SIGNAL_TAG:-"latest"} NETBIRD_MANAGEMENT_TAG=${NETBIRD_MANAGEMENT_TAG:-"latest"} COTURN_TAG=${COTURN_TAG:-"latest"} - +NETBIRD_RELAY_TAG=${NETBIRD_RELAY_TAG:-"latest"} # exports export NETBIRD_DOMAIN @@ -123,3 +129,7 @@ export NETBIRD_SIGNAL_TAG export NETBIRD_MANAGEMENT_TAG export COTURN_TAG export NETBIRD_TURN_EXTERNAL_IP +export NETBIRD_RELAY_DOMAIN +export NETBIRD_RELAY_PORT +export NETBIRD_RELAY_AUTH_SECRET +export NETBIRD_RELAY_TAG diff --git a/infrastructure_files/configure.sh b/infrastructure_files/configure.sh index bf021c9ac..ff33004b2 100755 --- a/infrastructure_files/configure.sh +++ b/infrastructure_files/configure.sh @@ -89,6 +89,11 @@ fi export TURN_EXTERNAL_IP_CONFIG +# if not provided, we generate a relay auth secret +if [[ "x-$NETBIRD_RELAY_AUTH_SECRET" == "x-" ]]; then + export NETBIRD_RELAY_AUTH_SECRET=$(openssl rand -base64 32 | sed 's/=//g') +fi + artifacts_path="./artifacts" mkdir -p $artifacts_path diff --git a/infrastructure_files/docker-compose.yml.tmpl b/infrastructure_files/docker-compose.yml.tmpl index 43c8b470c..ba68b3f8d 100644 --- a/infrastructure_files/docker-compose.yml.tmpl +++ b/infrastructure_files/docker-compose.yml.tmpl @@ -49,6 +49,23 @@ services: options: max-size: "500m" max-file: "2" + # Relay + relay: + image: netbirdio/relay:$NETBIRD_RELAY_TAG + restart: unless-stopped + environment: + - NB_LOG_LEVEL=info + - NB_LISTEN_ADDRESS=:$NETBIRD_RELAY_PORT + - NB_EXPOSED_ADDRESS=$NETBIRD_RELAY_DOMAIN:$NETBIRD_RELAY_PORT + # todo: change to a secure secret + - NB_AUTH_SECRET=$NETBIRD_RELAY_AUTH_SECRET + ports: + - $NETBIRD_RELAY_PORT:$NETBIRD_RELAY_PORT + logging: + driver: "json-file" + options: + max-size: "500m" + max-file: "2" # Management management: diff --git a/infrastructure_files/getting-started-with-zitadel.sh b/infrastructure_files/getting-started-with-zitadel.sh index 1aae212ee..c0275536b 100644 --- a/infrastructure_files/getting-started-with-zitadel.sh +++ b/infrastructure_files/getting-started-with-zitadel.sh @@ -103,13 +103,25 @@ wait_api() { INSTANCE_URL=$1 PAT=$2 set +e + counter=1 while true; do - curl -s --fail -o /dev/null "$INSTANCE_URL/auth/v1/users/me" -H "Authorization: Bearer $PAT" + FLAGS="-s" + if [[ $counter -eq 45 ]]; then + FLAGS="-v" + echo "" + fi + + curl $FLAGS --fail --connect-timeout 1 -o /dev/null "$INSTANCE_URL/auth/v1/users/me" -H "Authorization: Bearer $PAT" if [[ $? -eq 0 ]]; then break fi + if [[ $counter -eq 45 ]]; then + echo "" + echo "Unable to connect to Zitadel for more than 45s, please check the output above, your firewall rules and the caddy container logs to confirm if there are any issues provisioning TLS certificates" + fi echo -n " ." sleep 1 + counter=$((counter + 1)) done echo " done" set -e @@ -424,8 +436,10 @@ initEnvironment() { ZITADEL_MASTERKEY="$(openssl rand -base64 32 | head -c 32)" NETBIRD_PORT=80 NETBIRD_HTTP_PROTOCOL="http" + NETBIRD_RELAY_PROTO="rel" TURN_USER="self" TURN_PASSWORD=$(openssl rand -base64 32 | sed 's/=//g') + NETBIRD_RELAY_AUTH_SECRET=$(openssl rand -base64 32 | sed 's/=//g') TURN_MIN_PORT=49152 TURN_MAX_PORT=65535 TURN_EXTERNAL_IP_CONFIG=$(get_turn_external_ip) @@ -442,6 +456,7 @@ initEnvironment() { NETBIRD_PORT=443 CADDY_SECURE_DOMAIN=", $NETBIRD_DOMAIN:$NETBIRD_PORT" NETBIRD_HTTP_PROTOCOL="https" + NETBIRD_RELAY_PROTO="rels" fi if [[ "$OSTYPE" == "darwin"* ]]; then @@ -458,7 +473,7 @@ initEnvironment() { echo "Generated files already exist, if you want to reinitialize the environment, please remove them first." echo "You can use the following commands:" echo " $DOCKER_COMPOSE_COMMAND down --volumes # to remove all containers and volumes" - echo " rm -f docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json" + echo " rm -f docker-compose.yml Caddyfile zitadel.env dashboard.env machinekey/zitadel-admin-sa.token turnserver.conf management.json relay.env" echo "Be aware that this will remove all data from the database, and you will have to reconfigure the dashboard." exit 1 fi @@ -484,6 +499,7 @@ initEnvironment() { echo "" > dashboard.env echo "" > turnserver.conf echo "" > management.json + echo "" > relay.env mkdir -p machinekey chmod 777 machinekey @@ -498,6 +514,7 @@ initEnvironment() { renderTurnServerConf > turnserver.conf renderManagementJson > management.json renderDashboardEnv > dashboard.env + renderRelayEnv > relay.env echo -e "\nStarting NetBird services\n" $DOCKER_COMPOSE_COMMAND up -d @@ -559,6 +576,8 @@ renderCaddyfile() { :80${CADDY_SECURE_DOMAIN} { import security_headers + # relay + reverse_proxy /relay* relay:80 # Signal reverse_proxy /signalexchange.SignalExchange/* h2c://signal:10000 # Management @@ -629,6 +648,11 @@ renderManagementJson() { ], "TimeBasedCredentials": false }, + "Relay": { + "Addresses": ["$NETBIRD_RELAY_PROTO://$NETBIRD_DOMAIN:$NETBIRD_PORT"], + "CredentialsTTL": "24h", + "Secret": "$NETBIRD_RELAY_AUTH_SECRET" + }, "Signal": { "Proto": "$NETBIRD_HTTP_PROTOCOL", "URI": "$NETBIRD_DOMAIN:$NETBIRD_PORT" @@ -744,6 +768,15 @@ POSTGRES_PASSWORD=$POSTGRES_ROOT_PASSWORD EOF } +renderRelayEnv() { + cat < management.PeerSystemMeta 17, // 1: management.SyncResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig - 20, // 2: management.SyncResponse.peerConfig:type_name -> management.PeerConfig - 22, // 3: management.SyncResponse.remotePeers:type_name -> management.RemotePeerConfig - 21, // 4: management.SyncResponse.NetworkMap:type_name -> management.NetworkMap - 37, // 5: management.SyncResponse.Checks:type_name -> management.Checks + 21, // 2: management.SyncResponse.peerConfig:type_name -> management.PeerConfig + 23, // 3: management.SyncResponse.remotePeers:type_name -> management.RemotePeerConfig + 22, // 4: management.SyncResponse.NetworkMap:type_name -> management.NetworkMap + 38, // 5: management.SyncResponse.Checks:type_name -> management.Checks 13, // 6: management.SyncMetaRequest.meta:type_name -> management.PeerSystemMeta 13, // 7: management.LoginRequest.meta:type_name -> management.PeerSystemMeta 10, // 8: management.LoginRequest.peerKeys:type_name -> management.PeerKeys - 36, // 9: management.PeerSystemMeta.networkAddresses:type_name -> management.NetworkAddress + 37, // 9: management.PeerSystemMeta.networkAddresses:type_name -> management.NetworkAddress 11, // 10: management.PeerSystemMeta.environment:type_name -> management.Environment 12, // 11: management.PeerSystemMeta.files:type_name -> management.File 17, // 12: management.LoginResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig - 20, // 13: management.LoginResponse.peerConfig:type_name -> management.PeerConfig - 37, // 14: management.LoginResponse.Checks:type_name -> management.Checks - 38, // 15: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp + 21, // 13: management.LoginResponse.peerConfig:type_name -> management.PeerConfig + 38, // 14: management.LoginResponse.Checks:type_name -> management.Checks + 39, // 15: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp 18, // 16: management.WiretrusteeConfig.stuns:type_name -> management.HostConfig - 19, // 17: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig + 20, // 17: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig 18, // 18: management.WiretrusteeConfig.signal:type_name -> management.HostConfig - 0, // 19: management.HostConfig.protocol:type_name -> management.HostConfig.Protocol - 18, // 20: management.ProtectedHostConfig.hostConfig:type_name -> management.HostConfig - 23, // 21: management.PeerConfig.sshConfig:type_name -> management.SSHConfig - 20, // 22: management.NetworkMap.peerConfig:type_name -> management.PeerConfig - 22, // 23: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig - 29, // 24: management.NetworkMap.Routes:type_name -> management.Route - 30, // 25: management.NetworkMap.DNSConfig:type_name -> management.DNSConfig - 22, // 26: management.NetworkMap.offlinePeers:type_name -> management.RemotePeerConfig - 35, // 27: management.NetworkMap.FirewallRules:type_name -> management.FirewallRule - 23, // 28: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig - 1, // 29: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider - 28, // 30: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig - 28, // 31: management.PKCEAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig - 33, // 32: management.DNSConfig.NameServerGroups:type_name -> management.NameServerGroup - 31, // 33: management.DNSConfig.CustomZones:type_name -> management.CustomZone - 32, // 34: management.CustomZone.Records:type_name -> management.SimpleRecord - 34, // 35: management.NameServerGroup.NameServers:type_name -> management.NameServer - 2, // 36: management.FirewallRule.Direction:type_name -> management.FirewallRule.direction - 3, // 37: management.FirewallRule.Action:type_name -> management.FirewallRule.action - 4, // 38: management.FirewallRule.Protocol:type_name -> management.FirewallRule.protocol - 5, // 39: management.ManagementService.Login:input_type -> management.EncryptedMessage - 5, // 40: management.ManagementService.Sync:input_type -> management.EncryptedMessage - 16, // 41: management.ManagementService.GetServerKey:input_type -> management.Empty - 16, // 42: management.ManagementService.isHealthy:input_type -> management.Empty - 5, // 43: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage - 5, // 44: management.ManagementService.GetPKCEAuthorizationFlow:input_type -> management.EncryptedMessage - 5, // 45: management.ManagementService.SyncMeta:input_type -> management.EncryptedMessage - 5, // 46: management.ManagementService.Login:output_type -> management.EncryptedMessage - 5, // 47: management.ManagementService.Sync:output_type -> management.EncryptedMessage - 15, // 48: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse - 16, // 49: management.ManagementService.isHealthy:output_type -> management.Empty - 5, // 50: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage - 5, // 51: management.ManagementService.GetPKCEAuthorizationFlow:output_type -> management.EncryptedMessage - 16, // 52: management.ManagementService.SyncMeta:output_type -> management.Empty - 46, // [46:53] is the sub-list for method output_type - 39, // [39:46] is the sub-list for method input_type - 39, // [39:39] is the sub-list for extension type_name - 39, // [39:39] is the sub-list for extension extendee - 0, // [0:39] is the sub-list for field type_name + 19, // 19: management.WiretrusteeConfig.relay:type_name -> management.RelayConfig + 0, // 20: management.HostConfig.protocol:type_name -> management.HostConfig.Protocol + 18, // 21: management.ProtectedHostConfig.hostConfig:type_name -> management.HostConfig + 24, // 22: management.PeerConfig.sshConfig:type_name -> management.SSHConfig + 21, // 23: management.NetworkMap.peerConfig:type_name -> management.PeerConfig + 23, // 24: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig + 30, // 25: management.NetworkMap.Routes:type_name -> management.Route + 31, // 26: management.NetworkMap.DNSConfig:type_name -> management.DNSConfig + 23, // 27: management.NetworkMap.offlinePeers:type_name -> management.RemotePeerConfig + 36, // 28: management.NetworkMap.FirewallRules:type_name -> management.FirewallRule + 24, // 29: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig + 1, // 30: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider + 29, // 31: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig + 29, // 32: management.PKCEAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig + 34, // 33: management.DNSConfig.NameServerGroups:type_name -> management.NameServerGroup + 32, // 34: management.DNSConfig.CustomZones:type_name -> management.CustomZone + 33, // 35: management.CustomZone.Records:type_name -> management.SimpleRecord + 35, // 36: management.NameServerGroup.NameServers:type_name -> management.NameServer + 2, // 37: management.FirewallRule.Direction:type_name -> management.FirewallRule.direction + 3, // 38: management.FirewallRule.Action:type_name -> management.FirewallRule.action + 4, // 39: management.FirewallRule.Protocol:type_name -> management.FirewallRule.protocol + 5, // 40: management.ManagementService.Login:input_type -> management.EncryptedMessage + 5, // 41: management.ManagementService.Sync:input_type -> management.EncryptedMessage + 16, // 42: management.ManagementService.GetServerKey:input_type -> management.Empty + 16, // 43: management.ManagementService.isHealthy:input_type -> management.Empty + 5, // 44: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage + 5, // 45: management.ManagementService.GetPKCEAuthorizationFlow:input_type -> management.EncryptedMessage + 5, // 46: management.ManagementService.SyncMeta:input_type -> management.EncryptedMessage + 5, // 47: management.ManagementService.Login:output_type -> management.EncryptedMessage + 5, // 48: management.ManagementService.Sync:output_type -> management.EncryptedMessage + 15, // 49: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse + 16, // 50: management.ManagementService.isHealthy:output_type -> management.Empty + 5, // 51: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage + 5, // 52: management.ManagementService.GetPKCEAuthorizationFlow:output_type -> management.EncryptedMessage + 16, // 53: management.ManagementService.SyncMeta:output_type -> management.Empty + 47, // [47:54] is the sub-list for method output_type + 40, // [40:47] is the sub-list for method input_type + 40, // [40:40] is the sub-list for extension type_name + 40, // [40:40] is the sub-list for extension extendee + 0, // [0:40] is the sub-list for field type_name } func init() { file_management_proto_init() } @@ -3256,7 +3339,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProtectedHostConfig); i { + switch v := v.(*RelayConfig); i { case 0: return &v.state case 1: @@ -3268,7 +3351,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerConfig); i { + switch v := v.(*ProtectedHostConfig); i { case 0: return &v.state case 1: @@ -3280,7 +3363,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NetworkMap); i { + switch v := v.(*PeerConfig); i { case 0: return &v.state case 1: @@ -3292,7 +3375,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemotePeerConfig); i { + switch v := v.(*NetworkMap); i { case 0: return &v.state case 1: @@ -3304,7 +3387,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SSHConfig); i { + switch v := v.(*RemotePeerConfig); i { case 0: return &v.state case 1: @@ -3316,7 +3399,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeviceAuthorizationFlowRequest); i { + switch v := v.(*SSHConfig); i { case 0: return &v.state case 1: @@ -3328,7 +3411,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeviceAuthorizationFlow); i { + switch v := v.(*DeviceAuthorizationFlowRequest); i { case 0: return &v.state case 1: @@ -3340,7 +3423,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PKCEAuthorizationFlowRequest); i { + switch v := v.(*DeviceAuthorizationFlow); i { case 0: return &v.state case 1: @@ -3352,7 +3435,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PKCEAuthorizationFlow); i { + switch v := v.(*PKCEAuthorizationFlowRequest); i { case 0: return &v.state case 1: @@ -3364,7 +3447,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProviderConfig); i { + switch v := v.(*PKCEAuthorizationFlow); i { case 0: return &v.state case 1: @@ -3376,7 +3459,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Route); i { + switch v := v.(*ProviderConfig); i { case 0: return &v.state case 1: @@ -3388,7 +3471,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DNSConfig); i { + switch v := v.(*Route); i { case 0: return &v.state case 1: @@ -3400,7 +3483,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CustomZone); i { + switch v := v.(*DNSConfig); i { case 0: return &v.state case 1: @@ -3412,7 +3495,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SimpleRecord); i { + switch v := v.(*CustomZone); i { case 0: return &v.state case 1: @@ -3424,7 +3507,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NameServerGroup); i { + switch v := v.(*SimpleRecord); i { case 0: return &v.state case 1: @@ -3436,7 +3519,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NameServer); i { + switch v := v.(*NameServerGroup); i { case 0: return &v.state case 1: @@ -3448,7 +3531,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FirewallRule); i { + switch v := v.(*NameServer); i { case 0: return &v.state case 1: @@ -3460,7 +3543,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NetworkAddress); i { + switch v := v.(*FirewallRule); i { case 0: return &v.state case 1: @@ -3472,6 +3555,18 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NetworkAddress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Checks); i { case 0: return &v.state @@ -3490,7 +3585,7 @@ func file_management_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_management_proto_rawDesc, NumEnums: 5, - NumMessages: 33, + NumMessages: 34, NumExtensions: 0, NumServices: 1, }, diff --git a/management/proto/management.proto b/management/proto/management.proto index 06b243773..c5646820f 100644 --- a/management/proto/management.proto +++ b/management/proto/management.proto @@ -177,6 +177,8 @@ message WiretrusteeConfig { // a Signal server config HostConfig signal = 3; + + RelayConfig relay = 4; } // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) @@ -193,6 +195,13 @@ message HostConfig { DTLS = 4; } } + +message RelayConfig { + repeated string urls = 1; + string tokenPayload = 2; + string tokenSignature = 3; +} + // ProtectedHostConfig is similar to HostConfig but has additional user and password // Mostly used for TURN servers message ProtectedHostConfig { diff --git a/management/server/config.go b/management/server/config.go index 4efe4fe74..2f7e49766 100644 --- a/management/server/config.go +++ b/management/server/config.go @@ -34,6 +34,7 @@ const ( type Config struct { Stuns []*Host TURNConfig *TURNConfig + Relay *Relay Signal *Host Datadir string @@ -75,6 +76,12 @@ type TURNConfig struct { Turns []*Host } +type Relay struct { + Addresses []string + CredentialsTTL util.Duration + Secret string +} + // HttpServerConfig is a config of the HTTP Management service server type HttpServerConfig struct { LetsEncryptDomain string diff --git a/management/server/grpcserver.go b/management/server/grpcserver.go index ead4a29d6..5d7094b6a 100644 --- a/management/server/grpcserver.go +++ b/management/server/grpcserver.go @@ -16,13 +16,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - nbContext "github.com/netbirdio/netbird/management/server/context" - "github.com/netbirdio/netbird/management/server/posture" - "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/management/proto" + nbContext "github.com/netbirdio/netbird/management/server/context" "github.com/netbirdio/netbird/management/server/jwtclaims" nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/posture" internalStatus "github.com/netbirdio/netbird/management/server/status" "github.com/netbirdio/netbird/management/server/telemetry" ) @@ -32,17 +31,25 @@ type GRPCServer struct { accountManager AccountManager wgKey wgtypes.Key proto.UnimplementedManagementServiceServer - peersUpdateManager *PeersUpdateManager - config *Config - turnCredentialsManager TURNCredentialsManager - jwtValidator *jwtclaims.JWTValidator - jwtClaimsExtractor *jwtclaims.ClaimsExtractor - appMetrics telemetry.AppMetrics - ephemeralManager *EphemeralManager + peersUpdateManager *PeersUpdateManager + config *Config + secretsManager SecretsManager + jwtValidator *jwtclaims.JWTValidator + jwtClaimsExtractor *jwtclaims.ClaimsExtractor + appMetrics telemetry.AppMetrics + ephemeralManager *EphemeralManager } // NewServer creates a new Management server -func NewServer(ctx context.Context, config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnCredentialsManager TURNCredentialsManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) { +func NewServer( + ctx context.Context, + config *Config, + accountManager AccountManager, + peersUpdateManager *PeersUpdateManager, + secretsManager SecretsManager, + appMetrics telemetry.AppMetrics, + ephemeralManager *EphemeralManager, +) (*GRPCServer, error) { key, err := wgtypes.GeneratePrivateKey() if err != nil { return nil, err @@ -88,14 +95,14 @@ func NewServer(ctx context.Context, config *Config, accountManager AccountManage return &GRPCServer{ wgKey: key, // peerKey -> event channel - peersUpdateManager: peersUpdateManager, - accountManager: accountManager, - config: config, - turnCredentialsManager: turnCredentialsManager, - jwtValidator: jwtValidator, - jwtClaimsExtractor: jwtClaimsExtractor, - appMetrics: appMetrics, - ephemeralManager: ephemeralManager, + peersUpdateManager: peersUpdateManager, + accountManager: accountManager, + config: config, + secretsManager: secretsManager, + jwtValidator: jwtValidator, + jwtClaimsExtractor: jwtClaimsExtractor, + appMetrics: appMetrics, + ephemeralManager: ephemeralManager, }, nil } @@ -177,9 +184,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi s.ephemeralManager.OnPeerConnected(ctx, peer) - if s.config.TURNConfig.TimeBasedCredentials { - s.turnCredentialsManager.SetupRefresh(ctx, peer.ID) - } + s.secretsManager.SetupRefresh(ctx, peer.ID) if s.appMetrics != nil { s.appMetrics.GRPCMetrics().CountSyncRequestDuration(time.Since(reqStart)) @@ -241,7 +246,7 @@ func (s *GRPCServer) sendUpdate(ctx context.Context, accountID string, peerKey w func (s *GRPCServer) cancelPeerRoutines(ctx context.Context, accountID string, peer *nbpeer.Peer) { s.peersUpdateManager.CloseChannel(ctx, peer.ID) - s.turnCredentialsManager.CancelRefresh(peer.ID) + s.secretsManager.CancelRefresh(peer.ID) _ = s.accountManager.OnPeerDisconnected(ctx, accountID, peer.Key) s.ephemeralManager.OnPeerDisconnected(ctx, peer) } @@ -427,9 +432,17 @@ func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*p s.ephemeralManager.OnPeerDisconnected(ctx, peer) } + var relayToken *Token + if s.config.Relay != nil && len(s.config.Relay.Addresses) > 0 { + relayToken, err = s.secretsManager.GenerateRelayToken() + if err != nil { + log.Errorf("failed generating Relay token: %v", err) + } + } + // if peer has reached this point then it has logged in loginResp := &proto.LoginResponse{ - WiretrusteeConfig: toWiretrusteeConfig(s.config, nil), + WiretrusteeConfig: toWiretrusteeConfig(s.config, nil, relayToken), PeerConfig: toPeerConfig(peer, netMap.Network, s.accountManager.GetDNSDomain()), Checks: toProtocolChecks(ctx, postureChecks), } @@ -487,10 +500,11 @@ func ToResponseProto(configProto Protocol) proto.HostConfig_Protocol { } } -func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *proto.WiretrusteeConfig { +func toWiretrusteeConfig(config *Config, turnCredentials *Token, relayToken *Token) *proto.WiretrusteeConfig { if config == nil { return nil } + var stuns []*proto.HostConfig for _, stun := range config.Stuns { stuns = append(stuns, &proto.HostConfig{ @@ -498,25 +512,40 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot Protocol: ToResponseProto(stun.Proto), }) } + var turns []*proto.ProtectedHostConfig - for _, turn := range config.TURNConfig.Turns { - var username string - var password string - if turnCredentials != nil { - username = turnCredentials.Username - password = turnCredentials.Password - } else { - username = turn.Username - password = turn.Password + if config.TURNConfig != nil { + for _, turn := range config.TURNConfig.Turns { + var username string + var password string + if turnCredentials != nil { + username = turnCredentials.Payload + password = turnCredentials.Signature + } else { + username = turn.Username + password = turn.Password + } + turns = append(turns, &proto.ProtectedHostConfig{ + HostConfig: &proto.HostConfig{ + Uri: turn.URI, + Protocol: ToResponseProto(turn.Proto), + }, + User: username, + Password: password, + }) + } + } + + var relayCfg *proto.RelayConfig + if config.Relay != nil && len(config.Relay.Addresses) > 0 { + relayCfg = &proto.RelayConfig{ + Urls: config.Relay.Addresses, + } + + if relayToken != nil { + relayCfg.TokenPayload = relayToken.Payload + relayCfg.TokenSignature = relayToken.Signature } - turns = append(turns, &proto.ProtectedHostConfig{ - HostConfig: &proto.HostConfig{ - Uri: turn.URI, - Protocol: ToResponseProto(turn.Proto), - }, - User: username, - Password: password, - }) } return &proto.WiretrusteeConfig{ @@ -526,6 +555,7 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot Uri: config.Signal.URI, Protocol: ToResponseProto(config.Signal.Proto), }, + Relay: relayCfg, } } @@ -539,9 +569,9 @@ func toPeerConfig(peer *nbpeer.Peer, network *Network, dnsName string) *proto.Pe } } -func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *TURNCredentials, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse { +func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *Token, relayCredentials *Token, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse { response := &proto.SyncResponse{ - WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials), + WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials, relayCredentials), PeerConfig: toPeerConfig(peer, networkMap.Network, dnsName), NetworkMap: &proto.NetworkMap{ Serial: networkMap.Network.CurrentSerial(), @@ -588,15 +618,25 @@ func (s *GRPCServer) IsHealthy(ctx context.Context, req *proto.Empty) (*proto.Em // sendInitialSync sends initial proto.SyncResponse to the peer requesting synchronization func (s *GRPCServer) sendInitialSync(ctx context.Context, peerKey wgtypes.Key, peer *nbpeer.Peer, networkMap *NetworkMap, postureChecks []*posture.Checks, srv proto.ManagementService_SyncServer) error { - // make secret time based TURN credentials optional - var turnCredentials *TURNCredentials - if s.config.TURNConfig.TimeBasedCredentials { - creds := s.turnCredentialsManager.GenerateCredentials() - turnCredentials = &creds - } else { - turnCredentials = nil + var err error + + var turnToken *Token + if s.config.TURNConfig != nil && s.config.TURNConfig.TimeBasedCredentials { + turnToken, err = s.secretsManager.GenerateTurnToken() + if err != nil { + log.Errorf("failed generating TURN token: %v", err) + } } - plainResp := toSyncResponse(ctx, s.config, peer, turnCredentials, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil) + + var relayToken *Token + if s.config.Relay != nil && len(s.config.Relay.Addresses) > 0 { + relayToken, err = s.secretsManager.GenerateRelayToken() + if err != nil { + log.Errorf("failed generating Relay token: %v", err) + } + } + + plainResp := toSyncResponse(ctx, s.config, peer, turnToken, relayToken, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil) encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, plainResp) if err != nil { diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index d48e1f513..00ee4bda2 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -439,10 +439,11 @@ func startManagementForTest(t TestingT, config *Config) (*grpc.Server, *DefaultA if err != nil { return nil, nil, "", err } - turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + + secretsManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay) ephemeralMgr := NewEphemeralManager(store, accountManager) - mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, ephemeralMgr) + mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, ephemeralMgr) if err != nil { return nil, nil, "", err } diff --git a/management/server/management_test.go b/management/server/management_test.go index 62e7f5a05..3956d96b1 100644 --- a/management/server/management_test.go +++ b/management/server/management_test.go @@ -552,8 +552,9 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) { if err != nil { log.Fatalf("failed creating a manager: %v", err) } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) - mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) + + secretsManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay) + mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, nil) Expect(err).NotTo(HaveOccurred()) mgmtProto.RegisterManagementServiceServer(s, mgmtServer) go func() { diff --git a/management/server/peer.go b/management/server/peer.go index 6926ef6bc..5fc6352ee 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -964,7 +964,7 @@ func (am *DefaultAccountManager) updateAccountPeers(ctx context.Context, account postureChecks := am.getPeerPostureChecks(account, p) remotePeerNetworkMap := account.GetPeerNetworkMap(ctx, p.ID, customZone, approvedPeersMap, am.metrics.AccountManagerMetrics()) - update := toSyncResponse(ctx, nil, p, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache) + update := toSyncResponse(ctx, nil, p, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache) am.peersUpdateManager.SendUpdate(ctx, p.ID, &UpdateMessage{Update: update}) }(peer) } diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 918436515..448e83a08 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -848,9 +848,9 @@ func TestToSyncResponse(t *testing.T) { DNSLabel: "peer1", SSHKey: "peer1-ssh-key", } - turnCredentials := &TURNCredentials{ - Username: "turn-user", - Password: "turn-pass", + turnRelayToken := &Token{ + Payload: "turn-user", + Signature: "turn-pass", } networkMap := &NetworkMap{ Network: &Network{Net: *ipnet, Serial: 1000}, @@ -916,7 +916,7 @@ func TestToSyncResponse(t *testing.T) { } dnsCache := &DNSConfigCache{} - response := toSyncResponse(context.Background(), config, peer, turnCredentials, networkMap, dnsName, checks, dnsCache) + response := toSyncResponse(context.Background(), config, peer, turnRelayToken, turnRelayToken, networkMap, dnsName, checks, dnsCache) assert.NotNil(t, response) // assert peer config diff --git a/management/server/token_mgr.go b/management/server/token_mgr.go new file mode 100644 index 000000000..8a6648a3a --- /dev/null +++ b/management/server/token_mgr.go @@ -0,0 +1,222 @@ +package server + +import ( + "context" + "crypto/sha1" + "crypto/sha256" + "fmt" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/management/proto" + auth "github.com/netbirdio/netbird/relay/auth/hmac" +) + +const defaultDuration = 12 * time.Hour + +// SecretsManager used to manage TURN and relay secrets +type SecretsManager interface { + GenerateTurnToken() (*Token, error) + GenerateRelayToken() (*Token, error) + SetupRefresh(ctx context.Context, peerKey string) + CancelRefresh(peerKey string) +} + +// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server +type TimeBasedAuthSecretsManager struct { + mux sync.Mutex + turnCfg *TURNConfig + relayCfg *Relay + turnHmacToken *auth.TimedHMAC + relayHmacToken *auth.TimedHMAC + updateManager *PeersUpdateManager + turnCancelMap map[string]chan struct{} + relayCancelMap map[string]chan struct{} +} + +type Token auth.Token + +func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayCfg *Relay) *TimeBasedAuthSecretsManager { + mgr := &TimeBasedAuthSecretsManager{ + updateManager: updateManager, + turnCfg: turnCfg, + relayCfg: relayCfg, + turnCancelMap: make(map[string]chan struct{}), + relayCancelMap: make(map[string]chan struct{}), + } + + if turnCfg != nil { + duration := turnCfg.CredentialsTTL.Duration + if turnCfg.CredentialsTTL.Duration <= 0 { + log.Warnf("TURN credentials TTL is not set or invalid, using default value %s", defaultDuration) + duration = defaultDuration + } + mgr.turnHmacToken = auth.NewTimedHMAC(turnCfg.Secret, duration) + } + + if relayCfg != nil { + duration := relayCfg.CredentialsTTL.Duration + if relayCfg.CredentialsTTL.Duration <= 0 { + log.Warnf("Relay credentials TTL is not set or invalid, using default value %s", defaultDuration) + duration = defaultDuration + } + + mgr.relayHmacToken = auth.NewTimedHMAC(relayCfg.Secret, duration) + } + + return mgr +} + +// GenerateTurnToken generates new time-based secret credentials for TURN +func (m *TimeBasedAuthSecretsManager) GenerateTurnToken() (*Token, error) { + if m.turnHmacToken == nil { + return nil, fmt.Errorf("TURN configuration is not set") + } + turnToken, err := m.turnHmacToken.GenerateToken(sha1.New) + if err != nil { + return nil, fmt.Errorf("failed to generate TURN token: %s", err) + } + return (*Token)(turnToken), nil +} + +// GenerateRelayToken generates new time-based secret credentials for relay +func (m *TimeBasedAuthSecretsManager) GenerateRelayToken() (*Token, error) { + if m.relayHmacToken == nil { + return nil, fmt.Errorf("relay configuration is not set") + } + relayToken, err := m.relayHmacToken.GenerateToken(sha256.New) + if err != nil { + return nil, fmt.Errorf("failed to generate relay token: %s", err) + } + return (*Token)(relayToken), nil +} + +func (m *TimeBasedAuthSecretsManager) cancelTURN(peerID string) { + if channel, ok := m.turnCancelMap[peerID]; ok { + close(channel) + delete(m.turnCancelMap, peerID) + } +} + +func (m *TimeBasedAuthSecretsManager) cancelRelay(peerID string) { + if channel, ok := m.relayCancelMap[peerID]; ok { + close(channel) + delete(m.relayCancelMap, peerID) + } +} + +// CancelRefresh cancels scheduled peer credentials refresh +func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) { + m.mux.Lock() + defer m.mux.Unlock() + m.cancelTURN(peerID) + m.cancelRelay(peerID) +} + +// SetupRefresh starts peer credentials refresh +func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) { + m.mux.Lock() + defer m.mux.Unlock() + + m.cancelTURN(peerID) + m.cancelRelay(peerID) + + if m.turnCfg != nil && m.turnCfg.TimeBasedCredentials { + turnCancel := make(chan struct{}, 1) + m.turnCancelMap[peerID] = turnCancel + go m.refreshTURNTokens(ctx, peerID, turnCancel) + log.WithContext(ctx).Debugf("starting TURN refresh for %s", peerID) + } + + if m.relayCfg != nil { + relayCancel := make(chan struct{}, 1) + m.relayCancelMap[peerID] = relayCancel + go m.refreshRelayTokens(ctx, peerID, relayCancel) + log.WithContext(ctx).Debugf("starting relay refresh for %s", peerID) + } +} + +func (m *TimeBasedAuthSecretsManager) refreshTURNTokens(ctx context.Context, peerID string, cancel chan struct{}) { + ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3) + defer ticker.Stop() + + for { + select { + case <-cancel: + log.WithContext(ctx).Debugf("stopping TURN refresh for %s", peerID) + return + case <-ticker.C: + m.pushNewTURNTokens(ctx, peerID) + } + } +} + +func (m *TimeBasedAuthSecretsManager) refreshRelayTokens(ctx context.Context, peerID string, cancel chan struct{}) { + ticker := time.NewTicker(m.relayCfg.CredentialsTTL.Duration / 4 * 3) + defer ticker.Stop() + + for { + select { + case <-cancel: + log.WithContext(ctx).Debugf("stopping relay refresh for %s", peerID) + return + case <-ticker.C: + m.pushNewRelayTokens(ctx, peerID) + } + } +} + +func (m *TimeBasedAuthSecretsManager) pushNewTURNTokens(ctx context.Context, peerID string) { + turnToken, err := m.turnHmacToken.GenerateToken(sha1.New) + if err != nil { + log.Errorf("failed to generate token for peer '%s': %s", peerID, err) + return + } + + var turns []*proto.ProtectedHostConfig + for _, host := range m.turnCfg.Turns { + turn := &proto.ProtectedHostConfig{ + HostConfig: &proto.HostConfig{ + Uri: host.URI, + Protocol: ToResponseProto(host.Proto), + }, + User: turnToken.Payload, + Password: turnToken.Signature, + } + turns = append(turns, turn) + } + + update := &proto.SyncResponse{ + WiretrusteeConfig: &proto.WiretrusteeConfig{ + Turns: turns, + // omit Relay to avoid updates there + }, + } + + log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID) + m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update}) +} + +func (m *TimeBasedAuthSecretsManager) pushNewRelayTokens(ctx context.Context, peerID string) { + relayToken, err := m.relayHmacToken.GenerateToken(sha256.New) + if err != nil { + log.Errorf("failed to generate relay token for peer '%s': %s", peerID, err) + return + } + + update := &proto.SyncResponse{ + WiretrusteeConfig: &proto.WiretrusteeConfig{ + Relay: &proto.RelayConfig{ + Urls: m.relayCfg.Addresses, + TokenPayload: relayToken.Payload, + TokenSignature: relayToken.Signature, + }, + // omit Turns to avoid updates there + }, + } + + log.WithContext(ctx).Debugf("sending new relay credentials to peer %s", peerID) + m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update}) +} diff --git a/management/server/token_mgr_test.go b/management/server/token_mgr_test.go new file mode 100644 index 000000000..d59fd3a3f --- /dev/null +++ b/management/server/token_mgr_test.go @@ -0,0 +1,218 @@ +package server + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "hash" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/management/proto" + "github.com/netbirdio/netbird/util" +) + +var TurnTestHost = &Host{ + Proto: UDP, + URI: "turn:turn.wiretrustee.com:77777", + Username: "username", + Password: "", +} + +func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) { + ttl := util.Duration{Duration: time.Hour} + secret := "some_secret" + peersManager := NewPeersUpdateManager(nil) + + rc := &Relay{ + Addresses: []string{"localhost:0"}, + CredentialsTTL: ttl, + Secret: secret, + } + + tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ + CredentialsTTL: ttl, + Secret: secret, + Turns: []*Host{TurnTestHost}, + TimeBasedCredentials: true, + }, rc) + + turnCredentials, err := tested.GenerateTurnToken() + require.NoError(t, err) + + if turnCredentials.Payload == "" { + t.Errorf("expected generated TURN username not to be empty, got empty") + } + if turnCredentials.Signature == "" { + t.Errorf("expected generated TURN password not to be empty, got empty") + } + + validateMAC(t, sha1.New, turnCredentials.Payload, turnCredentials.Signature, []byte(secret)) + + relayCredentials, err := tested.GenerateRelayToken() + require.NoError(t, err) + + if relayCredentials.Payload == "" { + t.Errorf("expected generated relay payload not to be empty, got empty") + } + if relayCredentials.Signature == "" { + t.Errorf("expected generated relay signature not to be empty, got empty") + } + + validateMAC(t, sha256.New, relayCredentials.Payload, relayCredentials.Signature, []byte(secret)) +} + +func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) { + ttl := util.Duration{Duration: 2 * time.Second} + secret := "some_secret" + peersManager := NewPeersUpdateManager(nil) + peer := "some_peer" + updateChannel := peersManager.CreateChannel(context.Background(), peer) + + rc := &Relay{ + Addresses: []string{"localhost:0"}, + CredentialsTTL: ttl, + Secret: secret, + } + tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ + CredentialsTTL: ttl, + Secret: secret, + Turns: []*Host{TurnTestHost}, + TimeBasedCredentials: true, + }, rc) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tested.SetupRefresh(ctx, peer) + + if _, ok := tested.turnCancelMap[peer]; !ok { + t.Errorf("expecting peer to be present in the turn cancel map, got not present") + } + + if _, ok := tested.relayCancelMap[peer]; !ok { + t.Errorf("expecting peer to be present in the relay cancel map, got not present") + } + + var updates []*UpdateMessage + +loop: + for timeout := time.After(5 * time.Second); ; { + select { + case update := <-updateChannel: + updates = append(updates, update) + case <-timeout: + break loop + } + + if len(updates) >= 2 { + break loop + } + } + + if len(updates) < 2 { + t.Errorf("expecting at least 2 peer credentials updates, got %v", len(updates)) + } + + var turnUpdates, relayUpdates int + var firstTurnUpdate, secondTurnUpdate *proto.ProtectedHostConfig + var firstRelayUpdate, secondRelayUpdate *proto.RelayConfig + + for _, update := range updates { + if turns := update.Update.GetWiretrusteeConfig().GetTurns(); len(turns) > 0 { + turnUpdates++ + if turnUpdates == 1 { + firstTurnUpdate = turns[0] + } else { + secondTurnUpdate = turns[0] + } + } + if relay := update.Update.GetWiretrusteeConfig().GetRelay(); relay != nil { + relayUpdates++ + if relayUpdates == 1 { + firstRelayUpdate = relay + } else { + secondRelayUpdate = relay + } + } + } + + if turnUpdates < 1 { + t.Errorf("expecting at least 1 TURN credential update, got %v", turnUpdates) + } + if relayUpdates < 1 { + t.Errorf("expecting at least 1 relay credential update, got %v", relayUpdates) + } + + if firstTurnUpdate != nil && secondTurnUpdate != nil { + if firstTurnUpdate.Password == secondTurnUpdate.Password { + t.Errorf("expecting first TURN credential update password %v to be different from second, got equal", firstTurnUpdate.Password) + } + } + + if firstRelayUpdate != nil && secondRelayUpdate != nil { + if firstRelayUpdate.TokenSignature == secondRelayUpdate.TokenSignature { + t.Errorf("expecting first relay credential update signature %v to be different from second, got equal", firstRelayUpdate.TokenSignature) + } + } +} + +func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) { + ttl := util.Duration{Duration: time.Hour} + secret := "some_secret" + peersManager := NewPeersUpdateManager(nil) + peer := "some_peer" + + rc := &Relay{ + Addresses: []string{"localhost:0"}, + CredentialsTTL: ttl, + Secret: secret, + } + tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ + CredentialsTTL: ttl, + Secret: secret, + Turns: []*Host{TurnTestHost}, + TimeBasedCredentials: true, + }, rc) + + tested.SetupRefresh(context.Background(), peer) + if _, ok := tested.turnCancelMap[peer]; !ok { + t.Errorf("expecting peer to be present in turn cancel map, got not present") + } + if _, ok := tested.relayCancelMap[peer]; !ok { + t.Errorf("expecting peer to be present in relay cancel map, got not present") + } + + tested.CancelRefresh(peer) + if _, ok := tested.turnCancelMap[peer]; ok { + t.Errorf("expecting peer to be not present in turn cancel map, got present") + } + if _, ok := tested.relayCancelMap[peer]; ok { + t.Errorf("expecting peer to be not present in relay cancel map, got present") + } +} + +func validateMAC(t *testing.T, algo func() hash.Hash, username string, actualMAC string, key []byte) { + t.Helper() + mac := hmac.New(algo, key) + + _, err := mac.Write([]byte(username)) + if err != nil { + t.Fatal(err) + } + + expectedMAC := mac.Sum(nil) + decodedMAC, err := base64.StdEncoding.DecodeString(actualMAC) + if err != nil { + t.Fatal(err) + } + equal := hmac.Equal(decodedMAC, expectedMAC) + + if !equal { + t.Errorf("expected password MAC to be %s. got %s", expectedMAC, decodedMAC) + } +} diff --git a/management/server/turncredentials.go b/management/server/turncredentials.go deleted file mode 100644 index 79f42e882..000000000 --- a/management/server/turncredentials.go +++ /dev/null @@ -1,126 +0,0 @@ -package server - -import ( - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "fmt" - "sync" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/management/proto" -) - -// TURNCredentialsManager used to manage TURN credentials -type TURNCredentialsManager interface { - GenerateCredentials() TURNCredentials - SetupRefresh(ctx context.Context, peerKey string) - CancelRefresh(peerKey string) -} - -// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server -type TimeBasedAuthSecretsManager struct { - mux sync.Mutex - config *TURNConfig - updateManager *PeersUpdateManager - cancelMap map[string]chan struct{} -} - -type TURNCredentials struct { - Username string - Password string -} - -func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, config *TURNConfig) *TimeBasedAuthSecretsManager { - return &TimeBasedAuthSecretsManager{ - mux: sync.Mutex{}, - config: config, - updateManager: updateManager, - cancelMap: make(map[string]chan struct{}), - } -} - -// GenerateCredentials generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret -func (m *TimeBasedAuthSecretsManager) GenerateCredentials() TURNCredentials { - mac := hmac.New(sha1.New, []byte(m.config.Secret)) - - timeAuth := time.Now().Add(m.config.CredentialsTTL.Duration).Unix() - - username := fmt.Sprint(timeAuth) - - _, err := mac.Write([]byte(username)) - if err != nil { - log.Errorln("Generating turn password failed with error: ", err) - } - - bytePassword := mac.Sum(nil) - password := base64.StdEncoding.EncodeToString(bytePassword) - - return TURNCredentials{ - Username: username, - Password: password, - } - -} - -func (m *TimeBasedAuthSecretsManager) cancel(peerID string) { - if channel, ok := m.cancelMap[peerID]; ok { - close(channel) - delete(m.cancelMap, peerID) - } -} - -// CancelRefresh cancels scheduled peer credentials refresh -func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) { - m.mux.Lock() - defer m.mux.Unlock() - m.cancel(peerID) -} - -// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer. -// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone. -func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) { - m.mux.Lock() - defer m.mux.Unlock() - m.cancel(peerID) - cancel := make(chan struct{}, 1) - m.cancelMap[peerID] = cancel - log.WithContext(ctx).Debugf("starting turn refresh for %s", peerID) - - go func() { - // we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL) - ticker := time.NewTicker(m.config.CredentialsTTL.Duration / 4 * 3) - - for { - select { - case <-cancel: - log.WithContext(ctx).Debugf("stopping turn refresh for %s", peerID) - return - case <-ticker.C: - c := m.GenerateCredentials() - var turns []*proto.ProtectedHostConfig - for _, host := range m.config.Turns { - turns = append(turns, &proto.ProtectedHostConfig{ - HostConfig: &proto.HostConfig{ - Uri: host.URI, - Protocol: ToResponseProto(host.Proto), - }, - User: c.Username, - Password: c.Password, - }) - } - - update := &proto.SyncResponse{ - WiretrusteeConfig: &proto.WiretrusteeConfig{ - Turns: turns, - }, - } - log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID) - m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update}) - } - } - }() -} diff --git a/management/server/turncredentials_test.go b/management/server/turncredentials_test.go deleted file mode 100644 index 667dccbb5..000000000 --- a/management/server/turncredentials_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package server - -import ( - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "testing" - "time" - - "github.com/netbirdio/netbird/util" -) - -var TurnTestHost = &Host{ - Proto: UDP, - URI: "turn:turn.wiretrustee.com:77777", - Username: "username", - Password: "", -} - -func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) { - ttl := util.Duration{Duration: time.Hour} - secret := "some_secret" - peersManager := NewPeersUpdateManager(nil) - - tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ - CredentialsTTL: ttl, - Secret: secret, - Turns: []*Host{TurnTestHost}, - }) - - credentials := tested.GenerateCredentials() - - if credentials.Username == "" { - t.Errorf("expected generated TURN username not to be empty, got empty") - } - if credentials.Password == "" { - t.Errorf("expected generated TURN password not to be empty, got empty") - } - - validateMAC(t, credentials.Username, credentials.Password, []byte(secret)) - -} - -func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) { - ttl := util.Duration{Duration: 2 * time.Second} - secret := "some_secret" - peersManager := NewPeersUpdateManager(nil) - peer := "some_peer" - updateChannel := peersManager.CreateChannel(context.Background(), peer) - - tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ - CredentialsTTL: ttl, - Secret: secret, - Turns: []*Host{TurnTestHost}, - }) - - tested.SetupRefresh(context.Background(), peer) - - if _, ok := tested.cancelMap[peer]; !ok { - t.Errorf("expecting peer to be present in a cancel map, got not present") - } - - var updates []*UpdateMessage - -loop: - for timeout := time.After(5 * time.Second); ; { - - select { - case update := <-updateChannel: - updates = append(updates, update) - case <-timeout: - break loop - } - - if len(updates) >= 2 { - break loop - } - } - - if len(updates) < 2 { - t.Errorf("expecting 2 peer credentials updates, got %v", len(updates)) - } - - firstUpdate := updates[0].Update.GetWiretrusteeConfig().Turns[0] - secondUpdate := updates[1].Update.GetWiretrusteeConfig().Turns[0] - - if firstUpdate.Password == secondUpdate.Password { - t.Errorf("expecting first credential update password %v to be diffeerent from second, got equal", firstUpdate.Password) - } - -} - -func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) { - ttl := util.Duration{Duration: time.Hour} - secret := "some_secret" - peersManager := NewPeersUpdateManager(nil) - peer := "some_peer" - - tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ - CredentialsTTL: ttl, - Secret: secret, - Turns: []*Host{TurnTestHost}, - }) - - tested.SetupRefresh(context.Background(), peer) - if _, ok := tested.cancelMap[peer]; !ok { - t.Errorf("expecting peer to be present in a cancel map, got not present") - } - - tested.CancelRefresh(peer) - if _, ok := tested.cancelMap[peer]; ok { - t.Errorf("expecting peer to be not present in a cancel map, got present") - } -} - -func validateMAC(t *testing.T, username string, actualMAC string, key []byte) { - t.Helper() - mac := hmac.New(sha1.New, key) - - _, err := mac.Write([]byte(username)) - if err != nil { - t.Fatal(err) - } - - expectedMAC := mac.Sum(nil) - decodedMAC, err := base64.StdEncoding.DecodeString(actualMAC) - if err != nil { - t.Fatal(err) - } - equal := hmac.Equal(decodedMAC, expectedMAC) - - if !equal { - t.Errorf("expected password MAC to be %s. got %s", expectedMAC, decodedMAC) - } -} diff --git a/relay/Dockerfile b/relay/Dockerfile new file mode 100644 index 000000000..f750027c3 --- /dev/null +++ b/relay/Dockerfile @@ -0,0 +1,4 @@ +FROM gcr.io/distroless/base:debug +ENTRYPOINT [ "/go/bin/netbird-relay" ] +ENV NB_LOG_FILE=console +COPY netbird-relay /go/bin/netbird-relay diff --git a/relay/auth/allow/allow_all.go b/relay/auth/allow/allow_all.go new file mode 100644 index 000000000..92845818b --- /dev/null +++ b/relay/auth/allow/allow_all.go @@ -0,0 +1,12 @@ +package allow + +import "hash" + +// Auth is a Validator that allows all connections. +// Used this for testing purposes only. +type Auth struct { +} + +func (a *Auth) Validate(func() hash.Hash, any) error { + return nil +} diff --git a/relay/auth/doc.go b/relay/auth/doc.go new file mode 100644 index 000000000..b3e8dbb08 --- /dev/null +++ b/relay/auth/doc.go @@ -0,0 +1,26 @@ +/* +Package auth manages the authentication process with the relay server. + +Key Components: + +Validator: The Validator interface defines the Validate method. Any type that provides this method can be used as a +Validator. + +Methods: + +Validate(func() hash.Hash, any): This method is defined in the Validator interface and is used to validate the authentication. + +Usage: + +To create a new AllowAllAuth validator, simply instantiate it: + + validator := &allow.Auth{} + +To validate the authentication, use the Validate method: + + err := validator.Validate(sha256.New, any) + +This package provides a simple and effective way to manage authentication with the relay server, ensuring that the +peers are authenticated properly. +*/ +package auth diff --git a/relay/auth/hmac/doc.go b/relay/auth/hmac/doc.go new file mode 100644 index 000000000..a1b135aa6 --- /dev/null +++ b/relay/auth/hmac/doc.go @@ -0,0 +1,8 @@ +/* +This package uses a similar HMAC method for authentication with the TURN server. The Management server provides the +tokens for the peers. The peers manage these tokens in the token store. The token store is a simple thread safe store +that keeps the tokens in memory. These tokens are used to authenticate the peers with the Relay server in the hello +message. +*/ + +package hmac diff --git a/relay/auth/hmac/store.go b/relay/auth/hmac/store.go new file mode 100644 index 000000000..36c195a7b --- /dev/null +++ b/relay/auth/hmac/store.go @@ -0,0 +1,36 @@ +package hmac + +import ( + "sync" + + log "github.com/sirupsen/logrus" +) + +// TokenStore is a simple in-memory store for token +// With this can update the token in thread safe way +type TokenStore struct { + mu sync.Mutex + token []byte +} + +func (a *TokenStore) UpdateToken(token *Token) error { + a.mu.Lock() + defer a.mu.Unlock() + if token == nil { + return nil + } + + t, err := marshalToken(*token) + if err != nil { + log.Debugf("failed to marshal token: %s", err) + return err + } + a.token = t + return nil +} + +func (a *TokenStore) TokenBinary() []byte { + a.mu.Lock() + defer a.mu.Unlock() + return a.token +} diff --git a/relay/auth/hmac/token.go b/relay/auth/hmac/token.go new file mode 100644 index 000000000..e2e62b84e --- /dev/null +++ b/relay/auth/hmac/token.go @@ -0,0 +1,105 @@ +package hmac + +import ( + "bytes" + "crypto/hmac" + "encoding/base64" + "encoding/gob" + "fmt" + "hash" + "strconv" + "time" + + log "github.com/sirupsen/logrus" +) + +type Token struct { + Payload string + Signature string +} + +func marshalToken(token Token) ([]byte, error) { + var buffer bytes.Buffer + encoder := gob.NewEncoder(&buffer) + err := encoder.Encode(token) + if err != nil { + log.Debugf("failed to marshal token: %s", err) + return nil, fmt.Errorf("failed to marshal token: %w", err) + } + return buffer.Bytes(), nil +} + +func unmarshalToken(payload []byte) (Token, error) { + var creds Token + buffer := bytes.NewBuffer(payload) + decoder := gob.NewDecoder(buffer) + err := decoder.Decode(&creds) + return creds, err +} + +// TimedHMAC generates a token with TTL and uses a pre-shared secret known to the relay server +type TimedHMAC struct { + secret string + timeToLive time.Duration +} + +// NewTimedHMAC creates a new TimedHMAC instance +func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC { + return &TimedHMAC{ + secret: secret, + timeToLive: timeToLive, + } +} + +// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC +// hash of a timestamp with a preshared TURN secret +func (m *TimedHMAC) GenerateToken(algo func() hash.Hash) (*Token, error) { + timeAuth := time.Now().Add(m.timeToLive).Unix() + timeStamp := strconv.FormatInt(timeAuth, 10) + + checksum, err := m.generate(algo, timeStamp) + if err != nil { + return nil, err + } + + return &Token{ + Payload: timeStamp, + Signature: base64.StdEncoding.EncodeToString(checksum), + }, nil +} + +// Validate checks if the token is valid +func (m *TimedHMAC) Validate(algo func() hash.Hash, token Token) error { + expectedMAC, err := m.generate(algo, token.Payload) + if err != nil { + return err + } + + expectedSignature := base64.StdEncoding.EncodeToString(expectedMAC) + + if !hmac.Equal([]byte(expectedSignature), []byte(token.Signature)) { + return fmt.Errorf("signature mismatch") + } + + timeAuthInt, err := strconv.ParseInt(token.Payload, 10, 64) + if err != nil { + return fmt.Errorf("invalid payload: %w", err) + } + + if time.Now().Unix() > timeAuthInt { + return fmt.Errorf("expired token") + } + + return nil +} + +func (m *TimedHMAC) generate(algo func() hash.Hash, payload string) ([]byte, error) { + mac := hmac.New(algo, []byte(m.secret)) + _, err := mac.Write([]byte(payload)) + if err != nil { + log.Debugf("failed to generate token: %s", err) + return nil, fmt.Errorf("failed to generate token: %w", err) + } + + return mac.Sum(nil), nil +} diff --git a/relay/auth/hmac/token_test.go b/relay/auth/hmac/token_test.go new file mode 100644 index 000000000..e629eab97 --- /dev/null +++ b/relay/auth/hmac/token_test.go @@ -0,0 +1,105 @@ +package hmac + +import ( + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "strconv" + "testing" + "time" +) + +func TestGenerateCredentials(t *testing.T) { + secret := "secret" + timeToLive := 1 * time.Hour + v := NewTimedHMAC(secret, timeToLive) + + creds, err := v.GenerateToken(sha1.New) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if creds.Payload == "" { + t.Fatalf("expected non-empty payload") + } + + _, err = strconv.ParseInt(creds.Payload, 10, 64) + if err != nil { + t.Fatalf("expected payload to be a valid unix timestamp, got %v", err) + } + + _, err = base64.StdEncoding.DecodeString(creds.Signature) + if err != nil { + t.Fatalf("expected signature to be base64 encoded, got %v", err) + } +} + +func TestValidateCredentials(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + manager := NewTimedHMAC(secret, timeToLive) + + // Test valid token + creds, err := manager.GenerateToken(sha1.New) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err := manager.Validate(sha1.New, *creds); err != nil { + t.Fatalf("expected valid token: %s", err) + } +} + +func TestInvalidSignature(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + manager := NewTimedHMAC(secret, timeToLive) + + creds, err := manager.GenerateToken(sha256.New) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + invalidCreds := &Token{ + Payload: creds.Payload, + Signature: "invalidsignature", + } + + if err = manager.Validate(sha1.New, *invalidCreds); err == nil { + t.Fatalf("expected invalid token due to signature mismatch") + } +} + +func TestExpired(t *testing.T) { + secret := "supersecret" + v := NewTimedHMAC(secret, -1*time.Hour) + expiredCreds, err := v.GenerateToken(sha256.New) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err = v.Validate(sha1.New, *expiredCreds); err == nil { + t.Fatalf("expected invalid token due to expiration") + } +} + +func TestInvalidPayload(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + v := NewTimedHMAC(secret, timeToLive) + + creds, err := v.GenerateToken(sha256.New) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + // Test invalid payload + invalidPayloadCreds := &Token{ + Payload: "invalidtimestamp", + Signature: creds.Signature, + } + + if err = v.Validate(sha1.New, *invalidPayloadCreds); err == nil { + t.Fatalf("expected invalid token due to invalid payload") + } +} diff --git a/relay/auth/hmac/validator.go b/relay/auth/hmac/validator.go new file mode 100644 index 000000000..6ddd89c19 --- /dev/null +++ b/relay/auth/hmac/validator.go @@ -0,0 +1,33 @@ +package hmac + +import ( + "fmt" + "hash" + "time" + + log "github.com/sirupsen/logrus" +) + +type TimedHMACValidator struct { + *TimedHMAC +} + +func NewTimedHMACValidator(secret string, duration time.Duration) *TimedHMACValidator { + ta := NewTimedHMAC(secret, duration) + return &TimedHMACValidator{ + ta, + } +} + +func (a *TimedHMACValidator) Validate(algo func() hash.Hash, credentials any) error { + b, ok := credentials.([]byte) + if !ok { + return fmt.Errorf("invalid credentials type") + } + c, err := unmarshalToken(b) + if err != nil { + log.Debugf("failed to unmarshal token: %s", err) + return err + } + return a.TimedHMAC.Validate(algo, c) +} diff --git a/relay/auth/validator.go b/relay/auth/validator.go new file mode 100644 index 000000000..078811f3d --- /dev/null +++ b/relay/auth/validator.go @@ -0,0 +1,8 @@ +package auth + +import "hash" + +// Validator is an interface that defines the Validate method. +type Validator interface { + Validate(func() hash.Hash, any) error +} diff --git a/relay/client/addr.go b/relay/client/addr.go new file mode 100644 index 000000000..af4f459f8 --- /dev/null +++ b/relay/client/addr.go @@ -0,0 +1,13 @@ +package client + +type RelayAddr struct { + addr string +} + +func (a RelayAddr) Network() string { + return "relay" +} + +func (a RelayAddr) String() string { + return a.addr +} diff --git a/relay/client/client.go b/relay/client/client.go new file mode 100644 index 000000000..1160d1c9e --- /dev/null +++ b/relay/client/client.go @@ -0,0 +1,553 @@ +package client + +import ( + "context" + "fmt" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + auth "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client/dialer/ws" + "github.com/netbirdio/netbird/relay/healthcheck" + "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/messages/address" + auth2 "github.com/netbirdio/netbird/relay/messages/auth" +) + +const ( + bufferSize = 8820 + serverResponseTimeout = 8 * time.Second +) + +var ( + ErrConnAlreadyExists = fmt.Errorf("connection already exists") +) + +type internalStopFlag struct { + sync.Mutex + stop bool +} + +func newInternalStopFlag() *internalStopFlag { + return &internalStopFlag{} +} + +func (isf *internalStopFlag) set() { + isf.Lock() + defer isf.Unlock() + isf.stop = true +} + +func (isf *internalStopFlag) isSet() bool { + isf.Lock() + defer isf.Unlock() + return isf.stop +} + +// Msg carry the payload from the server to the client. With this struct, the net.Conn can free the buffer. +type Msg struct { + Payload []byte + + bufPool *sync.Pool + bufPtr *[]byte +} + +func (m *Msg) Free() { + m.bufPool.Put(m.bufPtr) +} + +type connContainer struct { + conn *Conn + messages chan Msg + msgChanLock sync.Mutex + closed bool // flag to check if channel is closed +} + +func newConnContainer(conn *Conn, messages chan Msg) *connContainer { + return &connContainer{ + conn: conn, + messages: messages, + } +} + +func (cc *connContainer) writeMsg(msg Msg) { + cc.msgChanLock.Lock() + defer cc.msgChanLock.Unlock() + if cc.closed { + return + } + cc.messages <- msg +} + +func (cc *connContainer) close() { + cc.msgChanLock.Lock() + defer cc.msgChanLock.Unlock() + if cc.closed { + return + } + close(cc.messages) + cc.closed = true +} + +// Client is a client for the relay server. It is responsible for establishing a connection to the relay server and +// managing connections to other peers. All exported functions are safe to call concurrently. After close the connection, +// the client can be reused by calling Connect again. When the client is closed, all connections are closed too. +// While the Connect is in progress, the OpenConn function will block until the connection is established with relay server. +type Client struct { + log *log.Entry + parentCtx context.Context + connectionURL string + authTokenStore *auth.TokenStore + hashedID []byte + + bufPool *sync.Pool + + relayConn net.Conn + conns map[string]*connContainer + serviceIsRunning bool + mu sync.Mutex // protect serviceIsRunning and conns + readLoopMutex sync.Mutex + wgReadLoop sync.WaitGroup + instanceURL *RelayAddr + muInstanceURL sync.Mutex + + onDisconnectListener func() + listenerMutex sync.Mutex +} + +// NewClient creates a new client for the relay server. The client is not connected to the server until the Connect +func NewClient(ctx context.Context, serverURL string, authTokenStore *auth.TokenStore, peerID string) *Client { + hashedID, hashedStringId := messages.HashID(peerID) + return &Client{ + log: log.WithField("client_id", hashedStringId), + parentCtx: ctx, + connectionURL: serverURL, + authTokenStore: authTokenStore, + hashedID: hashedID, + bufPool: &sync.Pool{ + New: func() any { + buf := make([]byte, bufferSize) + return &buf + }, + }, + conns: make(map[string]*connContainer), + } +} + +// Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. +func (c *Client) Connect() error { + c.log.Infof("connecting to relay server: %s", c.connectionURL) + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + + c.mu.Lock() + defer c.mu.Unlock() + + if c.serviceIsRunning { + return nil + } + + err := c.connect() + if err != nil { + return err + } + + c.serviceIsRunning = true + + c.wgReadLoop.Add(1) + go c.readLoop(c.relayConn) + + c.log.Infof("relay connection established with: %s", c.connectionURL) + return nil +} + +// OpenConn create a new net.Conn for the destination peer ID. In case if the connection is in progress +// to the relay server, the function will block until the connection is established or timed out. Otherwise, +// it will return immediately. +// todo: what should happen if call with the same peerID with multiple times? +func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.serviceIsRunning { + return nil, fmt.Errorf("relay connection is not established") + } + + hashedID, hashedStringID := messages.HashID(dstPeerID) + _, ok := c.conns[hashedStringID] + if ok { + return nil, ErrConnAlreadyExists + } + + log.Infof("open connection to peer: %s", hashedStringID) + msgChannel := make(chan Msg, 2) + conn := NewConn(c, hashedID, hashedStringID, msgChannel, c.instanceURL) + + c.conns[hashedStringID] = newConnContainer(conn, msgChannel) + return conn, nil +} + +// ServerInstanceURL returns the address of the relay server. It could change after the close and reopen the connection. +func (c *Client) ServerInstanceURL() (string, error) { + c.muInstanceURL.Lock() + defer c.muInstanceURL.Unlock() + if c.instanceURL == nil { + return "", fmt.Errorf("relay connection is not established") + } + return c.instanceURL.String(), nil +} + +// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. +func (c *Client) SetOnDisconnectListener(fn func()) { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + c.onDisconnectListener = fn +} + +// HasConns returns true if there are connections. +func (c *Client) HasConns() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.conns) > 0 +} + +// Close closes the connection to the relay server and all connections to other peers. +func (c *Client) Close() error { + return c.close(true) +} + +func (c *Client) connect() error { + conn, err := ws.Dial(c.connectionURL) + if err != nil { + return err + } + c.relayConn = conn + + err = c.handShake() + if err != nil { + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection: %s", cErr) + } + return err + } + + return nil +} + +func (c *Client) handShake() error { + authMsg := &auth2.Msg{ + AuthAlgorithm: auth2.AlgoHMACSHA256, + AdditionalData: c.authTokenStore.TokenBinary(), + } + + authData, err := authMsg.Marshal() + if err != nil { + return fmt.Errorf("marshal auth message: %w", err) + } + + msg, err := messages.MarshalHelloMsg(c.hashedID, authData) + if err != nil { + log.Errorf("failed to marshal hello message: %s", err) + return err + } + + _, err = c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to send hello message: %s", err) + return err + } + buf := make([]byte, messages.MaxHandshakeSize) + n, err := c.readWithTimeout(buf) + if err != nil { + log.Errorf("failed to read hello response: %s", err) + return err + } + + _, err = messages.ValidateVersion(buf[:n]) + if err != nil { + return fmt.Errorf("validate version: %w", err) + } + + msgType, err := messages.DetermineServerMessageType(buf[messages.SizeOfVersionByte:n]) + if err != nil { + log.Errorf("failed to determine message type: %s", err) + return err + } + + if msgType != messages.MsgTypeHelloResponse { + log.Errorf("unexpected message type: %s", msgType) + return fmt.Errorf("unexpected message type") + } + + additionalData, err := messages.UnmarshalHelloResponse(buf[messages.SizeOfProtoHeader:n]) + if err != nil { + return err + } + + addr, err := address.Unmarshal(additionalData) + if err != nil { + return fmt.Errorf("unmarshal address: %w", err) + } + + c.muInstanceURL.Lock() + c.instanceURL = &RelayAddr{addr: addr.URL} + c.muInstanceURL.Unlock() + return nil +} + +func (c *Client) readLoop(relayConn net.Conn) { + internallyStoppedFlag := newInternalStopFlag() + hc := healthcheck.NewReceiver() + go c.listenForStopEvents(hc, relayConn, internallyStoppedFlag) + + var ( + errExit error + n int + ) + for { + bufPtr := c.bufPool.Get().(*[]byte) + buf := *bufPtr + n, errExit = relayConn.Read(buf) + if errExit != nil { + c.mu.Lock() + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Debugf("failed to read message from relay server: %s", errExit) + } + c.mu.Unlock() + break + } + + _, err := messages.ValidateVersion(buf[:n]) + if err != nil { + c.log.Errorf("failed to validate protocol version: %s", err) + c.bufPool.Put(bufPtr) + continue + } + + msgType, err := messages.DetermineServerMessageType(buf[messages.SizeOfVersionByte:n]) + if err != nil { + c.log.Errorf("failed to determine message type: %s", err) + c.bufPool.Put(bufPtr) + continue + } + + if !c.handleMsg(msgType, buf[messages.SizeOfProtoHeader:n], bufPtr, hc, internallyStoppedFlag) { + break + } + } + + hc.Stop() + + c.muInstanceURL.Lock() + c.instanceURL = nil + c.muInstanceURL.Unlock() + + c.notifyDisconnected() + c.wgReadLoop.Done() + _ = c.close(false) +} + +func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) { + switch msgType { + case messages.MsgTypeHealthCheck: + c.handleHealthCheck(hc, internallyStoppedFlag) + c.bufPool.Put(bufPtr) + case messages.MsgTypeTransport: + return c.handleTransportMsg(buf, bufPtr, internallyStoppedFlag) + case messages.MsgTypeClose: + log.Debugf("relay connection close by server") + c.bufPool.Put(bufPtr) + return false + } + + return true +} + +func (c *Client) handleHealthCheck(hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) { + msg := messages.MarshalHealthcheck() + _, wErr := c.relayConn.Write(msg) + if wErr != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to send heartbeat: %s", wErr) + } + } + hc.Heartbeat() +} + +func (c *Client) handleTransportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool { + peerID, payload, err := messages.UnmarshalTransportMsg(buf) + if err != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to parse transport message: %v", err) + } + + c.bufPool.Put(bufPtr) + return true + } + + stringID := messages.HashIDToString(peerID) + + c.mu.Lock() + if !c.serviceIsRunning { + c.mu.Unlock() + c.bufPool.Put(bufPtr) + return false + } + container, ok := c.conns[stringID] + c.mu.Unlock() + if !ok { + c.log.Errorf("peer not found: %s", stringID) + c.bufPool.Put(bufPtr) + return true + } + msg := Msg{ + bufPool: c.bufPool, + bufPtr: bufPtr, + Payload: payload, + } + container.writeMsg(msg) + return true +} + +func (c *Client) writeTo(connReference *Conn, id string, dstID []byte, payload []byte) (int, error) { + c.mu.Lock() + conn, ok := c.conns[id] + c.mu.Unlock() + if !ok { + return 0, io.EOF + } + + if conn.conn != connReference { + return 0, io.EOF + } + + // todo: use buffer pool instead of create new transport msg. + msg, err := messages.MarshalTransportMsg(dstID, payload) + if err != nil { + log.Errorf("failed to marshal transport message: %s", err) + return 0, err + } + + // the write always return with 0 length because the underling does not support the size feedback. + _, err = c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to write transport message: %s", err) + } + return len(payload), err +} + +func (c *Client) listenForStopEvents(hc *healthcheck.Receiver, conn net.Conn, internalStopFlag *internalStopFlag) { + for { + select { + case _, ok := <-hc.OnTimeout: + if !ok { + return + } + c.log.Errorf("health check timeout") + internalStopFlag.set() + _ = conn.Close() // ignore the err because the readLoop will handle it + return + case <-c.parentCtx.Done(): + err := c.close(true) + if err != nil { + log.Errorf("failed to teardown connection: %s", err) + } + return + } + } +} + +func (c *Client) closeAllConns() { + for _, container := range c.conns { + container.close() + } + c.conns = make(map[string]*connContainer) +} + +func (c *Client) closeConn(connReference *Conn, id string) error { + c.mu.Lock() + defer c.mu.Unlock() + + container, ok := c.conns[id] + if !ok { + return fmt.Errorf("connection already closed") + } + + if container.conn != connReference { + return fmt.Errorf("conn reference mismatch") + } + container.close() + delete(c.conns, id) + + return nil +} + +func (c *Client) close(gracefullyExit bool) error { + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + + c.mu.Lock() + var err error + if !c.serviceIsRunning { + c.mu.Unlock() + return nil + } + + c.serviceIsRunning = false + c.closeAllConns() + if gracefullyExit { + c.writeCloseMsg() + } + err = c.relayConn.Close() + c.mu.Unlock() + + c.wgReadLoop.Wait() + c.log.Infof("relay connection closed with: %s", c.connectionURL) + return err +} + +func (c *Client) notifyDisconnected() { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + + if c.onDisconnectListener == nil { + return + } + go c.onDisconnectListener() +} + +func (c *Client) writeCloseMsg() { + msg := messages.MarshalCloseMsg() + _, err := c.relayConn.Write(msg) + if err != nil { + c.log.Errorf("failed to send close message: %s", err) + } +} + +func (c *Client) readWithTimeout(buf []byte) (int, error) { + ctx, cancel := context.WithTimeout(c.parentCtx, serverResponseTimeout) + defer cancel() + + readDone := make(chan struct{}) + var ( + n int + err error + ) + + go func() { + n, err = c.relayConn.Read(buf) + close(readDone) + }() + + select { + case <-ctx.Done(): + return 0, fmt.Errorf("read operation timed out") + case <-readDone: + return n, err + } +} diff --git a/relay/client/client_test.go b/relay/client/client_test.go new file mode 100644 index 000000000..b7f1a63ca --- /dev/null +++ b/relay/client/client_test.go @@ -0,0 +1,631 @@ +package client + +import ( + "context" + "net" + "os" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/auth/allow" + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/util" + + "github.com/netbirdio/netbird/relay/server" +) + +var ( + av = &allow.Auth{} + hmacTokenStore = &hmac.TokenStore{} + serverListenAddr = "127.0.0.1:1234" + serverURL = "rel://127.0.0.1:1234" +) + +func TestMain(m *testing.M) { + _ = util.InitLog("error", "console") + code := m.Run() + os.Exit(code) +} + +func TestClient(t *testing.T) { + ctx := context.Background() + + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + listenCfg := server.ListenerConfig{Address: serverListenAddr} + err := srv.Listen(listenCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + t.Log("alice connecting to server") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientAlice.Close() + + t.Log("placeholder connecting to server") + clientPlaceHolder := NewClient(ctx, serverURL, hmacTokenStore, "clientPlaceHolder") + err = clientPlaceHolder.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientPlaceHolder.Close() + + t.Log("Bob connecting to server") + clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") + err = clientBob.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientBob.Close() + + t.Log("Alice open connection to Bob") + connAliceToBob, err := clientAlice.OpenConn("bob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + t.Log("Bob open connection to Alice") + connBobToAlice, err := clientBob.OpenConn("alice") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + log.Debugf("alice sent message to bob") + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + log.Debugf("on new message from alice to bob") + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestRegistration(t *testing.T) { + ctx := context.Background() + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + _ = srv.Shutdown(ctx) + t.Fatalf("failed to connect to server: %s", err) + } + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } + err = srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } +} + +func TestRegistrationTimeout(t *testing.T) { + ctx := context.Background() + fakeUDPListener, err := net.ListenUDP("udp", &net.UDPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + }) + if err != nil { + t.Fatalf("failed to bind UDP server: %s", err) + } + defer func(fakeUDPListener *net.UDPConn) { + _ = fakeUDPListener.Close() + }(fakeUDPListener) + + fakeTCPListener, err := net.ListenTCP("tcp", &net.TCPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + }) + if err != nil { + t.Fatalf("failed to bind TCP server: %s", err) + } + defer func(fakeTCPListener *net.TCPListener) { + _ = fakeTCPListener.Close() + }(fakeTCPListener) + + clientAlice := NewClient(ctx, "127.0.0.1:1234", hmacTokenStore, "alice") + err = clientAlice.Connect() + if err == nil { + t.Errorf("failed to connect to server: %s", err) + } + log.Debugf("%s", err) + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } +} + +func TestEcho(t *testing.T) { + ctx := context.Background() + idAlice := "alice" + idBob := "bob" + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice) + err = clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err := clientAlice.Close() + if err != nil { + t.Errorf("failed to close Alice client: %s", err) + } + }() + + clientBob := NewClient(ctx, serverURL, hmacTokenStore, idBob) + err = clientBob.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err := clientBob.Close() + if err != nil { + t.Errorf("failed to close Bob client: %s", err) + } + }() + + connAliceToBob, err := clientAlice.OpenConn(idBob) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + connBobToAlice, err := clientBob.OpenConn(idAlice) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + _, err = connBobToAlice.Write(buf[:n]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + n, err = connAliceToBob.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestBindToUnavailabePeer(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + _, err = clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } +} + +func TestBindReconnect(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + _, err = clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") + err = clientBob.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + chBob, err := clientBob.OpenConn("alice") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing client Alice") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + + clientAlice = NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + chAlice, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + testString := "hello alice, I am bob" + _, err = chBob.Write([]byte(testString)) + if err != nil { + t.Errorf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := chAlice.Read(buf) + if err != nil { + t.Errorf("failed to read from channel: %s", err) + } + + if testString != string(buf[:n]) { + t.Errorf("expected %s, got %s", testString, string(buf[:n])) + } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } +} + +func TestCloseConn(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing connection") + err = conn.Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + _, err = conn.Write([]byte("hello")) + if err == nil { + t.Errorf("unexpected writing from closed connection") + } +} + +func TestCloseRelayConn(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + _ = clientAlice.relayConn.Close() + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + _, err = clientAlice.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } +} + +func TestCloseByServer(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv1, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + + go func() { + err := srv1.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) + err = relayClient.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + + disconnected := make(chan struct{}) + relayClient.SetOnDisconnectListener(func() { + log.Infof("client disconnected") + close(disconnected) + }) + + err = srv1.Shutdown(ctx) + if err != nil { + t.Fatalf("failed to close server: %s", err) + } + + select { + case <-disconnected: + case <-time.After(3 * time.Second): + log.Fatalf("timeout waiting for client to disconnect") + } + + _, err = relayClient.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } +} + +func TestCloseByClient(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) + err = relayClient.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + + err = relayClient.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + + _, err = relayClient.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } + + err = srv.Shutdown(ctx) + if err != nil { + t.Fatalf("failed to close server: %s", err) + } +} + +func waitForServerToStart(errChan chan error) error { + select { + case err := <-errChan: + if err != nil { + return err + } + case <-time.After(300 * time.Millisecond): + return nil + } + return nil +} diff --git a/relay/client/conn.go b/relay/client/conn.go new file mode 100644 index 000000000..b4ff903e8 --- /dev/null +++ b/relay/client/conn.go @@ -0,0 +1,76 @@ +package client + +import ( + "io" + "net" + "time" +) + +// Conn represent a connection to a relayed remote peer. +type Conn struct { + client *Client + dstID []byte + dstStringID string + messageChan chan Msg + instanceURL *RelayAddr +} + +// NewConn creates a new connection to a relayed remote peer. +// client: the client instance, it used to send messages to the destination peer +// dstID: the destination peer ID +// dstStringID: the destination peer ID in string format +// messageChan: the channel where the messages will be received +// instanceURL: the relay instance URL, it used to get the proper server instance address for the remote peer +func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg, instanceURL *RelayAddr) *Conn { + c := &Conn{ + client: client, + dstID: dstID, + dstStringID: dstStringID, + messageChan: messageChan, + instanceURL: instanceURL, + } + + return c +} + +func (c *Conn) Write(p []byte) (n int, err error) { + return c.client.writeTo(c, c.dstStringID, c.dstID, p) +} + +func (c *Conn) Read(b []byte) (n int, err error) { + msg, ok := <-c.messageChan + if !ok { + return 0, io.EOF + } + + n = copy(b, msg.Payload) + msg.Free() + return n, nil +} + +func (c *Conn) Close() error { + return c.client.closeConn(c, c.dstStringID) +} + +func (c *Conn) LocalAddr() net.Addr { + return c.client.relayConn.LocalAddr() +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.instanceURL +} + +func (c *Conn) SetDeadline(t time.Time) error { + //TODO implement me + panic("SetDeadline is not implemented") +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + //TODO implement me + panic("SetReadDeadline is not implemented") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + //TODO implement me + panic("SetReadDeadline is not implemented") +} diff --git a/relay/client/dialer/ws/addr.go b/relay/client/dialer/ws/addr.go new file mode 100644 index 000000000..43f5dd6af --- /dev/null +++ b/relay/client/dialer/ws/addr.go @@ -0,0 +1,13 @@ +package ws + +type WebsocketAddr struct { + addr string +} + +func (a WebsocketAddr) Network() string { + return "websocket" +} + +func (a WebsocketAddr) String() string { + return a.addr +} diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go new file mode 100644 index 000000000..e7f771b8d --- /dev/null +++ b/relay/client/dialer/ws/conn.go @@ -0,0 +1,66 @@ +package ws + +import ( + "context" + "fmt" + "net" + "time" + + "nhooyr.io/websocket" +) + +type Conn struct { + ctx context.Context + *websocket.Conn + remoteAddr WebsocketAddr +} + +func NewConn(wsConn *websocket.Conn, serverAddress string) net.Conn { + return &Conn{ + ctx: context.Background(), + Conn: wsConn, + remoteAddr: WebsocketAddr{serverAddress}, + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, ioReader, err := c.Conn.Reader(c.ctx) + if err != nil { + return 0, err + } + + if t != websocket.MessageBinary { + return 0, fmt.Errorf("unexpected message type") + } + + return ioReader.Read(b) +} + +func (c *Conn) Write(b []byte) (n int, err error) { + err = c.Conn.Write(c.ctx, websocket.MessageBinary, b) + return 0, err +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.remoteAddr +} + +func (c *Conn) LocalAddr() net.Addr { + return WebsocketAddr{addr: "unknown"} +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + return fmt.Errorf("SetReadDeadline is not implemented") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + return fmt.Errorf("SetWriteDeadline is not implemented") +} + +func (c *Conn) SetDeadline(t time.Time) error { + return fmt.Errorf("SetDeadline is not implemented") +} + +func (c *Conn) Close() error { + return c.Conn.CloseNow() +} diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go new file mode 100644 index 000000000..d9388aafd --- /dev/null +++ b/relay/client/dialer/ws/ws.go @@ -0,0 +1,67 @@ +package ws + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "strings" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" + + "github.com/netbirdio/netbird/relay/server/listener/ws" + nbnet "github.com/netbirdio/netbird/util/net" +) + +func Dial(address string) (net.Conn, error) { + wsURL, err := prepareURL(address) + if err != nil { + return nil, err + } + + opts := &websocket.DialOptions{ + HTTPClient: httpClientNbDialer(), + } + + parsedURL, err := url.Parse(wsURL) + if err != nil { + return nil, err + } + parsedURL.Path = ws.URLPath + + wsConn, resp, err := websocket.Dial(context.Background(), parsedURL.String(), opts) + if err != nil { + log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err) + return nil, err + } + if resp.Body != nil { + _ = resp.Body.Close() + } + + conn := NewConn(wsConn, address) + return conn, nil +} + +func prepareURL(address string) (string, error) { + if !strings.HasPrefix(address, "rel:") && !strings.HasPrefix(address, "rels:") { + return "", fmt.Errorf("unsupported scheme: %s", address) + } + + return strings.Replace(address, "rel", "ws", 1), nil +} + +func httpClientNbDialer() *http.Client { + customDialer := nbnet.NewDialer() + + customTransport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return customDialer.DialContext(ctx, network, addr) + }, + } + + return &http.Client{ + Transport: customTransport, + } +} diff --git a/relay/client/doc.go b/relay/client/doc.go new file mode 100644 index 000000000..1339251d9 --- /dev/null +++ b/relay/client/doc.go @@ -0,0 +1,12 @@ +/* +Package client contains the implementation of the Relay client. + +The Relay client is responsible for establishing a connection with the Relay server and sending and receiving messages, +Keep persistent connection with the Relay server and handle the connection issues. +It uses the WebSocket protocol for communication and optionally supports TLS (Transport Layer Security). + +If a peer wants to communicate with a peer on a different relay server, the manager will establish a new connection to +the relay server. The connection with these relay servers will be closed if there is no active connection. The peers +negotiate the common relay instance via signaling service. +*/ +package client diff --git a/relay/client/guard.go b/relay/client/guard.go new file mode 100644 index 000000000..f826cf1b6 --- /dev/null +++ b/relay/client/guard.go @@ -0,0 +1,48 @@ +package client + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" +) + +var ( + reconnectingTimeout = 5 * time.Second +) + +// Guard manage the reconnection tries to the Relay server in case of disconnection event. +type Guard struct { + ctx context.Context + relayClient *Client +} + +// NewGuard creates a new guard for the relay client. +func NewGuard(context context.Context, relayClient *Client) *Guard { + g := &Guard{ + ctx: context, + relayClient: relayClient, + } + return g +} + +// OnDisconnected is called when the relay client is disconnected from the relay server. It will trigger the reconnection +// todo prevent multiple reconnection instances. In the current usage it should not happen, but it is better to prevent +func (g *Guard) OnDisconnected() { + ticker := time.NewTicker(reconnectingTimeout) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + err := g.relayClient.Connect() + if err != nil { + log.Errorf("failed to reconnect to relay server: %s", err) + continue + } + return + case <-g.ctx.Done(): + return + } + } +} diff --git a/relay/client/manager.go b/relay/client/manager.go new file mode 100644 index 000000000..3e152a963 --- /dev/null +++ b/relay/client/manager.go @@ -0,0 +1,365 @@ +package client + +import ( + "container/list" + "context" + "errors" + "fmt" + "net" + "reflect" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + relayAuth "github.com/netbirdio/netbird/relay/auth/hmac" +) + +var ( + relayCleanupInterval = 60 * time.Second + connectionTimeout = 30 * time.Second + maxConcurrentServers = 7 + + ErrRelayClientNotConnected = fmt.Errorf("relay client not connected") +) + +// RelayTrack hold the relay clients for the foreign relay servers. +// With the mutex can ensure we can open new connection in case the relay connection has been established with +// the relay server. +type RelayTrack struct { + sync.RWMutex + relayClient *Client +} + +func NewRelayTrack() *RelayTrack { + return &RelayTrack{} +} + +type OnServerCloseListener func() + +// ManagerService is the interface for the relay manager. +type ManagerService interface { + Serve() error + OpenConn(serverAddress, peerKey string) (net.Conn, error) + AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error + RelayInstanceAddress() (string, error) + ServerURLs() []string + HasRelayAddress() bool + UpdateToken(token *relayAuth.Token) error +} + +// Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL +// and automatically reconnect to them in case disconnection. +// The manager also manage temporary relay connection. If a client wants to communicate with a client on a +// different relay server, the manager will establish a new connection to the relay server. The connection with these +// relay servers will be closed if there is no active connection. Periodically the manager will check if there is any +// unused relay connection and close it. +type Manager struct { + ctx context.Context + serverURLs []string + peerID string + tokenStore *relayAuth.TokenStore + + relayClient *Client + reconnectGuard *Guard + + relayClients map[string]*RelayTrack + relayClientsMutex sync.RWMutex + + onDisconnectedListeners map[string]*list.List + listenerLock sync.Mutex +} + +// NewManager creates a new manager instance. +// The serverURL address can be empty. In this case, the manager will not serve. +func NewManager(ctx context.Context, serverURLs []string, peerID string) *Manager { + return &Manager{ + ctx: ctx, + serverURLs: serverURLs, + peerID: peerID, + tokenStore: &relayAuth.TokenStore{}, + relayClients: make(map[string]*RelayTrack), + onDisconnectedListeners: make(map[string]*list.List), + } +} + +// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop for +// the unused relay connections. The manager will automatically reconnect to the relay server in case of disconnection. +func (m *Manager) Serve() error { + if m.relayClient != nil { + return fmt.Errorf("manager already serving") + } + log.Debugf("starting relay client manager with %v relay servers", m.serverURLs) + + totalServers := len(m.serverURLs) + + successChan := make(chan *Client, 1) + errChan := make(chan error, len(m.serverURLs)) + + ctx, cancel := context.WithTimeout(m.ctx, connectionTimeout) + defer cancel() + + sem := make(chan struct{}, maxConcurrentServers) + + for _, url := range m.serverURLs { + sem <- struct{}{} + go func(url string) { + defer func() { <-sem }() + m.connect(m.ctx, url, successChan, errChan) + }(url) + } + + var errCount int + + for { + select { + case client := <-successChan: + log.Infof("Successfully connected to relay server: %s", client.connectionURL) + + m.relayClient = client + + m.reconnectGuard = NewGuard(m.ctx, m.relayClient) + m.relayClient.SetOnDisconnectListener(func() { + m.onServerDisconnected(client.connectionURL) + }) + m.startCleanupLoop() + return nil + case err := <-errChan: + errCount++ + log.Warnf("Connection attempt failed: %v", err) + if errCount == totalServers { + return errors.New("failed to connect to any relay server: all attempts failed") + } + case <-ctx.Done(): + return fmt.Errorf("failed to connect to any relay server: %w", ctx.Err()) + } + } +} + +func (m *Manager) connect(ctx context.Context, serverURL string, successChan chan<- *Client, errChan chan<- error) { + // TODO: abort the connection if another connection was successful + relayClient := NewClient(ctx, serverURL, m.tokenStore, m.peerID) + if err := relayClient.Connect(); err != nil { + errChan <- fmt.Errorf("failed to connect to %s: %w", serverURL, err) + return + } + + select { + case successChan <- relayClient: + // This client was the first to connect successfully + default: + if err := relayClient.Close(); err != nil { + log.Debugf("failed to close relay client: %s", err) + } + } +} + +// OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be +// established via the relay server. If the peer is on a different relay server, the manager will establish a new +// connection to the relay server. It returns back with a net.Conn what represent the remote peer connection. +func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { + if m.relayClient == nil { + return nil, ErrRelayClientNotConnected + } + + foreign, err := m.isForeignServer(serverAddress) + if err != nil { + return nil, err + } + + var ( + netConn net.Conn + ) + if !foreign { + log.Debugf("open peer connection via permanent server: %s", peerKey) + netConn, err = m.relayClient.OpenConn(peerKey) + } else { + log.Debugf("open peer connection via foreign server: %s", serverAddress) + netConn, err = m.openConnVia(serverAddress, peerKey) + } + if err != nil { + return nil, err + } + + return netConn, err +} + +// AddCloseListener adds a listener to the given server instance address. The listener will be called if the connection +// closed. +func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error { + foreign, err := m.isForeignServer(serverAddress) + if err != nil { + return err + } + + var listenerAddr string + if foreign { + listenerAddr = serverAddress + } else { + listenerAddr = m.relayClient.connectionURL + } + m.addListener(listenerAddr, onClosedListener) + return nil +} + +// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is +// lost. This address will be sent to the target peer to choose the common relay server for the communication. +func (m *Manager) RelayInstanceAddress() (string, error) { + if m.relayClient == nil { + return "", ErrRelayClientNotConnected + } + return m.relayClient.ServerInstanceURL() +} + +// ServerURLs returns the addresses of the relay servers. +func (m *Manager) ServerURLs() []string { + return m.serverURLs +} + +// HasRelayAddress returns true if the manager is serving. With this method can check if the peer can communicate with +// Relay service. +func (m *Manager) HasRelayAddress() bool { + return len(m.serverURLs) > 0 +} + +// UpdateToken updates the token in the token store. +func (m *Manager) UpdateToken(token *relayAuth.Token) error { + return m.tokenStore.UpdateToken(token) +} + +func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { + // check if already has a connection to the desired relay server + m.relayClientsMutex.RLock() + rt, ok := m.relayClients[serverAddress] + if ok { + rt.RLock() + m.relayClientsMutex.RUnlock() + defer rt.RUnlock() + return rt.relayClient.OpenConn(peerKey) + } + m.relayClientsMutex.RUnlock() + + // if not, establish a new connection but check it again (because changed the lock type) before starting the + // connection + m.relayClientsMutex.Lock() + rt, ok = m.relayClients[serverAddress] + if ok { + rt.RLock() + m.relayClientsMutex.Unlock() + defer rt.RUnlock() + return rt.relayClient.OpenConn(peerKey) + } + + // create a new relay client and store it in the relayClients map + rt = NewRelayTrack() + rt.Lock() + m.relayClients[serverAddress] = rt + m.relayClientsMutex.Unlock() + + relayClient := NewClient(m.ctx, serverAddress, m.tokenStore, m.peerID) + err := relayClient.Connect() + if err != nil { + rt.Unlock() + m.relayClientsMutex.Lock() + delete(m.relayClients, serverAddress) + m.relayClientsMutex.Unlock() + return nil, err + } + // if connection closed then delete the relay client from the list + relayClient.SetOnDisconnectListener(func() { + m.onServerDisconnected(serverAddress) + }) + rt.relayClient = relayClient + rt.Unlock() + + conn, err := relayClient.OpenConn(peerKey) + if err != nil { + return nil, err + } + return conn, nil +} + +func (m *Manager) onServerDisconnected(serverAddress string) { + if serverAddress == m.relayClient.connectionURL { + go m.reconnectGuard.OnDisconnected() + } + + m.notifyOnDisconnectListeners(serverAddress) +} + +func (m *Manager) isForeignServer(address string) (bool, error) { + rAddr, err := m.relayClient.ServerInstanceURL() + if err != nil { + return false, fmt.Errorf("relay client not connected") + } + return rAddr != address, nil +} + +func (m *Manager) startCleanupLoop() { + if m.ctx.Err() != nil { + return + } + + ticker := time.NewTicker(relayCleanupInterval) + go func() { + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.cleanUpUnusedRelays() + } + } + }() +} + +func (m *Manager) cleanUpUnusedRelays() { + m.relayClientsMutex.Lock() + defer m.relayClientsMutex.Unlock() + + for addr, rt := range m.relayClients { + rt.Lock() + if rt.relayClient.HasConns() { + rt.Unlock() + continue + } + rt.relayClient.SetOnDisconnectListener(nil) + go func() { + _ = rt.relayClient.Close() + }() + log.Debugf("clean up unused relay server connection: %s", addr) + delete(m.relayClients, addr) + rt.Unlock() + } +} + +func (m *Manager) addListener(serverAddress string, onClosedListener OnServerCloseListener) { + m.listenerLock.Lock() + defer m.listenerLock.Unlock() + l, ok := m.onDisconnectedListeners[serverAddress] + if !ok { + l = list.New() + } + for e := l.Front(); e != nil; e = e.Next() { + if reflect.ValueOf(e.Value).Pointer() == reflect.ValueOf(onClosedListener).Pointer() { + return + } + } + l.PushBack(onClosedListener) + m.onDisconnectedListeners[serverAddress] = l +} + +func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { + m.listenerLock.Lock() + defer m.listenerLock.Unlock() + + l, ok := m.onDisconnectedListeners[serverAddress] + if !ok { + return + } + for e := l.Front(); e != nil; e = e.Next() { + go e.Value.(OnServerCloseListener)() + } + delete(m.onDisconnectedListeners, serverAddress) +} diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go new file mode 100644 index 000000000..e9cc2c581 --- /dev/null +++ b/relay/client/manager_test.go @@ -0,0 +1,432 @@ +package client + +import ( + "context" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/server" +) + +func TestEmptyURL(t *testing.T) { + mgr := NewManager(context.Background(), nil, "alice") + err := mgr.Serve() + if err == nil { + t.Errorf("expected error, got nil") + } +} + +func TestForeignConn(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + srvCfg2 := server.ListenerConfig{ + Address: "localhost:2234", + } + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan2 := make(chan error, 1) + go func() { + err := srv2.Listen(srvCfg2) + if err != nil { + errChan2 <- err + } + }() + + defer func() { + err := srv2.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + idBob := "bob" + log.Debugf("connect by bob") + clientBob := NewManager(mCtx, toURL(srvCfg2), idBob) + err = clientBob.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + bobsSrvAddr, err := clientBob.RelayInstanceAddress() + if err != nil { + t.Fatalf("failed to get relay address: %s", err) + } + connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + _, err = connBobToAlice.Write(buf[:n]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + n, err = connAliceToBob.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestForeginConnClose(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + srvCfg2 := server.ListenerConfig{ + Address: "localhost:2234", + } + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan2 := make(chan error, 1) + go func() { + err := srv2.Listen(srvCfg2) + if err != nil { + errChan2 <- err + } + }() + + defer func() { + err := srv2.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = mgr.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + conn, err := mgr.OpenConn(toURL(srvCfg2)[0], "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } +} + +func TestForeginAutoClose(t *testing.T) { + ctx := context.Background() + relayCleanupInterval = 1 * time.Second + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + t.Log("binding server 1.") + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + t.Logf("closing server 1.") + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + t.Logf("server 1. closed") + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + srvCfg2 := server.ListenerConfig{ + Address: "localhost:2234", + } + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan2 := make(chan error, 1) + go func() { + t.Log("binding server 2.") + err := srv2.Listen(srvCfg2) + if err != nil { + errChan2 <- err + } + }() + defer func() { + t.Logf("closing server 2.") + err := srv2.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + t.Logf("server 2 closed.") + }() + + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + t.Log("connect to server 1.") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = mgr.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + t.Log("open connection to another peer") + conn, err := mgr.OpenConn(toURL(srvCfg2)[0], "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + t.Log("close conn") + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } + + t.Logf("waiting for relay cleanup: %s", relayCleanupInterval+1*time.Second) + time.Sleep(relayCleanupInterval + 1*time.Second) + if len(mgr.relayClients) != 0 { + t.Errorf("expected 0, got %d", len(mgr.relayClients)) + } + + t.Logf("closing manager") +} + +func TestAutoReconnect(t *testing.T) { + ctx := context.Background() + reconnectingTimeout = 2 * time.Second + + srvCfg := server.ListenerConfig{ + Address: "localhost:1234", + } + srv, err := server.NewServer(otel.Meter(""), srvCfg.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg), "alice") + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + ra, err := clientAlice.RelayInstanceAddress() + if err != nil { + t.Errorf("failed to get relay address: %s", err) + } + conn, err := clientAlice.OpenConn(ra, "bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + t.Log("closing client relay connection") + // todo figure out moc server + _ = clientAlice.relayClient.relayConn.Close() + t.Log("start test reading") + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + log.Infof("waiting for reconnection") + time.Sleep(reconnectingTimeout + 1*time.Second) + + log.Infof("reopent the connection") + _, err = clientAlice.OpenConn(ra, "bob") + if err != nil { + t.Errorf("failed to open channel: %s", err) + } +} + +func TestNotifierDoubleAdd(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + conn1, err := clientAlice.OpenConn(clientAlice.ServerURLs()[0], "idBob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + fnCloseListener := OnServerCloseListener(func() { + log.Infof("close listener") + }) + + err = clientAlice.AddCloseListener(clientAlice.ServerURLs()[0], fnCloseListener) + if err != nil { + t.Fatalf("failed to add close listener: %s", err) + } + + err = clientAlice.AddCloseListener(clientAlice.ServerURLs()[0], fnCloseListener) + if err != nil { + t.Fatalf("failed to add close listener: %s", err) + } + + err = conn1.Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + +} + +func toURL(address server.ListenerConfig) []string { + return []string{"rel://" + address.Address} +} diff --git a/relay/cmd/env.go b/relay/cmd/env.go new file mode 100644 index 000000000..3c15ebe1f --- /dev/null +++ b/relay/cmd/env.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "os" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// setFlagsFromEnvVars reads and updates flag values from environment variables with prefix NB_ +func setFlagsFromEnvVars(cmd *cobra.Command) { + flags := cmd.PersistentFlags() + flags.VisitAll(func(f *pflag.Flag) { + newEnvVar := flagNameToEnvVar(f.Name, "NB_") + value, present := os.LookupEnv(newEnvVar) + if !present { + return + } + + err := flags.Set(f.Name, value) + if err != nil { + log.Infof("unable to configure flag %s using variable %s, err: %v", f.Name, newEnvVar, err) + } + }) +} + +// flagNameToEnvVar converts flag name to environment var name adding a prefix, +// replacing dashes and making all uppercase (e.g. setup-keys is converted to NB_SETUP_KEYS according to the input prefix) +func flagNameToEnvVar(cmdFlag string, prefix string) string { + parsed := strings.ReplaceAll(cmdFlag, "-", "_") + upper := strings.ToUpper(parsed) + return prefix + upper +} diff --git a/relay/cmd/root.go b/relay/cmd/root.go new file mode 100644 index 000000000..784b42c1a --- /dev/null +++ b/relay/cmd/root.go @@ -0,0 +1,214 @@ +package cmd + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/netbirdio/netbird/encryption" + auth "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/signal/metrics" + "github.com/netbirdio/netbird/util" +) + +const ( + metricsPort = 9090 +) + +type Config struct { + ListenAddress string + // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection + // it is a domain:port or ip:port + ExposedAddress string + LetsencryptEmail string + LetsencryptDataDir string + LetsencryptDomains []string + // in case of using Route 53 for DNS challenge the credentials should be provided in the environment variables or + // in the AWS credentials file + LetsencryptAWSRoute53 bool + TlsCertFile string + TlsKeyFile string + AuthSecret string + LogLevel string + LogFile string +} + +func (c Config) Validate() error { + if c.ExposedAddress == "" { + return fmt.Errorf("exposed address is required") + } + if c.AuthSecret == "" { + return fmt.Errorf("auth secret is required") + } + return nil +} + +func (c Config) HasCertConfig() bool { + return c.TlsCertFile != "" && c.TlsKeyFile != "" +} + +func (c Config) HasLetsEncrypt() bool { + return c.LetsencryptDataDir != "" && c.LetsencryptDomains != nil && len(c.LetsencryptDomains) > 0 +} + +var ( + cobraConfig *Config + rootCmd = &cobra.Command{ + Use: "relay", + Short: "Relay service", + Long: "Relay service for Netbird agents", + SilenceUsage: true, + SilenceErrors: true, + RunE: execute, + } +) + +func init() { + _ = util.InitLog("trace", "console") + cobraConfig = &Config{} + rootCmd.PersistentFlags().StringVarP(&cobraConfig.ListenAddress, "listen-address", "l", ":443", "listen address") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.ExposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.LetsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") + rootCmd.PersistentFlags().StringSliceVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration") + rootCmd.PersistentFlags().BoolVar(&cobraConfig.LetsencryptAWSRoute53, "letsencrypt-aws-route53", false, "use AWS Route 53 for Let's Encrypt DNS challenge") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsCertFile, "tls-cert-file", "c", "", "") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsKeyFile, "tls-key-file", "k", "", "") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "auth secret") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LogLevel, "log-level", "info", "log level") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LogFile, "log-file", "console", "log file") + + setFlagsFromEnvVars(rootCmd) +} + +func Execute() error { + return rootCmd.Execute() +} + +func waitForExitSignal() { + osSigs := make(chan os.Signal, 1) + signal.Notify(osSigs, syscall.SIGINT, syscall.SIGTERM) + <-osSigs +} + +func execute(cmd *cobra.Command, args []string) error { + err := cobraConfig.Validate() + if err != nil { + log.Debugf("invalid config: %s", err) + return fmt.Errorf("invalid config: %s", err) + } + + err = util.InitLog(cobraConfig.LogLevel, cobraConfig.LogFile) + if err != nil { + log.Debugf("failed to initialize log: %s", err) + return fmt.Errorf("failed to initialize log: %s", err) + } + + metricsServer, err := metrics.NewServer(metricsPort, "") + if err != nil { + log.Debugf("setup metrics: %v", err) + return fmt.Errorf("setup metrics: %v", err) + } + + go func() { + log.Infof("running metrics server: %s%s", metricsServer.Addr, metricsServer.Endpoint) + if err := metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Failed to start metrics server: %v", err) + } + }() + + srvListenerCfg := server.ListenerConfig{ + Address: cobraConfig.ListenAddress, + } + + tlsConfig, tlsSupport, err := handleTLSConfig(cobraConfig) + if err != nil { + log.Debugf("failed to setup TLS config: %s", err) + return fmt.Errorf("failed to setup TLS config: %s", err) + } + srvListenerCfg.TLSConfig = tlsConfig + + authenticator := auth.NewTimedHMACValidator(cobraConfig.AuthSecret, 24*time.Hour) + srv, err := server.NewServer(metricsServer.Meter, cobraConfig.ExposedAddress, tlsSupport, authenticator) + if err != nil { + log.Debugf("failed to create relay server: %v", err) + return fmt.Errorf("failed to create relay server: %v", err) + } + log.Infof("server will be available on: %s", srv.InstanceURL()) + go func() { + if err := srv.Listen(srvListenerCfg); err != nil { + log.Fatalf("failed to bind server: %s", err) + } + }() + + // it will block until exit signal + waitForExitSignal() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var shutDownErrors error + if err := srv.Shutdown(ctx); err != nil { + shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close server: %s", err)) + } + + log.Infof("shutting down metrics server") + if err := metricsServer.Shutdown(ctx); err != nil { + shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err)) + } + return shutDownErrors +} + +func handleTLSConfig(cfg *Config) (*tls.Config, bool, error) { + if cfg.LetsencryptAWSRoute53 { + log.Debugf("using Let's Encrypt DNS resolver with Route 53 support") + r53 := encryption.Route53TLS{ + DataDir: cfg.LetsencryptDataDir, + Email: cfg.LetsencryptEmail, + Domains: cfg.LetsencryptDomains, + } + tlsCfg, err := r53.GetCertificate() + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + + if cfg.HasLetsEncrypt() { + log.Infof("setting up TLS with Let's Encrypt.") + tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...) + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + + if cfg.HasCertConfig() { + log.Debugf("using file based TLS config") + tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile) + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + return nil, false, nil +} + +func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) { + certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...) + if err != nil { + return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err) + } + return certManager.TLSConfig(), nil +} diff --git a/relay/doc.go b/relay/doc.go new file mode 100644 index 000000000..56e010e3e --- /dev/null +++ b/relay/doc.go @@ -0,0 +1,14 @@ +//Package main +/* +The `relay` package contains the implementation of the Relay server and client. The Relay server can be used to relay +messages between peers on a single network channel. In this implementation the transport layer is the WebSocket +protocol. + +Between the server and client communication has been design a custom protocol and message format. These messages are +transported over the WebSocket connection. Optionally the server can use TLS to secure the communication. + +The service can support multiple Relay server instances. For this purpose the peers must know the server instance URL. +This URL will be sent to the target peer to choose the common Relay server for the communication via Signal service. + +*/ +package main diff --git a/relay/healthcheck/doc.go b/relay/healthcheck/doc.go new file mode 100644 index 000000000..da9689c6b --- /dev/null +++ b/relay/healthcheck/doc.go @@ -0,0 +1,17 @@ +/* +The `healthcheck` package is responsible for managing the health checks between the client and the relay server. It +ensures that the connection between the client and the server are alive and functioning properly. + +The `Sender` struct is responsible for sending health check signals to the receiver. The receiver listens for these +signals and sends a new signal back to the sender to acknowledge that the signal has been received. If the sender does +not receive an acknowledgment signal within a certain time frame, it will send a timeout signal via timeout channel +and stop working. + +The `Receiver` struct is responsible for receiving the health check signals from the sender. If the receiver does not +receive a signal within a certain time frame, it will send a timeout signal via the OnTimeout channel and stop working. + +In the Relay usage the signal is sent to the peer in message type Healthcheck. In case of timeout the connection is +closed and the peer is removed from the relay. +*/ + +package healthcheck diff --git a/relay/healthcheck/receiver.go b/relay/healthcheck/receiver.go new file mode 100644 index 000000000..2b9c9e2e0 --- /dev/null +++ b/relay/healthcheck/receiver.go @@ -0,0 +1,82 @@ +package healthcheck + +import ( + "context" + "time" +) + +var ( + heartbeatTimeout = healthCheckInterval + 3*time.Second +) + +// Receiver is a healthcheck receiver +// It will listen for heartbeat and check if the heartbeat is not received in a certain time +// If the heartbeat is not received in a certain time, it will send a timeout signal and stop to work +// The heartbeat timeout is a bit longer than the sender's healthcheck interval +type Receiver struct { + OnTimeout chan struct{} + + ctx context.Context + ctxCancel context.CancelFunc + heartbeat chan struct{} + alive bool +} + +// NewReceiver creates a new healthcheck receiver and start the timer in the background +func NewReceiver() *Receiver { + ctx, ctxCancel := context.WithCancel(context.Background()) + + r := &Receiver{ + OnTimeout: make(chan struct{}, 1), + ctx: ctx, + ctxCancel: ctxCancel, + heartbeat: make(chan struct{}, 1), + } + + go r.waitForHealthcheck() + return r +} + +// Heartbeat acknowledge the heartbeat has been received +func (r *Receiver) Heartbeat() { + select { + case r.heartbeat <- struct{}{}: + default: + } +} + +// Stop check the timeout and do not send new notifications +func (r *Receiver) Stop() { + r.ctxCancel() +} + +func (r *Receiver) waitForHealthcheck() { + ticker := time.NewTicker(heartbeatTimeout) + defer ticker.Stop() + defer r.ctxCancel() + defer close(r.OnTimeout) + + for { + select { + case <-r.heartbeat: + r.alive = true + case <-ticker.C: + if r.alive { + r.alive = false + continue + } + + r.notifyTimeout() + return + case <-r.ctx.Done(): + return + } + } +} + +func (r *Receiver) notifyTimeout() { + select { + case r.OnTimeout <- struct{}{}: + default: + } +} diff --git a/relay/healthcheck/receiver_test.go b/relay/healthcheck/receiver_test.go new file mode 100644 index 000000000..4b4123416 --- /dev/null +++ b/relay/healthcheck/receiver_test.go @@ -0,0 +1,42 @@ +package healthcheck + +import ( + "testing" + "time" +) + +func TestNewReceiver(t *testing.T) { + heartbeatTimeout = 5 * time.Second + r := NewReceiver() + + select { + case <-r.OnTimeout: + t.Error("unexpected timeout") + case <-time.After(1 * time.Second): + + } +} + +func TestNewReceiverNotReceive(t *testing.T) { + heartbeatTimeout = 1 * time.Second + r := NewReceiver() + + select { + case <-r.OnTimeout: + case <-time.After(2 * time.Second): + t.Error("timeout not received") + } +} + +func TestNewReceiverAck(t *testing.T) { + heartbeatTimeout = 2 * time.Second + r := NewReceiver() + + r.Heartbeat() + + select { + case <-r.OnTimeout: + t.Error("unexpected timeout") + case <-time.After(3 * time.Second): + } +} diff --git a/relay/healthcheck/sender.go b/relay/healthcheck/sender.go new file mode 100644 index 000000000..ec0560ef2 --- /dev/null +++ b/relay/healthcheck/sender.go @@ -0,0 +1,68 @@ +package healthcheck + +import ( + "context" + "time" +) + +var ( + healthCheckInterval = 25 * time.Second + healthCheckTimeout = 5 * time.Second +) + +// Sender is a healthcheck sender +// It will send healthcheck signal to the receiver +// If the receiver does not receive the signal in a certain time, it will send a timeout signal and stop to work +// It will also stop if the context is canceled +type Sender struct { + // HealthCheck is a channel to send health check signal to the peer + HealthCheck chan struct{} + // Timeout is a channel to the health check signal is not received in a certain time + Timeout chan struct{} + + ack chan struct{} +} + +// NewSender creates a new healthcheck sender +func NewSender() *Sender { + hc := &Sender{ + HealthCheck: make(chan struct{}, 1), + Timeout: make(chan struct{}, 1), + ack: make(chan struct{}, 1), + } + + return hc +} + +// OnHCResponse sends an acknowledgment signal to the sender +func (hc *Sender) OnHCResponse() { + select { + case hc.ack <- struct{}{}: + default: + } +} + +func (hc *Sender) StartHealthCheck(ctx context.Context) { + ticker := time.NewTicker(healthCheckInterval) + defer ticker.Stop() + + timeoutTimer := time.NewTimer(healthCheckInterval + healthCheckTimeout) + defer timeoutTimer.Stop() + + defer close(hc.HealthCheck) + defer close(hc.Timeout) + + for { + select { + case <-ticker.C: + hc.HealthCheck <- struct{}{} + case <-timeoutTimer.C: + hc.Timeout <- struct{}{} + return + case <-hc.ack: + timeoutTimer.Reset(healthCheckInterval + healthCheckTimeout) + case <-ctx.Done(): + return + } + } +} diff --git a/relay/healthcheck/sender_test.go b/relay/healthcheck/sender_test.go new file mode 100644 index 000000000..7a105c308 --- /dev/null +++ b/relay/healthcheck/sender_test.go @@ -0,0 +1,103 @@ +package healthcheck + +import ( + "context" + "os" + "testing" + "time" +) + +func TestMain(m *testing.M) { + // override the health check interval to speed up the test + healthCheckInterval = 2 * time.Second + healthCheckTimeout = 100 * time.Millisecond + code := m.Run() + os.Exit(code) +} + +func TestNewHealthPeriod(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender() + go hc.StartHealthCheck(ctx) + + iterations := 0 + for i := 0; i < 3; i++ { + select { + case <-hc.HealthCheck: + iterations++ + hc.OnHCResponse() + case <-hc.Timeout: + t.Fatalf("health check is timed out") + case <-time.After(healthCheckInterval + 100*time.Millisecond): + t.Fatalf("health check not received") + } + } +} + +func TestNewHealthFailed(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender() + go hc.StartHealthCheck(ctx) + + select { + case <-hc.Timeout: + case <-time.After(healthCheckInterval + healthCheckTimeout + 100*time.Millisecond): + t.Fatalf("health check is not timed out") + } +} + +func TestNewHealthcheckStop(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + hc := NewSender() + go hc.StartHealthCheck(ctx) + + time.Sleep(100 * time.Millisecond) + cancel() + + select { + case _, ok := <-hc.HealthCheck: + if ok { + t.Fatalf("health check on received") + } + case _, ok := <-hc.Timeout: + if ok { + t.Fatalf("health check on received") + } + case <-ctx.Done(): + // expected + case <-time.After(10 * time.Second): + t.Fatalf("is not exited") + } +} + +func TestTimeoutReset(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender() + go hc.StartHealthCheck(ctx) + + iterations := 0 + for i := 0; i < 3; i++ { + select { + case <-hc.HealthCheck: + iterations++ + hc.OnHCResponse() + case <-hc.Timeout: + t.Fatalf("health check is timed out") + case <-time.After(healthCheckInterval + 100*time.Millisecond): + t.Fatalf("health check not received") + } + } + + select { + case <-hc.HealthCheck: + case <-hc.Timeout: + // expected + case <-ctx.Done(): + t.Fatalf("context is done") + case <-time.After(10 * time.Second): + t.Fatalf("is not exited") + } +} diff --git a/relay/main.go b/relay/main.go new file mode 100644 index 000000000..e28f73603 --- /dev/null +++ b/relay/main.go @@ -0,0 +1,13 @@ +package main + +import ( + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/cmd" +) + +func main() { + if err := cmd.Execute(); err != nil { + log.Fatalf("failed to execute command: %v", err) + } +} diff --git a/relay/messages/address/address.go b/relay/messages/address/address.go new file mode 100644 index 000000000..829206294 --- /dev/null +++ b/relay/messages/address/address.go @@ -0,0 +1,30 @@ +package address + +import ( + "bytes" + "encoding/gob" + "fmt" +) + +type Address struct { + URL string +} + +func (addr *Address) Marshal() ([]byte, error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + if err := enc.Encode(addr); err != nil { + return nil, fmt.Errorf("encode Address: %w", err) + } + return buf.Bytes(), nil +} + +func Unmarshal(data []byte) (*Address, error) { + var addr Address + buf := bytes.NewBuffer(data) + dec := gob.NewDecoder(buf) + if err := dec.Decode(&addr); err != nil { + return nil, fmt.Errorf("decode Address: %w", err) + } + return &addr, nil +} diff --git a/relay/messages/auth/auth.go b/relay/messages/auth/auth.go new file mode 100644 index 000000000..8230bccf2 --- /dev/null +++ b/relay/messages/auth/auth.go @@ -0,0 +1,51 @@ +package auth + +import ( + "bytes" + "encoding/gob" + "fmt" +) + +type Algorithm int + +const ( + AlgoUnknown Algorithm = iota + AlgoHMACSHA256 + AlgoHMACSHA512 +) + +func (a Algorithm) String() string { + switch a { + case AlgoHMACSHA256: + return "HMAC-SHA256" + case AlgoHMACSHA512: + return "HMAC-SHA512" + default: + return "Unknown" + } +} + +type Msg struct { + AuthAlgorithm Algorithm + AdditionalData []byte +} + +func (msg *Msg) Marshal() ([]byte, error) { + var buf bytes.Buffer + enc := gob.NewEncoder(&buf) + if err := enc.Encode(msg); err != nil { + return nil, fmt.Errorf("encode Msg: %w", err) + } + return buf.Bytes(), nil +} + +func UnmarshalMsg(data []byte) (*Msg, error) { + var msg *Msg + + buf := bytes.NewBuffer(data) + dec := gob.NewDecoder(buf) + if err := dec.Decode(&msg); err != nil { + return nil, fmt.Errorf("decode Msg: %w", err) + } + return msg, nil +} diff --git a/relay/messages/doc.go b/relay/messages/doc.go new file mode 100644 index 000000000..4c719df3a --- /dev/null +++ b/relay/messages/doc.go @@ -0,0 +1,5 @@ +/* +Package messages provides the message types that are used to communicate between the relay and the client. +This package is used to determine the type of message that is being sent and received between the relay and the client. +*/ +package messages diff --git a/relay/messages/id.go b/relay/messages/id.go new file mode 100644 index 000000000..e2162cd3b --- /dev/null +++ b/relay/messages/id.go @@ -0,0 +1,31 @@ +package messages + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" +) + +const ( + prefixLength = 4 + IDSize = prefixLength + sha256.Size +) + +var ( + prefix = []byte("sha-") // 4 bytes +) + +// HashID generates a sha256 hash from the peerID and returns the hash and the human-readable string +func HashID(peerID string) ([]byte, string) { + idHash := sha256.Sum256([]byte(peerID)) + idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:]) + var prefixedHash []byte + prefixedHash = append(prefixedHash, prefix...) + prefixedHash = append(prefixedHash, idHash[:]...) + return prefixedHash, idHashString +} + +// HashIDToString converts a hash to a human-readable string +func HashIDToString(idHash []byte) string { + return fmt.Sprintf("%s%s", idHash[:prefixLength], base64.StdEncoding.EncodeToString(idHash[prefixLength:])) +} diff --git a/relay/messages/id_test.go b/relay/messages/id_test.go new file mode 100644 index 000000000..271a8f90d --- /dev/null +++ b/relay/messages/id_test.go @@ -0,0 +1,13 @@ +package messages + +import ( + "testing" +) + +func TestHashID(t *testing.T) { + hashedID, hashedStringId := HashID("alice") + enc := HashIDToString(hashedID) + if enc != hashedStringId { + t.Errorf("expected %s, got %s", hashedStringId, enc) + } +} diff --git a/relay/messages/message.go b/relay/messages/message.go new file mode 100644 index 000000000..cfcac3f72 --- /dev/null +++ b/relay/messages/message.go @@ -0,0 +1,239 @@ +package messages + +import ( + "bytes" + "errors" + "fmt" +) + +const ( + MsgTypeUnknown MsgType = 0 + MsgTypeHello MsgType = 1 + MsgTypeHelloResponse MsgType = 2 + MsgTypeTransport MsgType = 3 + MsgTypeClose MsgType = 4 + MsgTypeHealthCheck MsgType = 5 + + SizeOfVersionByte = 1 + SizeOfMsgType = 1 + + SizeOfProtoHeader = SizeOfVersionByte + SizeOfMsgType + + sizeOfMagicByte = 4 + + headerSizeTransport = IDSize + headerSizeHello = sizeOfMagicByte + IDSize + headerSizeHelloResp = 0 + + MaxHandshakeSize = 8192 + + CurrentProtocolVersion = 1 +) + +var ( + ErrInvalidMessageLength = errors.New("invalid message length") + ErrUnsupportedVersion = errors.New("unsupported version") + + magicHeader = []byte{0x21, 0x12, 0xA4, 0x42} + + healthCheckMsg = []byte{byte(CurrentProtocolVersion), byte(MsgTypeHealthCheck)} +) + +type MsgType byte + +func (m MsgType) String() string { + switch m { + case MsgTypeHello: + return "hello" + case MsgTypeHelloResponse: + return "hello response" + case MsgTypeTransport: + return "transport" + case MsgTypeClose: + return "close" + case MsgTypeHealthCheck: + return "health check" + default: + return "unknown" + } +} + +type HelloResponse struct { + InstanceAddress string +} + +// ValidateVersion checks if the given version is supported by the protocol +func ValidateVersion(msg []byte) (int, error) { + if len(msg) < SizeOfVersionByte { + return 0, ErrInvalidMessageLength + } + version := int(msg[0]) + if version != CurrentProtocolVersion { + return 0, fmt.Errorf("%d: %w", version, ErrUnsupportedVersion) + } + return version, nil +} + +// DetermineClientMessageType determines the message type from the first the message +func DetermineClientMessageType(msg []byte) (MsgType, error) { + if len(msg) < SizeOfMsgType { + return 0, ErrInvalidMessageLength + } + + msgType := MsgType(msg[0]) + switch msgType { + case + MsgTypeHello, + MsgTypeTransport, + MsgTypeClose, + MsgTypeHealthCheck: + return msgType, nil + default: + return MsgTypeUnknown, fmt.Errorf("invalid msg type %d", msgType) + } +} + +// DetermineServerMessageType determines the message type from the first the message +func DetermineServerMessageType(msg []byte) (MsgType, error) { + if len(msg) < SizeOfMsgType { + return 0, ErrInvalidMessageLength + } + + msgType := MsgType(msg[0]) + switch msgType { + case + MsgTypeHelloResponse, + MsgTypeTransport, + MsgTypeClose, + MsgTypeHealthCheck: + return msgType, nil + default: + return MsgTypeUnknown, fmt.Errorf("invalid msg type %d", msgType) + } +} + +// MarshalHelloMsg initial hello message +// The Hello message is the first message sent by a client after establishing a connection with the Relay server. This +// message is used to authenticate the client with the server. The authentication is done using an HMAC method. +// The protocol does not limit to use HMAC, it can be any other method. If the authentication failed the server will +// close the network connection without any response. +func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { + if len(peerID) != IDSize { + return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) + } + + msg := make([]byte, SizeOfProtoHeader+sizeOfMagicByte, SizeOfProtoHeader+headerSizeHello+len(additions)) + + msg[0] = byte(CurrentProtocolVersion) + msg[1] = byte(MsgTypeHello) + + copy(msg[SizeOfProtoHeader:SizeOfProtoHeader+sizeOfMagicByte], magicHeader) + + msg = append(msg, peerID...) + msg = append(msg, additions...) + + return msg, nil +} + +// UnmarshalHelloMsg extracts peerID and the additional data from the hello message. The Additional data is used to +// authenticate the client with the server. +func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { + if len(msg) < headerSizeHello { + return nil, nil, ErrInvalidMessageLength + } + if !bytes.Equal(msg[:sizeOfMagicByte], magicHeader) { + return nil, nil, errors.New("invalid magic header") + } + + return msg[sizeOfMagicByte:headerSizeHello], msg[headerSizeHello:], nil +} + +// MarshalHelloResponse creates a response message to the hello message. +// In case of success connection the server response with a Hello Response message. This message contains the server's +// instance URL. This URL will be used by choose the common Relay server in case if the peers are in different Relay +// servers. +func MarshalHelloResponse(additionalData []byte) ([]byte, error) { + msg := make([]byte, SizeOfProtoHeader, SizeOfProtoHeader+headerSizeHelloResp+len(additionalData)) + + msg[0] = byte(CurrentProtocolVersion) + msg[1] = byte(MsgTypeHelloResponse) + + msg = append(msg, additionalData...) + + return msg, nil +} + +// UnmarshalHelloResponse extracts the additional data from the hello response message. +func UnmarshalHelloResponse(msg []byte) ([]byte, error) { + if len(msg) < headerSizeHelloResp { + return nil, ErrInvalidMessageLength + } + return msg, nil +} + +// MarshalCloseMsg creates a close message. +// The close message is used to close the connection gracefully between the client and the server. The server and the +// client can send this message. After receiving this message, the server or client will close the connection. +func MarshalCloseMsg() []byte { + msg := make([]byte, SizeOfProtoHeader) + + msg[0] = byte(CurrentProtocolVersion) + msg[1] = byte(MsgTypeClose) + + return msg +} + +// MarshalTransportMsg creates a transport message. +// The transport message is used to exchange data between peers. The message contains the data to be exchanged and the +// destination peer hashed ID. +func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) { + if len(peerID) != IDSize { + return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) + } + + msg := make([]byte, SizeOfProtoHeader+headerSizeTransport, SizeOfProtoHeader+headerSizeTransport+len(payload)) + + msg[0] = byte(CurrentProtocolVersion) + msg[1] = byte(MsgTypeTransport) + + copy(msg[SizeOfProtoHeader:], peerID) + + msg = append(msg, payload...) + + return msg, nil +} + +// UnmarshalTransportMsg extracts the peerID and the payload from the transport message. +func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) { + if len(buf) < headerSizeTransport { + return nil, nil, ErrInvalidMessageLength + } + + return buf[:headerSizeTransport], buf[headerSizeTransport:], nil +} + +// UnmarshalTransportID extracts the peerID from the transport message. +func UnmarshalTransportID(buf []byte) ([]byte, error) { + if len(buf) < headerSizeTransport { + return nil, ErrInvalidMessageLength + } + return buf[:headerSizeTransport], nil +} + +// UpdateTransportMsg updates the peerID in the transport message. +// With this function the server can reuse the given byte slice to update the peerID in the transport message. So do +// need to allocate a new byte slice. +func UpdateTransportMsg(msg []byte, peerID []byte) error { + if len(msg) < len(peerID) { + return ErrInvalidMessageLength + } + copy(msg, peerID) + return nil +} + +// MarshalHealthcheck creates a health check message. +// Health check message is sent by the server periodically. The client will respond with a health check response +// message. If the client does not respond to the health check message, the server will close the connection. +func MarshalHealthcheck() []byte { + return healthCheckMsg +} diff --git a/relay/messages/message_test.go b/relay/messages/message_test.go new file mode 100644 index 000000000..a4e7d9fae --- /dev/null +++ b/relay/messages/message_test.go @@ -0,0 +1,43 @@ +package messages + +import ( + "testing" +) + +func TestMarshalHelloMsg(t *testing.T) { + peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") + bHello, err := MarshalHelloMsg(peerID, nil) + if err != nil { + t.Fatalf("error: %v", err) + } + + receivedPeerID, _, err := UnmarshalHelloMsg(bHello[SizeOfProtoHeader:]) + if err != nil { + t.Fatalf("error: %v", err) + } + if string(receivedPeerID) != string(peerID) { + t.Errorf("expected %s, got %s", peerID, receivedPeerID) + } +} + +func TestMarshalTransportMsg(t *testing.T) { + peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") + payload := []byte("payload") + msg, err := MarshalTransportMsg(peerID, payload) + if err != nil { + t.Fatalf("error: %v", err) + } + + id, respPayload, err := UnmarshalTransportMsg(msg[SizeOfProtoHeader:]) + if err != nil { + t.Fatalf("error: %v", err) + } + + if string(id) != string(peerID) { + t.Errorf("expected %s, got %s", peerID, id) + } + + if string(respPayload) != string(payload) { + t.Errorf("expected %s, got %s", payload, respPayload) + } +} diff --git a/relay/metrics/realy.go b/relay/metrics/realy.go new file mode 100644 index 000000000..80e12ee6b --- /dev/null +++ b/relay/metrics/realy.go @@ -0,0 +1,136 @@ +package metrics + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" +) + +const ( + idleTimeout = 30 * time.Second +) + +type Metrics struct { + metric.Meter + + TransferBytesSent metric.Int64Counter + TransferBytesRecv metric.Int64Counter + + peers metric.Int64UpDownCounter + peerActivityChan chan string + peerLastActive map[string]time.Time + mutexActivity sync.Mutex + ctx context.Context +} + +func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) { + bytesSent, err := meter.Int64Counter("relay_transfer_sent_bytes_total") + if err != nil { + return nil, err + } + + bytesRecv, err := meter.Int64Counter("relay_transfer_received_bytes_total") + if err != nil { + return nil, err + } + + peers, err := meter.Int64UpDownCounter("relay_peers") + if err != nil { + return nil, err + } + + peersActive, err := meter.Int64ObservableGauge("relay_peers_active") + if err != nil { + return nil, err + } + + peersIdle, err := meter.Int64ObservableGauge("relay_peers_idle") + if err != nil { + return nil, err + } + + m := &Metrics{ + Meter: meter, + TransferBytesSent: bytesSent, + TransferBytesRecv: bytesRecv, + peers: peers, + + ctx: ctx, + peerActivityChan: make(chan string, 10), + peerLastActive: make(map[string]time.Time), + } + + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + active, idle := m.calculateActiveIdleConnections() + o.ObserveInt64(peersActive, active) + o.ObserveInt64(peersIdle, idle) + return nil + }, + peersActive, peersIdle, + ) + if err != nil { + return nil, err + } + + go m.readPeerActivity() + return m, nil +} + +// PeerConnected increments the number of connected peers and increments number of idle connections +func (m *Metrics) PeerConnected(id string) { + m.peers.Add(m.ctx, 1) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + m.peerLastActive[id] = time.Time{} +} + +// PeerDisconnected decrements the number of connected peers and decrements number of idle or active connections +func (m *Metrics) PeerDisconnected(id string) { + m.peers.Add(m.ctx, -1) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + delete(m.peerLastActive, id) +} + +// PeerActivity increases the active connections +func (m *Metrics) PeerActivity(peerID string) { + select { + case m.peerActivityChan <- peerID: + default: + log.Errorf("peer activity channel is full, dropping activity metrics for peer %s", peerID) + } +} + +func (m *Metrics) calculateActiveIdleConnections() (int64, int64) { + active, idle := int64(0), int64(0) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + for _, lastActive := range m.peerLastActive { + if time.Since(lastActive) > idleTimeout { + idle++ + } else { + active++ + } + } + return active, idle +} + +func (m *Metrics) readPeerActivity() { + for { + select { + case peerID := <-m.peerActivityChan: + m.mutexActivity.Lock() + m.peerLastActive[peerID] = time.Now() + m.mutexActivity.Unlock() + case <-m.ctx.Done(): + return + } + } +} diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go new file mode 100644 index 000000000..535c8bcd9 --- /dev/null +++ b/relay/server/listener/listener.go @@ -0,0 +1,11 @@ +package listener + +import ( + "context" + "net" +) + +type Listener interface { + Listen(func(conn net.Conn)) error + Shutdown(ctx context.Context) error +} diff --git a/relay/server/listener/ws/conn.go b/relay/server/listener/ws/conn.go new file mode 100644 index 000000000..c248963b9 --- /dev/null +++ b/relay/server/listener/ws/conn.go @@ -0,0 +1,114 @@ +package ws + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" +) + +const ( + writeTimeout = 10 * time.Second +) + +type Conn struct { + *websocket.Conn + lAddr *net.TCPAddr + rAddr *net.TCPAddr + + closed bool + closedMu sync.Mutex + ctx context.Context +} + +func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn { + return &Conn{ + Conn: wsConn, + lAddr: lAddr, + rAddr: rAddr, + ctx: context.Background(), + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, r, err := c.Reader(c.ctx) + if err != nil { + return 0, c.ioErrHandling(err) + } + + if t != websocket.MessageBinary { + log.Errorf("unexpected message type: %d", t) + return 0, fmt.Errorf("unexpected message type") + } + + n, err = r.Read(b) + if err != nil { + return 0, c.ioErrHandling(err) + } + return n, err +} + +// Write writes a binary message with the given payload. +// It does not block until fill the internal buffer. +// If the buffer filled up, wait until the buffer is drained or timeout. +func (c *Conn) Write(b []byte) (int, error) { + ctx, ctxCancel := context.WithTimeout(c.ctx, writeTimeout) + defer ctxCancel() + + err := c.Conn.Write(ctx, websocket.MessageBinary, b) + return len(b), err +} + +func (c *Conn) LocalAddr() net.Addr { + return c.lAddr +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.rAddr +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + return fmt.Errorf("SetReadDeadline is not implemented") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + return fmt.Errorf("SetWriteDeadline is not implemented") +} + +func (c *Conn) SetDeadline(t time.Time) error { + return fmt.Errorf("SetDeadline is not implemented") +} + +func (c *Conn) Close() error { + c.closedMu.Lock() + c.closed = true + c.closedMu.Unlock() + return c.Conn.CloseNow() +} + +func (c *Conn) isClosed() bool { + c.closedMu.Lock() + defer c.closedMu.Unlock() + return c.closed +} + +func (c *Conn) ioErrHandling(err error) error { + if c.isClosed() { + return io.EOF + } + + var wErr *websocket.CloseError + if !errors.As(err, &wErr) { + return err + } + if wErr.Code == websocket.StatusNormalClosure { + return io.EOF + } + return err +} diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go new file mode 100644 index 000000000..10bfbe44d --- /dev/null +++ b/relay/server/listener/ws/listener.go @@ -0,0 +1,92 @@ +package ws + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" +) + +// URLPath is the path for the websocket connection. +const URLPath = "/relay" + +type Listener struct { + // Address is the address to listen on. + Address string + // TLSConfig is the TLS configuration for the server. + TLSConfig *tls.Config + + server *http.Server + acceptFn func(conn net.Conn) +} + +func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { + l.acceptFn = acceptFn + mux := http.NewServeMux() + mux.HandleFunc(URLPath, l.onAccept) + + l.server = &http.Server{ + Addr: l.Address, + Handler: mux, + TLSConfig: l.TLSConfig, + } + + log.Infof("WS server listening address: %s", l.Address) + var err error + if l.TLSConfig != nil { + err = l.server.ListenAndServeTLS("", "") + } else { + err = l.server.ListenAndServe() + } + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func (l *Listener) Shutdown(ctx context.Context) error { + if l.server == nil { + return nil + } + + log.Infof("stop WS listener") + if err := l.server.Shutdown(ctx); err != nil { + return fmt.Errorf("server shutdown failed: %v", err) + } + log.Infof("WS listener stopped") + return nil +} + +func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { + wsConn, err := websocket.Accept(w, r, nil) + if err != nil { + log.Errorf("failed to accept ws connection from %s: %s", r.RemoteAddr, err) + return + } + + rAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr) + if err != nil { + err = wsConn.Close(websocket.StatusInternalError, "internal error") + if err != nil { + log.Errorf("failed to close ws connection: %s", err) + } + return + } + + lAddr, err := net.ResolveTCPAddr("tcp", l.server.Addr) + if err != nil { + err = wsConn.Close(websocket.StatusInternalError, "internal error") + if err != nil { + log.Errorf("failed to close ws connection: %s", err) + } + return + } + + conn := NewConn(wsConn, lAddr, rAddr) + l.acceptFn(conn) +} diff --git a/relay/server/peer.go b/relay/server/peer.go new file mode 100644 index 000000000..a9583700a --- /dev/null +++ b/relay/server/peer.go @@ -0,0 +1,203 @@ +package server + +import ( + "context" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/healthcheck" + "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/metrics" +) + +const ( + bufferSize = 8820 +) + +// Peer represents a peer connection +type Peer struct { + metrics *metrics.Metrics + log *log.Entry + idS string + idB []byte + conn net.Conn + connMu sync.RWMutex + store *Store +} + +// NewPeer creates a new Peer instance and prepare custom logging +func NewPeer(metrics *metrics.Metrics, id []byte, conn net.Conn, store *Store) *Peer { + stringID := messages.HashIDToString(id) + return &Peer{ + metrics: metrics, + log: log.WithField("peer_id", stringID), + idS: stringID, + idB: id, + conn: conn, + store: store, + } +} + +// Work reads data from the connection +// It manages the protocol (healthcheck, transport, close). Read the message and determine the message type and handle +// the message accordingly. +func (p *Peer) Work() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hc := healthcheck.NewSender() + go hc.StartHealthCheck(ctx) + go p.handleHealthcheckEvents(ctx, hc) + + buf := make([]byte, bufferSize) + for { + n, err := p.conn.Read(buf) + if err != nil { + if err != io.EOF { + p.log.Errorf("failed to read message: %s", err) + } + return + } + + if n == 0 { + p.log.Errorf("received empty message") + return + } + + msg := buf[:n] + + _, err = messages.ValidateVersion(msg) + if err != nil { + p.log.Warnf("failed to validate protocol version: %s", err) + return + } + + msgType, err := messages.DetermineClientMessageType(msg[messages.SizeOfVersionByte:]) + if err != nil { + p.log.Errorf("failed to determine message type: %s", err) + return + } + + p.handleMsgType(ctx, msgType, hc, n, msg) + } +} + +func (p *Peer) handleMsgType(ctx context.Context, msgType messages.MsgType, hc *healthcheck.Sender, n int, msg []byte) { + switch msgType { + case messages.MsgTypeHealthCheck: + hc.OnHCResponse() + case messages.MsgTypeTransport: + p.metrics.TransferBytesRecv.Add(ctx, int64(n)) + p.metrics.PeerActivity(p.String()) + p.handleTransportMsg(msg) + case messages.MsgTypeClose: + p.log.Infof("peer exited gracefully") + if err := p.conn.Close(); err != nil { + log.Errorf("failed to close connection to peer: %s", err) + } + default: + p.log.Warnf("received unexpected message type: %s", msgType) + } +} + +// Write writes data to the connection +func (p *Peer) Write(b []byte) (int, error) { + p.connMu.RLock() + defer p.connMu.RUnlock() + return p.conn.Write(b) +} + +// CloseGracefully closes the connection with the peer gracefully. Send a close message to the client and close the +// connection. +func (p *Peer) CloseGracefully(ctx context.Context) { + p.connMu.Lock() + err := p.writeWithTimeout(ctx, messages.MarshalCloseMsg()) + if err != nil { + p.log.Errorf("failed to send close message to peer: %s", p.String()) + } + + err = p.conn.Close() + if err != nil { + p.log.Errorf("failed to close connection to peer: %s", err) + } + + defer p.connMu.Unlock() +} + +// String returns the peer ID +func (p *Peer) String() string { + return p.idS +} + +func (p *Peer) writeWithTimeout(ctx context.Context, buf []byte) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + writeDone := make(chan struct{}) + var err error + go func() { + _, err = p.conn.Write(buf) + close(writeDone) + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-writeDone: + return err + } +} + +func (p *Peer) handleHealthcheckEvents(ctx context.Context, hc *healthcheck.Sender) { + for { + select { + case <-hc.HealthCheck: + _, err := p.Write(messages.MarshalHealthcheck()) + if err != nil { + p.log.Errorf("failed to send healthcheck message: %s", err) + return + } + case <-hc.Timeout: + p.log.Errorf("peer healthcheck timeout") + err := p.conn.Close() + if err != nil { + p.log.Errorf("failed to close connection to peer: %s", err) + } + return + case <-ctx.Done(): + return + } + } +} + +func (p *Peer) handleTransportMsg(msg []byte) { + peerID, err := messages.UnmarshalTransportID(msg[messages.SizeOfProtoHeader:]) + if err != nil { + p.log.Errorf("failed to unmarshal transport message: %s", err) + return + } + + stringPeerID := messages.HashIDToString(peerID) + dp, ok := p.store.Peer(stringPeerID) + if !ok { + p.log.Errorf("peer not found: %s", stringPeerID) + return + } + + err = messages.UpdateTransportMsg(msg[messages.SizeOfProtoHeader:], p.idB) + if err != nil { + p.log.Errorf("failed to update transport message: %s", err) + return + } + + n, err := dp.Write(msg) + if err != nil { + p.log.Errorf("failed to write transport message to: %s", dp.String()) + return + } + p.metrics.TransferBytesSent.Add(context.Background(), int64(n)) +} diff --git a/relay/server/relay.go b/relay/server/relay.go new file mode 100644 index 000000000..6d88cbbb2 --- /dev/null +++ b/relay/server/relay.go @@ -0,0 +1,206 @@ +package server + +import ( + "context" + "crypto/sha256" + "fmt" + "net" + "net/url" + "strings" + "sync" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/messages/address" + authmsg "github.com/netbirdio/netbird/relay/messages/auth" + "github.com/netbirdio/netbird/relay/metrics" +) + +// Relay represents the relay server +type Relay struct { + metrics *metrics.Metrics + metricsCancel context.CancelFunc + validator auth.Validator + + store *Store + instanceURL string + + closed bool + closeMu sync.RWMutex +} + +// NewRelay creates a new Relay instance +// +// Parameters: +// meter: An instance of metric.Meter from the go.opentelemetry.io/otel/metric package. It is used to create and manage +// metrics for the relay server. +// exposedAddress: A string representing the address that the relay server is exposed on. The client will use this +// address as the relay server's instance URL. +// tlsSupport: A boolean indicating whether the relay server supports TLS (Transport Layer Security) or not. The +// instance URL depends on this value. +// validator: An instance of auth.Validator from the auth package. It is used to validate the authentication of the +// peers. +// +// Returns: +// A pointer to a Relay instance and an error. If the Relay instance is successfully created, the error is nil. +// Otherwise, the error contains the details of what went wrong. +func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, validator auth.Validator) (*Relay, error) { + ctx, metricsCancel := context.WithCancel(context.Background()) + m, err := metrics.NewMetrics(ctx, meter) + if err != nil { + metricsCancel() + return nil, fmt.Errorf("creating app metrics: %v", err) + } + + r := &Relay{ + metrics: m, + metricsCancel: metricsCancel, + validator: validator, + store: NewStore(), + } + + r.instanceURL, err = getInstanceURL(exposedAddress, tlsSupport) + if err != nil { + metricsCancel() + return nil, fmt.Errorf("get instance URL: %v", err) + } + + return r, nil +} + +// getInstanceURL checks if user supplied a URL scheme otherwise adds to the +// provided address according to TLS definition and parses the address before returning it +func getInstanceURL(exposedAddress string, tlsSupported bool) (string, error) { + addr := exposedAddress + split := strings.Split(exposedAddress, "://") + switch { + case len(split) == 1 && tlsSupported: + addr = "rels://" + exposedAddress + case len(split) == 1 && !tlsSupported: + addr = "rel://" + exposedAddress + case len(split) > 2: + return "", fmt.Errorf("invalid exposed address: %s", exposedAddress) + } + + parsedURL, err := url.ParseRequestURI(addr) + if err != nil { + return "", fmt.Errorf("invalid exposed address: %v", err) + } + + if parsedURL.Scheme != "rel" && parsedURL.Scheme != "rels" { + return "", fmt.Errorf("invalid scheme: %s", parsedURL.Scheme) + } + + return parsedURL.String(), nil +} + +// Accept start to handle a new peer connection +func (r *Relay) Accept(conn net.Conn) { + r.closeMu.RLock() + defer r.closeMu.RUnlock() + if r.closed { + return + } + + peerID, err := r.handshake(conn) + if err != nil { + log.Errorf("failed to handshake: %s", err) + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr) + } + return + } + + peer := NewPeer(r.metrics, peerID, conn, r.store) + peer.log.Infof("peer connected from: %s", conn.RemoteAddr()) + r.store.AddPeer(peer) + r.metrics.PeerConnected(peer.String()) + go func() { + peer.Work() + r.store.DeletePeer(peer) + peer.log.Debugf("relay connection closed") + r.metrics.PeerDisconnected(peer.String()) + }() +} + +// Shutdown closes the relay server +// It closes the connection with all peers in gracefully and stops accepting new connections. +func (r *Relay) Shutdown(ctx context.Context) { + log.Infof("close connection with all peers") + r.closeMu.Lock() + wg := sync.WaitGroup{} + peers := r.store.Peers() + for _, peer := range peers { + wg.Add(1) + go func(p *Peer) { + p.CloseGracefully(ctx) + wg.Done() + }(peer) + } + wg.Wait() + r.metricsCancel() + r.closeMu.Unlock() +} + +// InstanceURL returns the instance URL of the relay server +func (r *Relay) InstanceURL() string { + return r.instanceURL +} + +func (r *Relay) handshake(conn net.Conn) ([]byte, error) { + buf := make([]byte, messages.MaxHandshakeSize) + n, err := conn.Read(buf) + if err != nil { + return nil, fmt.Errorf("read from %s: %w", conn.RemoteAddr(), err) + } + + _, err = messages.ValidateVersion(buf[:n]) + if err != nil { + return nil, fmt.Errorf("validate version from %s: %w", conn.RemoteAddr(), err) + } + + msgType, err := messages.DetermineClientMessageType(buf[messages.SizeOfVersionByte:n]) + if err != nil { + return nil, fmt.Errorf("determine message type from %s: %w", conn.RemoteAddr(), err) + } + + if msgType != messages.MsgTypeHello { + return nil, fmt.Errorf("invalid message type from %s", conn.RemoteAddr()) + } + + peerID, authData, err := messages.UnmarshalHelloMsg(buf[messages.SizeOfProtoHeader:n]) + if err != nil { + return nil, fmt.Errorf("unmarshal hello message: %w", err) + } + + authMsg, err := authmsg.UnmarshalMsg(authData) + if err != nil { + return nil, fmt.Errorf("unmarshal auth message: %w", err) + } + + if err := r.validator.Validate(sha256.New, authMsg.AdditionalData); err != nil { + return nil, fmt.Errorf("validate %s (%s): %w", peerID, conn.RemoteAddr(), err) + } + + addr := &address.Address{URL: r.instanceURL} + addrData, err := addr.Marshal() + if err != nil { + return nil, fmt.Errorf("marshal addressc to %s (%s): %w", peerID, conn.RemoteAddr(), err) + } + + msg, err := messages.MarshalHelloResponse(addrData) + if err != nil { + return nil, fmt.Errorf("marshal hello response to %s (%s): %w", peerID, conn.RemoteAddr(), err) + } + + _, err = conn.Write(msg) + if err != nil { + return nil, fmt.Errorf("write to %s (%s): %w", peerID, conn.RemoteAddr(), err) + } + + return peerID, nil +} diff --git a/relay/server/relay_test.go b/relay/server/relay_test.go new file mode 100644 index 000000000..062039ab9 --- /dev/null +++ b/relay/server/relay_test.go @@ -0,0 +1,36 @@ +package server + +import "testing" + +func TestGetInstanceURL(t *testing.T) { + tests := []struct { + name string + exposedAddress string + tlsSupported bool + expectedURL string + expectError bool + }{ + {"Valid address with TLS", "example.com", true, "rels://example.com", false}, + {"Valid address without TLS", "example.com", false, "rel://example.com", false}, + {"Valid address with scheme", "rel://example.com", false, "rel://example.com", false}, + {"Valid address with non TLS scheme and TLS true", "rel://example.com", true, "rel://example.com", false}, + {"Valid address with TLS scheme", "rels://example.com", true, "rels://example.com", false}, + {"Valid address with TLS scheme and TLS false", "rels://example.com", false, "rels://example.com", false}, + {"Valid address with TLS scheme and custom port", "rels://example.com:9300", true, "rels://example.com:9300", false}, + {"Invalid address with multiple schemes", "rel://rels://example.com", false, "", true}, + {"Invalid address with unsupported scheme", "http://example.com", false, "", true}, + {"Invalid address format", "://example.com", false, "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + url, err := getInstanceURL(tt.exposedAddress, tt.tlsSupported) + if (err != nil) != tt.expectError { + t.Errorf("expected error: %v, got: %v", tt.expectError, err) + } + if url != tt.expectedURL { + t.Errorf("expected URL: %s, got: %s", tt.expectedURL, url) + } + }) + } +} diff --git a/relay/server/server.go b/relay/server/server.go new file mode 100644 index 000000000..0036e2390 --- /dev/null +++ b/relay/server/server.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "crypto/tls" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/server/listener" + "github.com/netbirdio/netbird/relay/server/listener/ws" +) + +// ListenerConfig is the configuration for the listener. +// Address: the address to bind the listener to. It could be an address behind a reverse proxy. +// TLSConfig: the TLS configuration for the listener. +type ListenerConfig struct { + Address string + TLSConfig *tls.Config +} + +// Server is the main entry point for the relay server. +// It is the gate between the WebSocket listener and the Relay server logic. +// In a new HTTP connection, the server will accept the connection and pass it to the Relay server via the Accept method. +type Server struct { + relay *Relay + wSListener listener.Listener +} + +// NewServer creates a new relay server instance. +// meter: the OpenTelemetry meter +// exposedAddress: this address will be used as the instance URL. It should be a domain:port format. +// tlsSupport: if true, the server will support TLS +// authValidator: the auth validator to use for the server +func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authValidator auth.Validator) (*Server, error) { + relay, err := NewRelay(meter, exposedAddress, tlsSupport, authValidator) + if err != nil { + return nil, err + } + return &Server{ + relay: relay, + }, nil +} + +// Listen starts the relay server. +func (r *Server) Listen(cfg ListenerConfig) error { + r.wSListener = &ws.Listener{ + Address: cfg.Address, + TLSConfig: cfg.TLSConfig, + } + + wslErr := r.wSListener.Listen(r.relay.Accept) + if wslErr != nil { + log.Errorf("failed to bind ws server: %s", wslErr) + } + + return wslErr +} + +// Shutdown stops the relay server. If there are active connections, they will be closed gracefully. In case of a context, +// the connections will be forcefully closed. +func (r *Server) Shutdown(ctx context.Context) (err error) { + // stop service new connections + if r.wSListener != nil { + err = r.wSListener.Shutdown(ctx) + } + + r.relay.Shutdown(ctx) + return +} + +// InstanceURL returns the instance URL of the relay server. +func (r *Server) InstanceURL() string { + return r.relay.instanceURL +} diff --git a/relay/server/store.go b/relay/server/store.go new file mode 100644 index 000000000..96879dae1 --- /dev/null +++ b/relay/server/store.go @@ -0,0 +1,64 @@ +package server + +import ( + "sync" +) + +// Store is a thread-safe store of peers +// It is used to store the peers that are connected to the relay server +type Store struct { + peers map[string]*Peer // consider to use [32]byte as key. The Peer(id string) would be faster + peersLock sync.RWMutex +} + +// NewStore creates a new Store instance +func NewStore() *Store { + return &Store{ + peers: make(map[string]*Peer), + } +} + +// AddPeer adds a peer to the store +// todo: consider to close peer conn if the peer already exists +func (s *Store) AddPeer(peer *Peer) { + s.peersLock.Lock() + defer s.peersLock.Unlock() + s.peers[peer.String()] = peer +} + +// DeletePeer deletes a peer from the store +func (s *Store) DeletePeer(peer *Peer) { + s.peersLock.Lock() + defer s.peersLock.Unlock() + + dp, ok := s.peers[peer.String()] + if !ok { + return + } + if dp != peer { + return + } + + delete(s.peers, peer.String()) +} + +// Peer returns a peer by its ID +func (s *Store) Peer(id string) (*Peer, bool) { + s.peersLock.RLock() + defer s.peersLock.RUnlock() + + p, ok := s.peers[id] + return p, ok +} + +// Peers returns all the peers in the store +func (s *Store) Peers() []*Peer { + s.peersLock.RLock() + defer s.peersLock.RUnlock() + + peers := make([]*Peer, 0, len(s.peers)) + for _, p := range s.peers { + peers = append(peers, p) + } + return peers +} diff --git a/relay/server/store_test.go b/relay/server/store_test.go new file mode 100644 index 000000000..4a30bc131 --- /dev/null +++ b/relay/server/store_test.go @@ -0,0 +1,40 @@ +package server + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/metrics" +) + +func TestStore_DeletePeer(t *testing.T) { + s := NewStore() + + m, _ := metrics.NewMetrics(context.Background(), otel.Meter("")) + + p := NewPeer(m, []byte("peer_one"), nil, nil) + s.AddPeer(p) + s.DeletePeer(p) + if _, ok := s.Peer(p.String()); ok { + t.Errorf("peer was not deleted") + } +} + +func TestStore_DeleteDeprecatedPeer(t *testing.T) { + s := NewStore() + + m, _ := metrics.NewMetrics(context.Background(), otel.Meter("")) + + p1 := NewPeer(m, []byte("peer_id"), nil, nil) + p2 := NewPeer(m, []byte("peer_id"), nil, nil) + + s.AddPeer(p1) + s.AddPeer(p2) + s.DeletePeer(p1) + + if _, ok := s.Peer(p2.String()); !ok { + t.Errorf("second peer was deleted") + } +} diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go new file mode 100644 index 000000000..ec2aa488c --- /dev/null +++ b/relay/test/benchmark_test.go @@ -0,0 +1,386 @@ +package test + +import ( + "context" + "crypto/rand" + "fmt" + "net" + "os" + "sync" + "testing" + "time" + + "github.com/pion/logging" + "github.com/pion/turn/v3" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/auth/allow" + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client" + "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/util" +) + +var ( + av = &allow.Auth{} + hmacTokenStore = &hmac.TokenStore{} + pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + dataSize = 1024 * 1024 * 10 +) + +func TestMain(m *testing.M) { + _ = util.InitLog("error", "console") + code := m.Run() + os.Exit(code) +} + +func TestRelayDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + transfer(t, testData, peerPairs) + }) + } +} + +// TestTurnDataTransfer run turn server: +// docker run --rm --name coturn -d --network=host coturn/coturn --user test:test +func TestTurnDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + runTurnTest(t, testData, peerPairs) + }) + } +} + +func transfer(t *testing.T, testData []byte, peerPairs int) { + t.Helper() + ctx := context.Background() + port := 35000 + peerPairs + serverAddress := fmt.Sprintf("127.0.0.1:%d", port) + serverConnURL := fmt.Sprintf("rel://%s", serverAddress) + + srv, err := server.NewServer(otel.Meter(""), serverConnURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + listenCfg := server.ListenerConfig{Address: serverAddress} + err := srv.Listen(listenCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientsSender := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + clientsReceiver := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsReceiver); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + + conn, err = clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + + var transferDuration []time.Duration + wg := sync.WaitGroup{} + var writeErr error + var readErr error + for i := 0; i < len(connsSender); i++ { + wg.Add(2) + start := time.Now() + go func(i int) { + defer wg.Done() + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr = connsSender[i].Write(testData[j:end]) + if writeErr != nil { + return + } + } + + }(i) + + go func(i int, start time.Time) { + defer wg.Done() + buf := make([]byte, 8192) + rcv := 0 + var n int + for receivedSize := 0; receivedSize < len(testData); { + + n, readErr = connsReceiver[i].Read(buf) + if readErr != nil { + return + } + + receivedSize += n + rcv += n + } + transferDuration = append(transferDuration, time.Since(start)) + }(i, start) + } + + wg.Wait() + + if writeErr != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + if readErr != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + // calculate the megabytes per second from the average transferDuration against the dataSize + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) + + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + err = connsReceiver[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + } +} + +func runTurnTest(t *testing.T, testData []byte, maxPairs int) { + t.Helper() + var transferDuration []time.Duration + var wg sync.WaitGroup + + for i := 0; i < maxPairs; i++ { + wg.Add(1) + go func() { + defer wg.Done() + d := runTurnDataTransfer(t, testData) + transferDuration = append(transferDuration, d) + }() + + } + wg.Wait() + + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) +} + +func runTurnDataTransfer(t *testing.T, testData []byte) time.Duration { + t.Helper() + testDataLen := len(testData) + relayAddress := "192.168.0.10:3478" + conn, err := net.Dial("tcp", relayAddress) + if err != nil { + t.Fatal(err) + } + defer func(conn net.Conn) { + _ = conn.Close() + }(conn) + + turnClient, err := getTurnClient(t, relayAddress, conn) + if err != nil { + t.Fatal(err) + } + defer turnClient.Close() + + relayConn, err := turnClient.Allocate() + if err != nil { + t.Fatal(err) + } + defer func(relayConn net.PacketConn) { + _ = relayConn.Close() + }(relayConn) + + receiverConn, err := net.Dial("udp", relayConn.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer func(receiverConn net.Conn) { + _ = receiverConn.Close() + }(receiverConn) + + var ( + tb int + start time.Time + timerInit bool + readDone = make(chan struct{}) + ack = make([]byte, 1) + ) + go func() { + defer func() { + readDone <- struct{}{} + }() + buff := make([]byte, 8192) + for { + n, e := receiverConn.Read(buff) + if e != nil { + return + } + if !timerInit { + start = time.Now() + timerInit = true + } + tb += n + _, _ = receiverConn.Write(ack) + + if tb >= testDataLen { + return + } + } + }() + + pieceSize := 1024 + ackBuff := make([]byte, 1) + pipelineSize := 10 + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, err := relayConn.WriteTo(testData[j:end], receiverConn.LocalAddr()) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + if pipelineSize == 0 { + _, _, _ = relayConn.ReadFrom(ackBuff) + } else { + pipelineSize-- + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + select { + case <-readDone: + if tb != testDataLen { + t.Fatalf("failed to read all data: %d/%d", tb, testDataLen) + } + case <-ctx.Done(): + t.Fatal("timeout") + } + return time.Since(start) +} + +func getTurnClient(t *testing.T, address string, conn net.Conn) (*turn.Client, error) { + t.Helper() + // Dial TURN Server + addrStr := fmt.Sprintf("%s:%d", address, 443) + + fac := logging.NewDefaultLoggerFactory() + //fac.DefaultLogLevel = logging.LogLevelTrace + + // Start a new TURN Client and wrap our net.Conn in a STUNConn + // This allows us to simulate datagram based communication over a net.Conn + cfg := &turn.ClientConfig{ + TURNServerAddr: address, + Conn: turn.NewSTUNConn(conn), + Username: "test", + Password: "test", + LoggerFactory: fac, + } + + client, err := turn.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err) + } + + // Start listening on the conn provided. + err = client.Listen() + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err) + } + + return client, nil +} + +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} + +func waitForServerToStart(errChan chan error) error { + select { + case err := <-errChan: + if err != nil { + return err + } + case <-time.After(300 * time.Millisecond): + return nil + } + return nil +} diff --git a/relay/testec2/main.go b/relay/testec2/main.go new file mode 100644 index 000000000..0c8099a5e --- /dev/null +++ b/relay/testec2/main.go @@ -0,0 +1,258 @@ +//go:build linux || darwin + +package main + +import ( + "crypto/rand" + "flag" + "fmt" + "net" + "os" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/util" +) + +const ( + errMsgFailedReadTCP = "failed to read from tcp: %s" +) + +var ( + dataSize = 1024 * 1024 * 50 // 50MB + pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + signalListenAddress = ":8081" + + relaySrvAddress string + turnSrvAddress string + signalURL string + udpListener string // used for TURN test +) + +type testResult struct { + numOfPairs int + duration time.Duration + speed float64 +} + +func (tr testResult) Speed() string { + speed := tr.speed + var unit string + + switch { + case speed < 1024: + unit = "B/s" + case speed < 1048576: + speed /= 1024 + unit = "KB/s" + case speed < 1073741824: + speed /= 1048576 + unit = "MB/s" + default: + speed /= 1073741824 + unit = "GB/s" + } + + return fmt.Sprintf("%.2f %s", speed, unit) +} + +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} + +func avg(transferDuration []time.Duration) (time.Duration, float64) { + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + bps := float64(dataSize) / avgDuration.Seconds() + return avgDuration, bps +} + +func RelayReceiverMain() []testResult { + testResults := make([]testResult, 0, len(pairs)) + for _, p := range pairs { + tr := testResult{numOfPairs: p} + td := relayReceive(relaySrvAddress, p) + tr.duration, tr.speed = avg(td) + + testResults = append(testResults, tr) + } + + return testResults +} + +func RelaySenderMain() { + log.Infof("starting sender") + log.Infof("starting seed phase") + + testData, err := seedRandomData(dataSize) + if err != nil { + log.Fatalf("failed to seed random data: %s", err) + } + + log.Infof("data size: %d", len(testData)) + + for n, p := range pairs { + log.Infof("running test with %d pairs", p) + relayTransfer(relaySrvAddress, testData, p) + + // grant time to prepare new receivers + if n < len(pairs)-1 { + time.Sleep(3 * time.Second) + } + } + +} + +// TRUNSenderMain is the sender +// - allocate turn clients +// - send relayed addresses to signal server in batch +// - wait for signal server to send back addresses in a map +// - send test data to each address in parallel +func TRUNSenderMain() { + log.Infof("starting TURN sender test") + + log.Infof("starting seed random data: %d", dataSize) + testData, err := seedRandomData(dataSize) + if err != nil { + log.Fatalf("failed to seed random data: %s", err) + } + + ss := SignalClient{signalURL} + + for _, p := range pairs { + log.Infof("running test with %d pairs", p) + turnSender := &TurnSender{} + + createTurnConns(p, turnSender) + + log.Infof("send addresses via signal server: %d", len(turnSender.addresses)) + clientAddresses, err := ss.SendAddress(turnSender.addresses) + if err != nil { + log.Fatalf("failed to send address: %s", err) + } + log.Infof("received addresses: %v", clientAddresses.Address) + + createSenderDevices(turnSender, clientAddresses) + + log.Infof("waiting for tcpListeners to be ready") + time.Sleep(2 * time.Second) + + tcpConns := make([]net.Conn, 0, len(turnSender.devices)) + for i := range turnSender.devices { + addr := fmt.Sprintf("10.0.%d.2:9999", i) + log.Infof("dialing: %s", addr) + tcpConn, err := net.Dial("tcp", addr) + if err != nil { + log.Fatalf("failed to dial tcp: %s", err) + } + tcpConns = append(tcpConns, tcpConn) + } + + log.Infof("start test data transfer for %d pairs", p) + testDataLen := len(testData) + wg := sync.WaitGroup{} + wg.Add(len(tcpConns)) + for i, tcpConn := range tcpConns { + log.Infof("sending test data to device: %d", i) + go runTurnWriting(tcpConn, testData, testDataLen, &wg) + } + wg.Wait() + + for _, d := range turnSender.devices { + _ = d.Close() + } + + log.Infof("test finished with %d pairs", p) + } +} + +func TURNReaderMain() []testResult { + log.Infof("starting TURN receiver test") + si := NewSignalService() + go func() { + log.Infof("starting signal server") + err := si.Listen(signalListenAddress) + if err != nil { + log.Errorf("failed to listen: %s", err) + } + }() + + testResults := make([]testResult, 0, len(pairs)) + for range pairs { + addresses := <-si.AddressesChan + instanceNumber := len(addresses) + log.Infof("received addresses: %d", instanceNumber) + + turnReceiver := &TurnReceiver{} + err := createDevices(addresses, turnReceiver) + if err != nil { + log.Fatalf("%s", err) + } + + // send client addresses back via signal server + si.ClientAddressChan <- turnReceiver.clientAddresses + + durations := make(chan time.Duration, instanceNumber) + for _, device := range turnReceiver.devices { + go runTurnReading(device, durations) + } + + durationsList := make([]time.Duration, 0, instanceNumber) + for d := range durations { + durationsList = append(durationsList, d) + if len(durationsList) == instanceNumber { + close(durations) + } + } + + avgDuration, avgSpeed := avg(durationsList) + ts := testResult{ + numOfPairs: len(durationsList), + duration: avgDuration, + speed: avgSpeed, + } + testResults = append(testResults, ts) + + for _, d := range turnReceiver.devices { + _ = d.Close() + } + } + return testResults +} + +func main() { + var mode string + + _ = util.InitLog("debug", "console") + flag.StringVar(&mode, "mode", "sender", "sender or receiver mode") + flag.Parse() + + relaySrvAddress = os.Getenv("TEST_RELAY_SERVER") // rel://ip:port + turnSrvAddress = os.Getenv("TEST_TURN_SERVER") // ip:3478 + signalURL = os.Getenv("TEST_SIGNAL_URL") // http://receiver_ip:8081 + udpListener = os.Getenv("TEST_UDP_LISTENER") // IP:0 + + if mode == "receiver" { + relayResult := RelayReceiverMain() + turnResults := TURNReaderMain() + for i := 0; i < len(turnResults); i++ { + log.Infof("pairs: %d,\tRelay speed:\t%s,\trelay duration:\t%s", relayResult[i].numOfPairs, relayResult[i].Speed(), relayResult[i].duration) + log.Infof("pairs: %d,\tTURN speed:\t%s,\tturn duration:\t%s", turnResults[i].numOfPairs, turnResults[i].Speed(), turnResults[i].duration) + } + } else { + RelaySenderMain() + // grant time for receiver to start + time.Sleep(3 * time.Second) + TRUNSenderMain() + } +} diff --git a/relay/testec2/relay.go b/relay/testec2/relay.go new file mode 100644 index 000000000..93d084387 --- /dev/null +++ b/relay/testec2/relay.go @@ -0,0 +1,176 @@ +//go:build linux || darwin + +package main + +import ( + "context" + "fmt" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client" +) + +var ( + hmacTokenStore = &hmac.TokenStore{} +) + +func relayTransfer(serverConnURL string, testData []byte, peerPairs int) { + connsSender := prepareConnsSender(serverConnURL, peerPairs) + defer func() { + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + log.Errorf("failed to close connection: %s", err) + } + } + }() + + wg := sync.WaitGroup{} + wg.Add(len(connsSender)) + for _, conn := range connsSender { + go func(conn net.Conn) { + defer wg.Done() + runWriter(conn, testData) + }(conn) + } + wg.Wait() +} + +func runWriter(conn net.Conn, testData []byte) { + si := NewStartInidication(time.Now(), len(testData)) + _, err := conn.Write(si) + if err != nil { + log.Errorf("failed to write to channel: %s", err) + return + } + log.Infof("sent start indication") + + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr := conn.Write(testData[j:end]) + if writeErr != nil { + log.Errorf("failed to write to channel: %s", writeErr) + return + } + } +} + +func prepareConnsSender(serverConnURL string, peerPairs int) []net.Conn { + ctx := context.Background() + clientsSender := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + if err := c.Connect(); err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + log.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + } + return connsSender +} + +func relayReceive(serverConnURL string, peerPairs int) []time.Duration { + connsReceiver := prepareConnsReceiver(serverConnURL, peerPairs) + defer func() { + for i := 0; i < len(connsReceiver); i++ { + if err := connsReceiver[i].Close(); err != nil { + log.Errorf("failed to close connection: %s", err) + } + } + }() + + durations := make(chan time.Duration, len(connsReceiver)) + wg := sync.WaitGroup{} + for _, conn := range connsReceiver { + wg.Add(1) + go func(conn net.Conn) { + defer wg.Done() + duration := runReader(conn) + durations <- duration + }(conn) + } + wg.Wait() + + durationsList := make([]time.Duration, 0, len(connsReceiver)) + for d := range durations { + durationsList = append(durationsList, d) + if len(durationsList) == len(connsReceiver) { + close(durations) + } + } + + return durationsList +} + +func runReader(conn net.Conn) time.Duration { + buf := make([]byte, 8192) + + n, readErr := conn.Read(buf) + if readErr != nil { + log.Errorf("failed to read from channel: %s", readErr) + return 0 + } + + si := DecodeStartIndication(buf[:n]) + log.Infof("received start indication: %v", si) + + receivedSize, err := conn.Read(buf) + if err != nil { + log.Fatalf("failed to read from relay: %s", err) + } + now := time.Now() + + rcv := 0 + for receivedSize < si.TransferSize { + n, readErr = conn.Read(buf) + if readErr != nil { + log.Errorf("failed to read from channel: %s", readErr) + return 0 + } + + receivedSize += n + rcv += n + } + return time.Since(now) +} + +func prepareConnsReceiver(serverConnURL string, peerPairs int) []net.Conn { + clientsReceiver := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsReceiver); i++ { + c := client.NewClient(context.Background(), serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsReceiver); i++ { + conn, err := clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + log.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + return connsReceiver +} diff --git a/relay/testec2/signal.go b/relay/testec2/signal.go new file mode 100644 index 000000000..fe93a2fe2 --- /dev/null +++ b/relay/testec2/signal.go @@ -0,0 +1,91 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "encoding/json" + "net/http" + + log "github.com/sirupsen/logrus" +) + +type PeerAddr struct { + Address []string +} + +type ClientPeerAddr struct { + Address map[string]string +} + +type Signal struct { + AddressesChan chan []string + ClientAddressChan chan map[string]string +} + +func NewSignalService() *Signal { + return &Signal{ + AddressesChan: make(chan []string), + ClientAddressChan: make(chan map[string]string), + } +} + +func (rs *Signal) Listen(listenAddr string) error { + http.HandleFunc("/", rs.onNewAddresses) + return http.ListenAndServe(listenAddr, nil) +} + +func (rs *Signal) onNewAddresses(w http.ResponseWriter, r *http.Request) { + var msg PeerAddr + err := json.NewDecoder(r.Body).Decode(&msg) + if err != nil { + log.Errorf("Error decoding message: %v", err) + } + + log.Infof("received addresses: %d", len(msg.Address)) + rs.AddressesChan <- msg.Address + clientAddresses := <-rs.ClientAddressChan + + respMsg := ClientPeerAddr{ + Address: clientAddresses, + } + data, err := json.Marshal(respMsg) + if err != nil { + log.Errorf("Error marshalling message: %v", err) + return + } + + _, err = w.Write(data) + if err != nil { + log.Errorf("Error writing response: %v", err) + } +} + +type SignalClient struct { + SignalURL string +} + +func (ss SignalClient) SendAddress(addresses []string) (*ClientPeerAddr, error) { + msg := PeerAddr{ + Address: addresses, + } + data, err := json.Marshal(msg) + if err != nil { + return nil, err + } + + response, err := http.Post(ss.SignalURL, "application/json", bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + defer response.Body.Close() + + log.Debugf("wait for signal response") + var respPeerAddress ClientPeerAddr + err = json.NewDecoder(response.Body).Decode(&respPeerAddress) + if err != nil { + return nil, err + } + return &respPeerAddress, nil +} diff --git a/relay/testec2/start_msg.go b/relay/testec2/start_msg.go new file mode 100644 index 000000000..19b65380b --- /dev/null +++ b/relay/testec2/start_msg.go @@ -0,0 +1,39 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "encoding/gob" + "time" + + log "github.com/sirupsen/logrus" +) + +type StartIndication struct { + Started time.Time + TransferSize int +} + +func NewStartInidication(started time.Time, transferSize int) []byte { + si := StartIndication{ + Started: started, + TransferSize: transferSize, + } + + var data bytes.Buffer + err := gob.NewEncoder(&data).Encode(si) + if err != nil { + log.Fatal("encode error:", err) + } + return data.Bytes() +} + +func DecodeStartIndication(data []byte) StartIndication { + var si StartIndication + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&si) + if err != nil { + log.Fatal("decode error:", err) + } + return si +} diff --git a/relay/testec2/tun/proxy.go b/relay/testec2/tun/proxy.go new file mode 100644 index 000000000..7d84bece7 --- /dev/null +++ b/relay/testec2/tun/proxy.go @@ -0,0 +1,72 @@ +//go:build linux || darwin + +package tun + +import ( + "net" + "sync/atomic" + + log "github.com/sirupsen/logrus" +) + +type Proxy struct { + Device *Device + PConn net.PacketConn + DstAddr net.Addr + shutdownFlag atomic.Bool +} + +func (p *Proxy) Start() { + go p.readFromDevice() + go p.readFromConn() +} + +func (p *Proxy) Close() { + p.shutdownFlag.Store(true) +} + +func (p *Proxy) readFromDevice() { + buf := make([]byte, 1500) + for { + n, err := p.Device.Read(buf) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to read from device: %s", err) + return + } + + _, err = p.PConn.WriteTo(buf[:n], p.DstAddr) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to write to conn: %s", err) + return + } + } +} + +func (p *Proxy) readFromConn() { + buf := make([]byte, 1500) + for { + n, _, err := p.PConn.ReadFrom(buf) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to read from conn: %s", err) + return + } + + _, err = p.Device.Write(buf[:n]) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to write to device: %s", err) + return + } + } +} diff --git a/relay/testec2/tun/tun.go b/relay/testec2/tun/tun.go new file mode 100644 index 000000000..5580785ce --- /dev/null +++ b/relay/testec2/tun/tun.go @@ -0,0 +1,110 @@ +//go:build linux || darwin + +package tun + +import ( + "net" + + log "github.com/sirupsen/logrus" + "github.com/songgao/water" + "github.com/vishvananda/netlink" +) + +type Device struct { + Name string + IP string + PConn net.PacketConn + DstAddr net.Addr + + iFace *water.Interface + proxy *Proxy +} + +func (d *Device) Up() error { + cfg := water.Config{ + DeviceType: water.TUN, + PlatformSpecificParams: water.PlatformSpecificParams{ + Name: d.Name, + }, + } + iFace, err := water.New(cfg) + if err != nil { + return err + } + d.iFace = iFace + + err = d.assignIP() + if err != nil { + return err + } + + err = d.bringUp() + if err != nil { + return err + } + + d.proxy = &Proxy{ + Device: d, + PConn: d.PConn, + DstAddr: d.DstAddr, + } + d.proxy.Start() + return nil +} + +func (d *Device) Close() error { + if d.proxy != nil { + d.proxy.Close() + } + if d.iFace != nil { + return d.iFace.Close() + } + return nil +} + +func (d *Device) Read(b []byte) (int, error) { + return d.iFace.Read(b) +} + +func (d *Device) Write(b []byte) (int, error) { + return d.iFace.Write(b) +} + +func (d *Device) assignIP() error { + iface, err := netlink.LinkByName(d.Name) + if err != nil { + log.Errorf("failed to get TUN device: %v", err) + return err + } + + ip := net.IPNet{ + IP: net.ParseIP(d.IP), + Mask: net.CIDRMask(24, 32), + } + + addr := &netlink.Addr{ + IPNet: &ip, + } + err = netlink.AddrAdd(iface, addr) + if err != nil { + log.Errorf("failed to add IP address: %v", err) + return err + } + return nil +} + +func (d *Device) bringUp() error { + iface, err := netlink.LinkByName(d.Name) + if err != nil { + log.Errorf("failed to get device: %v", err) + return err + } + + // Bring the interface up + err = netlink.LinkSetUp(iface) + if err != nil { + log.Errorf("failed to set device up: %v", err) + return err + } + return nil +} diff --git a/relay/testec2/turn.go b/relay/testec2/turn.go new file mode 100644 index 000000000..8beb40423 --- /dev/null +++ b/relay/testec2/turn.go @@ -0,0 +1,181 @@ +//go:build linux || darwin + +package main + +import ( + "fmt" + "net" + "sync" + "time" + + "github.com/netbirdio/netbird/relay/testec2/tun" + + log "github.com/sirupsen/logrus" +) + +type TurnReceiver struct { + conns []*net.UDPConn + clientAddresses map[string]string + devices []*tun.Device +} + +type TurnSender struct { + turnConns map[string]*TurnConn + addresses []string + devices []*tun.Device +} + +func runTurnWriting(tcpConn net.Conn, testData []byte, testDataLen int, wg *sync.WaitGroup) { + defer wg.Done() + defer tcpConn.Close() + + log.Infof("start to sending test data: %s", tcpConn.RemoteAddr()) + + si := NewStartInidication(time.Now(), testDataLen) + _, err := tcpConn.Write(si) + if err != nil { + log.Errorf("failed to write to tcp: %s", err) + return + } + + pieceSize := 1024 + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr := tcpConn.Write(testData[j:end]) + if writeErr != nil { + log.Errorf("failed to write to tcp conn: %s", writeErr) + return + } + } + + // grant time to flush out packages + time.Sleep(3 * time.Second) +} + +func createSenderDevices(sender *TurnSender, clientAddresses *ClientPeerAddr) { + var i int + devices := make([]*tun.Device, 0, len(clientAddresses.Address)) + for k, v := range clientAddresses.Address { + tc, ok := sender.turnConns[k] + if !ok { + log.Fatalf("failed to find turn conn: %s", k) + } + + addr, err := net.ResolveUDPAddr("udp", v) + if err != nil { + log.Fatalf("failed to resolve udp address: %s", err) + } + device := &tun.Device{ + Name: fmt.Sprintf("mtun-sender-%d", i), + IP: fmt.Sprintf("10.0.%d.1", i), + PConn: tc.relayConn, + DstAddr: addr, + } + + err = device.Up() + if err != nil { + log.Fatalf("failed to bring up device: %s", err) + } + + devices = append(devices, device) + i++ + } + sender.devices = devices +} + +func createTurnConns(p int, sender *TurnSender) { + turnConns := make(map[string]*TurnConn) + addresses := make([]string, 0, len(pairs)) + for i := 0; i < p; i++ { + tc := AllocateTurnClient(turnSrvAddress) + log.Infof("allocated turn client: %s", tc.Address().String()) + turnConns[tc.Address().String()] = tc + addresses = append(addresses, tc.Address().String()) + } + + sender.turnConns = turnConns + sender.addresses = addresses +} + +func runTurnReading(d *tun.Device, durations chan time.Duration) { + tcpListener, err := net.Listen("tcp", d.IP+":9999") + if err != nil { + log.Fatalf("failed to listen on tcp: %s", err) + } + log := log.WithField("device", tcpListener.Addr()) + + tcpConn, err := tcpListener.Accept() + if err != nil { + _ = tcpListener.Close() + log.Fatalf("failed to accept connection: %s", err) + } + log.Infof("remote peer connected") + + buf := make([]byte, 103) + n, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + + si := DecodeStartIndication(buf[:n]) + log.Infof("received start indication: %v, %d", si, n) + + buf = make([]byte, 8192) + i, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + now := time.Now() + for i < si.TransferSize { + n, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + i += n + } + durations <- time.Since(now) +} + +func createDevices(addresses []string, receiver *TurnReceiver) error { + receiver.conns = make([]*net.UDPConn, 0, len(addresses)) + receiver.clientAddresses = make(map[string]string, len(addresses)) + receiver.devices = make([]*tun.Device, 0, len(addresses)) + for i, addr := range addresses { + localAddr, err := net.ResolveUDPAddr("udp", udpListener) + if err != nil { + return fmt.Errorf("failed to resolve UDP address: %s", err) + } + + conn, err := net.ListenUDP("udp", localAddr) + if err != nil { + return fmt.Errorf("failed to create UDP connection: %s", err) + } + + receiver.conns = append(receiver.conns, conn) + receiver.clientAddresses[addr] = conn.LocalAddr().String() + + dstAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return fmt.Errorf("failed to resolve address: %s", err) + } + + device := &tun.Device{ + Name: fmt.Sprintf("mtun-%d", i), + IP: fmt.Sprintf("10.0.%d.2", i), + PConn: conn, + DstAddr: dstAddr, + } + + if err = device.Up(); err != nil { + return fmt.Errorf("failed to bring up device: %s, %s", device.Name, err) + } + receiver.devices = append(receiver.devices, device) + } + return nil +} diff --git a/relay/testec2/turn_allocator.go b/relay/testec2/turn_allocator.go new file mode 100644 index 000000000..fd86208df --- /dev/null +++ b/relay/testec2/turn_allocator.go @@ -0,0 +1,83 @@ +//go:build linux || darwin + +package main + +import ( + "fmt" + "net" + + "github.com/pion/logging" + "github.com/pion/turn/v3" + log "github.com/sirupsen/logrus" +) + +type TurnConn struct { + conn net.Conn + turnClient *turn.Client + relayConn net.PacketConn +} + +func (tc *TurnConn) Address() net.Addr { + return tc.relayConn.LocalAddr() +} + +func (tc *TurnConn) Close() { + _ = tc.relayConn.Close() + tc.turnClient.Close() + _ = tc.conn.Close() +} + +func AllocateTurnClient(serverAddr string) *TurnConn { + conn, err := net.Dial("tcp", serverAddr) + if err != nil { + log.Fatal(err) + } + + turnClient, err := getTurnClient(serverAddr, conn) + if err != nil { + log.Fatal(err) + } + + relayConn, err := turnClient.Allocate() + if err != nil { + log.Fatal(err) + } + + return &TurnConn{ + conn: conn, + turnClient: turnClient, + relayConn: relayConn, + } +} + +func getTurnClient(address string, conn net.Conn) (*turn.Client, error) { + // Dial TURN Server + addrStr := fmt.Sprintf("%s:%d", address, 443) + + fac := logging.NewDefaultLoggerFactory() + //fac.DefaultLogLevel = logging.LogLevelTrace + + // Start a new TURN Client and wrap our net.Conn in a STUNConn + // This allows us to simulate datagram based communication over a net.Conn + cfg := &turn.ClientConfig{ + TURNServerAddr: address, + Conn: turn.NewSTUNConn(conn), + Username: "test", + Password: "test", + LoggerFactory: fac, + } + + client, err := turn.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err) + } + + // Start listening on the conn provided. + err = client.Listen() + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err) + } + + return client, nil +} diff --git a/signal/client/client.go b/signal/client/client.go index 9d99b3677..ced3fb7d0 100644 --- a/signal/client/client.go +++ b/signal/client/client.go @@ -51,11 +51,10 @@ func UnMarshalCredential(msg *proto.Message) (*Credential, error) { } // MarshalCredential marshal a Credential instance and returns a Message object -func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type, - rosenpassPubKey []byte, rosenpassAddr string) (*proto.Message, error) { +func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey string, credential *Credential, t proto.Body_Type, rosenpassPubKey []byte, rosenpassAddr string, relaySrvAddress string) (*proto.Message, error) { return &proto.Message{ Key: myKey.PublicKey().String(), - RemoteKey: remoteKey.String(), + RemoteKey: remoteKey, Body: &proto.Body{ Type: t, Payload: fmt.Sprintf("%s:%s", credential.UFrag, credential.Pwd), @@ -65,6 +64,7 @@ func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, cre RosenpassPubKey: rosenpassPubKey, RosenpassServerAddr: rosenpassAddr, }, + RelayServerAddress: relaySrvAddress, }, }, nil } diff --git a/signal/proto/signalexchange.pb.go b/signal/proto/signalexchange.pb.go index 782c45da1..30f704c6f 100644 --- a/signal/proto/signalexchange.pb.go +++ b/signal/proto/signalexchange.pb.go @@ -1,15 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.12 // source: signalexchange.proto package proto import ( - _ "github.com/golang/protobuf/protoc-gen-go/descriptor" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/descriptorpb" reflect "reflect" sync "sync" ) @@ -225,6 +225,8 @@ type Body struct { FeaturesSupported []uint32 `protobuf:"varint,6,rep,packed,name=featuresSupported,proto3" json:"featuresSupported,omitempty"` // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig *RosenpassConfig `protobuf:"bytes,7,opt,name=rosenpassConfig,proto3" json:"rosenpassConfig,omitempty"` + // relayServerAddress is an IP:port of the relay server + RelayServerAddress string `protobuf:"bytes,8,opt,name=relayServerAddress,proto3" json:"relayServerAddress,omitempty"` } func (x *Body) Reset() { @@ -308,6 +310,13 @@ func (x *Body) GetRosenpassConfig() *RosenpassConfig { return nil } +func (x *Body) GetRelayServerAddress() string { + if x != nil { + return x.RelayServerAddress + } + return "" +} + // Mode indicates a connection mode type Mode struct { state protoimpl.MessageState @@ -431,7 +440,7 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x52, 0x09, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xf6, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xa6, 0x03, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, @@ -451,7 +460,10 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x52, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x12, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x22, 0x2e, diff --git a/signal/proto/signalexchange.proto b/signal/proto/signalexchange.proto index a8c4c309c..4431edd7c 100644 --- a/signal/proto/signalexchange.proto +++ b/signal/proto/signalexchange.proto @@ -60,6 +60,9 @@ message Body { // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig rosenpassConfig = 7; + + // relayServerAddress is url of the relay server + string relayServerAddress = 8; } // Mode indicates a connection mode diff --git a/util/net/dialer_nonios.go b/util/net/dialer_nonios.go index 7a5de7587..4032a75c0 100644 --- a/util/net/dialer_nonios.go +++ b/util/net/dialer_nonios.go @@ -49,6 +49,8 @@ func RemoveDialerHooks() { // DialContext wraps the net.Dialer's DialContext method to use the custom connection func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + log.Debugf("Dialing %s %s", network, address) + if CustomRoutingDisabled() { return d.Dialer.DialContext(ctx, network, address) }