diff --git a/client/cmd/status.go b/client/cmd/status.go index d9b7a9c91..1ef8b4913 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -31,9 +31,9 @@ type peerStateDetailOutput struct { Status string `json:"status" yaml:"status"` LastStatusUpdate time.Time `json:"lastStatusUpdate" yaml:"lastStatusUpdate"` ConnType string `json:"connectionType" yaml:"connectionType"` - Direct bool `json:"direct" yaml:"direct"` IceCandidateType iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"` IceCandidateEndpoint iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"` + RelayAddress string `json:"relayAddress" yaml:"relayAddress"` LastWireguardHandshake time.Time `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"` TransferReceived int64 `json:"transferReceived" yaml:"transferReceived"` TransferSent int64 `json:"transferSent" yaml:"transferSent"` @@ -335,16 +335,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput { func mapPeers(peers []*proto.PeerState) peersStateOutput { var peersStateDetail []peerStateDetailOutput - localICE := "" - remoteICE := "" - localICEEndpoint := "" - remoteICEEndpoint := "" - connType := "" peersConnected := 0 - lastHandshake := time.Time{} - transferReceived := int64(0) - transferSent := int64(0) for _, pbPeerState := range peers { + localICE := "" + remoteICE := "" + localICEEndpoint := "" + remoteICEEndpoint := "" + relayServerAddress := "" + connType := "" + lastHandshake := time.Time{} + transferReceived := int64(0) + transferSent := int64(0) + isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String() if skipDetailByFilters(pbPeerState, isPeerConnected) { continue @@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { if pbPeerState.Relayed { connType = "Relayed" } + relayServerAddress = pbPeerState.GetRelayAddress() lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local() transferReceived = pbPeerState.GetBytesRx() transferSent = pbPeerState.GetBytesTx() @@ -372,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { Status: pbPeerState.GetConnStatus(), LastStatusUpdate: timeLocal, ConnType: connType, - Direct: pbPeerState.GetDirect(), IceCandidateType: iceCandidateType{ Local: localICE, Remote: remoteICE, @@ -381,6 +383,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { Local: localICEEndpoint, Remote: remoteICEEndpoint, }, + RelayAddress: relayServerAddress, FQDN: pbPeerState.GetFqdn(), LastWireguardHandshake: lastHandshake, TransferReceived: transferReceived, @@ -641,9 +644,9 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo " Status: %s\n"+ " -- detail --\n"+ " Connection type: %s\n"+ - " Direct: %t\n"+ " ICE candidate (Local/Remote): %s/%s\n"+ " ICE candidate endpoints (Local/Remote): %s/%s\n"+ + " Relay server address: %s\n"+ " Last connection update: %s\n"+ " Last WireGuard handshake: %s\n"+ " Transfer status (received/sent) %s/%s\n"+ @@ -655,11 +658,11 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo peerState.PubKey, peerState.Status, peerState.ConnType, - peerState.Direct, localICE, remoteICE, localICEEndpoint, remoteICEEndpoint, + peerState.RelayAddress, timeAgo(peerState.LastStatusUpdate), timeAgo(peerState.LastWireguardHandshake), toIEC(peerState.TransferReceived), diff --git a/client/cmd/status_test.go b/client/cmd/status_test.go index 46620a956..ca43df8a5 100644 --- a/client/cmd/status_test.go +++ b/client/cmd/status_test.go @@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{ ConnStatus: "Connected", ConnStatusUpdate: timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)), Relayed: false, - Direct: true, LocalIceCandidateType: "", RemoteIceCandidateType: "", LocalIceCandidateEndpoint: "", @@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{ ConnStatus: "Connected", ConnStatusUpdate: timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)), Relayed: true, - Direct: false, LocalIceCandidateType: "relay", RemoteIceCandidateType: "prflx", LocalIceCandidateEndpoint: "10.0.0.1:10001", @@ -137,7 +135,6 @@ var overview = statusOutputOverview{ Status: "Connected", LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC), ConnType: "P2P", - Direct: true, IceCandidateType: iceCandidateType{ Local: "", Remote: "", @@ -161,7 +158,6 @@ var overview = statusOutputOverview{ Status: "Connected", LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC), ConnType: "Relayed", - Direct: false, IceCandidateType: iceCandidateType{ Local: "relay", Remote: "prflx", @@ -283,7 +279,6 @@ func TestParsingToJSON(t *testing.T) { "status": "Connected", "lastStatusUpdate": "2001-01-01T01:01:01Z", "connectionType": "P2P", - "direct": true, "iceCandidateType": { "local": "", "remote": "" @@ -292,6 +287,7 @@ func TestParsingToJSON(t *testing.T) { "local": "", "remote": "" }, + "relayAddress": "", "lastWireguardHandshake": "2001-01-01T01:01:02Z", "transferReceived": 200, "transferSent": 100, @@ -308,7 +304,6 @@ func TestParsingToJSON(t *testing.T) { "status": "Connected", "lastStatusUpdate": "2002-02-02T02:02:02Z", "connectionType": "Relayed", - "direct": false, "iceCandidateType": { "local": "relay", "remote": "prflx" @@ -317,6 +312,7 @@ func TestParsingToJSON(t *testing.T) { "local": "10.0.0.1:10001", "remote": "10.0.10.1:10002" }, + "relayAddress": "", "lastWireguardHandshake": "2002-02-02T02:02:03Z", "transferReceived": 2000, "transferSent": 1000, @@ -408,13 +404,13 @@ func TestParsingToYAML(t *testing.T) { status: Connected lastStatusUpdate: 2001-01-01T01:01:01Z connectionType: P2P - direct: true iceCandidateType: local: "" remote: "" iceCandidateEndpoint: local: "" remote: "" + relayAddress: "" lastWireguardHandshake: 2001-01-01T01:01:02Z transferReceived: 200 transferSent: 100 @@ -428,13 +424,13 @@ func TestParsingToYAML(t *testing.T) { status: Connected lastStatusUpdate: 2002-02-02T02:02:02Z connectionType: Relayed - direct: false iceCandidateType: local: relay remote: prflx iceCandidateEndpoint: local: 10.0.0.1:10001 remote: 10.0.10.1:10002 + relayAddress: "" lastWireguardHandshake: 2002-02-02T02:02:03Z transferReceived: 2000 transferSent: 1000 @@ -505,9 +501,9 @@ func TestParsingToDetail(t *testing.T) { Status: Connected -- detail -- Connection type: P2P - Direct: true ICE candidate (Local/Remote): -/- ICE candidate endpoints (Local/Remote): -/- + Relay server address: Last connection update: %s Last WireGuard handshake: %s Transfer status (received/sent) 200 B/100 B @@ -521,9 +517,9 @@ func TestParsingToDetail(t *testing.T) { Status: Connected -- detail -- Connection type: Relayed - Direct: false ICE candidate (Local/Remote): relay/prflx ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002 + Relay server address: Last connection update: %s Last WireGuard handshake: %s Transfer status (received/sent) 2.0 KiB/1000 B diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index 984aa6df7..d5add8516 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -98,7 +98,11 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + + rc := &mgmt.RelayConfig{ + Address: "localhost:0", + } + turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { t.Fatal(err) diff --git a/client/cmd/up.go b/client/cmd/up.go index 2ed6e41d2..b447f7141 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -168,7 +168,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error { ctx, cancel = context.WithCancel(ctx) SetupCloseHandler(ctx, cancel) - connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String())) + r := peer.NewRecorder(config.ManagementURL.String()) + r.GetFullStatus() + + connectClient := internal.NewConnectClient(ctx, config, r) return connectClient.Run() } diff --git a/client/internal/connect.go b/client/internal/connect.go index 1cfabe910..56fad2734 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -26,6 +26,8 @@ import ( "github.com/netbirdio/netbird/iface" mgm "github.com/netbirdio/netbird/management/client" mgmProto "github.com/netbirdio/netbird/management/proto" + "github.com/netbirdio/netbird/relay/auth/hmac" + relayClient "github.com/netbirdio/netbird/relay/client" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/util" "github.com/netbirdio/netbird/version" @@ -162,10 +164,8 @@ func (c *ConnectClient) run( defer c.statusRecorder.ClientStop() operation := func() error { // if context cancelled we not start new backoff cycle - select { - case <-c.ctx.Done(): + if c.isContextCancelled() { return nil - default: } state.Set(StatusConnecting) @@ -187,8 +187,7 @@ func (c *ConnectClient) run( log.Debugf("connected to the Management service %s", c.config.ManagementURL.Host) defer func() { - err = mgmClient.Close() - if err != nil { + if err = mgmClient.Close(); err != nil { log.Warnf("failed to close the Management service client %v", err) } }() @@ -211,7 +210,6 @@ func (c *ConnectClient) run( KernelInterface: iface.WireGuardModuleIsLoaded(), FQDN: loginResp.GetPeerConfig().GetFqdn(), } - c.statusRecorder.UpdateLocalPeerState(localPeerState) signalURL := fmt.Sprintf("%s://%s", @@ -244,6 +242,20 @@ func (c *ConnectClient) run( c.statusRecorder.MarkSignalConnected() + relayURL, token := parseRelayInfo(loginResp) + relayManager := relayClient.NewManager(engineCtx, relayURL, myPrivateKey.PublicKey().String()) + if relayURL != "" { + if token != nil { + relayManager.UpdateToken(token) + } + log.Infof("connecting to the Relay service %s", relayURL) + if err = relayManager.Serve(); err != nil { + log.Error(err) + return wrapErr(err) + } + c.statusRecorder.SetRelayMgr(relayManager) + } + peerConfig := loginResp.GetPeerConfig() engineConfig, err := createEngineConfig(myPrivateKey, c.config, peerConfig) @@ -255,11 +267,10 @@ func (c *ConnectClient) run( checks := loginResp.GetChecks() c.engineMutex.Lock() - c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe, checks) + c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe, checks) c.engineMutex.Unlock() - err = c.engine.Start() - if err != nil { + if err := c.engine.Start(); err != nil { log.Errorf("error while starting Netbird Connection Engine: %s", err) return wrapErr(err) } @@ -299,6 +310,25 @@ func (c *ConnectClient) run( return nil } +func parseRelayInfo(resp *mgmProto.LoginResponse) (string, *hmac.Token) { + msg := resp.GetWiretrusteeConfig().GetRelay() + if msg == nil { + return "", nil + } + + var url string + if msg.GetUrls() != nil && len(msg.GetUrls()) > 0 { + url = msg.GetUrls()[0] + } + + token := &hmac.Token{ + Payload: msg.GetTokenPayload(), + Signature: msg.GetTokenSignature(), + } + + return url, token +} + func (c *ConnectClient) Engine() *Engine { var e *Engine c.engineMutex.Lock() @@ -307,6 +337,15 @@ func (c *ConnectClient) Engine() *Engine { return e } +func (c *ConnectClient) isContextCancelled() bool { + select { + case <-c.ctx.Done(): + return true + default: + return false + } +} + // createEngineConfig converts configuration received from Management Service to EngineConfig func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) { nm := false diff --git a/client/internal/engine.go b/client/internal/engine.go index d65322d6a..3a868657b 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/pion/ice/v3" @@ -24,6 +25,7 @@ import ( "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/internal/acl" "github.com/netbirdio/netbird/client/internal/dns" + "github.com/netbirdio/netbird/client/internal/networkmonitor" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/relay" @@ -39,6 +41,8 @@ import ( mgm "github.com/netbirdio/netbird/management/client" "github.com/netbirdio/netbird/management/domain" mgmProto "github.com/netbirdio/netbird/management/proto" + auth "github.com/netbirdio/netbird/relay/auth/hmac" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" sProto "github.com/netbirdio/netbird/signal/proto" @@ -101,7 +105,8 @@ type EngineConfig struct { // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers. type Engine struct { // signal is a Signal Service client - signal signal.Client + signal signal.Client + signaler *peer.Signaler // mgmClient is a Management Service client mgmClient mgm.Client // peerConns is a map that holds all the peers that are known to this peer @@ -122,7 +127,8 @@ type Engine struct { // STUNs is a list of STUN servers used by ICE STUNs []*stun.URI // TURNs is a list of STUN servers used by ICE - TURNs []*stun.URI + TURNs []*stun.URI + StunTurn atomic.Value // clientRoutes is the most recent list of clientRoutes received from the Management Service clientRoutes route.HAMap @@ -134,7 +140,7 @@ type Engine struct { ctx context.Context cancel context.CancelFunc - wgInterface *iface.WGIface + wgInterface iface.IWGIface wgProxyFactory *wgproxy.Factory udpMux *bind.UniversalUDPMuxDefault @@ -160,10 +166,10 @@ type Engine struct { relayProbe *Probe wgProbe *Probe - wgConnWorker sync.WaitGroup - // checks are the client-applied posture checks that need to be evaluated on the client checks []*mgmProto.Checks + + relayManager *relayClient.Manager } // Peer is an instance of the Connection Peer @@ -178,6 +184,7 @@ func NewEngine( clientCancel context.CancelFunc, signalClient signal.Client, mgmClient mgm.Client, + relayManager *relayClient.Manager, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, @@ -188,6 +195,7 @@ func NewEngine( clientCancel, signalClient, mgmClient, + relayManager, config, mobileDep, statusRecorder, @@ -205,6 +213,7 @@ func NewEngineWithProbes( clientCancel context.CancelFunc, signalClient signal.Client, mgmClient mgm.Client, + relayManager *relayClient.Manager, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, @@ -214,12 +223,13 @@ func NewEngineWithProbes( wgProbe *Probe, checks []*mgmProto.Checks, ) *Engine { - return &Engine{ clientCtx: clientCtx, clientCancel: clientCancel, signal: signalClient, + signaler: peer.NewSignaler(signalClient, config.WgPrivateKey), mgmClient: mgmClient, + relayManager: relayManager, peerConns: make(map[string]*peer.Conn), syncMsgMux: &sync.Mutex{}, config: config, @@ -265,24 +275,8 @@ func (e *Engine) Stop() error { time.Sleep(500 * time.Millisecond) e.close() - e.wgConnWorker.Wait() - - maxWaitTime := 5 * time.Second - timeout := time.After(maxWaitTime) - - for { - if !e.IsWGIfaceUp() { - log.Infof("stopped Netbird Engine") - return nil - } - - select { - case <-timeout: - return fmt.Errorf("timeout when waiting for interface shutdown") - default: - time.Sleep(100 * time.Millisecond) - } - } + log.Infof("stopped Netbird Engine") + return nil } // Start creates a new WireGuard tunnel interface and listens to events from Signal and Management services @@ -480,80 +474,42 @@ func (e *Engine) removePeer(peerKey string) error { conn, exists := e.peerConns[peerKey] if exists { delete(e.peerConns, peerKey) - err := conn.Close() - if err != nil { - switch err.(type) { - case *peer.ConnectionAlreadyClosedError: - return nil - default: - return err - } - } + conn.Close() } return nil } -func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client) error { - err := s.Send(&sProto.Message{ - Key: myKey.PublicKey().String(), - RemoteKey: remoteKey.String(), - Body: &sProto.Body{ - Type: sProto.Body_CANDIDATE, - Payload: candidate.Marshal(), - }, - }) - if err != nil { - return err - } - - return nil -} - -func sendSignal(message *sProto.Message, s signal.Client) error { - return s.Send(message) -} - -// SignalOfferAnswer signals either an offer or an answer to remote peer -func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client, - isAnswer bool) error { - var t sProto.Body_Type - if isAnswer { - t = sProto.Body_ANSWER - } else { - t = sProto.Body_OFFER - } - - msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{ - UFrag: offerAnswer.IceCredentials.UFrag, - Pwd: offerAnswer.IceCredentials.Pwd, - }, t, offerAnswer.RosenpassPubKey, offerAnswer.RosenpassAddr) - if err != nil { - return err - } - - err = s.Send(msg) - if err != nil { - return err - } - - return nil -} - func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { e.syncMsgMux.Lock() defer e.syncMsgMux.Unlock() if update.GetWiretrusteeConfig() != nil { - err := e.updateTURNs(update.GetWiretrusteeConfig().GetTurns()) + wCfg := update.GetWiretrusteeConfig() + err := e.updateTURNs(wCfg.GetTurns()) if err != nil { return err } - err = e.updateSTUNs(update.GetWiretrusteeConfig().GetStuns()) + err = e.updateSTUNs(wCfg.GetStuns()) if err != nil { return err } + var stunTurn []*stun.URI + stunTurn = append(stunTurn, e.STUNs...) + stunTurn = append(stunTurn, e.TURNs...) + e.StunTurn.Store(stunTurn) + + relayMsg := wCfg.GetRelay() + if relayMsg != nil { + c := &auth.Token{ + Payload: relayMsg.GetTokenPayload(), + Signature: relayMsg.GetTokenSignature(), + } + e.relayManager.UpdateToken(c) + } + + // todo update relay address in the relay manager // todo update signal } @@ -949,68 +905,13 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err) } - e.wgConnWorker.Add(1) - go e.connWorker(conn, peerKey) + conn.Open() } return nil } -func (e *Engine) connWorker(conn *peer.Conn, peerKey string) { - defer e.wgConnWorker.Done() - for { - - // randomize starting time a bit - minValue := 500 - maxValue := 2000 - duration := time.Duration(rand.Intn(maxValue-minValue)+minValue) * time.Millisecond - select { - case <-e.ctx.Done(): - return - case <-time.After(duration): - } - - // if peer has been removed -> give up - if !e.peerExists(peerKey) { - log.Debugf("peer %s doesn't exist anymore, won't retry connection", peerKey) - return - } - - if !e.signal.Ready() { - log.Infof("signal client isn't ready, skipping connection attempt %s", peerKey) - continue - } - - // we might have received new STUN and TURN servers meanwhile, so update them - e.syncMsgMux.Lock() - conn.UpdateStunTurn(append(e.STUNs, e.TURNs...)) - e.syncMsgMux.Unlock() - - err := conn.Open(e.ctx) - if err != nil { - log.Debugf("connection to peer %s failed: %v", peerKey, err) - var connectionClosedError *peer.ConnectionClosedError - switch { - case errors.As(err, &connectionClosedError): - // conn has been forced to close, so we exit the loop - return - default: - } - } - } -} - -func (e *Engine) peerExists(peerKey string) bool { - e.syncMsgMux.Lock() - defer e.syncMsgMux.Unlock() - _, ok := e.peerConns[peerKey] - return ok -} - func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) { log.Debugf("creating peer connection %s", pubKey) - var stunTurn []*stun.URI - stunTurn = append(stunTurn, e.STUNs...) - stunTurn = append(stunTurn, e.TURNs...) wgConfig := peer.WgConfig{ RemoteKey: pubKey, @@ -1043,52 +944,29 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e // randomize connection timeout timeout := time.Duration(rand.Intn(PeerConnectionTimeoutMax-PeerConnectionTimeoutMin)+PeerConnectionTimeoutMin) * time.Millisecond config := peer.ConnConfig{ - Key: pubKey, - LocalKey: e.config.WgPrivateKey.PublicKey().String(), - StunTurn: stunTurn, - InterfaceBlackList: e.config.IFaceBlackList, - DisableIPv6Discovery: e.config.DisableIPv6Discovery, - Timeout: timeout, - UDPMux: e.udpMux.UDPMuxDefault, - UDPMuxSrflx: e.udpMux, - WgConfig: wgConfig, - LocalWgPort: e.config.WgPort, - NATExternalIPs: e.parseNATExternalIPMappings(), - RosenpassPubKey: e.getRosenpassPubKey(), - RosenpassAddr: e.getRosenpassAddr(), + Key: pubKey, + LocalKey: e.config.WgPrivateKey.PublicKey().String(), + Timeout: timeout, + WgConfig: wgConfig, + LocalWgPort: e.config.WgPort, + RosenpassPubKey: e.getRosenpassPubKey(), + RosenpassAddr: e.getRosenpassAddr(), + ICEConfig: peer.ICEConfig{ + StunTurn: &e.StunTurn, + InterfaceBlackList: e.config.IFaceBlackList, + DisableIPv6Discovery: e.config.DisableIPv6Discovery, + UDPMux: e.udpMux.UDPMuxDefault, + UDPMuxSrflx: e.udpMux, + NATExternalIPs: e.parseNATExternalIPMappings(), + }, } - peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover) + peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager) if err != nil { return nil, err } - wgPubKey, err := wgtypes.ParseKey(pubKey) - if err != nil { - return nil, err - } - - signalOffer := func(offerAnswer peer.OfferAnswer) error { - return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false) - } - - signalCandidate := func(candidate ice.Candidate) error { - return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal) - } - - signalAnswer := func(offerAnswer peer.OfferAnswer) error { - return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true) - } - - peerConn.SetSignalCandidate(signalCandidate) - peerConn.SetSignalOffer(signalOffer) - peerConn.SetSignalAnswer(signalAnswer) - peerConn.SetSendSignalMessage(func(message *sProto.Message) error { - return sendSignal(message, e.signal) - }) - if e.rpManager != nil { - peerConn.SetOnConnected(e.rpManager.OnConnected) peerConn.SetOnDisconnected(e.rpManager.OnDisconnected) } @@ -1131,6 +1009,7 @@ func (e *Engine) receiveSignalEvents() { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), }) case sProto.Body_ANSWER: remoteCred, err := signal.UnMarshalCredential(msg) @@ -1153,6 +1032,7 @@ func (e *Engine) receiveSignalEvents() { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), }) case sProto.Body_CANDIDATE: candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload) @@ -1161,7 +1041,7 @@ func (e *Engine) receiveSignalEvents() { return err } - conn.OnRemoteCandidate(candidate, e.GetClientRoutes()) + go conn.OnRemoteCandidate(candidate, e.GetClientRoutes()) case sProto.Body_MODE: } @@ -1457,7 +1337,7 @@ func (e *Engine) receiveProbeEvents() { for _, peer := range e.peerConns { key := peer.GetKey() - wgStats, err := peer.GetConf().WgConfig.WgInterface.GetStats(key) + wgStats, err := peer.WgConfig().WgInterface.GetStats(key) if err != nil { log.Debugf("failed to get wg stats for peer %s: %s", key, err) } diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index e0f85d211..d2ef14712 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -36,6 +36,7 @@ import ( mgmtProto "github.com/netbirdio/netbird/management/proto" "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/activity" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" @@ -58,6 +59,12 @@ var ( } ) +func TestMain(m *testing.M) { + _ = util.InitLog("debug", "console") + code := m.Run() + os.Exit(code) +} + func TestEngine_SSH(t *testing.T) { // todo resolve test execution on freebsd if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" { @@ -73,13 +80,23 @@ func TestEngine_SSH(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ - WgIfaceName: "utun101", - WgAddr: "100.64.0.1/24", - WgPrivateKey: key, - WgPort: 33100, - ServerSSHAllowed: true, - }, MobileDependency{}, peer.NewRecorder("https://mgm"), nil) + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine( + ctx, cancel, + &signal.MockClient{}, + &mgmt.MockClient{}, + relayMgr, + &EngineConfig{ + WgIfaceName: "utun101", + WgAddr: "100.64.0.1/24", + WgPrivateKey: key, + WgPort: 33100, + ServerSSHAllowed: true, + }, + MobileDependency{}, + peer.NewRecorder("https://mgm"), + nil, + ) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, @@ -208,20 +225,28 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ - WgIfaceName: "utun102", - WgAddr: "100.64.0.1/24", - WgPrivateKey: key, - WgPort: 33100, - }, MobileDependency{}, peer.NewRecorder("https://mgm"), nil) - newNet, err := stdnet.NewNet() - if err != nil { - t.Fatal(err) - } - engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil, nil) - if err != nil { - t.Fatal(err) + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine( + ctx, cancel, + &signal.MockClient{}, + &mgmt.MockClient{}, + relayMgr, + &EngineConfig{ + WgIfaceName: "utun102", + WgAddr: "100.64.0.1/24", + WgPrivateKey: key, + WgPort: 33100, + }, + MobileDependency{}, + peer.NewRecorder("https://mgm"), + nil) + + wgIface := &iface.MockWGIface{ + RemovePeerFunc: func(peerKey string) error { + return nil + }, } + engine.wgInterface = wgIface engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, nil) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, @@ -404,8 +429,8 @@ func TestEngine_Sync(t *testing.T) { } return nil } - - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, relayMgr, &EngineConfig{ WgIfaceName: "utun103", WgAddr: "100.64.0.1/24", WgPrivateKey: key, @@ -564,7 +589,8 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) { wgIfaceName := fmt.Sprintf("utun%d", 104+n) wgAddr := fmt.Sprintf("100.66.%d.1/24", n) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, WgPrivateKey: key, @@ -734,7 +760,8 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) { wgIfaceName := fmt.Sprintf("utun%d", 104+n) wgAddr := fmt.Sprintf("100.66.%d.1/24", n) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, WgPrivateKey: key, @@ -1010,7 +1037,8 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin WgPort: wgPort, } - e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, relayMgr, conf, MobileDependency{}, peer.NewRecorder("https://mgm"), nil), nil e.ctx = ctx return e, err } @@ -1078,7 +1106,10 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error) if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + rc := &server.RelayConfig{ + Address: "127.0.0.1:1234", + } + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { return nil, "", err diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 0d8fd932c..9f17a087d 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -2,570 +2,272 @@ package peer import ( "context" - "fmt" + "math/rand" "net" + "os" "runtime" "strings" "sync" "time" + "github.com/cenkalti/backoff/v4" "github.com/pion/ice/v3" - "github.com/pion/stun/v2" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" - "github.com/netbirdio/netbird/iface/bind" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" - sProto "github.com/netbirdio/netbird/signal/proto" nbnet "github.com/netbirdio/netbird/util/net" - "github.com/netbirdio/netbird/version" ) -const ( - iceKeepAliveDefault = 4 * time.Second - iceDisconnectedTimeoutDefault = 6 * time.Second - // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package - iceRelayAcceptanceMinWaitDefault = 2 * time.Second +type ConnPriority int +const ( defaultWgKeepAlive = 25 * time.Second + + connPriorityRelay ConnPriority = 1 + connPriorityICETurn ConnPriority = 1 + connPriorityICEP2P ConnPriority = 2 ) type WgConfig struct { WgListenPort int RemoteKey string - WgInterface *iface.WGIface + WgInterface iface.IWGIface AllowedIps string PreSharedKey *wgtypes.Key } // ConnConfig is a peer Connection configuration type ConnConfig struct { - // Key is a public key of a remote peer Key string // LocalKey is a public key of a local peer LocalKey string - // StunTurn is a list of STUN and TURN URLs - StunTurn []*stun.URI - - // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering - // (e.g. if eth0 is in the list, host candidate of this interface won't be used) - InterfaceBlackList []string - DisableIPv6Discovery bool - Timeout time.Duration WgConfig WgConfig - UDPMux ice.UDPMux - UDPMuxSrflx ice.UniversalUDPMux - LocalWgPort int - NATExternalIPs []string - // RosenpassPubKey is this peer's Rosenpass public key RosenpassPubKey []byte // RosenpassPubKey is this peer's RosenpassAddr server address (IP:port) RosenpassAddr string + + // ICEConfig ICE protocol configuration + ICEConfig ICEConfig } -// OfferAnswer represents a session establishment offer or answer -type OfferAnswer struct { - IceCredentials IceCredentials - // WgListenPort is a remote WireGuard listen port. - // This field is used when establishing a direct WireGuard connection without any proxy. - // We can set the remote peer's endpoint with this port. - WgListenPort int +type WorkerCallbacks struct { + OnRelayReadyCallback func(info RelayConnInfo) + OnRelayStatusChanged func(ConnStatus) - // Version of NetBird Agent - Version string - // RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message - // This value is the local Rosenpass server public key when sending the message - RosenpassPubKey []byte - // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message - // This value is the local Rosenpass server address when sending the message - RosenpassAddr string -} - -// IceCredentials ICE protocol credentials struct -type IceCredentials struct { - UFrag string - Pwd string + OnICEConnReadyCallback func(ConnPriority, ICEConnInfo) + OnICEStatusChanged func(ConnStatus) } type Conn struct { - config ConnConfig - mu sync.Mutex - - // signalCandidate is a handler function to signal remote peer about local connection candidate - signalCandidate func(candidate ice.Candidate) error - // signalOffer is a handler function to signal remote peer our connection offer (credentials) - signalOffer func(OfferAnswer) error - signalAnswer func(OfferAnswer) error - sendSignalMessage func(message *sProto.Message) error - onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) - onDisconnected func(remotePeer string, wgIP string) - - // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection - remoteOffersCh chan OfferAnswer - // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection - remoteAnswerCh chan OfferAnswer - closeCh chan struct{} - ctx context.Context - notifyDisconnected context.CancelFunc - - agent *ice.Agent - status ConnStatus - + log *log.Entry + mu sync.Mutex + ctx context.Context + ctxCancel context.CancelFunc + config ConnConfig statusRecorder *Status - wgProxyFactory *wgproxy.Factory - wgProxy wgproxy.Proxy + wgProxyICE wgproxy.Proxy + wgProxyRelay wgproxy.Proxy + signaler *Signaler + relayManager *relayClient.Manager + allowedIPsIP string + handshaker *Handshaker - adapter iface.TunAdapter - iFaceDiscover stdnet.ExternalIFaceDiscover - sentExtraSrflx bool + onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) + onDisconnected func(remotePeer string, wgIP string) - connID nbnet.ConnectionID + statusRelay ConnStatus + statusICE ConnStatus + currentConnPriority ConnPriority + opened bool // this flag is used to prevent close in case of not opened connection + + workerICE *WorkerICE + workerRelay *WorkerRelay + + connIDRelay nbnet.ConnectionID + connIDICE nbnet.ConnectionID beforeAddPeerHooks []nbnet.AddHookFunc afterRemovePeerHooks []nbnet.RemoveHookFunc -} -// GetConf returns the connection config -func (conn *Conn) GetConf() ConnConfig { - return conn.config -} + endpointRelay *net.UDPAddr -// WgConfig returns the WireGuard config -func (conn *Conn) WgConfig() WgConfig { - return conn.config.WgConfig -} - -// UpdateStunTurn update the turn and stun addresses -func (conn *Conn) UpdateStunTurn(turnStun []*stun.URI) { - conn.config.StunTurn = turnStun + // for reconnection operations + iCEDisconnected chan bool + relayDisconnected chan bool } // NewConn creates a new not opened Conn to the remote peer. // To establish a connection run Conn.Open -func NewConn(config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover) (*Conn, error) { - return &Conn{ - config: config, - mu: sync.Mutex{}, - status: StatusDisconnected, - closeCh: make(chan struct{}), - remoteOffersCh: make(chan OfferAnswer), - remoteAnswerCh: make(chan OfferAnswer), - statusRecorder: statusRecorder, - wgProxyFactory: wgProxyFactory, - adapter: adapter, - iFaceDiscover: iFaceDiscover, - }, nil +func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { + _, allowedIPsIP, err := net.ParseCIDR(config.WgConfig.AllowedIps) + if err != nil { + log.Errorf("failed to parse allowedIPS: %v", err) + return nil, err + } + + ctx, ctxCancel := context.WithCancel(engineCtx) + connLog := log.WithField("peer", config.Key) + + var conn = &Conn{ + log: connLog, + ctx: ctx, + ctxCancel: ctxCancel, + config: config, + statusRecorder: statusRecorder, + wgProxyFactory: wgProxyFactory, + signaler: signaler, + relayManager: relayManager, + allowedIPsIP: allowedIPsIP.String(), + statusRelay: StatusDisconnected, + statusICE: StatusDisconnected, + iCEDisconnected: make(chan bool, 1), + relayDisconnected: make(chan bool, 1), + } + + rFns := WorkerRelayCallbacks{ + OnConnReady: conn.relayConnectionIsReady, + OnDisconnected: conn.onWorkerRelayStateDisconnected, + } + + wFns := WorkerICECallbacks{ + OnConnReady: conn.iCEConnectionIsReady, + OnStatusChanged: conn.onWorkerICEStateDisconnected, + } + + conn.workerRelay = NewWorkerRelay(ctx, connLog, config, relayManager, rFns) + + relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() + conn.workerICE, err = NewWorkerICE(ctx, connLog, config, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns) + if err != nil { + return nil, err + } + + conn.handshaker = NewHandshaker(ctx, connLog, config, signaler, conn.workerICE, conn.workerRelay) + + conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer) + if os.Getenv("NB_FORCE_RELAY") != "true" { + conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer) + } + + go conn.handshaker.Listen() + + return conn, nil } -func (conn *Conn) reCreateAgent() error { +// Open opens connection to the remote peer +// It will try to establish a connection using ICE and in parallel with relay. The higher priority connection type will +// be used. +func (conn *Conn) Open() { + conn.log.Debugf("open connection to peer") conn.mu.Lock() defer conn.mu.Unlock() - - failedTimeout := 6 * time.Second - - var err error - transportNet, err := conn.newStdNet() - if err != nil { - log.Errorf("failed to create pion's stdnet: %s", err) - } - - iceKeepAlive := iceKeepAlive() - iceDisconnectedTimeout := iceDisconnectedTimeout() - iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() - - agentConfig := &ice.AgentConfig{ - MulticastDNSMode: ice.MulticastDNSModeDisabled, - NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: conn.config.StunTurn, - CandidateTypes: conn.candidateTypes(), - FailedTimeout: &failedTimeout, - InterfaceFilter: stdnet.InterfaceFilter(conn.config.InterfaceBlackList), - UDPMux: conn.config.UDPMux, - UDPMuxSrflx: conn.config.UDPMuxSrflx, - NAT1To1IPs: conn.config.NATExternalIPs, - Net: transportNet, - DisconnectedTimeout: &iceDisconnectedTimeout, - KeepaliveInterval: &iceKeepAlive, - RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, - } - - if conn.config.DisableIPv6Discovery { - agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} - } - - conn.agent, err = ice.NewAgent(agentConfig) - if err != nil { - return err - } - - err = conn.agent.OnCandidate(conn.onICECandidate) - if err != nil { - return err - } - - err = conn.agent.OnConnectionStateChange(conn.onICEConnectionStateChange) - if err != nil { - return err - } - - err = conn.agent.OnSelectedCandidatePairChange(conn.onICESelectedCandidatePair) - if err != nil { - return err - } - - err = conn.agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { - err := conn.statusRecorder.UpdateLatency(conn.config.Key, p.Latency()) - if err != nil { - log.Debugf("failed to update latency for peer %s: %s", conn.config.Key, err) - return - } - }) - if err != nil { - return fmt.Errorf("failed setting binding response callback: %w", err) - } - - return nil -} - -func (conn *Conn) candidateTypes() []ice.CandidateType { - if hasICEForceRelayConn() { - return []ice.CandidateType{ice.CandidateTypeRelay} - } - // TODO: remove this once we have refactored userspace proxy into the bind package - if runtime.GOOS == "ios" { - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} - } - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} -} - -// Open opens connection to the remote peer starting ICE candidate gathering process. -// Blocks until connection has been closed or connection timeout. -// ConnStatus will be set accordingly -func (conn *Conn) Open(ctx context.Context) error { - log.Debugf("trying to connect to peer %s", conn.config.Key) + conn.opened = true peerState := State{ PubKey: conn.config.Key, IP: strings.Split(conn.config.WgConfig.AllowedIps, "/")[0], ConnStatusUpdate: time.Now(), - ConnStatus: conn.status, + ConnStatus: StatusDisconnected, Mux: new(sync.RWMutex), } err := conn.statusRecorder.UpdatePeerState(peerState) if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + conn.log.Warnf("error while updating the state err: %v", err) } - defer func() { - err := conn.cleanup() - if err != nil { - log.Warnf("error while cleaning up peer connection %s: %v", conn.config.Key, err) - return - } - }() - - err = conn.reCreateAgent() - if err != nil { - return err - } - - err = conn.sendOffer() - if err != nil { - return err - } - - log.Debugf("connection offer sent to peer %s, waiting for the confirmation", conn.config.Key) - - // Only continue once we got a connection confirmation from the remote peer. - // The connection timeout could have happened before a confirmation received from the remote. - // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) - var remoteOfferAnswer OfferAnswer - select { - case remoteOfferAnswer = <-conn.remoteOffersCh: - // received confirmation from the remote peer -> ready to proceed - err = conn.sendAnswer() - if err != nil { - return err - } - case remoteOfferAnswer = <-conn.remoteAnswerCh: - case <-time.After(conn.config.Timeout): - return NewConnectionTimeoutError(conn.config.Key, conn.config.Timeout) - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - } - - log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", - conn.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) - - // at this point we received offer/answer and we are ready to gather candidates - conn.mu.Lock() - conn.status = StatusConnecting - conn.ctx, conn.notifyDisconnected = context.WithCancel(ctx) - defer conn.notifyDisconnected() - conn.mu.Unlock() - - peerState = State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) - } - - err = conn.agent.GatherCandidates() - if err != nil { - return fmt.Errorf("gather candidates: %v", err) - } - - // will block until connection succeeded - // but it won't release if ICE Agent went into Disconnected or Failed state, - // so we have to cancel it with the provided context once agent detected a broken connection - isControlling := conn.config.LocalKey > conn.config.Key - var remoteConn *ice.Conn - if isControlling { - remoteConn, err = conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) - } else { - remoteConn, err = conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) - } - if err != nil { - return err - } - - // dynamically set remote WireGuard port if other side specified a different one from the default one - remoteWgPort := iface.DefaultWgPort - if remoteOfferAnswer.WgListenPort != 0 { - remoteWgPort = remoteOfferAnswer.WgListenPort - } - - // the ice connection has been established successfully so we are ready to start the proxy - remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort, remoteOfferAnswer.RosenpassPubKey, - remoteOfferAnswer.RosenpassAddr) - if err != nil { - return err - } - - log.Infof("connected to peer %s, endpoint address: %s", conn.config.Key, remoteAddr.String()) - - // wait until connection disconnected or has been closed externally (upper layer, e.g. engine) - select { - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - case <-conn.ctx.Done(): - // disconnected from the remote peer - return NewConnectionDisconnectedError(conn.config.Key) - } + go conn.startHandshakeAndReconnect() } -func isRelayCandidate(candidate ice.Candidate) bool { - return candidate.Type() == ice.CandidateTypeRelay +func (conn *Conn) startHandshakeAndReconnect() { + conn.waitInitialRandomSleepTime() + + err := conn.handshaker.sendOffer() + if err != nil { + conn.log.Errorf("failed to send initial offer: %v", err) + } + + if conn.workerRelay.IsController() { + conn.reconnectLoopWithRetry() + } else { + conn.reconnectLoopForOnDisconnectedEvent() + } + +} + +// Close closes this peer Conn issuing a close event to the Conn closeCh +func (conn *Conn) Close() { + conn.mu.Lock() + defer conn.mu.Unlock() + + conn.ctxCancel() + + if !conn.opened { + log.Debugf("ignore close connection to peer") + return + } + + if conn.wgProxyRelay != nil { + err := conn.wgProxyRelay.CloseConn() + if err != nil { + conn.log.Errorf("failed to close wg proxy for relay: %v", err) + } + conn.wgProxyRelay = nil + } + + if conn.wgProxyICE != nil { + err := conn.wgProxyICE.CloseConn() + if err != nil { + conn.log.Errorf("failed to close wg proxy for ice: %v", err) + } + conn.wgProxyICE = nil + } + + err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if err != nil { + conn.log.Errorf("failed to remove wg endpoint: %v", err) + } + + conn.freeUpConnID() + + if conn.evalStatus() == StatusConnected && conn.onDisconnected != nil { + conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) + } + + conn.setStatusToDisconnected() +} + +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { + conn.log.Debugf("OnRemoteAnswer, status ICE: %s, status relay: %s", conn.statusICE, conn.statusRelay) + return conn.handshaker.OnRemoteAnswer(answer) +} + +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + conn.workerICE.OnRemoteCandidate(candidate, haRoutes) } func (conn *Conn) AddBeforeAddPeerHook(hook nbnet.AddHookFunc) { conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook) } - func (conn *Conn) AddAfterRemovePeerHook(hook nbnet.RemoveHookFunc) { conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook) } -// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected -func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) (net.Addr, error) { - conn.mu.Lock() - defer conn.mu.Unlock() - - pair, err := conn.agent.GetSelectedCandidatePair() - if err != nil { - return nil, err - } - - var endpoint net.Addr - if isRelayCandidate(pair.Local) { - log.Debugf("setup relay connection") - conn.wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - endpoint, err = conn.wgProxy.AddTurnConn(remoteConn) - if err != nil { - return nil, err - } - } else { - // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port - go conn.punchRemoteWGPort(pair, remoteWgPort) - endpoint = remoteConn.RemoteAddr() - } - - endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) - log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) - - conn.connID = nbnet.GenerateConnID() - for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { - log.Errorf("Before add peer hook failed: %v", err) - } - } - - err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) - if err != nil { - if conn.wgProxy != nil { - if err := conn.wgProxy.CloseConn(); err != nil { - log.Warnf("Failed to close turn connection: %v", err) - } - } - return nil, fmt.Errorf("update peer: %w", err) - } - - conn.status = StatusConnected - rosenpassEnabled := false - if remoteRosenpassPubKey != nil { - rosenpassEnabled = true - } - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - LocalIceCandidateType: pair.Local.Type().String(), - RemoteIceCandidateType: pair.Remote.Type().String(), - LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), - RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), - Direct: !isRelayCandidate(pair.Local), - RosenpassEnabled: rosenpassEnabled, - Mux: new(sync.RWMutex), - } - if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { - peerState.Relayed = true - } - - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("unable to save peer's state, got error: %v", err) - } - - _, ipNet, err := net.ParseCIDR(conn.config.WgConfig.AllowedIps) - if err != nil { - return nil, err - } - - if runtime.GOOS == "ios" { - runtime.GC() - } - - if conn.onConnected != nil { - conn.onConnected(conn.config.Key, remoteRosenpassPubKey, ipNet.IP.String(), remoteRosenpassAddr) - } - - return endpoint, nil -} - -func (conn *Conn) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { - // wait local endpoint configuration - time.Sleep(time.Second) - addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) - if err != nil { - log.Warnf("got an error while resolving the udp address, err: %s", err) - return - } - - mux, ok := conn.config.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) - if !ok { - log.Warn("invalid udp mux conversion") - return - } - _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) - if err != nil { - log.Warnf("got an error while sending the punch packet, err: %s", err) - } -} - -// cleanup closes all open resources and sets status to StatusDisconnected -func (conn *Conn) cleanup() error { - log.Debugf("trying to cleanup %s", conn.config.Key) - conn.mu.Lock() - defer conn.mu.Unlock() - - conn.sentExtraSrflx = false - - var err1, err2, err3 error - if conn.agent != nil { - err1 = conn.agent.Close() - if err1 == nil { - conn.agent = nil - } - } - - if conn.wgProxy != nil { - err2 = conn.wgProxy.CloseConn() - conn.wgProxy = nil - } - - // todo: is it problem if we try to remove a peer what is never existed? - err3 = conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) - - if conn.connID != "" { - for _, hook := range conn.afterRemovePeerHooks { - if err := hook(conn.connID); err != nil { - log.Errorf("After remove peer hook failed: %v", err) - } - } - } - conn.connID = "" - - if conn.notifyDisconnected != nil { - conn.notifyDisconnected() - conn.notifyDisconnected = nil - } - - if conn.status == StatusConnected && conn.onDisconnected != nil { - conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) - } - - conn.status = StatusDisconnected - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err := conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - // pretty common error because by that time Engine can already remove the peer and status won't be available. - // todo rethink status updates - log.Debugf("error while updating peer's %s state, err: %v", conn.config.Key, err) - } - if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { - log.Debugf("failed to reset wireguard stats for peer %s: %s", conn.config.Key, err) - } - - log.Debugf("cleaned up connection to peer %s", conn.config.Key) - if err1 != nil { - return err1 - } - if err2 != nil { - return err2 - } - return err3 -} - -// SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer -func (conn *Conn) SetSignalOffer(handler func(offer OfferAnswer) error) { - conn.signalOffer = handler -} - // SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) { conn.onConnected = handler @@ -576,218 +278,499 @@ func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string) conn.onDisconnected = handler } -// SetSignalAnswer sets a handler function to be triggered by Conn when a new connection answer has to be signalled to the remote peer -func (conn *Conn) SetSignalAnswer(handler func(answer OfferAnswer) error) { - conn.signalAnswer = handler +func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { + conn.log.Debugf("OnRemoteOffer, on status ICE: %s, status Relay: %s", conn.statusICE, conn.statusRelay) + return conn.handshaker.OnRemoteOffer(offer) } -// SetSignalCandidate sets a handler function to be triggered by Conn when a new ICE local connection candidate has to be signalled to the remote peer -func (conn *Conn) SetSignalCandidate(handler func(candidate ice.Candidate) error) { - conn.signalCandidate = handler -} - -// SetSendSignalMessage sets a handler function to be triggered by Conn when there is new message to send via signal -func (conn *Conn) SetSendSignalMessage(handler func(message *sProto.Message) error) { - conn.sendSignalMessage = handler -} - -// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates -// and then signals them to the remote peer -func (conn *Conn) onICECandidate(candidate ice.Candidate) { - // nil means candidate gathering has been ended - if candidate == nil { - return - } - - // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored - log.Debugf("discovered local candidate %s", candidate.String()) - go func() { - err := conn.signalCandidate(candidate) - if err != nil { - log.Errorf("failed signaling candidate to the remote peer %s %s", conn.config.Key, err) - } - }() - - if !conn.shouldSendExtraSrflxCandidate(candidate) { - return - } - - // sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port) - // this is useful when network has an existing port forwarding rule for the wireguard port and this peer - extraSrflx, err := extraSrflxCandidate(candidate) - if err != nil { - log.Errorf("failed creating extra server reflexive candidate %s", err) - return - } - conn.sentExtraSrflx = true - - go func() { - err = conn.signalCandidate(extraSrflx) - if err != nil { - log.Errorf("failed signaling the extra server reflexive candidate to the remote peer %s: %s", conn.config.Key, err) - } - }() -} - -func (conn *Conn) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { - log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), - conn.config.Key) -} - -// onICEConnectionStateChange registers callback of an ICE Agent to track connection state -func (conn *Conn) onICEConnectionStateChange(state ice.ConnectionState) { - log.Debugf("peer %s ICE ConnectionState has changed to %s", conn.config.Key, state.String()) - if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { - conn.notifyDisconnected() - } -} - -func (conn *Conn) sendAnswer() error { - conn.mu.Lock() - defer conn.mu.Unlock() - - localUFrag, localPwd, err := conn.agent.GetLocalUserCredentials() - if err != nil { - return err - } - - log.Debugf("sending answer to %s", conn.config.Key) - err = conn.signalAnswer(OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, - WgListenPort: conn.config.LocalWgPort, - Version: version.NetbirdVersion(), - RosenpassPubKey: conn.config.RosenpassPubKey, - RosenpassAddr: conn.config.RosenpassAddr, - }) - if err != nil { - return err - } - - return nil -} - -// sendOffer prepares local user credentials and signals them to the remote peer -func (conn *Conn) sendOffer() error { - conn.mu.Lock() - defer conn.mu.Unlock() - - localUFrag, localPwd, err := conn.agent.GetLocalUserCredentials() - if err != nil { - return err - } - err = conn.signalOffer(OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, - WgListenPort: conn.config.LocalWgPort, - Version: version.NetbirdVersion(), - RosenpassPubKey: conn.config.RosenpassPubKey, - RosenpassAddr: conn.config.RosenpassAddr, - }) - if err != nil { - return err - } - return nil -} - -// Close closes this peer Conn issuing a close event to the Conn closeCh -func (conn *Conn) Close() error { - conn.mu.Lock() - defer conn.mu.Unlock() - select { - case conn.closeCh <- struct{}{}: - return nil - default: - // probably could happen when peer has been added and removed right after not even starting to connect - // todo further investigate - // this really happens due to unordered messages coming from management - // more importantly it causes inconsistency -> 2 Conn objects for the same peer - // e.g. this flow: - // update from management has peers: [1,2,3,4] - // engine creates a Conn for peers: [1,2,3,4] and schedules Open in ~1sec - // before conn.Open() another update from management arrives with peers: [1,2,3] - // engine removes peer 4 and calls conn.Close() which does nothing (this default clause) - // before conn.Open() another update from management arrives with peers: [1,2,3,4,5] - // engine adds a new Conn for 4 and 5 - // therefore peer 4 has 2 Conn objects - log.Warnf("Connection has been already closed or attempted closing not started connection %s", conn.config.Key) - return NewConnectionAlreadyClosed(conn.config.Key) - } +// WgConfig returns the WireGuard config +func (conn *Conn) WgConfig() WgConfig { + return conn.config.WgConfig } // Status returns current status of the Conn func (conn *Conn) Status() ConnStatus { conn.mu.Lock() defer conn.mu.Unlock() - return conn.status -} - -// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { - log.Debugf("OnRemoteOffer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteOffersCh <- offer: - return true - default: - log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - // connection might not be ready yet to receive so we ignore the message - return false - } -} - -// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { - log.Debugf("OnRemoteAnswer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteAnswerCh <- answer: - return true - default: - // connection might not be ready yet to receive so we ignore the message - log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - return false - } -} - -// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. -func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) - go func() { - conn.mu.Lock() - defer conn.mu.Unlock() - - if conn.agent == nil { - return - } - - err := conn.agent.AddRemoteCandidate(candidate) - if err != nil { - log.Errorf("error while handling remote candidate from peer %s", conn.config.Key) - return - } - }() + return conn.evalStatus() } func (conn *Conn) GetKey() string { return conn.config.Key } -func (conn *Conn) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { - if !conn.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { - return true +func (conn *Conn) reconnectLoopWithRetry() { + // Give chance to the peer to establish the initial connection. + // With it, we can decrease to send necessary offer + select { + case <-conn.ctx.Done(): + case <-time.After(3 * time.Second): + } + + ticker := conn.prepareExponentTicker() + defer ticker.Stop() + time.Sleep(1 * time.Second) + for { + select { + case t := <-ticker.C: + if t.IsZero() { + // in case if the ticker has been canceled by context then avoid the temporary loop + return + } + + if conn.workerRelay.IsRelayConnectionSupportedWithPeer() { + if conn.statusRelay == StatusDisconnected || conn.statusICE == StatusDisconnected { + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + } + } else { + if conn.statusICE == StatusDisconnected { + conn.log.Tracef("ticker timedout, ice state: %s", conn.statusICE) + } + } + + // checks if there is peer connection is established via relay or ice + if conn.isConnected() { + continue + } + + err := conn.handshaker.sendOffer() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, reset reconnect timer") + ticker.Stop() + ticker = conn.prepareExponentTicker() + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, reset reconnect timer") + ticker.Stop() + ticker = conn.prepareExponentTicker() + case <-conn.ctx.Done(): + return + } } - return false } -func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { - relatedAdd := candidate.RelatedAddress() - return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ - Network: candidate.NetworkType().String(), - Address: candidate.Address(), - Port: relatedAdd.Port, - Component: candidate.Component(), - RelAddr: relatedAdd.Address, - RelPort: relatedAdd.Port, - }) +func (conn *Conn) prepareExponentTicker() *backoff.Ticker { + bo := backoff.WithContext(&backoff.ExponentialBackOff{ + InitialInterval: 800 * time.Millisecond, + RandomizationFactor: 0.01, + Multiplier: 2, + MaxInterval: conn.config.Timeout, + MaxElapsedTime: 0, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + }, conn.ctx) + + ticker := backoff.NewTicker(bo) + <-ticker.C // consume the initial tick what is happening right after the ticker has been created + + return ticker +} + +// reconnectLoopForOnDisconnectedEvent is used when the peer is not a controller and it should reconnect to the peer +// when the connection is lost. It will try to establish a connection only once time if before the connection was established +// It track separately the ice and relay connection status. Just because a lover priority connection reestablished it does not +// mean that to switch to it. We always force to use the higher priority connection. +func (conn *Conn) reconnectLoopForOnDisconnectedEvent() { + for { + select { + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, try to send new offer") + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, try to send new offer") + case <-conn.ctx.Done(): + return + } + + err := conn.handshaker.SendOffer() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + } +} + +// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected +func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICEConnInfo) { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + conn.log.Debugf("ICE connection is ready") + + conn.statusICE = StatusConnected + + defer conn.updateIceState(iceConnInfo) + + if conn.currentConnPriority > priority { + return + } + + conn.log.Infof("set ICE to active connection") + + endpoint, wgProxy, err := conn.getEndpointForICEConnInfo(iceConnInfo) + if err != nil { + return + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.log.Debugf("Conn resolved IP is %s for endopint %s", endpoint, endpointUdpAddr.IP) + + conn.connIDICE = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connIDICE, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) + } + } + + err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) + if err != nil { + if wgProxy != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close turn connection: %v", err) + } + } + conn.log.Warnf("Failed to update wg peer configuration: %v", err) + return + } + wgConfigWorkaround() + + if conn.wgProxyICE != nil { + if err := conn.wgProxyICE.CloseConn(); err != nil { + conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err) + } + } + conn.wgProxyICE = wgProxy + + conn.currentConnPriority = priority + + conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) +} + +// todo review to make sense to handle connecting and disconnected status also? +func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { + conn.mu.Lock() + defer conn.mu.Unlock() + + conn.log.Tracef("ICE connection state changed to %s", newState) + + // switch back to relay connection + if conn.endpointRelay != nil && conn.currentConnPriority != connPriorityRelay { + conn.log.Debugf("ICE disconnected, set Relay to active connection") + err := conn.configureWGEndpoint(conn.endpointRelay) + if err != nil { + conn.log.Errorf("failed to switch to relay conn: %v", err) + } + } + + changed := conn.statusICE != newState && newState != StatusConnecting + conn.statusICE = newState + + select { + case conn.iCEDisconnected <- changed: + default: + } + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), + ConnStatusUpdate: time.Now(), + } + + err := conn.statusRecorder.UpdatePeerICEStateToDisconnected(peerState) + if err != nil { + conn.log.Warnf("unable to set peer's state to disconnected ice, got error: %v", err) + } +} + +func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + conn.log.Debugf("Relay connection is ready to use") + conn.statusRelay = StatusConnected + + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + endpoint, err := wgProxy.AddTurnConn(rci.relayedConn) + if err != nil { + conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err) + return + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.endpointRelay = endpointUdpAddr + conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + + defer conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey) + + if conn.currentConnPriority > connPriorityRelay { + if conn.statusICE == StatusConnected { + log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnPriority) + return + } + } + + conn.connIDRelay = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connIDRelay, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) + } + } + + err = conn.configureWGEndpoint(endpointUdpAddr) + if err != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close relay connection: %v", err) + } + conn.log.Errorf("Failed to update wg peer configuration: %v", err) + return + } + wgConfigWorkaround() + + if conn.wgProxyRelay != nil { + if err := conn.wgProxyRelay.CloseConn(); err != nil { + conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err) + } + } + conn.wgProxyRelay = wgProxy + conn.currentConnPriority = connPriorityRelay + + conn.log.Infof("start to communicate with peer via relay") + conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr) +} + +func (conn *Conn) onWorkerRelayStateDisconnected() { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.wgProxyRelay != nil { + log.Debugf("relayed connection is closed, clean up WireGuard config") + err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if err != nil { + conn.log.Errorf("failed to remove wg endpoint: %v", err) + } + + conn.endpointRelay = nil + _ = conn.wgProxyRelay.CloseConn() + conn.wgProxyRelay = nil + } + + changed := conn.statusRelay != StatusDisconnected + conn.statusRelay = StatusDisconnected + + select { + case conn.relayDisconnected <- changed: + default: + } + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), + ConnStatusUpdate: time.Now(), + } + + err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's state to Relay disconnected, got error: %v", err) + } +} + +func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error { + return conn.config.WgConfig.WgInterface.UpdatePeer( + conn.config.WgConfig.RemoteKey, + conn.config.WgConfig.AllowedIps, + defaultWgKeepAlive, + addr, + conn.config.WgConfig.PreSharedKey, + ) +} + +func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []byte) { + peerState := State{ + PubKey: conn.config.Key, + ConnStatusUpdate: time.Now(), + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), + RelayServerAddress: relayServerAddr, + RosenpassEnabled: isRosenpassEnabled(rosenpassPubKey), + } + + err := conn.statusRecorder.UpdatePeerRelayedState(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's Relay state, got error: %v", err) + } +} + +func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) { + peerState := State{ + PubKey: conn.config.Key, + ConnStatusUpdate: time.Now(), + ConnStatus: conn.evalStatus(), + Relayed: iceConnInfo.Relayed, + LocalIceCandidateType: iceConnInfo.LocalIceCandidateType, + RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, + LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, + RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, + RosenpassEnabled: isRosenpassEnabled(iceConnInfo.RosenpassPubKey), + } + + err := conn.statusRecorder.UpdatePeerICEState(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's ICE state, got error: %v", err) + } +} + +func (conn *Conn) setStatusToDisconnected() { + conn.statusRelay = StatusDisconnected + conn.statusICE = StatusDisconnected + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: StatusDisconnected, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + err := conn.statusRecorder.UpdatePeerState(peerState) + if err != nil { + // pretty common error because by that time Engine can already remove the peer and status won't be available. + // todo rethink status updates + conn.log.Debugf("error while updating peer's state, err: %v", err) + } + if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { + conn.log.Debugf("failed to reset wireguard stats for peer: %s", err) + } +} + +func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { + if runtime.GOOS == "ios" { + runtime.GC() + } + + if conn.onConnected != nil { + conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIPsIP, remoteRosenpassAddr) + } +} + +func (conn *Conn) waitInitialRandomSleepTime() { + minWait := 100 + maxWait := 800 + duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + + timeout := time.NewTimer(duration) + defer timeout.Stop() + + select { + case <-conn.ctx.Done(): + case <-timeout.C: + } +} + +func (conn *Conn) isRelayed() bool { + if conn.statusRelay == StatusDisconnected && (conn.statusICE == StatusDisconnected || conn.statusICE == StatusConnecting) { + return false + } + + if conn.currentConnPriority == connPriorityICEP2P { + return false + } + + return true +} + +func (conn *Conn) evalStatus() ConnStatus { + if conn.statusRelay == StatusConnected || conn.statusICE == StatusConnected { + return StatusConnected + } + + if conn.statusRelay == StatusConnecting || conn.statusICE == StatusConnecting { + return StatusConnecting + } + + return StatusDisconnected +} + +func (conn *Conn) isConnected() bool { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.statusICE != StatusConnected && conn.statusICE != StatusConnecting { + return false + } + + if conn.workerRelay.IsRelayConnectionSupportedWithPeer() { + if conn.statusRelay != StatusConnected { + return false + } + } + + return true +} + +func (conn *Conn) freeUpConnID() { + if conn.connIDRelay != "" { + for _, hook := range conn.afterRemovePeerHooks { + if err := hook(conn.connIDRelay); err != nil { + conn.log.Errorf("After remove peer hook failed: %v", err) + } + } + conn.connIDRelay = "" + } + + if conn.connIDICE != "" { + for _, hook := range conn.afterRemovePeerHooks { + if err := hook(conn.connIDICE); err != nil { + conn.log.Errorf("After remove peer hook failed: %v", err) + } + } + conn.connIDICE = "" + } +} + +func (conn *Conn) getEndpointForICEConnInfo(iceConnInfo ICEConnInfo) (net.Addr, wgproxy.Proxy, error) { + if !iceConnInfo.RelayedOnLocal { + return iceConnInfo.RemoteConn.RemoteAddr(), nil, nil + } + conn.log.Debugf("setup ice turn connection") + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + ep, err := wgProxy.AddTurnConn(iceConnInfo.RemoteConn) + if err != nil { + conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) + err = wgProxy.CloseConn() + if err != nil { + conn.log.Warnf("failed to close turn proxy connection: %v", err) + } + return nil, nil, err + } + return ep, wgProxy, nil +} + +func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { + return remoteRosenpassPubKey != nil +} + +// wgConfigWorkaround is a workaround for the issue with WireGuard configuration update +// When update a peer configuration in near to each other time, the second update can be ignored by WireGuard +func wgConfigWorkaround() { + time.Sleep(100 * time.Millisecond) } diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index b608a5929..59f249b82 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -2,25 +2,33 @@ package peer import ( "context" + "os" "sync" "testing" "time" "github.com/magiconair/properties/assert" - "github.com/pion/stun/v2" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" + "github.com/netbirdio/netbird/util" ) var connConf = ConnConfig{ - Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - StunTurn: []*stun.URI{}, - InterfaceBlackList: nil, - Timeout: time.Second, - LocalWgPort: 51820, + Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + Timeout: time.Second, + LocalWgPort: 51820, + ICEConfig: ICEConfig{ + InterfaceBlackList: nil, + }, +} + +func TestMain(m *testing.M) { + _ = util.InitLog("trace", "console") + code := m.Run() + os.Exit(code) } func TestNewConn_interfaceFilter(t *testing.T) { @@ -40,7 +48,7 @@ func TestConn_GetKey(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -55,7 +63,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -63,7 +71,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) go func() { - <-conn.remoteOffersCh + <-conn.handshaker.remoteOffersCh wg.Done() }() @@ -92,7 +100,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -100,7 +108,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) go func() { - <-conn.remoteAnswerCh + <-conn.handshaker.remoteAnswerCh wg.Done() }() @@ -128,58 +136,33 @@ func TestConn_Status(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } tables := []struct { - name string - status ConnStatus - want ConnStatus + name string + statusIce ConnStatus + statusRelay ConnStatus + want ConnStatus }{ - {"StatusConnected", StatusConnected, StatusConnected}, - {"StatusDisconnected", StatusDisconnected, StatusDisconnected}, - {"StatusConnecting", StatusConnecting, StatusConnecting}, + {"StatusConnected", StatusConnected, StatusConnected, StatusConnected}, + {"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected}, + {"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting}, + {"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting}, + {"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected}, + {"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting}, + {"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected}, } for _, table := range tables { t.Run(table.name, func(t *testing.T) { - conn.status = table.status + conn.statusICE = table.statusIce + conn.statusRelay = table.statusRelay got := conn.Status() assert.Equal(t, got, table.want, "they should be equal") }) } } - -func TestConn_Close(t *testing.T) { - wgProxyFactory := wgproxy.NewFactory(context.Background(), false, connConf.LocalWgPort) - defer func() { - _ = wgProxyFactory.Free() - }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) - if err != nil { - return - } - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - <-conn.closeCh - wg.Done() - }() - - go func() { - for { - err := conn.Close() - if err != nil { - continue - } else { - return - } - } - }() - - wg.Wait() -} diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go new file mode 100644 index 000000000..d5ed10357 --- /dev/null +++ b/client/internal/peer/handshaker.go @@ -0,0 +1,191 @@ +package peer + +import ( + "context" + "errors" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/version" +) + +var ( + ErrSignalIsNotReady = errors.New("signal is not ready") +) + +// IceCredentials ICE protocol credentials struct +type IceCredentials struct { + UFrag string + Pwd string +} + +// OfferAnswer represents a session establishment offer or answer +type OfferAnswer struct { + IceCredentials IceCredentials + // WgListenPort is a remote WireGuard listen port. + // This field is used when establishing a direct WireGuard connection without any proxy. + // We can set the remote peer's endpoint with this port. + WgListenPort int + + // Version of NetBird Agent + Version string + // RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message + // This value is the local Rosenpass server public key when sending the message + RosenpassPubKey []byte + // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message + // This value is the local Rosenpass server address when sending the message + RosenpassAddr string + + // relay server address + RelaySrvAddress string +} + +type Handshaker struct { + mu sync.Mutex + ctx context.Context + log *log.Entry + config ConnConfig + signaler *Signaler + ice *WorkerICE + relay *WorkerRelay + onNewOfferListeners []func(*OfferAnswer) + + // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection + remoteOffersCh chan OfferAnswer + // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection + remoteAnswerCh chan OfferAnswer +} + +func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker { + return &Handshaker{ + ctx: ctx, + log: log, + config: config, + signaler: signaler, + ice: ice, + relay: relay, + remoteOffersCh: make(chan OfferAnswer), + remoteAnswerCh: make(chan OfferAnswer), + } +} + +func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAnswer)) { + h.onNewOfferListeners = append(h.onNewOfferListeners, offer) +} + +func (h *Handshaker) Listen() { + for { + h.log.Debugf("wait for remote offer confirmation") + remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() + if err != nil { + if _, ok := err.(*ConnectionClosedError); ok { + h.log.Tracef("stop handshaker") + return + } + h.log.Errorf("failed to received remote offer confirmation: %s", err) + continue + } + + h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) + for _, listener := range h.onNewOfferListeners { + go listener(remoteOfferAnswer) + } + } +} + +func (h *Handshaker) SendOffer() error { + h.mu.Lock() + defer h.mu.Unlock() + return h.sendOffer() +} + +// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { + select { + case h.remoteOffersCh <- offer: + return true + default: + h.log.Debugf("OnRemoteOffer skipping message because is not ready") + // connection might not be ready yet to receive so we ignore the message + return false + } +} + +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { + select { + case h.remoteAnswerCh <- answer: + return true + default: + // connection might not be ready yet to receive so we ignore the message + h.log.Debugf("OnRemoteAnswer skipping message because is not ready") + return false + } +} + +func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { + select { + case remoteOfferAnswer := <-h.remoteOffersCh: + // received confirmation from the remote peer -> ready to proceed + err := h.sendAnswer() + if err != nil { + return nil, err + } + return &remoteOfferAnswer, nil + case remoteOfferAnswer := <-h.remoteAnswerCh: + return &remoteOfferAnswer, nil + case <-h.ctx.Done(): + // closed externally + return nil, NewConnectionClosedError(h.config.Key) + } +} + +// sendOffer prepares local user credentials and signals them to the remote peer +func (h *Handshaker) sendOffer() error { + if !h.signaler.Ready() { + return ErrSignalIsNotReady + } + + iceUFrag, icePwd := h.ice.GetLocalUserCredentials() + offer := OfferAnswer{ + IceCredentials: IceCredentials{iceUFrag, icePwd}, + WgListenPort: h.config.LocalWgPort, + Version: version.NetbirdVersion(), + RosenpassPubKey: h.config.RosenpassPubKey, + RosenpassAddr: h.config.RosenpassAddr, + } + + addr, err := h.relay.RelayInstanceAddress() + if err == nil { + offer.RelaySrvAddress = addr + } + + return h.signaler.SignalOffer(offer, h.config.Key) +} + +func (h *Handshaker) sendAnswer() error { + h.log.Debugf("sending answer") + uFrag, pwd := h.ice.GetLocalUserCredentials() + + answer := OfferAnswer{ + IceCredentials: IceCredentials{uFrag, pwd}, + WgListenPort: h.config.LocalWgPort, + Version: version.NetbirdVersion(), + RosenpassPubKey: h.config.RosenpassPubKey, + RosenpassAddr: h.config.RosenpassAddr, + } + addr, err := h.relay.RelayInstanceAddress() + if err == nil { + answer.RelaySrvAddress = addr + } + + err = h.signaler.SignalAnswer(answer, h.config.Key) + if err != nil { + return err + } + + return nil +} diff --git a/client/internal/peer/signaler.go b/client/internal/peer/signaler.go new file mode 100644 index 000000000..713123e5d --- /dev/null +++ b/client/internal/peer/signaler.go @@ -0,0 +1,70 @@ +package peer + +import ( + "github.com/pion/ice/v3" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + signal "github.com/netbirdio/netbird/signal/client" + sProto "github.com/netbirdio/netbird/signal/proto" +) + +type Signaler struct { + signal signal.Client + wgPrivateKey wgtypes.Key +} + +func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler { + return &Signaler{ + signal: signal, + wgPrivateKey: wgPrivateKey, + } +} + +func (s *Signaler) SignalOffer(offer OfferAnswer, remoteKey string) error { + return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER) +} + +func (s *Signaler) SignalAnswer(offer OfferAnswer, remoteKey string) error { + return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER) +} + +func (s *Signaler) SignalICECandidate(candidate ice.Candidate, remoteKey string) error { + return s.signal.Send(&sProto.Message{ + Key: s.wgPrivateKey.PublicKey().String(), + RemoteKey: remoteKey, + Body: &sProto.Body{ + Type: sProto.Body_CANDIDATE, + Payload: candidate.Marshal(), + }, + }) +} + +func (s *Signaler) Ready() bool { + return s.signal.Ready() +} + +// SignalOfferAnswer signals either an offer or an answer to remote peer +func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error { + msg, err := signal.MarshalCredential( + s.wgPrivateKey, + offerAnswer.WgListenPort, + remoteKey, + &signal.Credential{ + UFrag: offerAnswer.IceCredentials.UFrag, + Pwd: offerAnswer.IceCredentials.Pwd, + }, + bodyType, + offerAnswer.RosenpassPubKey, + offerAnswer.RosenpassAddr, + offerAnswer.RelaySrvAddress) + if err != nil { + return err + } + + err = s.signal.Send(msg) + if err != nil { + return err + } + + return nil +} diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index a7cfb95c4..44b6083a6 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -13,6 +13,7 @@ import ( "github.com/netbirdio/netbird/client/internal/relay" "github.com/netbirdio/netbird/iface" "github.com/netbirdio/netbird/management/domain" + relayClient "github.com/netbirdio/netbird/relay/client" ) // State contains the latest state of a peer @@ -24,11 +25,11 @@ type State struct { ConnStatus ConnStatus ConnStatusUpdate time.Time Relayed bool - Direct bool LocalIceCandidateType string RemoteIceCandidateType string LocalIceCandidateEndpoint string RemoteIceCandidateEndpoint string + RelayServerAddress string LastWireguardHandshake time.Time BytesTx int64 BytesRx int64 @@ -142,6 +143,8 @@ type Status struct { // Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events // set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications() peerListChangedForNotification bool + + relayMgr *relayClient.Manager } // NewRecorder returns a new Status instance @@ -156,6 +159,12 @@ func NewRecorder(mgmAddress string) *Status { } } +func (d *Status) SetRelayMgr(manager *relayClient.Manager) { + d.mux.Lock() + defer d.mux.Unlock() + d.relayMgr = manager +} + // ReplaceOfflinePeers replaces func (d *Status) ReplaceOfflinePeers(replacement []State) { d.mux.Lock() @@ -231,17 +240,17 @@ func (d *Status) UpdatePeerState(receivedState State) error { peerState.SetRoutes(receivedState.GetRoutes()) } - skipNotification := shouldSkipNotify(receivedState, peerState) + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) if receivedState.ConnStatus != peerState.ConnStatus { peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + peerState.RelayServerAddress = receivedState.RelayServerAddress peerState.RosenpassEnabled = receivedState.RosenpassEnabled } @@ -261,6 +270,146 @@ func (d *Status) UpdatePeerState(receivedState State) error { return nil } +func (d *Status) UpdatePeerICEState(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + if receivedState.IP != "" { + peerState.IP = receivedState.IP + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.Relayed = receivedState.Relayed + peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType + peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType + peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint + peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + peerState.RosenpassEnabled = receivedState.RosenpassEnabled + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerRelayedState(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.Relayed = receivedState.Relayed + peerState.RelayServerAddress = receivedState.RelayServerAddress + peerState.RosenpassEnabled = receivedState.RosenpassEnabled + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.Relayed = receivedState.Relayed + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.RelayServerAddress = "" + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.Relayed = receivedState.Relayed + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType + peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType + peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint + peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + // UpdateWireGuardPeerState updates the WireGuard bits of the peer state func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) error { d.mux.Lock() @@ -280,13 +429,13 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) return nil } -func shouldSkipNotify(received, curr State) bool { +func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool { switch { - case received.ConnStatus == StatusConnecting: + case receivedConnStatus == StatusConnecting: return true - case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting: + case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting: return true - case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected: + case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected: return curr.IP != "" default: return false @@ -503,7 +652,28 @@ func (d *Status) GetSignalState() SignalState { } func (d *Status) GetRelayStates() []relay.ProbeResult { - return d.relayStates + if d.relayMgr == nil { + return d.relayStates + } + + // extend the list of stun, turn servers with relay address + relaysState := make([]relay.ProbeResult, len(d.relayStates), len(d.relayStates)+1) + copy(relaysState, d.relayStates) + + relayState := relay.ProbeResult{} + + // if the server connection is not established then we will use the general address + // in case of connection we will use the instance specific address + instanceAddr, err := d.relayMgr.RelayInstanceAddress() + if err != nil { + relayState.URI = d.relayMgr.ServerURL() + relayState.Err = err + } else { + relayState.URI = instanceAddr + } + + relaysState = append(relaysState, relayState) + return relaysState } func (d *Status) GetDNSStates() []NSGroupState { @@ -535,7 +705,6 @@ func (d *Status) GetFullStatus() FullStatus { } fullStatus.Peers = append(fullStatus.Peers, d.offlinePeers...) - return fullStatus } diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index a4a6e6081..1d283433b 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -2,8 +2,8 @@ package peer import ( "errors" - "testing" "sync" + "testing" "github.com/stretchr/testify/assert" ) @@ -43,7 +43,7 @@ func TestUpdatePeerState(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -64,7 +64,7 @@ func TestStatus_UpdatePeerFQDN(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -83,7 +83,7 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -108,7 +108,7 @@ func TestRemovePeer(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState diff --git a/client/internal/peer/stdnet.go b/client/internal/peer/stdnet.go index 13f5886f5..ae31ebbf0 100644 --- a/client/internal/peer/stdnet.go +++ b/client/internal/peer/stdnet.go @@ -6,6 +6,6 @@ import ( "github.com/netbirdio/netbird/client/internal/stdnet" ) -func (conn *Conn) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNet(conn.config.InterfaceBlackList) +func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList) } diff --git a/client/internal/peer/stdnet_android.go b/client/internal/peer/stdnet_android.go index 8a2454371..b411405bb 100644 --- a/client/internal/peer/stdnet_android.go +++ b/client/internal/peer/stdnet_android.go @@ -2,6 +2,6 @@ package peer import "github.com/netbirdio/netbird/client/internal/stdnet" -func (conn *Conn) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.config.InterfaceBlackList) +func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList) } diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go new file mode 100644 index 000000000..0edb124cf --- /dev/null +++ b/client/internal/peer/worker_ice.go @@ -0,0 +1,457 @@ +package peer + +import ( + "context" + "fmt" + "net" + "net/netip" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/pion/ice/v3" + "github.com/pion/randutil" + "github.com/pion/stun/v2" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/iface" + "github.com/netbirdio/netbird/iface/bind" + "github.com/netbirdio/netbird/route" +) + +const ( + iceKeepAliveDefault = 4 * time.Second + iceDisconnectedTimeoutDefault = 6 * time.Second + // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package + iceRelayAcceptanceMinWaitDefault = 2 * time.Second + + lenUFrag = 16 + lenPwd = 32 + runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +var ( + failedTimeout = 6 * time.Second +) + +type ICEConfig struct { + // StunTurn is a list of STUN and TURN URLs + StunTurn *atomic.Value // []*stun.URI + + // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering + // (e.g. if eth0 is in the list, host candidate of this interface won't be used) + InterfaceBlackList []string + DisableIPv6Discovery bool + + UDPMux ice.UDPMux + UDPMuxSrflx ice.UniversalUDPMux + + NATExternalIPs []string +} + +type ICEConnInfo struct { + RemoteConn net.Conn + RosenpassPubKey []byte + RosenpassAddr string + LocalIceCandidateType string + RemoteIceCandidateType string + RemoteIceCandidateEndpoint string + LocalIceCandidateEndpoint string + Relayed bool + RelayedOnLocal bool +} + +type WorkerICECallbacks struct { + OnConnReady func(ConnPriority, ICEConnInfo) + OnStatusChanged func(ConnStatus) +} + +type WorkerICE struct { + ctx context.Context + log *log.Entry + config ConnConfig + signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + statusRecorder *Status + hasRelayOnLocally bool + conn WorkerICECallbacks + + selectedPriority ConnPriority + + agent *ice.Agent + muxAgent sync.Mutex + + StunTurn []*stun.URI + + sentExtraSrflx bool + + localUfrag string + localPwd string +} + +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) { + w := &WorkerICE{ + ctx: ctx, + log: log, + config: config, + signaler: signaler, + iFaceDiscover: ifaceDiscover, + statusRecorder: statusRecorder, + hasRelayOnLocally: hasRelayOnLocally, + conn: callBacks, + } + + localUfrag, localPwd, err := generateICECredentials() + if err != nil { + return nil, err + } + w.localUfrag = localUfrag + w.localPwd = localPwd + return w, nil +} + +func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { + w.log.Debugf("OnNewOffer for ICE") + w.muxAgent.Lock() + + if w.agent != nil { + w.log.Debugf("agent already exists, skipping the offer") + w.muxAgent.Unlock() + return + } + + var preferredCandidateTypes []ice.CandidateType + if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { + w.selectedPriority = connPriorityICEP2P + preferredCandidateTypes = candidateTypesP2P() + } else { + w.selectedPriority = connPriorityICETurn + preferredCandidateTypes = candidateTypes() + } + + w.log.Debugf("recreate ICE agent") + agentCtx, agentCancel := context.WithCancel(w.ctx) + agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes) + if err != nil { + w.log.Errorf("failed to recreate ICE Agent: %s", err) + w.muxAgent.Unlock() + return + } + w.agent = agent + w.muxAgent.Unlock() + + w.log.Debugf("gather candidates") + err = w.agent.GatherCandidates() + if err != nil { + w.log.Debugf("failed to gather candidates: %s", err) + return + } + + // will block until connection succeeded + // but it won't release if ICE Agent went into Disconnected or Failed state, + // so we have to cancel it with the provided context once agent detected a broken connection + w.log.Debugf("turn agent dial") + remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer) + if err != nil { + w.log.Debugf("failed to dial the remote peer: %s", err) + return + } + w.log.Debugf("agent dial succeeded") + + pair, err := w.agent.GetSelectedCandidatePair() + if err != nil { + return + } + + if !isRelayCandidate(pair.Local) { + // dynamically set remote WireGuard port if other side specified a different one from the default one + remoteWgPort := iface.DefaultWgPort + if remoteOfferAnswer.WgListenPort != 0 { + remoteWgPort = remoteOfferAnswer.WgListenPort + } + + // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port + go w.punchRemoteWGPort(pair, remoteWgPort) + } + + ci := ICEConnInfo{ + RemoteConn: remoteConn, + RosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + RosenpassAddr: remoteOfferAnswer.RosenpassAddr, + LocalIceCandidateType: pair.Local.Type().String(), + RemoteIceCandidateType: pair.Remote.Type().String(), + LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), + RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), + Relayed: isRelayed(pair), + RelayedOnLocal: isRelayCandidate(pair.Local), + } + w.log.Debugf("on ICE conn read to use ready") + go w.conn.OnConnReady(w.selectedPriority, ci) +} + +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String()) + if w.agent == nil { + w.log.Warnf("ICE Agent is not initialized yet") + return + } + + if candidateViaRoutes(candidate, haRoutes) { + return + } + + err := w.agent.AddRemoteCandidate(candidate) + if err != nil { + w.log.Errorf("error while handling remote candidate") + return + } +} + +func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) { + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + return w.localUfrag, w.localPwd +} + +func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { + transportNet, err := w.newStdNet() + if err != nil { + w.log.Errorf("failed to create pion's stdnet: %s", err) + } + + iceKeepAlive := iceKeepAlive() + iceDisconnectedTimeout := iceDisconnectedTimeout() + iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() + + agentConfig := &ice.AgentConfig{ + MulticastDNSMode: ice.MulticastDNSModeDisabled, + NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, + Urls: w.config.ICEConfig.StunTurn.Load().([]*stun.URI), + CandidateTypes: relaySupport, + InterfaceFilter: stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList), + UDPMux: w.config.ICEConfig.UDPMux, + UDPMuxSrflx: w.config.ICEConfig.UDPMuxSrflx, + NAT1To1IPs: w.config.ICEConfig.NATExternalIPs, + Net: transportNet, + FailedTimeout: &failedTimeout, + DisconnectedTimeout: &iceDisconnectedTimeout, + KeepaliveInterval: &iceKeepAlive, + RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, + LocalUfrag: w.localUfrag, + LocalPwd: w.localPwd, + } + + if w.config.ICEConfig.DisableIPv6Discovery { + agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} + } + + w.sentExtraSrflx = false + agent, err := ice.NewAgent(agentConfig) + if err != nil { + return nil, err + } + + err = agent.OnCandidate(w.onICECandidate) + if err != nil { + return nil, err + } + + err = agent.OnConnectionStateChange(func(state ice.ConnectionState) { + w.log.Debugf("ICE ConnectionState has changed to %s", state.String()) + if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { + w.conn.OnStatusChanged(StatusDisconnected) + + w.muxAgent.Lock() + agentCancel() + _ = agent.Close() + w.agent = nil + + w.muxAgent.Unlock() + } + }) + if err != nil { + return nil, err + } + + err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair) + if err != nil { + return nil, err + } + + err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { + err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency()) + if err != nil { + w.log.Debugf("failed to update latency for peer: %s", err) + return + } + }) + if err != nil { + return nil, fmt.Errorf("failed setting binding response callback: %w", err) + } + + return agent, nil +} + +func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { + // wait local endpoint configuration + time.Sleep(time.Second) + addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) + if err != nil { + w.log.Warnf("got an error while resolving the udp address, err: %s", err) + return + } + + mux, ok := w.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) + if !ok { + w.log.Warn("invalid udp mux conversion") + return + } + _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) + if err != nil { + w.log.Warnf("got an error while sending the punch packet, err: %s", err) + } +} + +// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates +// and then signals them to the remote peer +func (w *WorkerICE) onICECandidate(candidate ice.Candidate) { + // nil means candidate gathering has been ended + if candidate == nil { + return + } + + // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored + w.log.Debugf("discovered local candidate %s", candidate.String()) + go func() { + err := w.signaler.SignalICECandidate(candidate, w.config.Key) + if err != nil { + w.log.Errorf("failed signaling candidate to the remote peer %s %s", w.config.Key, err) + } + }() + + if !w.shouldSendExtraSrflxCandidate(candidate) { + return + } + + // sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port) + // this is useful when network has an existing port forwarding rule for the wireguard port and this peer + extraSrflx, err := extraSrflxCandidate(candidate) + if err != nil { + w.log.Errorf("failed creating extra server reflexive candidate %s", err) + return + } + w.sentExtraSrflx = true + + go func() { + err = w.signaler.SignalICECandidate(extraSrflx, w.config.Key) + if err != nil { + w.log.Errorf("failed signaling the extra server reflexive candidate: %s", err) + } + }() +} + +func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { + w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), + w.config.Key) +} + +func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { + if !w.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { + return true + } + return false +} + +func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { + isControlling := w.config.LocalKey > w.config.Key + if isControlling { + return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } else { + return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } +} + +func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { + relatedAdd := candidate.RelatedAddress() + return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ + Network: candidate.NetworkType().String(), + Address: candidate.Address(), + Port: relatedAdd.Port, + Component: candidate.Component(), + RelAddr: relatedAdd.Address, + RelPort: relatedAdd.Port, + }) +} + +func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool { + var routePrefixes []netip.Prefix + for _, routes := range clientRoutes { + if len(routes) > 0 && routes[0] != nil { + routePrefixes = append(routePrefixes, routes[0].Network) + } + } + + addr, err := netip.ParseAddr(candidate.Address()) + if err != nil { + log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err) + return false + } + + for _, prefix := range routePrefixes { + // default route is + if prefix.Bits() == 0 { + continue + } + + if prefix.Contains(addr) { + log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix) + return true + } + } + return false +} + +func candidateTypes() []ice.CandidateType { + if hasICEForceRelayConn() { + return []ice.CandidateType{ice.CandidateTypeRelay} + } + // TODO: remove this once we have refactored userspace proxy into the bind package + if runtime.GOOS == "ios" { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} + } + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} +} + +func candidateTypesP2P() []ice.CandidateType { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} +} + +func isRelayCandidate(candidate ice.Candidate) bool { + return candidate.Type() == ice.CandidateTypeRelay +} + +func isRelayed(pair *ice.CandidatePair) bool { + if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { + return true + } + return false +} + +func generateICECredentials() (string, string, error) { + ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha) + if err != nil { + return "", "", err + } + + pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha) + if err != nil { + return "", "", err + } + return ufrag, pwd, nil + +} diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go new file mode 100644 index 000000000..f03626e14 --- /dev/null +++ b/client/internal/peer/worker_relay.go @@ -0,0 +1,173 @@ +package peer + +import ( + "context" + "errors" + "net" + "sync/atomic" + "time" + + log "github.com/sirupsen/logrus" + + relayClient "github.com/netbirdio/netbird/relay/client" +) + +var ( + wgHandshakePeriod = 2 * time.Minute + wgHandshakeOvertime = 30000 * time.Millisecond +) + +type RelayConnInfo struct { + relayedConn net.Conn + rosenpassPubKey []byte + rosenpassAddr string +} + +type WorkerRelayCallbacks struct { + OnConnReady func(RelayConnInfo) + OnDisconnected func() +} + +type WorkerRelay struct { + parentCtx context.Context + log *log.Entry + config ConnConfig + relayManager relayClient.ManagerService + conn WorkerRelayCallbacks + + ctxCancel context.CancelFunc + relaySupportedOnRemotePeer atomic.Bool +} + +func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay { + r := &WorkerRelay{ + parentCtx: ctx, + log: log, + config: config, + relayManager: relayManager, + conn: callbacks, + } + return r +} + +func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { + if !w.isRelaySupported(remoteOfferAnswer) { + w.log.Infof("Relay is not supported by remote peer") + w.relaySupportedOnRemotePeer.Store(false) + return + } + w.relaySupportedOnRemotePeer.Store(true) + + // the relayManager will return with error in case if the connection has lost with relay server + currentRelayAddress, err := w.relayManager.RelayInstanceAddress() + if err != nil { + w.log.Errorf("failed to handle new offer: %s", err) + return + } + + srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress) + + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) + if err != nil { + // todo handle all type errors + if errors.Is(err, relayClient.ErrConnAlreadyExists) { + w.log.Infof("do not need to reopen relay connection") + return + } + w.log.Errorf("failed to open connection via Relay: %s", err) + return + } + + ctx, ctxCancel := context.WithCancel(w.parentCtx) + w.ctxCancel = ctxCancel + + err = w.relayManager.AddCloseListener(srv, w.disconnected) + if err != nil { + log.Errorf("failed to add close listener: %s", err) + _ = relayedConn.Close() + ctxCancel() + return + } + + go w.wgStateCheck(ctx, relayedConn) + + w.log.Debugf("peer conn opened via Relay: %s", srv) + go w.conn.OnConnReady(RelayConnInfo{ + relayedConn: relayedConn, + rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + rosenpassAddr: remoteOfferAnswer.RosenpassAddr, + }) +} + +func (w *WorkerRelay) RelayInstanceAddress() (string, error) { + return w.relayManager.RelayInstanceAddress() +} + +func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool { + return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally() +} + +func (w *WorkerRelay) IsController() bool { + return w.config.LocalKey > w.config.Key +} + +func (w *WorkerRelay) RelayIsSupportedLocally() bool { + return w.relayManager.HasRelayAddress() +} + +// wgStateCheck help to check the state of the wireguard handshake and relay connection +func (w *WorkerRelay) wgStateCheck(ctx context.Context, conn net.Conn) { + timer := time.NewTimer(wgHandshakeOvertime) + defer timer.Stop() + for { + select { + case <-timer.C: + lastHandshake, err := w.wgState() + if err != nil { + w.log.Errorf("failed to read wg stats: %v", err) + continue + } + w.log.Tracef("last handshake: %v", lastHandshake) + + if time.Since(lastHandshake) > wgHandshakePeriod { + w.log.Infof("Wireguard handshake timed out, closing relay connection") + _ = conn.Close() + w.conn.OnDisconnected() + return + } + resetTime := time.Until(lastHandshake.Add(wgHandshakeOvertime + wgHandshakePeriod)) + timer.Reset(resetTime) + case <-ctx.Done(): + return + } + } +} + +func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { + if !w.relayManager.HasRelayAddress() { + return false + } + return answer.RelaySrvAddress != "" +} + +func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { + if w.IsController() { + return myRelayAddress + } + return remoteRelayAddress +} + +func (w *WorkerRelay) wgState() (time.Time, error) { + wgState, err := w.config.WgConfig.WgInterface.GetStats(w.config.Key) + if err != nil { + return time.Time{}, err + } + return wgState.LastHandshake, nil +} + +func (w *WorkerRelay) disconnected() { + if w.ctxCancel != nil { + w.ctxCancel() + } + w.conn.OnDisconnected() +} diff --git a/client/internal/relay/relay.go b/client/internal/relay/relay.go index 4542a37fe..7d98a6060 100644 --- a/client/internal/relay/relay.go +++ b/client/internal/relay/relay.go @@ -17,7 +17,7 @@ import ( // ProbeResult holds the info about the result of a relay probe request type ProbeResult struct { - URI *stun.URI + URI string Err error Addr string } @@ -176,7 +176,7 @@ func ProbeAll( wg.Add(1) go func(res *ProbeResult, stunURI *stun.URI) { defer wg.Done() - res.URI = stunURI + res.URI = stunURI.String() res.Addr, res.Err = fn(ctx, stunURI) }(&results[i], uri) } diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index 1566d10dd..f49dd115a 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -22,7 +22,6 @@ import ( type routerPeerStatus struct { connected bool relayed bool - direct bool latency time.Duration } @@ -44,7 +43,7 @@ type clientNetwork struct { ctx context.Context cancel context.CancelFunc statusRecorder *peer.Status - wgInterface *iface.WGIface + wgInterface iface.IWGIface routes map[route.ID]*route.Route routeUpdate chan routesUpdate peerStateUpdate chan struct{} @@ -54,7 +53,7 @@ type clientNetwork struct { updateSerial uint64 } -func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface *iface.WGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork { +func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface iface.IWGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork { ctx, cancel := context.WithCancel(ctx) client := &clientNetwork{ @@ -82,7 +81,6 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { routePeerStatuses[r.ID] = routerPeerStatus{ connected: peerStatus.ConnStatus == peer.StatusConnected, relayed: peerStatus.Relayed, - direct: peerStatus.Direct, latency: peerStatus.Latency, } } @@ -137,10 +135,6 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID] tempScore++ } - if peerStatus.direct { - tempScore++ - } - if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") { chosen = r.ID chosenScore = tempScore @@ -384,7 +378,7 @@ func (c *clientNetwork) peersStateAndUpdateWatcher() { } } -func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface *iface.WGIface) RouteHandler { +func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface iface.IWGIface) RouteHandler { if rt.IsDynamic() { dns := nbdns.NewServiceViaMemory(wgInterface) return dynamic.NewRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouterInteval, statusRecorder, wgInterface, fmt.Sprintf("%s:%d", dns.RuntimeIP(), dns.RuntimePort())) diff --git a/client/internal/routemanager/client_test.go b/client/internal/routemanager/client_test.go index 0ae10e568..583156e4d 100644 --- a/client/internal/routemanager/client_test.go +++ b/client/internal/routemanager/client_test.go @@ -24,7 +24,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -43,7 +42,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: true, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -62,7 +60,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: true, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -81,7 +78,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: false, relayed: false, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -100,12 +96,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: false, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -129,41 +123,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: true, - direct: true, - }, - }, - existingRoutes: map[route.ID]*route.Route{ - "route1": { - ID: "route1", - Metric: route.MaxMetric, - Peer: "peer1", - }, - "route2": { - ID: "route2", - Metric: route.MaxMetric, - Peer: "peer2", - }, - }, - currentRoute: "", - expectedRouteID: "route1", - }, - { - name: "multiple connected peers with one direct", - statuses: map[route.ID]routerPeerStatus{ - "route1": { - connected: true, - relayed: false, - direct: true, - }, - "route2": { - connected: true, - relayed: false, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -241,13 +204,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 15 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, @@ -272,13 +233,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 200 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, @@ -303,13 +262,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 20 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, diff --git a/client/internal/routemanager/dynamic/route.go b/client/internal/routemanager/dynamic/route.go index 3296f3ddf..5897031e7 100644 --- a/client/internal/routemanager/dynamic/route.go +++ b/client/internal/routemanager/dynamic/route.go @@ -48,7 +48,7 @@ type Route struct { currentPeerKey string cancel context.CancelFunc statusRecorder *peer.Status - wgInterface *iface.WGIface + wgInterface iface.IWGIface resolverAddr string } @@ -58,7 +58,7 @@ func NewRoute( allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, interval time.Duration, statusRecorder *peer.Status, - wgInterface *iface.WGIface, + wgInterface iface.IWGIface, resolverAddr string, ) *Route { return &Route{ diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 0b10dbe33..597eddd51 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -49,7 +49,7 @@ type DefaultManager struct { serverRouter serverRouter sysOps *systemops.SysOps statusRecorder *peer.Status - wgInterface *iface.WGIface + wgInterface iface.IWGIface pubKey string notifier *notifier.Notifier routeRefCounter *refcounter.RouteRefCounter @@ -61,7 +61,7 @@ func NewManager( ctx context.Context, pubKey string, dnsRouteInterval time.Duration, - wgInterface *iface.WGIface, + wgInterface iface.IWGIface, statusRecorder *peer.Status, initialRoutes []*route.Route, ) *DefaultManager { diff --git a/client/internal/routemanager/server_android.go b/client/internal/routemanager/server_android.go index b4065bca6..2057b9cc8 100644 --- a/client/internal/routemanager/server_android.go +++ b/client/internal/routemanager/server_android.go @@ -11,6 +11,6 @@ import ( "github.com/netbirdio/netbird/iface" ) -func newServerRouter(context.Context, *iface.WGIface, firewall.Manager, *peer.Status) (serverRouter, error) { +func newServerRouter(context.Context, iface.IWGIface, firewall.Manager, *peer.Status) (serverRouter, error) { return nil, fmt.Errorf("server route not supported on this os") } diff --git a/client/internal/routemanager/server_nonandroid.go b/client/internal/routemanager/server_nonandroid.go index 8470934c2..43a266cd2 100644 --- a/client/internal/routemanager/server_nonandroid.go +++ b/client/internal/routemanager/server_nonandroid.go @@ -22,11 +22,11 @@ type defaultServerRouter struct { ctx context.Context routes map[route.ID]*route.Route firewall firewall.Manager - wgInterface *iface.WGIface + wgInterface iface.IWGIface statusRecorder *peer.Status } -func newServerRouter(ctx context.Context, wgInterface *iface.WGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) { +func newServerRouter(ctx context.Context, wgInterface iface.IWGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) { return &defaultServerRouter{ ctx: ctx, routes: make(map[route.ID]*route.Route), diff --git a/client/internal/routemanager/sysctl/sysctl_linux.go b/client/internal/routemanager/sysctl/sysctl_linux.go index 43394a823..13e1229f8 100644 --- a/client/internal/routemanager/sysctl/sysctl_linux.go +++ b/client/internal/routemanager/sysctl/sysctl_linux.go @@ -23,7 +23,7 @@ const ( ) // Setup configures sysctl settings for RP filtering and source validation. -func Setup(wgIface *iface.WGIface) (map[string]int, error) { +func Setup(wgIface iface.IWGIface) (map[string]int, error) { keys := map[string]int{} var result *multierror.Error diff --git a/client/internal/routemanager/systemops/systemops.go b/client/internal/routemanager/systemops/systemops.go index cddd7e7e2..ae27b0123 100644 --- a/client/internal/routemanager/systemops/systemops.go +++ b/client/internal/routemanager/systemops/systemops.go @@ -19,7 +19,7 @@ type ExclusionCounter = refcounter.Counter[any, Nexthop] type SysOps struct { refCounter *ExclusionCounter - wgInterface *iface.WGIface + wgInterface iface.IWGIface // prefixes is tracking all the current added prefixes im memory // (this is used in iOS as all route updates require a full table update) //nolint @@ -30,7 +30,7 @@ type SysOps struct { notifier *notifier.Notifier } -func NewSysOps(wgInterface *iface.WGIface, notifier *notifier.Notifier) *SysOps { +func NewSysOps(wgInterface iface.IWGIface, notifier *notifier.Notifier) *SysOps { return &SysOps{ wgInterface: wgInterface, notifier: notifier, diff --git a/client/internal/routemanager/systemops/systemops_generic.go b/client/internal/routemanager/systemops/systemops_generic.go index 671545b86..d76824c10 100644 --- a/client/internal/routemanager/systemops/systemops_generic.go +++ b/client/internal/routemanager/systemops/systemops_generic.go @@ -122,7 +122,7 @@ func (r *SysOps) addRouteForCurrentDefaultGateway(prefix netip.Prefix) error { // addRouteToNonVPNIntf adds a new route to the routing table for the given prefix and returns the next hop and interface. // If the next hop or interface is pointing to the VPN interface, it will return the initial values. -func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf *iface.WGIface, initialNextHop Nexthop) (Nexthop, error) { +func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf iface.IWGIface, initialNextHop Nexthop) (Nexthop, error) { addr := prefix.Addr() switch { case addr.IsLoopback(), diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index bbd00d6e2..d385cc4ca 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -181,7 +181,7 @@ func (p *WGEBPFProxy) proxyToRemote() { conn, ok := p.turnConnStore[uint16(addr.Port)] p.turnConnMutex.Unlock() if !ok { - log.Infof("turn conn not found by port: %d", addr.Port) + log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port) continue } @@ -206,7 +206,7 @@ func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) { } func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) { - log.Tracef("remove turn conn from store by port: %d", turnConnID) + log.Debugf("remove turn conn from store by port: %d", turnConnID) p.turnConnMutex.Lock() defer p.turnConnMutex.Unlock() delete(p.turnConnStore, turnConnID) diff --git a/client/internal/wgproxy/proxy_userspace.go b/client/internal/wgproxy/proxy_userspace.go index 234ea2a42..c2c8a9b51 100644 --- a/client/internal/wgproxy/proxy_userspace.go +++ b/client/internal/wgproxy/proxy_userspace.go @@ -3,6 +3,7 @@ package wgproxy import ( "context" "fmt" + "io" "net" log "github.com/sirupsen/logrus" @@ -64,7 +65,6 @@ func (p *WGUserSpaceProxy) Free() error { // proxyToRemote proxies everything from Wireguard to the RemoteKey peer // blocks func (p *WGUserSpaceProxy) proxyToRemote() { - buf := make([]byte, 1500) for { select { @@ -73,11 +73,17 @@ func (p *WGUserSpaceProxy) proxyToRemote() { default: n, err := p.localConn.Read(buf) if err != nil { + log.Debugf("failed to read from wg interface conn: %s", err) continue } _, err = p.remoteConn.Write(buf[:n]) if err != nil { + if err == io.EOF { + p.cancel() + } else { + log.Debugf("failed to write to remote conn: %s", err) + } continue } } @@ -96,11 +102,17 @@ func (p *WGUserSpaceProxy) proxyToLocal() { default: n, err := p.remoteConn.Read(buf) if err != nil { + if err == io.EOF { + p.cancel() + return + } + log.Errorf("failed to read from remote conn: %s", err) continue } _, err = p.localConn.Write(buf[:n]) if err != nil { + log.Debugf("failed to write to wg interface conn: %s", err) continue } } diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index 779c27a4d..dc13706bf 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -168,7 +168,6 @@ func (c *Client) GetStatusDetails() *StatusDetails { BytesTx: p.BytesTx, ConnStatus: p.ConnStatus.String(), ConnStatusUpdate: p.ConnStatusUpdate.Format("2006-01-02 15:04:05"), - Direct: p.Direct, LastWireguardHandshake: p.LastWireguardHandshake.String(), Relayed: p.Relayed, RosenpassEnabled: p.RosenpassEnabled, diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index fb10a38d3..b942d8b6e 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v4.23.4 +// protoc v3.21.12 // source: daemon.proto package proto @@ -899,7 +899,6 @@ type PeerState struct { ConnStatus string `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"` ConnStatusUpdate *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"` Relayed bool `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"` - Direct bool `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"` LocalIceCandidateType string `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"` RemoteIceCandidateType string `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"` Fqdn string `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"` @@ -911,6 +910,7 @@ type PeerState struct { RosenpassEnabled bool `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` Routes []string `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"` Latency *durationpb.Duration `protobuf:"bytes,17,opt,name=latency,proto3" json:"latency,omitempty"` + RelayAddress string `protobuf:"bytes,18,opt,name=relayAddress,proto3" json:"relayAddress,omitempty"` } func (x *PeerState) Reset() { @@ -980,13 +980,6 @@ func (x *PeerState) GetRelayed() bool { return false } -func (x *PeerState) GetDirect() bool { - if x != nil { - return x.Direct - } - return false -} - func (x *PeerState) GetLocalIceCandidateType() string { if x != nil { return x.LocalIceCandidateType @@ -1064,6 +1057,13 @@ func (x *PeerState) GetLatency() *durationpb.Duration { return nil } +func (x *PeerState) GetRelayAddress() string { + if x != nil { + return x.RelayAddress + } + return "" +} + // LocalPeerState contains the latest state of the local peer type LocalPeerState struct { state protoimpl.MessageState @@ -2243,7 +2243,7 @@ var file_daemon_proto_rawDesc = []byte{ 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xce, 0x05, 0x0a, 0x09, 0x50, 0x65, + 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xda, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, @@ -2255,209 +2255,210 @@ var file_daemon_proto_rawDesc = []byte{ 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, - 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, + 0x64, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, + 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, - 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, - 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, - 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, - 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, - 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, - 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, - 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, - 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, - 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, - 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, - 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, - 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, - 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, - 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, - 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, - 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, - 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, - 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, - 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, - 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, - 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, - 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, - 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, - 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, - 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, - 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, - 0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, - 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, - 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, - 0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, - 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, - 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, - 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x1e, 0x0a, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, - 0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, - 0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, - 0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, - 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, - 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, - 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, - 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, - 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, - 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, - 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, - 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, - 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, - 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, - 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, - 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, - 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, + 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, + 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, + 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, + 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, + 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, + 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, + 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, + 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, + 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, + 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, + 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, + 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, + 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, + 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, + 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, + 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, + 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, + 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, + 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, + 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, + 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, + 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, + 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, + 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, + 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a, + 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a, + 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, + 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, + 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, + 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x29, 0x0a, 0x13, 0x44, + 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, + 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e, + 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12, + 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, + 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09, + 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, + 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, + 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, + 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, + 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, + 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 43c379fb5..384bc0e62 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -168,7 +168,6 @@ message PeerState { string connStatus = 3; google.protobuf.Timestamp connStatusUpdate = 4; bool relayed = 5; - bool direct = 6; string localIceCandidateType = 7; string remoteIceCandidateType = 8; string fqdn = 9; @@ -180,6 +179,7 @@ message PeerState { bool rosenpassEnabled = 15; repeated string routes = 16; google.protobuf.Duration latency = 17; + string relayAddress = 18; } // LocalPeerState contains the latest state of the local peer diff --git a/client/server/debug.go b/client/server/debug.go index 1187f3187..5ed43293b 100644 --- a/client/server/debug.go +++ b/client/server/debug.go @@ -369,8 +369,8 @@ func seedFromStatus(a *anonymize.Anonymizer, status *peer.FullStatus) { } for _, relay := range status.Relays { - if relay.URI != nil { - a.AnonymizeURI(relay.URI.String()) + if relay.URI != "" { + a.AnonymizeURI(relay.URI) } } } diff --git a/client/server/server.go b/client/server/server.go index 8173d0741..aa70f2404 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -745,11 +745,11 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { ConnStatus: peerState.ConnStatus.String(), ConnStatusUpdate: timestamppb.New(peerState.ConnStatusUpdate), Relayed: peerState.Relayed, - Direct: peerState.Direct, LocalIceCandidateType: peerState.LocalIceCandidateType, RemoteIceCandidateType: peerState.RemoteIceCandidateType, LocalIceCandidateEndpoint: peerState.LocalIceCandidateEndpoint, RemoteIceCandidateEndpoint: peerState.RemoteIceCandidateEndpoint, + RelayAddress: peerState.RelayServerAddress, Fqdn: peerState.FQDN, LastWireguardHandshake: timestamppb.New(peerState.LastWireguardHandshake), BytesRx: peerState.BytesRx, @@ -763,7 +763,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { for _, relayState := range fullStatus.Relays { pbRelayState := &proto.RelayState{ - URI: relayState.URI.String(), + URI: relayState.URI, Available: relayState.Err == nil, } if err := relayState.Err; err != nil { diff --git a/client/server/server_test.go b/client/server/server_test.go index 6a3de774c..85d04c1f3 100644 --- a/client/server/server_test.go +++ b/client/server/server_test.go @@ -129,7 +129,10 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + rc := &server.RelayConfig{ + Address: "localhost:0", + } + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { return nil, "", err diff --git a/encryption/cert.go b/encryption/cert.go new file mode 100644 index 000000000..3f6d5c679 --- /dev/null +++ b/encryption/cert.go @@ -0,0 +1,19 @@ +package encryption + +import "crypto/tls" + +func LoadTLSConfig(certFile, keyFile string) (*tls.Config, error) { + serverCert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.NoClientCert, + NextProtos: []string{ + "h2", "http/1.1", // enable HTTP/2 + }, + } + return config, nil +} diff --git a/encryption/letsencrypt.go b/encryption/letsencrypt.go index cfe54ec5a..27a5e3110 100644 --- a/encryption/letsencrypt.go +++ b/encryption/letsencrypt.go @@ -9,7 +9,7 @@ import ( ) // CreateCertManager wraps common logic of generating Let's encrypt certificate. -func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Manager, error) { +func CreateCertManager(datadir string, letsencryptDomain ...string) (*autocert.Manager, error) { certDir := filepath.Join(datadir, "letsencrypt") if _, err := os.Stat(certDir); os.IsNotExist(err) { @@ -24,7 +24,7 @@ func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Mana certManager := &autocert.Manager{ Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(certDir), - HostPolicy: autocert.HostWhitelist(letsencryptDomain), + HostPolicy: autocert.HostWhitelist(letsencryptDomain...), } return certManager, nil diff --git a/encryption/route53.go b/encryption/route53.go new file mode 100644 index 000000000..3c81ab103 --- /dev/null +++ b/encryption/route53.go @@ -0,0 +1,87 @@ +package encryption + +import ( + "context" + "crypto/tls" + "fmt" + "os" + "strings" + + "github.com/caddyserver/certmagic" + "github.com/libdns/route53" + log "github.com/sirupsen/logrus" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/crypto/acme" +) + +// Route53TLS by default, loads the AWS configuration from the environment. +// env variables: AWS_REGION, AWS_PROFILE, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN +type Route53TLS struct { + DataDir string + Email string + Domains []string + CA string +} + +func (r *Route53TLS) GetCertificate() (*tls.Config, error) { + if len(r.Domains) == 0 { + return nil, fmt.Errorf("no domains provided") + } + + certmagic.Default.Logger = logger() + certmagic.Default.Storage = &certmagic.FileStorage{Path: r.DataDir} + certmagic.DefaultACME.Agreed = true + if r.Email != "" { + certmagic.DefaultACME.Email = r.Email + } else { + certmagic.DefaultACME.Email = emailFromDomain(r.Domains[0]) + } + + if r.CA == "" { + certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA + } else { + certmagic.DefaultACME.CA = r.CA + } + + certmagic.DefaultACME.DNS01Solver = &certmagic.DNS01Solver{ + DNSManager: certmagic.DNSManager{ + DNSProvider: &route53.Provider{}, + }, + } + cm := certmagic.NewDefault() + if err := cm.ManageSync(context.Background(), r.Domains); err != nil { + log.Errorf("failed to manage certificate: %v", err) + return nil, err + } + + tlsConfig := &tls.Config{ + GetCertificate: cm.GetCertificate, + NextProtos: []string{"h2", "http/1.1", acme.ALPNProto}, + } + + return tlsConfig, nil +} + +func emailFromDomain(domain string) string { + if domain == "" { + return "" + } + + parts := strings.Split(domain, ".") + if len(parts) < 2 { + return "" + } + if parts[0] == "" { + return "" + } + return fmt.Sprintf("admin@%s.%s", parts[len(parts)-2], parts[len(parts)-1]) +} + +func logger() *zap.Logger { + return zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()), + os.Stderr, + zap.ErrorLevel, + )) +} diff --git a/encryption/route53_test.go b/encryption/route53_test.go new file mode 100644 index 000000000..765b60f84 --- /dev/null +++ b/encryption/route53_test.go @@ -0,0 +1,84 @@ +package encryption + +import ( + "context" + "io" + "net/http" + "os" + "testing" + "time" +) + +func TestRoute53TLSConfig(t *testing.T) { + t.SkipNow() // This test requires AWS credentials + exampleString := "Hello, world!" + rtls := &Route53TLS{ + DataDir: t.TempDir(), + Email: os.Getenv("LE_EMAIL_ROUTE53"), + Domains: []string{os.Getenv("DOMAIN")}, + } + tlsConfig, err := rtls.GetCertificate() + if err != nil { + t.Errorf("Route53TLSConfig failed: %v", err) + } + + server := &http.Server{ + Addr: ":8443", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(exampleString)) + }), + TLSConfig: tlsConfig, + } + + go func() { + err := server.ListenAndServeTLS("", "") + if err != http.ErrServerClosed { + t.Errorf("Failed to start server: %v", err) + } + }() + defer func() { + if err := server.Shutdown(context.Background()); err != nil { + t.Errorf("Failed to shutdown server: %v", err) + } + }() + + time.Sleep(1 * time.Second) + resp, err := http.Get("https://relay.godevltd.com:8443") + if err != nil { + t.Errorf("Failed to get response: %v", err) + return + } + defer func() { + _ = resp.Body.Close() + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("Failed to read response body: %v", err) + } + if string(body) != exampleString { + t.Errorf("Unexpected response: %s", body) + } +} + +func Test_emailFromDomain(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"example.com", "admin@example.com"}, + {"x.example.com", "admin@example.com"}, + {"x.x.example.com", "admin@example.com"}, + {"*.example.com", "admin@example.com"}, + {"example", ""}, + {"", ""}, + {".com", ""}, + } + for _, tt := range tests { + t.Run("domain test", func(t *testing.T) { + if got := emailFromDomain(tt.input); got != tt.want { + t.Errorf("emailFromDomain() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/go.mod b/go.mod index fc7d8ddb9..235fdaec6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.23.0 + github.com/onsi/gomega v1.27.6 github.com/pion/ice/v3 v3.0.2 github.com/rs/cors v1.8.0 github.com/sirupsen/logrus v1.9.3 @@ -34,6 +34,7 @@ require ( fyne.io/systray v1.11.0 github.com/TheJumpCloud/jcapi-go v3.0.0+incompatible github.com/c-robinson/iplib v1.0.3 + github.com/caddyserver/certmagic v0.21.3 github.com/cilium/ebpf v0.15.0 github.com/coreos/go-iptables v0.7.0 github.com/creack/pty v1.1.18 @@ -50,11 +51,12 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 github.com/hashicorp/go-version v1.6.0 + github.com/libdns/route53 v1.5.0 github.com/libp2p/go-netroute v0.2.1 github.com/magiconair/properties v1.8.7 github.com/mattn/go-sqlite3 v1.14.19 github.com/mdlayher/socket v0.4.1 - github.com/miekg/dns v1.1.43 + github.com/miekg/dns v1.1.59 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/nadoo/ipset v0.5.0 github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e @@ -63,6 +65,7 @@ require ( github.com/oschwald/maxminddb-golang v1.12.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pion/logging v0.2.2 + github.com/pion/randutil v0.1.0 github.com/pion/stun/v2 v2.0.0 github.com/pion/transport/v3 v3.0.1 github.com/pion/turn/v3 v3.0.1 @@ -70,6 +73,7 @@ require ( github.com/rs/xid v1.3.0 github.com/shirou/gopsutil/v3 v3.24.4 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.31.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0 @@ -81,6 +85,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.26.0 go.opentelemetry.io/otel/sdk/metric v1.26.0 + go.uber.org/zap v1.27.0 goauthentik.io/api/v3 v3.2023051.3 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a @@ -93,6 +98,7 @@ require ( gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.3 gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde + nhooyr.io/websocket v1.8.11 ) require ( @@ -106,8 +112,23 @@ require ( github.com/Microsoft/hcsshim v0.12.3 // indirect github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/containerd v1.7.16 // indirect github.com/containerd/log v0.1.0 // indirect @@ -140,7 +161,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -149,13 +170,17 @@ require ( github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/mdlayher/genetlink v1.3.2 // indirect github.com/mdlayher/netlink v1.7.2 // indirect + github.com/mholt/acmez/v2 v2.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect @@ -164,12 +189,12 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/nicksnyder/go-i18n/v2 v2.4.0 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pegasus-kv/thrift v0.13.0 // indirect github.com/pion/dtls/v2 v2.2.10 // indirect github.com/pion/mdns v0.0.12 // indirect - github.com/pion/randutil v0.1.0 // indirect github.com/pion/transport/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -186,10 +211,12 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/yuin/goldmark v1.7.1 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index 59decfa31..17bc41311 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,34 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= +github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 h1:MmLCRqP4U4Cw9gJ4bNrCG0mWqEtBlmAVleyelcHARMU= +github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3/go.mod h1:AMPjK2YnRh0YgOID3PqhJA1BRNfXDfGOnSsKHtAe8yA= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -87,6 +115,10 @@ github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwel github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= +github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0= +github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -207,6 +239,8 @@ github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZs github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-text/render v0.1.0 h1:osrmVDZNHuP1RSu3pNG7Z77Sd2xSbcb/xWytAj9kyVs= github.com/go-text/render v0.1.0/go.mod h1:jqEuNMenrmj6QRnkdpeaP0oKGFLDNhDkVKwGjsWWYU4= github.com/go-text/typesetting v0.1.0 h1:vioSaLPYcHwPEPLT7gsjCGDCoYSbljxoHJzMnKwVvHw= @@ -350,8 +384,9 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -382,6 +417,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -401,6 +440,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -413,6 +455,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= +github.com/libdns/route53 v1.5.0 h1:2SKdpPFl/qgWsXQvsLNJJAoX7rSxlk7zgoL4jnWdXVA= +github.com/libdns/route53 v1.5.0/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -431,9 +477,11 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k= +github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -494,14 +542,14 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -592,6 +640,8 @@ github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrx github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -660,6 +710,12 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -695,8 +751,14 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4= goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -890,7 +952,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1187,6 +1248,8 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/iface/iface_moc.go b/iface/iface_moc.go new file mode 100644 index 000000000..fab3054a0 --- /dev/null +++ b/iface/iface_moc.go @@ -0,0 +1,103 @@ +package iface + +import ( + "net" + "time" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/iface/bind" +) + +type MockWGIface struct { + CreateFunc func() error + CreateOnAndroidFunc func(routeRange []string, ip string, domains []string) error + IsUserspaceBindFunc func() bool + NameFunc func() string + AddressFunc func() WGAddress + ToInterfaceFunc func() *net.Interface + UpFunc func() (*bind.UniversalUDPMuxDefault, error) + UpdateAddrFunc func(newAddr string) error + UpdatePeerFunc func(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeerFunc func(peerKey string) error + AddAllowedIPFunc func(peerKey string, allowedIP string) error + RemoveAllowedIPFunc func(peerKey string, allowedIP string) error + CloseFunc func() error + SetFilterFunc func(filter PacketFilter) error + GetFilterFunc func() PacketFilter + GetDeviceFunc func() *DeviceWrapper + GetStatsFunc func(peerKey string) (WGStats, error) + GetInterfaceGUIDStringFunc func() (string, error) +} + +func (m *MockWGIface) GetInterfaceGUIDString() (string, error) { + return m.GetInterfaceGUIDStringFunc() +} + +func (m *MockWGIface) Create() error { + return m.CreateFunc() +} + +func (m *MockWGIface) CreateOnAndroid(routeRange []string, ip string, domains []string) error { + return m.CreateOnAndroidFunc(routeRange, ip, domains) +} + +func (m *MockWGIface) IsUserspaceBind() bool { + return m.IsUserspaceBindFunc() +} + +func (m *MockWGIface) Name() string { + return m.NameFunc() +} + +func (m *MockWGIface) Address() WGAddress { + return m.AddressFunc() +} + +func (m *MockWGIface) ToInterface() *net.Interface { + return m.ToInterfaceFunc() +} + +func (m *MockWGIface) Up() (*bind.UniversalUDPMuxDefault, error) { + return m.UpFunc() +} + +func (m *MockWGIface) UpdateAddr(newAddr string) error { + return m.UpdateAddrFunc(newAddr) +} + +func (m *MockWGIface) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { + return m.UpdatePeerFunc(peerKey, allowedIps, keepAlive, endpoint, preSharedKey) +} + +func (m *MockWGIface) RemovePeer(peerKey string) error { + return m.RemovePeerFunc(peerKey) +} + +func (m *MockWGIface) AddAllowedIP(peerKey string, allowedIP string) error { + return m.AddAllowedIPFunc(peerKey, allowedIP) +} + +func (m *MockWGIface) RemoveAllowedIP(peerKey string, allowedIP string) error { + return m.RemoveAllowedIPFunc(peerKey, allowedIP) +} + +func (m *MockWGIface) Close() error { + return m.CloseFunc() +} + +func (m *MockWGIface) SetFilter(filter PacketFilter) error { + return m.SetFilterFunc(filter) +} + +func (m *MockWGIface) GetFilter() PacketFilter { + return m.GetFilterFunc() +} + +func (m *MockWGIface) GetDevice() *DeviceWrapper { + return m.GetDeviceFunc() +} + +func (m *MockWGIface) GetStats(peerKey string) (WGStats, error) { + return m.GetStatsFunc(peerKey) +} diff --git a/iface/iwginterface.go b/iface/iwginterface.go new file mode 100644 index 000000000..501f51d2b --- /dev/null +++ b/iface/iwginterface.go @@ -0,0 +1,32 @@ +//go:build !windows + +package iface + +import ( + "net" + "time" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/iface/bind" +) + +type IWGIface interface { + Create() error + CreateOnAndroid(routeRange []string, ip string, domains []string) error + IsUserspaceBind() bool + Name() string + Address() WGAddress + ToInterface() *net.Interface + Up() (*bind.UniversalUDPMuxDefault, error) + UpdateAddr(newAddr string) error + UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeer(peerKey string) error + AddAllowedIP(peerKey string, allowedIP string) error + RemoveAllowedIP(peerKey string, allowedIP string) error + Close() error + SetFilter(filter PacketFilter) error + GetFilter() PacketFilter + GetDevice() *DeviceWrapper + GetStats(peerKey string) (WGStats, error) +} diff --git a/iface/iwginterface_windows.go b/iface/iwginterface_windows.go new file mode 100644 index 000000000..b5053474e --- /dev/null +++ b/iface/iwginterface_windows.go @@ -0,0 +1,31 @@ +package iface + +import ( + "net" + "time" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/iface/bind" +) + +type IWGIface interface { + Create() error + CreateOnAndroid(routeRange []string, ip string, domains []string) error + IsUserspaceBind() bool + Name() string + Address() WGAddress + ToInterface() *net.Interface + Up() (*bind.UniversalUDPMuxDefault, error) + UpdateAddr(newAddr string) error + UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeer(peerKey string) error + AddAllowedIP(peerKey string, allowedIP string) error + RemoveAllowedIP(peerKey string, allowedIP string) error + Close() error + SetFilter(filter PacketFilter) error + GetFilter() PacketFilter + GetDevice() *DeviceWrapper + GetStats(peerKey string) (WGStats, error) + GetInterfaceGUIDString() (string, error) +} diff --git a/management/client/client_test.go b/management/client/client_test.go index cec3e77f2..95fc6b724 100644 --- a/management/client/client_test.go +++ b/management/client/client_test.go @@ -82,7 +82,10 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) { if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + rc := &mgmt.RelayConfig{ + Address: "localhost:0", + } + turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { t.Fatal(err) diff --git a/management/cmd/management.go b/management/cmd/management.go index a0176c548..ce8ee291f 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -195,7 +195,7 @@ var ( return fmt.Errorf("failed to build default manager: %v", err) } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnRelayTokenManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.RelayConfig) trustedPeers := config.ReverseProxy.TrustedPeers defaultTrustedPeers := []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0")} @@ -271,7 +271,7 @@ var ( ephemeralManager.LoadInitialPeers(ctx) gRPCAPIHandler := grpc.NewServer(gRPCOpts...) - srv, err := server.NewServer(ctx, config, accountManager, peersUpdateManager, turnManager, appMetrics, ephemeralManager) + srv, err := server.NewServer(ctx, config, accountManager, peersUpdateManager, turnRelayTokenManager, appMetrics, ephemeralManager) if err != nil { return fmt.Errorf("failed creating gRPC API handler: %v", err) } @@ -538,6 +538,10 @@ func loadMgmtConfig(ctx context.Context, mgmtConfigPath string) (*server.Config, } } + if loadedConfig.RelayConfig != nil { + log.Infof("Relay address: %v", loadedConfig.RelayConfig.Address) + } + return loadedConfig, err } diff --git a/management/cmd/management_test.go b/management/cmd/management_test.go new file mode 100644 index 000000000..ae6ac978f --- /dev/null +++ b/management/cmd/management_test.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "context" + "os" + "testing" +) + +const ( + exampleConfig = `{ + "RelayConfig": { + "Address": "rels://relay.stage.npeer.io" + }, + "HttpConfig": { + "AuthAudience": "https://stageapp/", + "AuthIssuer": "https://something.eu.auth0.com/", + "OIDCConfigEndpoint": "https://something.eu.auth0.com/.well-known/openid-configuration" + } + }` +) + +func Test_loadMgmtConfig(t *testing.T) { + tmpFile, err := createConfig() + if err != nil { + t.Fatalf("failed to create config: %s", err) + } + + cfg, err := loadMgmtConfig(context.Background(), tmpFile) + if err != nil { + t.Fatalf("failed to load management config: %s", err) + } + if cfg.RelayConfig == nil { + t.Fatalf("config is nil") + } + if cfg.RelayConfig.Address == "" { + t.Fatalf("relay address is empty") + } +} + +func createConfig() (string, error) { + tmpfile, err := os.CreateTemp("", "config.json") + if err != nil { + return "", err + } + _, err = tmpfile.Write([]byte(exampleConfig)) + if err != nil { + return "", err + } + + if err := tmpfile.Close(); err != nil { + return "", err + } + return tmpfile.Name(), nil +} diff --git a/management/proto/management.pb.go b/management/proto/management.pb.go index ecf738ea5..48f048c4c 100644 --- a/management/proto/management.pb.go +++ b/management/proto/management.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v4.23.4 +// protoc v3.21.12 // source: management.proto package proto @@ -116,7 +116,7 @@ func (x DeviceAuthorizationFlowProvider) Number() protoreflect.EnumNumber { // Deprecated: Use DeviceAuthorizationFlowProvider.Descriptor instead. func (DeviceAuthorizationFlowProvider) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{20, 0} + return file_management_proto_rawDescGZIP(), []int{21, 0} } type FirewallRuleDirection int32 @@ -162,7 +162,7 @@ func (x FirewallRuleDirection) Number() protoreflect.EnumNumber { // Deprecated: Use FirewallRuleDirection.Descriptor instead. func (FirewallRuleDirection) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{30, 0} + return file_management_proto_rawDescGZIP(), []int{31, 0} } type FirewallRuleAction int32 @@ -208,7 +208,7 @@ func (x FirewallRuleAction) Number() protoreflect.EnumNumber { // Deprecated: Use FirewallRuleAction.Descriptor instead. func (FirewallRuleAction) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{30, 1} + return file_management_proto_rawDescGZIP(), []int{31, 1} } type FirewallRuleProtocol int32 @@ -263,7 +263,7 @@ func (x FirewallRuleProtocol) Number() protoreflect.EnumNumber { // Deprecated: Use FirewallRuleProtocol.Descriptor instead. func (FirewallRuleProtocol) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{30, 2} + return file_management_proto_rawDescGZIP(), []int{31, 2} } type EncryptedMessage struct { @@ -1130,7 +1130,8 @@ type WiretrusteeConfig struct { // a list of TURN servers Turns []*ProtectedHostConfig `protobuf:"bytes,2,rep,name=turns,proto3" json:"turns,omitempty"` // a Signal server config - Signal *HostConfig `protobuf:"bytes,3,opt,name=signal,proto3" json:"signal,omitempty"` + Signal *HostConfig `protobuf:"bytes,3,opt,name=signal,proto3" json:"signal,omitempty"` + Relay *RelayConfig `protobuf:"bytes,4,opt,name=relay,proto3" json:"relay,omitempty"` } func (x *WiretrusteeConfig) Reset() { @@ -1186,6 +1187,13 @@ func (x *WiretrusteeConfig) GetSignal() *HostConfig { return nil } +func (x *WiretrusteeConfig) GetRelay() *RelayConfig { + if x != nil { + return x.Relay + } + return nil +} + // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) type HostConfig struct { state protoimpl.MessageState @@ -1243,6 +1251,69 @@ func (x *HostConfig) GetProtocol() HostConfig_Protocol { return HostConfig_UDP } +type RelayConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Urls []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"` + TokenPayload string `protobuf:"bytes,2,opt,name=tokenPayload,proto3" json:"tokenPayload,omitempty"` + TokenSignature string `protobuf:"bytes,3,opt,name=tokenSignature,proto3" json:"tokenSignature,omitempty"` +} + +func (x *RelayConfig) Reset() { + *x = RelayConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RelayConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RelayConfig) ProtoMessage() {} + +func (x *RelayConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RelayConfig.ProtoReflect.Descriptor instead. +func (*RelayConfig) Descriptor() ([]byte, []int) { + return file_management_proto_rawDescGZIP(), []int{14} +} + +func (x *RelayConfig) GetUrls() []string { + if x != nil { + return x.Urls + } + return nil +} + +func (x *RelayConfig) GetTokenPayload() string { + if x != nil { + return x.TokenPayload + } + return "" +} + +func (x *RelayConfig) GetTokenSignature() string { + if x != nil { + return x.TokenSignature + } + return "" +} + // ProtectedHostConfig is similar to HostConfig but has additional user and password // Mostly used for TURN servers type ProtectedHostConfig struct { @@ -1258,7 +1329,7 @@ type ProtectedHostConfig struct { func (x *ProtectedHostConfig) Reset() { *x = ProtectedHostConfig{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[14] + mi := &file_management_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1271,7 +1342,7 @@ func (x *ProtectedHostConfig) String() string { func (*ProtectedHostConfig) ProtoMessage() {} func (x *ProtectedHostConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[14] + mi := &file_management_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1284,7 +1355,7 @@ func (x *ProtectedHostConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use ProtectedHostConfig.ProtoReflect.Descriptor instead. func (*ProtectedHostConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{14} + return file_management_proto_rawDescGZIP(), []int{15} } func (x *ProtectedHostConfig) GetHostConfig() *HostConfig { @@ -1328,7 +1399,7 @@ type PeerConfig struct { func (x *PeerConfig) Reset() { *x = PeerConfig{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[15] + mi := &file_management_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1341,7 +1412,7 @@ func (x *PeerConfig) String() string { func (*PeerConfig) ProtoMessage() {} func (x *PeerConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[15] + mi := &file_management_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1354,7 +1425,7 @@ func (x *PeerConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use PeerConfig.ProtoReflect.Descriptor instead. func (*PeerConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{15} + return file_management_proto_rawDescGZIP(), []int{16} } func (x *PeerConfig) GetAddress() string { @@ -1416,7 +1487,7 @@ type NetworkMap struct { func (x *NetworkMap) Reset() { *x = NetworkMap{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[16] + mi := &file_management_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1429,7 +1500,7 @@ func (x *NetworkMap) String() string { func (*NetworkMap) ProtoMessage() {} func (x *NetworkMap) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[16] + mi := &file_management_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1442,7 +1513,7 @@ func (x *NetworkMap) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkMap.ProtoReflect.Descriptor instead. func (*NetworkMap) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{16} + return file_management_proto_rawDescGZIP(), []int{17} } func (x *NetworkMap) GetSerial() uint64 { @@ -1528,7 +1599,7 @@ type RemotePeerConfig struct { func (x *RemotePeerConfig) Reset() { *x = RemotePeerConfig{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[17] + mi := &file_management_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1541,7 +1612,7 @@ func (x *RemotePeerConfig) String() string { func (*RemotePeerConfig) ProtoMessage() {} func (x *RemotePeerConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[17] + mi := &file_management_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1554,7 +1625,7 @@ func (x *RemotePeerConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use RemotePeerConfig.ProtoReflect.Descriptor instead. func (*RemotePeerConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{17} + return file_management_proto_rawDescGZIP(), []int{18} } func (x *RemotePeerConfig) GetWgPubKey() string { @@ -1601,7 +1672,7 @@ type SSHConfig struct { func (x *SSHConfig) Reset() { *x = SSHConfig{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[18] + mi := &file_management_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1614,7 +1685,7 @@ func (x *SSHConfig) String() string { func (*SSHConfig) ProtoMessage() {} func (x *SSHConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[18] + mi := &file_management_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1627,7 +1698,7 @@ func (x *SSHConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use SSHConfig.ProtoReflect.Descriptor instead. func (*SSHConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{18} + return file_management_proto_rawDescGZIP(), []int{19} } func (x *SSHConfig) GetSshEnabled() bool { @@ -1654,7 +1725,7 @@ type DeviceAuthorizationFlowRequest struct { func (x *DeviceAuthorizationFlowRequest) Reset() { *x = DeviceAuthorizationFlowRequest{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[19] + mi := &file_management_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1667,7 +1738,7 @@ func (x *DeviceAuthorizationFlowRequest) String() string { func (*DeviceAuthorizationFlowRequest) ProtoMessage() {} func (x *DeviceAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[19] + mi := &file_management_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1680,7 +1751,7 @@ func (x *DeviceAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeviceAuthorizationFlowRequest.ProtoReflect.Descriptor instead. func (*DeviceAuthorizationFlowRequest) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{19} + return file_management_proto_rawDescGZIP(), []int{20} } // DeviceAuthorizationFlow represents Device Authorization Flow information @@ -1699,7 +1770,7 @@ type DeviceAuthorizationFlow struct { func (x *DeviceAuthorizationFlow) Reset() { *x = DeviceAuthorizationFlow{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[20] + mi := &file_management_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1712,7 +1783,7 @@ func (x *DeviceAuthorizationFlow) String() string { func (*DeviceAuthorizationFlow) ProtoMessage() {} func (x *DeviceAuthorizationFlow) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[20] + mi := &file_management_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1725,7 +1796,7 @@ func (x *DeviceAuthorizationFlow) ProtoReflect() protoreflect.Message { // Deprecated: Use DeviceAuthorizationFlow.ProtoReflect.Descriptor instead. func (*DeviceAuthorizationFlow) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{20} + return file_management_proto_rawDescGZIP(), []int{21} } func (x *DeviceAuthorizationFlow) GetProvider() DeviceAuthorizationFlowProvider { @@ -1752,7 +1823,7 @@ type PKCEAuthorizationFlowRequest struct { func (x *PKCEAuthorizationFlowRequest) Reset() { *x = PKCEAuthorizationFlowRequest{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[21] + mi := &file_management_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1765,7 +1836,7 @@ func (x *PKCEAuthorizationFlowRequest) String() string { func (*PKCEAuthorizationFlowRequest) ProtoMessage() {} func (x *PKCEAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[21] + mi := &file_management_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1778,7 +1849,7 @@ func (x *PKCEAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PKCEAuthorizationFlowRequest.ProtoReflect.Descriptor instead. func (*PKCEAuthorizationFlowRequest) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{21} + return file_management_proto_rawDescGZIP(), []int{22} } // PKCEAuthorizationFlow represents Authorization Code Flow information @@ -1795,7 +1866,7 @@ type PKCEAuthorizationFlow struct { func (x *PKCEAuthorizationFlow) Reset() { *x = PKCEAuthorizationFlow{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[22] + mi := &file_management_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1808,7 +1879,7 @@ func (x *PKCEAuthorizationFlow) String() string { func (*PKCEAuthorizationFlow) ProtoMessage() {} func (x *PKCEAuthorizationFlow) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[22] + mi := &file_management_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1821,7 +1892,7 @@ func (x *PKCEAuthorizationFlow) ProtoReflect() protoreflect.Message { // Deprecated: Use PKCEAuthorizationFlow.ProtoReflect.Descriptor instead. func (*PKCEAuthorizationFlow) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{22} + return file_management_proto_rawDescGZIP(), []int{23} } func (x *PKCEAuthorizationFlow) GetProviderConfig() *ProviderConfig { @@ -1863,7 +1934,7 @@ type ProviderConfig struct { func (x *ProviderConfig) Reset() { *x = ProviderConfig{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[23] + mi := &file_management_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1876,7 +1947,7 @@ func (x *ProviderConfig) String() string { func (*ProviderConfig) ProtoMessage() {} func (x *ProviderConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[23] + mi := &file_management_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1889,7 +1960,7 @@ func (x *ProviderConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use ProviderConfig.ProtoReflect.Descriptor instead. func (*ProviderConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{23} + return file_management_proto_rawDescGZIP(), []int{24} } func (x *ProviderConfig) GetClientID() string { @@ -1982,7 +2053,7 @@ type Route struct { func (x *Route) Reset() { *x = Route{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[24] + mi := &file_management_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1995,7 +2066,7 @@ func (x *Route) String() string { func (*Route) ProtoMessage() {} func (x *Route) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[24] + mi := &file_management_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2008,7 +2079,7 @@ func (x *Route) ProtoReflect() protoreflect.Message { // Deprecated: Use Route.ProtoReflect.Descriptor instead. func (*Route) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{24} + return file_management_proto_rawDescGZIP(), []int{25} } func (x *Route) GetID() string { @@ -2088,7 +2159,7 @@ type DNSConfig struct { func (x *DNSConfig) Reset() { *x = DNSConfig{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[25] + mi := &file_management_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2101,7 +2172,7 @@ func (x *DNSConfig) String() string { func (*DNSConfig) ProtoMessage() {} func (x *DNSConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[25] + mi := &file_management_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2114,7 +2185,7 @@ func (x *DNSConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use DNSConfig.ProtoReflect.Descriptor instead. func (*DNSConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{25} + return file_management_proto_rawDescGZIP(), []int{26} } func (x *DNSConfig) GetServiceEnable() bool { @@ -2151,7 +2222,7 @@ type CustomZone struct { func (x *CustomZone) Reset() { *x = CustomZone{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[26] + mi := &file_management_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2164,7 +2235,7 @@ func (x *CustomZone) String() string { func (*CustomZone) ProtoMessage() {} func (x *CustomZone) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[26] + mi := &file_management_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2177,7 +2248,7 @@ func (x *CustomZone) ProtoReflect() protoreflect.Message { // Deprecated: Use CustomZone.ProtoReflect.Descriptor instead. func (*CustomZone) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{26} + return file_management_proto_rawDescGZIP(), []int{27} } func (x *CustomZone) GetDomain() string { @@ -2210,7 +2281,7 @@ type SimpleRecord struct { func (x *SimpleRecord) Reset() { *x = SimpleRecord{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[27] + mi := &file_management_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2223,7 +2294,7 @@ func (x *SimpleRecord) String() string { func (*SimpleRecord) ProtoMessage() {} func (x *SimpleRecord) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[27] + mi := &file_management_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2236,7 +2307,7 @@ func (x *SimpleRecord) ProtoReflect() protoreflect.Message { // Deprecated: Use SimpleRecord.ProtoReflect.Descriptor instead. func (*SimpleRecord) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{27} + return file_management_proto_rawDescGZIP(), []int{28} } func (x *SimpleRecord) GetName() string { @@ -2289,7 +2360,7 @@ type NameServerGroup struct { func (x *NameServerGroup) Reset() { *x = NameServerGroup{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[28] + mi := &file_management_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2302,7 +2373,7 @@ func (x *NameServerGroup) String() string { func (*NameServerGroup) ProtoMessage() {} func (x *NameServerGroup) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[28] + mi := &file_management_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2315,7 +2386,7 @@ func (x *NameServerGroup) ProtoReflect() protoreflect.Message { // Deprecated: Use NameServerGroup.ProtoReflect.Descriptor instead. func (*NameServerGroup) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{28} + return file_management_proto_rawDescGZIP(), []int{29} } func (x *NameServerGroup) GetNameServers() []*NameServer { @@ -2360,7 +2431,7 @@ type NameServer struct { func (x *NameServer) Reset() { *x = NameServer{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[29] + mi := &file_management_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2373,7 +2444,7 @@ func (x *NameServer) String() string { func (*NameServer) ProtoMessage() {} func (x *NameServer) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[29] + mi := &file_management_proto_msgTypes[30] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2386,7 +2457,7 @@ func (x *NameServer) ProtoReflect() protoreflect.Message { // Deprecated: Use NameServer.ProtoReflect.Descriptor instead. func (*NameServer) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{29} + return file_management_proto_rawDescGZIP(), []int{30} } func (x *NameServer) GetIP() string { @@ -2426,7 +2497,7 @@ type FirewallRule struct { func (x *FirewallRule) Reset() { *x = FirewallRule{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[30] + mi := &file_management_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2439,7 +2510,7 @@ func (x *FirewallRule) String() string { func (*FirewallRule) ProtoMessage() {} func (x *FirewallRule) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[30] + mi := &file_management_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2452,7 +2523,7 @@ func (x *FirewallRule) ProtoReflect() protoreflect.Message { // Deprecated: Use FirewallRule.ProtoReflect.Descriptor instead. func (*FirewallRule) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{30} + return file_management_proto_rawDescGZIP(), []int{31} } func (x *FirewallRule) GetPeerIP() string { @@ -2502,7 +2573,7 @@ type NetworkAddress struct { func (x *NetworkAddress) Reset() { *x = NetworkAddress{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[31] + mi := &file_management_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2515,7 +2586,7 @@ func (x *NetworkAddress) String() string { func (*NetworkAddress) ProtoMessage() {} func (x *NetworkAddress) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[31] + mi := &file_management_proto_msgTypes[32] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2528,7 +2599,7 @@ func (x *NetworkAddress) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkAddress.ProtoReflect.Descriptor instead. func (*NetworkAddress) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{31} + return file_management_proto_rawDescGZIP(), []int{32} } func (x *NetworkAddress) GetNetIP() string { @@ -2556,7 +2627,7 @@ type Checks struct { func (x *Checks) Reset() { *x = Checks{} if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[32] + mi := &file_management_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2569,7 +2640,7 @@ func (x *Checks) String() string { func (*Checks) ProtoMessage() {} func (x *Checks) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[32] + mi := &file_management_proto_msgTypes[33] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2582,7 +2653,7 @@ func (x *Checks) ProtoReflect() protoreflect.Message { // Deprecated: Use Checks.ProtoReflect.Descriptor instead. func (*Checks) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{32} + return file_management_proto_rawDescGZIP(), []int{33} } func (x *Checks) GetFiles() []string { @@ -2718,7 +2789,7 @@ var file_management_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x07, 0x0a, 0x05, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0xa8, 0x01, 0x0a, 0x11, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0xd7, 0x01, 0x0a, 0x11, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2c, 0x0a, 0x05, 0x73, 0x74, 0x75, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, @@ -2728,241 +2799,251 @@ var file_management_proto_rawDesc = []byte{ 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x12, 0x2e, 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x22, - 0x98, 0x01, 0x0a, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x10, - 0x0a, 0x03, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, - 0x12, 0x3b, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x3b, 0x0a, - 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, - 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, - 0x54, 0x54, 0x50, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x54, 0x54, 0x50, 0x53, 0x10, 0x03, - 0x12, 0x08, 0x0a, 0x04, 0x44, 0x54, 0x4c, 0x53, 0x10, 0x04, 0x22, 0x7d, 0x0a, 0x13, 0x50, 0x72, - 0x6f, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x68, - 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, - 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, - 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x0a, 0x50, 0x65, - 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, - 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x64, 0x6e, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, - 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, - 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0xe2, 0x03, - 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x06, - 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, 0x65, - 0x72, 0x69, 0x61, 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, - 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x29, 0x0a, 0x06, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, - 0x06, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x40, 0x0a, 0x0c, - 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x0c, 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x3e, - 0x0a, 0x0d, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, - 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x52, - 0x0d, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x32, - 0x0a, 0x14, 0x66, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, - 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x66, 0x69, - 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x22, 0x97, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, - 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, - 0x4b, 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, - 0x49, 0x70, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, - 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0x49, 0x0a, 0x09, - 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, - 0x73, 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, - 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, - 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, - 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, - 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, - 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, - 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, - 0x0a, 0x0a, 0x06, 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x1e, 0x0a, 0x1c, 0x50, - 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x46, 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5b, 0x0a, 0x15, 0x50, - 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xea, 0x02, 0x0a, 0x0e, 0x50, 0x72, 0x6f, - 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x12, - 0x2e, 0x0a, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, - 0x24, 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, - 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x55, - 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0a, 0x55, 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x41, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x41, 0x75, 0x74, 0x68, - 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, 0x4c, - 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x55, 0x52, 0x4c, 0x73, 0x22, 0xed, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, - 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, - 0x18, 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, - 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, - 0x65, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x12, - 0x16, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, - 0x65, 0x72, 0x61, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, 0x73, - 0x71, 0x75, 0x65, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x12, 0x18, 0x0a, - 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, - 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x52, - 0x6f, 0x75, 0x74, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6b, 0x65, 0x65, 0x70, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x22, 0xb4, 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x4e, 0x61, 0x6d, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x52, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x73, 0x12, 0x38, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x52, - 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x22, 0x58, 0x0a, 0x0a, - 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x12, + 0x2d, 0x0a, 0x05, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6c, 0x61, + 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x22, 0x98, + 0x01, 0x0a, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x10, 0x0a, + 0x03, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, + 0x3b, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, + 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x3b, 0x0a, 0x08, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, + 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x54, + 0x54, 0x50, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x54, 0x54, 0x50, 0x53, 0x10, 0x03, 0x12, + 0x08, 0x0a, 0x04, 0x44, 0x54, 0x4c, 0x53, 0x10, 0x04, 0x22, 0x6d, 0x0a, 0x0b, 0x52, 0x65, 0x6c, + 0x61, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x22, 0x0a, 0x0c, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x12, 0x26, 0x0a, 0x0e, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x7d, 0x0a, 0x13, 0x50, 0x72, 0x6f, 0x74, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x36, 0x0a, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x68, 0x6f, 0x73, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x70, + 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, + 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x12, 0x10, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, + 0x6e, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, + 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0xe2, 0x03, 0x0a, 0x0a, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x65, + 0x72, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, 0x65, 0x72, 0x69, + 0x61, 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, + 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, + 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x29, 0x0a, 0x06, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x40, 0x0a, 0x0c, 0x6f, 0x66, + 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, + 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x3e, 0x0a, 0x0d, + 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x52, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x74, 0x0a, 0x0c, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, - 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, - 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x43, - 0x6c, 0x61, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x54, 0x54, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x22, 0xb3, 0x01, 0x0a, - 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x0b, 0x4e, - 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, - 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x69, - 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x32, - 0x0a, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, - 0x65, 0x64, 0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, - 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xf0, 0x02, 0x0a, - 0x0c, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x16, 0x0a, - 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, - 0x65, 0x65, 0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, - 0x6c, 0x65, 0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x44, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, - 0x65, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x3d, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, - 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, - 0x6f, 0x72, 0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x06, 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4f, 0x55, 0x54, 0x10, - 0x01, 0x22, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, - 0x43, 0x43, 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x52, 0x4f, 0x50, 0x10, - 0x01, 0x22, 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x0b, 0x0a, - 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, - 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, - 0x55, 0x44, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x43, 0x4d, 0x50, 0x10, 0x04, 0x22, - 0x38, 0x0a, 0x0e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x63, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, 0x22, 0x1e, 0x0a, 0x06, 0x43, 0x68, 0x65, - 0x63, 0x6b, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x32, 0x90, 0x04, 0x0a, 0x11, 0x4d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x0d, 0x46, + 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x14, + 0x66, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x66, 0x69, 0x72, 0x65, + 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x22, 0x97, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, + 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, + 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x68, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0x49, 0x0a, 0x09, 0x53, 0x53, + 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x73, 0x68, + 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, + 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, 0x50, + 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, + 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, 0x76, 0x69, + 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, + 0x6c, 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, + 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x42, 0x0a, + 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0a, 0x0a, + 0x06, 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x4b, 0x43, + 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, + 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5b, 0x0a, 0x15, 0x50, 0x4b, 0x43, + 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, + 0x6f, 0x77, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xea, 0x02, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, + 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, + 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2e, 0x0a, + 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, + 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x24, 0x0a, + 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, 0x65, + 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x55, + 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x41, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, + 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, 0x4c, 0x73, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, + 0x52, 0x4c, 0x73, 0x22, 0xed, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x4e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, + 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x12, 0x16, 0x0a, + 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, 0x72, + 0x61, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, + 0x65, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x44, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, + 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x6b, 0x65, 0x65, 0x70, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x22, 0xb4, 0x01, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x47, 0x0a, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, + 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x10, + 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, + 0x12, 0x38, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x52, 0x0b, 0x43, + 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x22, 0x58, 0x0a, 0x0a, 0x43, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, + 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x12, 0x32, 0x0a, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, + 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x52, 0x65, 0x63, + 0x6f, 0x72, 0x64, 0x73, 0x22, 0x74, 0x0a, 0x0c, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x43, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x43, 0x6c, 0x61, + 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x54, 0x54, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x03, 0x54, 0x54, 0x4c, 0x12, 0x14, 0x0a, 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x22, 0xb3, 0x01, 0x0a, 0x0f, 0x4e, + 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x38, + 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x0b, 0x4e, 0x61, 0x6d, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x6d, + 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, + 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, + 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x53, 0x65, 0x61, 0x72, + 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, + 0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x0e, + 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, + 0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, + 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0xf0, 0x02, 0x0a, 0x0c, 0x46, + 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50, + 0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, + 0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, + 0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x44, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, + 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, + 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, + 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a, + 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x6f, 0x72, + 0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x06, + 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x22, + 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x43, + 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, 0x52, 0x4f, 0x50, 0x10, 0x01, 0x22, + 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, + 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, + 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x43, 0x4d, 0x50, 0x10, 0x04, 0x22, 0x38, 0x0a, + 0x0e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x6e, 0x65, 0x74, 0x49, 0x50, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x63, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, 0x22, 0x1e, 0x0a, 0x06, 0x43, 0x68, 0x65, 0x63, 0x6b, + 0x73, 0x12, 0x14, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x05, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x32, 0x90, 0x04, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, + 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, - 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, + 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, - 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x11, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, - 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x44, 0x65, - 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, + 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, + 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, + 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69, + 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, + 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, - 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, - 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, + 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, + 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, + 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, - 0x08, 0x53, 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x08, 0x53, + 0x79, 0x6e, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2978,7 +3059,7 @@ func file_management_proto_rawDescGZIP() []byte { } var file_management_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_management_proto_msgTypes = make([]protoimpl.MessageInfo, 33) +var file_management_proto_msgTypes = make([]protoimpl.MessageInfo, 34) var file_management_proto_goTypes = []interface{}{ (HostConfig_Protocol)(0), // 0: management.HostConfig.Protocol (DeviceAuthorizationFlowProvider)(0), // 1: management.DeviceAuthorizationFlow.provider @@ -2999,86 +3080,88 @@ var file_management_proto_goTypes = []interface{}{ (*Empty)(nil), // 16: management.Empty (*WiretrusteeConfig)(nil), // 17: management.WiretrusteeConfig (*HostConfig)(nil), // 18: management.HostConfig - (*ProtectedHostConfig)(nil), // 19: management.ProtectedHostConfig - (*PeerConfig)(nil), // 20: management.PeerConfig - (*NetworkMap)(nil), // 21: management.NetworkMap - (*RemotePeerConfig)(nil), // 22: management.RemotePeerConfig - (*SSHConfig)(nil), // 23: management.SSHConfig - (*DeviceAuthorizationFlowRequest)(nil), // 24: management.DeviceAuthorizationFlowRequest - (*DeviceAuthorizationFlow)(nil), // 25: management.DeviceAuthorizationFlow - (*PKCEAuthorizationFlowRequest)(nil), // 26: management.PKCEAuthorizationFlowRequest - (*PKCEAuthorizationFlow)(nil), // 27: management.PKCEAuthorizationFlow - (*ProviderConfig)(nil), // 28: management.ProviderConfig - (*Route)(nil), // 29: management.Route - (*DNSConfig)(nil), // 30: management.DNSConfig - (*CustomZone)(nil), // 31: management.CustomZone - (*SimpleRecord)(nil), // 32: management.SimpleRecord - (*NameServerGroup)(nil), // 33: management.NameServerGroup - (*NameServer)(nil), // 34: management.NameServer - (*FirewallRule)(nil), // 35: management.FirewallRule - (*NetworkAddress)(nil), // 36: management.NetworkAddress - (*Checks)(nil), // 37: management.Checks - (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*RelayConfig)(nil), // 19: management.RelayConfig + (*ProtectedHostConfig)(nil), // 20: management.ProtectedHostConfig + (*PeerConfig)(nil), // 21: management.PeerConfig + (*NetworkMap)(nil), // 22: management.NetworkMap + (*RemotePeerConfig)(nil), // 23: management.RemotePeerConfig + (*SSHConfig)(nil), // 24: management.SSHConfig + (*DeviceAuthorizationFlowRequest)(nil), // 25: management.DeviceAuthorizationFlowRequest + (*DeviceAuthorizationFlow)(nil), // 26: management.DeviceAuthorizationFlow + (*PKCEAuthorizationFlowRequest)(nil), // 27: management.PKCEAuthorizationFlowRequest + (*PKCEAuthorizationFlow)(nil), // 28: management.PKCEAuthorizationFlow + (*ProviderConfig)(nil), // 29: management.ProviderConfig + (*Route)(nil), // 30: management.Route + (*DNSConfig)(nil), // 31: management.DNSConfig + (*CustomZone)(nil), // 32: management.CustomZone + (*SimpleRecord)(nil), // 33: management.SimpleRecord + (*NameServerGroup)(nil), // 34: management.NameServerGroup + (*NameServer)(nil), // 35: management.NameServer + (*FirewallRule)(nil), // 36: management.FirewallRule + (*NetworkAddress)(nil), // 37: management.NetworkAddress + (*Checks)(nil), // 38: management.Checks + (*timestamppb.Timestamp)(nil), // 39: google.protobuf.Timestamp } var file_management_proto_depIdxs = []int32{ 13, // 0: management.SyncRequest.meta:type_name -> management.PeerSystemMeta 17, // 1: management.SyncResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig - 20, // 2: management.SyncResponse.peerConfig:type_name -> management.PeerConfig - 22, // 3: management.SyncResponse.remotePeers:type_name -> management.RemotePeerConfig - 21, // 4: management.SyncResponse.NetworkMap:type_name -> management.NetworkMap - 37, // 5: management.SyncResponse.Checks:type_name -> management.Checks + 21, // 2: management.SyncResponse.peerConfig:type_name -> management.PeerConfig + 23, // 3: management.SyncResponse.remotePeers:type_name -> management.RemotePeerConfig + 22, // 4: management.SyncResponse.NetworkMap:type_name -> management.NetworkMap + 38, // 5: management.SyncResponse.Checks:type_name -> management.Checks 13, // 6: management.SyncMetaRequest.meta:type_name -> management.PeerSystemMeta 13, // 7: management.LoginRequest.meta:type_name -> management.PeerSystemMeta 10, // 8: management.LoginRequest.peerKeys:type_name -> management.PeerKeys - 36, // 9: management.PeerSystemMeta.networkAddresses:type_name -> management.NetworkAddress + 37, // 9: management.PeerSystemMeta.networkAddresses:type_name -> management.NetworkAddress 11, // 10: management.PeerSystemMeta.environment:type_name -> management.Environment 12, // 11: management.PeerSystemMeta.files:type_name -> management.File 17, // 12: management.LoginResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig - 20, // 13: management.LoginResponse.peerConfig:type_name -> management.PeerConfig - 37, // 14: management.LoginResponse.Checks:type_name -> management.Checks - 38, // 15: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp + 21, // 13: management.LoginResponse.peerConfig:type_name -> management.PeerConfig + 38, // 14: management.LoginResponse.Checks:type_name -> management.Checks + 39, // 15: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp 18, // 16: management.WiretrusteeConfig.stuns:type_name -> management.HostConfig - 19, // 17: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig + 20, // 17: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig 18, // 18: management.WiretrusteeConfig.signal:type_name -> management.HostConfig - 0, // 19: management.HostConfig.protocol:type_name -> management.HostConfig.Protocol - 18, // 20: management.ProtectedHostConfig.hostConfig:type_name -> management.HostConfig - 23, // 21: management.PeerConfig.sshConfig:type_name -> management.SSHConfig - 20, // 22: management.NetworkMap.peerConfig:type_name -> management.PeerConfig - 22, // 23: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig - 29, // 24: management.NetworkMap.Routes:type_name -> management.Route - 30, // 25: management.NetworkMap.DNSConfig:type_name -> management.DNSConfig - 22, // 26: management.NetworkMap.offlinePeers:type_name -> management.RemotePeerConfig - 35, // 27: management.NetworkMap.FirewallRules:type_name -> management.FirewallRule - 23, // 28: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig - 1, // 29: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider - 28, // 30: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig - 28, // 31: management.PKCEAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig - 33, // 32: management.DNSConfig.NameServerGroups:type_name -> management.NameServerGroup - 31, // 33: management.DNSConfig.CustomZones:type_name -> management.CustomZone - 32, // 34: management.CustomZone.Records:type_name -> management.SimpleRecord - 34, // 35: management.NameServerGroup.NameServers:type_name -> management.NameServer - 2, // 36: management.FirewallRule.Direction:type_name -> management.FirewallRule.direction - 3, // 37: management.FirewallRule.Action:type_name -> management.FirewallRule.action - 4, // 38: management.FirewallRule.Protocol:type_name -> management.FirewallRule.protocol - 5, // 39: management.ManagementService.Login:input_type -> management.EncryptedMessage - 5, // 40: management.ManagementService.Sync:input_type -> management.EncryptedMessage - 16, // 41: management.ManagementService.GetServerKey:input_type -> management.Empty - 16, // 42: management.ManagementService.isHealthy:input_type -> management.Empty - 5, // 43: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage - 5, // 44: management.ManagementService.GetPKCEAuthorizationFlow:input_type -> management.EncryptedMessage - 5, // 45: management.ManagementService.SyncMeta:input_type -> management.EncryptedMessage - 5, // 46: management.ManagementService.Login:output_type -> management.EncryptedMessage - 5, // 47: management.ManagementService.Sync:output_type -> management.EncryptedMessage - 15, // 48: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse - 16, // 49: management.ManagementService.isHealthy:output_type -> management.Empty - 5, // 50: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage - 5, // 51: management.ManagementService.GetPKCEAuthorizationFlow:output_type -> management.EncryptedMessage - 16, // 52: management.ManagementService.SyncMeta:output_type -> management.Empty - 46, // [46:53] is the sub-list for method output_type - 39, // [39:46] is the sub-list for method input_type - 39, // [39:39] is the sub-list for extension type_name - 39, // [39:39] is the sub-list for extension extendee - 0, // [0:39] is the sub-list for field type_name + 19, // 19: management.WiretrusteeConfig.relay:type_name -> management.RelayConfig + 0, // 20: management.HostConfig.protocol:type_name -> management.HostConfig.Protocol + 18, // 21: management.ProtectedHostConfig.hostConfig:type_name -> management.HostConfig + 24, // 22: management.PeerConfig.sshConfig:type_name -> management.SSHConfig + 21, // 23: management.NetworkMap.peerConfig:type_name -> management.PeerConfig + 23, // 24: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig + 30, // 25: management.NetworkMap.Routes:type_name -> management.Route + 31, // 26: management.NetworkMap.DNSConfig:type_name -> management.DNSConfig + 23, // 27: management.NetworkMap.offlinePeers:type_name -> management.RemotePeerConfig + 36, // 28: management.NetworkMap.FirewallRules:type_name -> management.FirewallRule + 24, // 29: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig + 1, // 30: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider + 29, // 31: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig + 29, // 32: management.PKCEAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig + 34, // 33: management.DNSConfig.NameServerGroups:type_name -> management.NameServerGroup + 32, // 34: management.DNSConfig.CustomZones:type_name -> management.CustomZone + 33, // 35: management.CustomZone.Records:type_name -> management.SimpleRecord + 35, // 36: management.NameServerGroup.NameServers:type_name -> management.NameServer + 2, // 37: management.FirewallRule.Direction:type_name -> management.FirewallRule.direction + 3, // 38: management.FirewallRule.Action:type_name -> management.FirewallRule.action + 4, // 39: management.FirewallRule.Protocol:type_name -> management.FirewallRule.protocol + 5, // 40: management.ManagementService.Login:input_type -> management.EncryptedMessage + 5, // 41: management.ManagementService.Sync:input_type -> management.EncryptedMessage + 16, // 42: management.ManagementService.GetServerKey:input_type -> management.Empty + 16, // 43: management.ManagementService.isHealthy:input_type -> management.Empty + 5, // 44: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage + 5, // 45: management.ManagementService.GetPKCEAuthorizationFlow:input_type -> management.EncryptedMessage + 5, // 46: management.ManagementService.SyncMeta:input_type -> management.EncryptedMessage + 5, // 47: management.ManagementService.Login:output_type -> management.EncryptedMessage + 5, // 48: management.ManagementService.Sync:output_type -> management.EncryptedMessage + 15, // 49: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse + 16, // 50: management.ManagementService.isHealthy:output_type -> management.Empty + 5, // 51: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage + 5, // 52: management.ManagementService.GetPKCEAuthorizationFlow:output_type -> management.EncryptedMessage + 16, // 53: management.ManagementService.SyncMeta:output_type -> management.Empty + 47, // [47:54] is the sub-list for method output_type + 40, // [40:47] is the sub-list for method input_type + 40, // [40:40] is the sub-list for extension type_name + 40, // [40:40] is the sub-list for extension extendee + 0, // [0:40] is the sub-list for field type_name } func init() { file_management_proto_init() } @@ -3256,7 +3339,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProtectedHostConfig); i { + switch v := v.(*RelayConfig); i { case 0: return &v.state case 1: @@ -3268,7 +3351,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerConfig); i { + switch v := v.(*ProtectedHostConfig); i { case 0: return &v.state case 1: @@ -3280,7 +3363,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NetworkMap); i { + switch v := v.(*PeerConfig); i { case 0: return &v.state case 1: @@ -3292,7 +3375,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemotePeerConfig); i { + switch v := v.(*NetworkMap); i { case 0: return &v.state case 1: @@ -3304,7 +3387,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SSHConfig); i { + switch v := v.(*RemotePeerConfig); i { case 0: return &v.state case 1: @@ -3316,7 +3399,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeviceAuthorizationFlowRequest); i { + switch v := v.(*SSHConfig); i { case 0: return &v.state case 1: @@ -3328,7 +3411,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeviceAuthorizationFlow); i { + switch v := v.(*DeviceAuthorizationFlowRequest); i { case 0: return &v.state case 1: @@ -3340,7 +3423,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PKCEAuthorizationFlowRequest); i { + switch v := v.(*DeviceAuthorizationFlow); i { case 0: return &v.state case 1: @@ -3352,7 +3435,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PKCEAuthorizationFlow); i { + switch v := v.(*PKCEAuthorizationFlowRequest); i { case 0: return &v.state case 1: @@ -3364,7 +3447,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProviderConfig); i { + switch v := v.(*PKCEAuthorizationFlow); i { case 0: return &v.state case 1: @@ -3376,7 +3459,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Route); i { + switch v := v.(*ProviderConfig); i { case 0: return &v.state case 1: @@ -3388,7 +3471,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DNSConfig); i { + switch v := v.(*Route); i { case 0: return &v.state case 1: @@ -3400,7 +3483,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CustomZone); i { + switch v := v.(*DNSConfig); i { case 0: return &v.state case 1: @@ -3412,7 +3495,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SimpleRecord); i { + switch v := v.(*CustomZone); i { case 0: return &v.state case 1: @@ -3424,7 +3507,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NameServerGroup); i { + switch v := v.(*SimpleRecord); i { case 0: return &v.state case 1: @@ -3436,7 +3519,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NameServer); i { + switch v := v.(*NameServerGroup); i { case 0: return &v.state case 1: @@ -3448,7 +3531,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FirewallRule); i { + switch v := v.(*NameServer); i { case 0: return &v.state case 1: @@ -3460,7 +3543,7 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NetworkAddress); i { + switch v := v.(*FirewallRule); i { case 0: return &v.state case 1: @@ -3472,6 +3555,18 @@ func file_management_proto_init() { } } file_management_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NetworkAddress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Checks); i { case 0: return &v.state @@ -3490,7 +3585,7 @@ func file_management_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_management_proto_rawDesc, NumEnums: 5, - NumMessages: 33, + NumMessages: 34, NumExtensions: 0, NumServices: 1, }, diff --git a/management/proto/management.proto b/management/proto/management.proto index 06b243773..c5646820f 100644 --- a/management/proto/management.proto +++ b/management/proto/management.proto @@ -177,6 +177,8 @@ message WiretrusteeConfig { // a Signal server config HostConfig signal = 3; + + RelayConfig relay = 4; } // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) @@ -193,6 +195,13 @@ message HostConfig { DTLS = 4; } } + +message RelayConfig { + repeated string urls = 1; + string tokenPayload = 2; + string tokenSignature = 3; +} + // ProtectedHostConfig is similar to HostConfig but has additional user and password // Mostly used for TURN servers message ProtectedHostConfig { diff --git a/management/server/config.go b/management/server/config.go index 4efe4fe74..96f0c7ffd 100644 --- a/management/server/config.go +++ b/management/server/config.go @@ -32,9 +32,10 @@ const ( // Config of the Management service type Config struct { - Stuns []*Host - TURNConfig *TURNConfig - Signal *Host + Stuns []*Host + TURNConfig *TURNConfig + RelayConfig *RelayConfig + Signal *Host Datadir string DataStoreEncryptionKey string @@ -75,6 +76,10 @@ type TURNConfig struct { Turns []*Host } +type RelayConfig struct { + Address string +} + // HttpServerConfig is a config of the HTTP Management service server type HttpServerConfig struct { LetsEncryptDomain string diff --git a/management/server/grpcserver.go b/management/server/grpcserver.go index ff7a71cfd..147a3d8d6 100644 --- a/management/server/grpcserver.go +++ b/management/server/grpcserver.go @@ -16,13 +16,12 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - nbContext "github.com/netbirdio/netbird/management/server/context" - "github.com/netbirdio/netbird/management/server/posture" - "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/management/proto" + nbContext "github.com/netbirdio/netbird/management/server/context" "github.com/netbirdio/netbird/management/server/jwtclaims" nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/posture" internalStatus "github.com/netbirdio/netbird/management/server/status" "github.com/netbirdio/netbird/management/server/telemetry" ) @@ -32,17 +31,17 @@ type GRPCServer struct { accountManager AccountManager wgKey wgtypes.Key proto.UnimplementedManagementServiceServer - peersUpdateManager *PeersUpdateManager - config *Config - turnCredentialsManager TURNCredentialsManager - jwtValidator *jwtclaims.JWTValidator - jwtClaimsExtractor *jwtclaims.ClaimsExtractor - appMetrics telemetry.AppMetrics - ephemeralManager *EphemeralManager + peersUpdateManager *PeersUpdateManager + config *Config + turnRelayTokenManager TURNRelayTokenManager + jwtValidator *jwtclaims.JWTValidator + jwtClaimsExtractor *jwtclaims.ClaimsExtractor + appMetrics telemetry.AppMetrics + ephemeralManager *EphemeralManager } // NewServer creates a new Management server -func NewServer(ctx context.Context, config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnCredentialsManager TURNCredentialsManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) { +func NewServer(ctx context.Context, config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnRelayTokenManager TURNRelayTokenManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) { key, err := wgtypes.GeneratePrivateKey() if err != nil { return nil, err @@ -88,14 +87,14 @@ func NewServer(ctx context.Context, config *Config, accountManager AccountManage return &GRPCServer{ wgKey: key, // peerKey -> event channel - peersUpdateManager: peersUpdateManager, - accountManager: accountManager, - config: config, - turnCredentialsManager: turnCredentialsManager, - jwtValidator: jwtValidator, - jwtClaimsExtractor: jwtClaimsExtractor, - appMetrics: appMetrics, - ephemeralManager: ephemeralManager, + peersUpdateManager: peersUpdateManager, + accountManager: accountManager, + config: config, + turnRelayTokenManager: turnRelayTokenManager, + jwtValidator: jwtValidator, + jwtClaimsExtractor: jwtClaimsExtractor, + appMetrics: appMetrics, + ephemeralManager: ephemeralManager, }, nil } @@ -172,7 +171,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi s.ephemeralManager.OnPeerConnected(ctx, peer) if s.config.TURNConfig.TimeBasedCredentials { - s.turnCredentialsManager.SetupRefresh(ctx, peer.ID) + s.turnRelayTokenManager.SetupRefresh(ctx, peer.ID) } if s.appMetrics != nil { @@ -235,7 +234,7 @@ func (s *GRPCServer) sendUpdate(ctx context.Context, accountID string, peerKey w func (s *GRPCServer) cancelPeerRoutines(ctx context.Context, accountID string, peer *nbpeer.Peer) { s.peersUpdateManager.CloseChannel(ctx, peer.ID) - s.turnCredentialsManager.CancelRefresh(peer.ID) + s.turnRelayTokenManager.CancelRefresh(peer.ID) _ = s.accountManager.OnPeerDisconnected(ctx, accountID, peer.Key) s.ephemeralManager.OnPeerDisconnected(ctx, peer) } @@ -421,9 +420,14 @@ func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*p s.ephemeralManager.OnPeerDisconnected(ctx, peer) } + trt, err := s.turnRelayTokenManager.Generate() + if err != nil { + log.Errorf("failed generating TURN and Relay token: %v", err) + } + // if peer has reached this point then it has logged in loginResp := &proto.LoginResponse{ - WiretrusteeConfig: toWiretrusteeConfig(s.config, nil), + WiretrusteeConfig: toWiretrusteeConfig(s.config, nil, trt), PeerConfig: toPeerConfig(peer, netMap.Network, s.accountManager.GetDNSDomain()), Checks: toProtocolChecks(ctx, postureChecks), } @@ -481,7 +485,7 @@ func ToResponseProto(configProto Protocol) proto.HostConfig_Protocol { } } -func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *proto.WiretrusteeConfig { +func toWiretrusteeConfig(config *Config, turnCredentials *TURNRelayToken, relayToken *TURNRelayToken) *proto.WiretrusteeConfig { if config == nil { return nil } @@ -497,8 +501,8 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot var username string var password string if turnCredentials != nil { - username = turnCredentials.Username - password = turnCredentials.Password + username = turnCredentials.Payload + password = turnCredentials.Signature } else { username = turn.Username password = turn.Password @@ -513,6 +517,18 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot }) } + var relayCfg *proto.RelayConfig + if config.RelayConfig != nil && config.RelayConfig.Address != "" { + relayCfg = &proto.RelayConfig{ + Urls: []string{config.RelayConfig.Address}, + } + + if relayToken != nil { + relayCfg.TokenPayload = relayToken.Payload + relayCfg.TokenSignature = relayToken.Signature + } + } + return &proto.WiretrusteeConfig{ Stuns: stuns, Turns: turns, @@ -520,6 +536,7 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot Uri: config.Signal.URI, Protocol: ToResponseProto(config.Signal.Proto), }, + Relay: relayCfg, } } @@ -533,9 +550,9 @@ func toPeerConfig(peer *nbpeer.Peer, network *Network, dnsName string) *proto.Pe } } -func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *TURNCredentials, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse { +func toSyncResponse(ctx context.Context, config *Config, peer *nbpeer.Peer, turnCredentials *TURNRelayToken, relayCredentials *TURNRelayToken, networkMap *NetworkMap, dnsName string, checks []*posture.Checks, dnsCache *DNSConfigCache) *proto.SyncResponse { response := &proto.SyncResponse{ - WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials), + WiretrusteeConfig: toWiretrusteeConfig(config, turnCredentials, relayCredentials), PeerConfig: toPeerConfig(peer, networkMap.Network, dnsName), NetworkMap: &proto.NetworkMap{ Serial: networkMap.Network.CurrentSerial(), @@ -583,14 +600,15 @@ func (s *GRPCServer) IsHealthy(ctx context.Context, req *proto.Empty) (*proto.Em // sendInitialSync sends initial proto.SyncResponse to the peer requesting synchronization func (s *GRPCServer) sendInitialSync(ctx context.Context, peerKey wgtypes.Key, peer *nbpeer.Peer, networkMap *NetworkMap, postureChecks []*posture.Checks, srv proto.ManagementService_SyncServer) error { // make secret time based TURN credentials optional - var turnCredentials *TURNCredentials - if s.config.TURNConfig.TimeBasedCredentials { - creds := s.turnCredentialsManager.GenerateCredentials() - turnCredentials = &creds - } else { - turnCredentials = nil + var turnCredentials *TURNRelayToken + trt, err := s.turnRelayTokenManager.Generate() + if err != nil { + log.Errorf("failed generating TURN and Relay token: %v", err) } - plainResp := toSyncResponse(ctx, s.config, peer, turnCredentials, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil) + if s.config.TURNConfig.TimeBasedCredentials { + turnCredentials = trt + } + plainResp := toSyncResponse(ctx, s.config, peer, turnCredentials, trt, networkMap, s.accountManager.GetDNSDomain(), postureChecks, nil) encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, plainResp) if err != nil { diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index fe1e36d47..52ef86e08 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -429,7 +429,11 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, *DefaultAccoun if err != nil { return nil, nil, "", err } - turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + + rc := &RelayConfig{ + Address: "localhost:0", + } + turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) ephemeralMgr := NewEphemeralManager(store, accountManager) mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, ephemeralMgr) diff --git a/management/server/management_test.go b/management/server/management_test.go index 62e7f5a05..352297baf 100644 --- a/management/server/management_test.go +++ b/management/server/management_test.go @@ -552,7 +552,11 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) { if err != nil { log.Fatalf("failed creating a manager: %v", err) } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + + rc := &server.RelayConfig{ + Address: "localhost:0", + } + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) Expect(err).NotTo(HaveOccurred()) mgmtProto.RegisterManagementServiceServer(s, mgmtServer) diff --git a/management/server/peer.go b/management/server/peer.go index 93234d9de..030d38471 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -964,7 +964,7 @@ func (am *DefaultAccountManager) updateAccountPeers(ctx context.Context, account postureChecks := am.getPeerPostureChecks(account, p) remotePeerNetworkMap := account.GetPeerNetworkMap(ctx, p.ID, customZone, approvedPeersMap, am.metrics.AccountManagerMetrics()) - update := toSyncResponse(ctx, nil, p, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache) + update := toSyncResponse(ctx, nil, p, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache) am.peersUpdateManager.SendUpdate(ctx, p.ID, &UpdateMessage{Update: update}) }(peer) } diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 918436515..773ce72ef 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -848,9 +848,9 @@ func TestToSyncResponse(t *testing.T) { DNSLabel: "peer1", SSHKey: "peer1-ssh-key", } - turnCredentials := &TURNCredentials{ - Username: "turn-user", - Password: "turn-pass", + turnRelayToken := &TURNRelayToken{ + Payload: "turn-user", + Signature: "turn-pass", } networkMap := &NetworkMap{ Network: &Network{Net: *ipnet, Serial: 1000}, @@ -916,7 +916,7 @@ func TestToSyncResponse(t *testing.T) { } dnsCache := &DNSConfigCache{} - response := toSyncResponse(context.Background(), config, peer, turnCredentials, networkMap, dnsName, checks, dnsCache) + response := toSyncResponse(context.Background(), config, peer, turnRelayToken, turnRelayToken, networkMap, dnsName, checks, dnsCache) assert.NotNil(t, response) // assert peer config diff --git a/management/server/token_mgr.go b/management/server/token_mgr.go new file mode 100644 index 000000000..c84f815e1 --- /dev/null +++ b/management/server/token_mgr.go @@ -0,0 +1,132 @@ +package server + +import ( + "context" + "fmt" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/management/proto" + auth "github.com/netbirdio/netbird/relay/auth/hmac" +) + +// TURNRelayTokenManager used to manage TURN credentials +type TURNRelayTokenManager interface { + Generate() (*TURNRelayToken, error) + SetupRefresh(ctx context.Context, peerKey string) + CancelRefresh(peerKey string) +} + +// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server +type TimeBasedAuthSecretsManager struct { + mux sync.Mutex + turnCfg *TURNConfig + relayAddr string + hmacToken *auth.TimedHMAC + updateManager *PeersUpdateManager + cancelMap map[string]chan struct{} +} + +type TURNRelayToken auth.Token + +func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayConfig *RelayConfig) *TimeBasedAuthSecretsManager { + + var relayAddr string + if relayConfig != nil { + relayAddr = relayConfig.Address + } + return &TimeBasedAuthSecretsManager{ + mux: sync.Mutex{}, + updateManager: updateManager, + turnCfg: turnCfg, + relayAddr: relayAddr, + hmacToken: auth.NewTimedHMAC(turnCfg.Secret, turnCfg.CredentialsTTL.Duration), + cancelMap: make(map[string]chan struct{}), + } +} + +// Generate generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret +func (m *TimeBasedAuthSecretsManager) Generate() (*TURNRelayToken, error) { + token, err := m.hmacToken.GenerateToken() + if err != nil { + return nil, fmt.Errorf("failed to generate token: %s", err) + } + + return (*TURNRelayToken)(token), nil +} + +func (m *TimeBasedAuthSecretsManager) cancel(peerID string) { + if channel, ok := m.cancelMap[peerID]; ok { + close(channel) + delete(m.cancelMap, peerID) + } +} + +// CancelRefresh cancels scheduled peer credentials refresh +func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) { + m.mux.Lock() + defer m.mux.Unlock() + m.cancel(peerID) +} + +// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer. +// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone. +func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) { + m.mux.Lock() + defer m.mux.Unlock() + m.cancel(peerID) + cancel := make(chan struct{}, 1) + m.cancelMap[peerID] = cancel + log.WithContext(ctx).Debugf("starting turn refresh for %s", peerID) + + go func() { + // we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL) + ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3) + defer ticker.Stop() + + for { + select { + case <-cancel: + log.WithContext(ctx).Debugf("stopping turn refresh for %s", peerID) + return + case <-ticker.C: + m.pushNewTokens(ctx, peerID) + } + } + }() +} + +func (m *TimeBasedAuthSecretsManager) pushNewTokens(ctx context.Context, peerID string) { + token, err := m.hmacToken.GenerateToken() + if err != nil { + log.Errorf("failed to generate token for peer '%s': %s", peerID, err) + return + } + + var turns []*proto.ProtectedHostConfig + for _, host := range m.turnCfg.Turns { + turns = append(turns, &proto.ProtectedHostConfig{ + HostConfig: &proto.HostConfig{ + Uri: host.URI, + Protocol: ToResponseProto(host.Proto), + }, + User: token.Payload, + Password: token.Signature, + }) + } + + update := &proto.SyncResponse{ + WiretrusteeConfig: &proto.WiretrusteeConfig{ + Turns: turns, + Relay: &proto.RelayConfig{ + Urls: []string{m.relayAddr}, + TokenPayload: token.Payload, + TokenSignature: token.Signature, + }, + }, + } + log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID) + m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update}) +} diff --git a/management/server/turncredentials_test.go b/management/server/token_mgr_test.go similarity index 90% rename from management/server/turncredentials_test.go rename to management/server/token_mgr_test.go index 667dccbb5..debd798d0 100644 --- a/management/server/turncredentials_test.go +++ b/management/server/token_mgr_test.go @@ -23,22 +23,25 @@ func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) { secret := "some_secret" peersManager := NewPeersUpdateManager(nil) + rc := &RelayConfig{ + Address: "localhost:0", + } tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }) + }, rc) - credentials := tested.GenerateCredentials() + credentials, _ := tested.Generate() - if credentials.Username == "" { + if credentials.Payload == "" { t.Errorf("expected generated TURN username not to be empty, got empty") } - if credentials.Password == "" { + if credentials.Signature == "" { t.Errorf("expected generated TURN password not to be empty, got empty") } - validateMAC(t, credentials.Username, credentials.Password, []byte(secret)) + validateMAC(t, credentials.Payload, credentials.Signature, []byte(secret)) } @@ -49,11 +52,14 @@ func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) { peer := "some_peer" updateChannel := peersManager.CreateChannel(context.Background(), peer) + rc := &RelayConfig{ + Address: "localhost:0", + } tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }) + }, rc) tested.SetupRefresh(context.Background(), peer) @@ -97,11 +103,14 @@ func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) { peersManager := NewPeersUpdateManager(nil) peer := "some_peer" + rc := &RelayConfig{ + Address: "localhost:0", + } tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }) + }, rc) tested.SetupRefresh(context.Background(), peer) if _, ok := tested.cancelMap[peer]; !ok { diff --git a/management/server/turncredentials.go b/management/server/turncredentials.go deleted file mode 100644 index 79f42e882..000000000 --- a/management/server/turncredentials.go +++ /dev/null @@ -1,126 +0,0 @@ -package server - -import ( - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "fmt" - "sync" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/management/proto" -) - -// TURNCredentialsManager used to manage TURN credentials -type TURNCredentialsManager interface { - GenerateCredentials() TURNCredentials - SetupRefresh(ctx context.Context, peerKey string) - CancelRefresh(peerKey string) -} - -// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server -type TimeBasedAuthSecretsManager struct { - mux sync.Mutex - config *TURNConfig - updateManager *PeersUpdateManager - cancelMap map[string]chan struct{} -} - -type TURNCredentials struct { - Username string - Password string -} - -func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, config *TURNConfig) *TimeBasedAuthSecretsManager { - return &TimeBasedAuthSecretsManager{ - mux: sync.Mutex{}, - config: config, - updateManager: updateManager, - cancelMap: make(map[string]chan struct{}), - } -} - -// GenerateCredentials generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret -func (m *TimeBasedAuthSecretsManager) GenerateCredentials() TURNCredentials { - mac := hmac.New(sha1.New, []byte(m.config.Secret)) - - timeAuth := time.Now().Add(m.config.CredentialsTTL.Duration).Unix() - - username := fmt.Sprint(timeAuth) - - _, err := mac.Write([]byte(username)) - if err != nil { - log.Errorln("Generating turn password failed with error: ", err) - } - - bytePassword := mac.Sum(nil) - password := base64.StdEncoding.EncodeToString(bytePassword) - - return TURNCredentials{ - Username: username, - Password: password, - } - -} - -func (m *TimeBasedAuthSecretsManager) cancel(peerID string) { - if channel, ok := m.cancelMap[peerID]; ok { - close(channel) - delete(m.cancelMap, peerID) - } -} - -// CancelRefresh cancels scheduled peer credentials refresh -func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) { - m.mux.Lock() - defer m.mux.Unlock() - m.cancel(peerID) -} - -// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer. -// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone. -func (m *TimeBasedAuthSecretsManager) SetupRefresh(ctx context.Context, peerID string) { - m.mux.Lock() - defer m.mux.Unlock() - m.cancel(peerID) - cancel := make(chan struct{}, 1) - m.cancelMap[peerID] = cancel - log.WithContext(ctx).Debugf("starting turn refresh for %s", peerID) - - go func() { - // we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL) - ticker := time.NewTicker(m.config.CredentialsTTL.Duration / 4 * 3) - - for { - select { - case <-cancel: - log.WithContext(ctx).Debugf("stopping turn refresh for %s", peerID) - return - case <-ticker.C: - c := m.GenerateCredentials() - var turns []*proto.ProtectedHostConfig - for _, host := range m.config.Turns { - turns = append(turns, &proto.ProtectedHostConfig{ - HostConfig: &proto.HostConfig{ - Uri: host.URI, - Protocol: ToResponseProto(host.Proto), - }, - User: c.Username, - Password: c.Password, - }) - } - - update := &proto.SyncResponse{ - WiretrusteeConfig: &proto.WiretrusteeConfig{ - Turns: turns, - }, - } - log.WithContext(ctx).Debugf("sending new TURN credentials to peer %s", peerID) - m.updateManager.SendUpdate(ctx, peerID, &UpdateMessage{Update: update}) - } - } - }() -} diff --git a/relay/auth/allow_all.go b/relay/auth/allow_all.go new file mode 100644 index 000000000..9a3f3cebd --- /dev/null +++ b/relay/auth/allow_all.go @@ -0,0 +1,10 @@ +package auth + +// AllowAllAuth is a Validator that allows all connections. +// Used this for testing purposes only. +type AllowAllAuth struct { +} + +func (a *AllowAllAuth) Validate(any) error { + return nil +} diff --git a/relay/auth/doc.go b/relay/auth/doc.go new file mode 100644 index 000000000..6bfb6cdf9 --- /dev/null +++ b/relay/auth/doc.go @@ -0,0 +1,26 @@ +/* +Package auth manages the authentication process with the relay server. + +Key Components: + +Validator: The Validator interface defines the Validate method. Any type that provides this method can be used as a +Validator. + +Methods: + +Validate(any): This method is defined in the Validator interface and is used to validate the authentication. + +Usage: + +To create a new AllowAllAuth validator, simply instantiate it: + + validator := &auth.AllowAllAuth{} + +To validate the authentication, use the Validate method: + + err := validator.Validate(any) + +This package provides a simple and effective way to manage authentication with the relay server, ensuring that the +peers are authenticated properly. +*/ +package auth diff --git a/relay/auth/hmac/doc.go b/relay/auth/hmac/doc.go new file mode 100644 index 000000000..a1b135aa6 --- /dev/null +++ b/relay/auth/hmac/doc.go @@ -0,0 +1,8 @@ +/* +This package uses a similar HMAC method for authentication with the TURN server. The Management server provides the +tokens for the peers. The peers manage these tokens in the token store. The token store is a simple thread safe store +that keeps the tokens in memory. These tokens are used to authenticate the peers with the Relay server in the hello +message. +*/ + +package hmac diff --git a/relay/auth/hmac/store.go b/relay/auth/hmac/store.go new file mode 100644 index 000000000..cba5b4f30 --- /dev/null +++ b/relay/auth/hmac/store.go @@ -0,0 +1,34 @@ +package hmac + +import ( + "sync" + + log "github.com/sirupsen/logrus" +) + +// TokenStore is a simple in-memory store for token +// With this can update the token in thread safe way +type TokenStore struct { + mu sync.Mutex + token []byte +} + +func (a *TokenStore) UpdateToken(token *Token) { + a.mu.Lock() + defer a.mu.Unlock() + if token == nil { + return + } + + t, err := marshalToken(*token) + if err != nil { + log.Errorf("failed to marshal token: %s", err) + } + a.token = t +} + +func (a *TokenStore) TokenBinary() []byte { + a.mu.Lock() + defer a.mu.Unlock() + return a.token +} diff --git a/relay/auth/hmac/token.go b/relay/auth/hmac/token.go new file mode 100644 index 000000000..b647ae319 --- /dev/null +++ b/relay/auth/hmac/token.go @@ -0,0 +1,105 @@ +package hmac + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/gob" + "fmt" + "strconv" + "time" + + log "github.com/sirupsen/logrus" +) + +type Token struct { + Payload string + Signature string +} + +func marshalToken(token Token) ([]byte, error) { + buffer := bytes.NewBuffer([]byte{}) + encoder := gob.NewEncoder(buffer) + err := encoder.Encode(token) + if err != nil { + log.Errorf("failed to marshal token: %s", err) + return nil, err + } + return buffer.Bytes(), nil +} + +func unmarshalToken(payload []byte) (Token, error) { + var creds Token + buffer := bytes.NewBuffer(payload) + decoder := gob.NewDecoder(buffer) + err := decoder.Decode(&creds) + return creds, err +} + +// TimedHMAC generates token with TTL and using pre-shared secret known to TURN server +type TimedHMAC struct { + secret string + timeToLive time.Duration +} + +// NewTimedHMAC creates a new TimedHMAC instance +func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC { + return &TimedHMAC{ + secret: secret, + timeToLive: timeToLive, + } +} + +// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC +// hash of a timestamp with a preshared TURN secret +func (m *TimedHMAC) GenerateToken() (*Token, error) { + timeAuth := time.Now().Add(m.timeToLive).Unix() + timeStamp := fmt.Sprint(timeAuth) + + checksum, err := m.generate(timeStamp) + if err != nil { + return nil, err + } + + return &Token{ + Payload: timeStamp, + Signature: base64.StdEncoding.EncodeToString(checksum), + }, nil +} + +// Validate checks if the token is valid +func (m *TimedHMAC) Validate(token Token) error { + expectedMAC, err := m.generate(token.Payload) + if err != nil { + return err + } + + expectedSignature := base64.StdEncoding.EncodeToString(expectedMAC) + + if !hmac.Equal([]byte(expectedSignature), []byte(token.Signature)) { + return fmt.Errorf("signature mismatch") + } + + timeAuthInt, err := strconv.ParseInt(token.Payload, 10, 64) + if err != nil { + return fmt.Errorf("invalid payload: %s", err) + } + + if time.Now().Unix() > timeAuthInt { + return fmt.Errorf("expired token") + } + + return nil +} + +func (m *TimedHMAC) generate(payload string) ([]byte, error) { + mac := hmac.New(sha1.New, []byte(m.secret)) + _, err := mac.Write([]byte(payload)) + if err != nil { + log.Errorf("failed to generate token: %s", err) + return nil, err + } + + return mac.Sum(nil), nil +} diff --git a/relay/auth/hmac/token_test.go b/relay/auth/hmac/token_test.go new file mode 100644 index 000000000..cbe36d5a7 --- /dev/null +++ b/relay/auth/hmac/token_test.go @@ -0,0 +1,103 @@ +package hmac + +import ( + "encoding/base64" + "strconv" + "testing" + "time" +) + +func TestGenerateCredentials(t *testing.T) { + secret := "secret" + timeToLive := 1 * time.Hour + v := NewTimedHMAC(secret, timeToLive) + + creds, err := v.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if creds.Payload == "" { + t.Fatalf("expected non-empty payload") + } + + _, err = strconv.ParseInt(creds.Payload, 10, 64) + if err != nil { + t.Fatalf("expected payload to be a valid unix timestamp, got %v", err) + } + + _, err = base64.StdEncoding.DecodeString(creds.Signature) + if err != nil { + t.Fatalf("expected signature to be base64 encoded, got %v", err) + } +} + +func TestValidateCredentials(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + manager := NewTimedHMAC(secret, timeToLive) + + // Test valid token + creds, err := manager.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err := manager.Validate(*creds); err != nil { + t.Fatalf("expected valid token: %s", err) + } +} + +func TestInvalidSignature(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + manager := NewTimedHMAC(secret, timeToLive) + + creds, err := manager.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + invalidCreds := &Token{ + Payload: creds.Payload, + Signature: "invalidsignature", + } + + if err = manager.Validate(*invalidCreds); err == nil { + t.Fatalf("expected invalid token due to signature mismatch") + } +} + +func TestExpired(t *testing.T) { + secret := "supersecret" + v := NewTimedHMAC(secret, -1*time.Hour) + expiredCreds, err := v.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err = v.Validate(*expiredCreds); err == nil { + t.Fatalf("expected invalid token due to expiration") + } +} + +func TestInvalidPayload(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + v := NewTimedHMAC(secret, timeToLive) + + creds, err := v.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + // Test invalid payload + invalidPayloadCreds := &Token{ + Payload: "invalidtimestamp", + Signature: creds.Signature, + } + + if err = v.Validate(*invalidPayloadCreds); err == nil { + t.Fatalf("expected invalid token due to invalid payload") + } +} diff --git a/relay/auth/hmac/validator.go b/relay/auth/hmac/validator.go new file mode 100644 index 000000000..92669cce6 --- /dev/null +++ b/relay/auth/hmac/validator.go @@ -0,0 +1,27 @@ +package hmac + +import ( + log "github.com/sirupsen/logrus" + "time" +) + +type TimedHMACValidator struct { + *TimedHMAC +} + +func NewTimedHMACValidator(secret string, duration time.Duration) *TimedHMACValidator { + ta := NewTimedHMAC(secret, duration) + return &TimedHMACValidator{ + ta, + } +} + +func (a *TimedHMACValidator) Validate(credentials any) error { + b := credentials.([]byte) + c, err := unmarshalToken(b) + if err != nil { + log.Errorf("failed to unmarshal token: %s", err) + return err + } + return a.TimedHMAC.Validate(c) +} diff --git a/relay/auth/validator.go b/relay/auth/validator.go new file mode 100644 index 000000000..067a42268 --- /dev/null +++ b/relay/auth/validator.go @@ -0,0 +1,6 @@ +package auth + +// Validator is an interface that defines the Validate method. +type Validator interface { + Validate(any) error +} diff --git a/relay/client/addr.go b/relay/client/addr.go new file mode 100644 index 000000000..af4f459f8 --- /dev/null +++ b/relay/client/addr.go @@ -0,0 +1,13 @@ +package client + +type RelayAddr struct { + addr string +} + +func (a RelayAddr) Network() string { + return "relay" +} + +func (a RelayAddr) String() string { + return a.addr +} diff --git a/relay/client/client.go b/relay/client/client.go new file mode 100644 index 000000000..aba940b41 --- /dev/null +++ b/relay/client/client.go @@ -0,0 +1,523 @@ +package client + +import ( + "context" + "fmt" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + auth "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client/dialer/ws" + "github.com/netbirdio/netbird/relay/healthcheck" + "github.com/netbirdio/netbird/relay/messages" +) + +const ( + bufferSize = 8820 + serverResponseTimeout = 8 * time.Second +) + +var ( + ErrConnAlreadyExists = fmt.Errorf("connection already exists") +) + +type internalStopFlag struct { + sync.Mutex + stop bool +} + +func newInternalStopFlag() *internalStopFlag { + return &internalStopFlag{} +} + +func (isf *internalStopFlag) set() { + isf.Lock() + defer isf.Unlock() + isf.stop = true +} + +func (isf *internalStopFlag) isSet() bool { + isf.Lock() + defer isf.Unlock() + return isf.stop +} + +// Msg carry the payload from the server to the client. With this struct, the net.Conn can free the buffer. +type Msg struct { + Payload []byte + + bufPool *sync.Pool + bufPtr *[]byte +} + +func (m *Msg) Free() { + m.bufPool.Put(m.bufPtr) +} + +type connContainer struct { + conn *Conn + messages chan Msg + msgChanLock sync.Mutex + closed bool // flag to check if channel is closed +} + +func newConnContainer(conn *Conn, messages chan Msg) *connContainer { + return &connContainer{ + conn: conn, + messages: messages, + } +} + +func (cc *connContainer) writeMsg(msg Msg) { + cc.msgChanLock.Lock() + defer cc.msgChanLock.Unlock() + if cc.closed { + return + } + cc.messages <- msg +} + +func (cc *connContainer) close() { + cc.msgChanLock.Lock() + defer cc.msgChanLock.Unlock() + if cc.closed { + return + } + close(cc.messages) + cc.closed = true +} + +// Client is a client for the relay server. It is responsible for establishing a connection to the relay server and +// managing connections to other peers. All exported functions are safe to call concurrently. After close the connection, +// the client can be reused by calling Connect again. When the client is closed, all connections are closed too. +// While the Connect is in progress, the OpenConn function will block until the connection is established with relay server. +type Client struct { + log *log.Entry + parentCtx context.Context + connectionURL string + authTokenStore *auth.TokenStore + hashedID []byte + + bufPool *sync.Pool + + relayConn net.Conn + conns map[string]*connContainer + serviceIsRunning bool + mu sync.Mutex // protect serviceIsRunning and conns + readLoopMutex sync.Mutex + wgReadLoop sync.WaitGroup + instanceURL *RelayAddr + muInstanceURL sync.Mutex + + onDisconnectListener func() + listenerMutex sync.Mutex +} + +// NewClient creates a new client for the relay server. The client is not connected to the server until the Connect +func NewClient(ctx context.Context, serverURL string, authTokenStore *auth.TokenStore, peerID string) *Client { + hashedID, hashedStringId := messages.HashID(peerID) + return &Client{ + log: log.WithField("client_id", hashedStringId), + parentCtx: ctx, + connectionURL: serverURL, + authTokenStore: authTokenStore, + hashedID: hashedID, + bufPool: &sync.Pool{ + New: func() any { + buf := make([]byte, bufferSize) + return &buf + }, + }, + conns: make(map[string]*connContainer), + } +} + +// Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. +func (c *Client) Connect() error { + c.log.Infof("connecting to relay server: %s", c.connectionURL) + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + + c.mu.Lock() + defer c.mu.Unlock() + + if c.serviceIsRunning { + return nil + } + + err := c.connect() + if err != nil { + return err + } + + c.serviceIsRunning = true + + c.wgReadLoop.Add(1) + go c.readLoop(c.relayConn) + + log.Infof("relay connection established with: %s", c.connectionURL) + return nil +} + +// OpenConn create a new net.Conn for the destination peer ID. In case if the connection is in progress +// to the relay server, the function will block until the connection is established or timed out. Otherwise, +// it will return immediately. +// todo: what should happen if call with the same peerID with multiple times? +func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.serviceIsRunning { + return nil, fmt.Errorf("relay connection is not established") + } + + hashedID, hashedStringID := messages.HashID(dstPeerID) + _, ok := c.conns[hashedStringID] + if ok { + return nil, ErrConnAlreadyExists + } + + log.Infof("open connection to peer: %s", hashedStringID) + msgChannel := make(chan Msg, 2) + conn := NewConn(c, hashedID, hashedStringID, msgChannel, c.instanceURL) + + c.conns[hashedStringID] = newConnContainer(conn, msgChannel) + return conn, nil +} + +// ServerInstanceURL returns the address of the relay server. It could change after the close and reopen the connection. +func (c *Client) ServerInstanceURL() (string, error) { + c.muInstanceURL.Lock() + defer c.muInstanceURL.Unlock() + if c.instanceURL == nil { + return "", fmt.Errorf("relay connection is not established") + } + return c.instanceURL.String(), nil +} + +// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. +func (c *Client) SetOnDisconnectListener(fn func()) { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + c.onDisconnectListener = fn +} + +// HasConns returns true if there are connections. +func (c *Client) HasConns() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.conns) > 0 +} + +// Close closes the connection to the relay server and all connections to other peers. +func (c *Client) Close() error { + return c.close(true) +} + +func (c *Client) connect() error { + conn, err := ws.Dial(c.connectionURL) + if err != nil { + return err + } + c.relayConn = conn + + err = c.handShake() + if err != nil { + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection: %s", cErr) + } + c.relayConn = nil + return err + } + + return nil +} + +func (c *Client) handShake() error { + tb := c.authTokenStore.TokenBinary() + + msg, err := messages.MarshalHelloMsg(c.hashedID, tb) + if err != nil { + log.Errorf("failed to marshal hello message: %s", err) + return err + } + _, err = c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to send hello message: %s", err) + return err + } + buf := make([]byte, messages.MaxHandshakeSize) + n, err := c.readWithTimeout(buf) + if err != nil { + log.Errorf("failed to read hello response: %s", err) + return err + } + + msgType, err := messages.DetermineServerMsgType(buf[:n]) + if err != nil { + log.Errorf("failed to determine message type: %s", err) + return err + } + + if msgType != messages.MsgTypeHelloResponse { + log.Errorf("unexpected message type: %s", msgType) + return fmt.Errorf("unexpected message type") + } + + ia, err := messages.UnmarshalHelloResponse(buf[:n]) + if err != nil { + return err + } + c.muInstanceURL.Lock() + c.instanceURL = &RelayAddr{addr: ia} + c.muInstanceURL.Unlock() + return nil +} + +func (c *Client) readLoop(relayConn net.Conn) { + internallyStoppedFlag := newInternalStopFlag() + hc := healthcheck.NewReceiver() + go c.listenForStopEvents(hc, relayConn, internallyStoppedFlag) + + var ( + errExit error + n int + ) + for { + bufPtr := c.bufPool.Get().(*[]byte) + buf := *bufPtr + n, errExit = relayConn.Read(buf) + if errExit != nil { + c.mu.Lock() + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Debugf("failed to read message from relay server: %s", errExit) + } + c.mu.Unlock() + break + } + + msgType, err := messages.DetermineServerMsgType(buf[:n]) + if err != nil { + c.log.Errorf("failed to determine message type: %s", err) + continue + } + + if !c.handleMsg(msgType, buf[:n], bufPtr, hc, internallyStoppedFlag) { + break + } + } + + hc.Stop() + + c.muInstanceURL.Lock() + c.instanceURL = nil + c.muInstanceURL.Unlock() + + c.notifyDisconnected() + c.wgReadLoop.Done() + _ = c.close(false) +} + +func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) { + switch msgType { + case messages.MsgTypeHealthCheck: + c.handleHealthCheck(hc, internallyStoppedFlag) + c.bufPool.Put(bufPtr) + case messages.MsgTypeTransport: + return c.handleTransportMsg(buf, bufPtr, internallyStoppedFlag) + case messages.MsgTypeClose: + log.Debugf("relay connection close by server") + c.bufPool.Put(bufPtr) + return false + } + + return true +} + +func (c *Client) handleHealthCheck(hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) { + msg := messages.MarshalHealthcheck() + _, wErr := c.relayConn.Write(msg) + if wErr != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to send heartbeat: %s", wErr) + } + } + hc.Heartbeat() +} + +func (c *Client) handleTransportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool { + peerID, payload, err := messages.UnmarshalTransportMsg(buf) + if err != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to parse transport message: %v", err) + } + + c.bufPool.Put(bufPtr) + return true + } + stringID := messages.HashIDToString(peerID) + + c.mu.Lock() + if !c.serviceIsRunning { + c.mu.Unlock() + c.bufPool.Put(bufPtr) + return false + } + container, ok := c.conns[stringID] + c.mu.Unlock() + if !ok { + c.log.Errorf("peer not found: %s", stringID) + c.bufPool.Put(bufPtr) + return true + } + msg := Msg{ + bufPool: c.bufPool, + bufPtr: bufPtr, + Payload: payload, + } + container.writeMsg(msg) + return true +} + +func (c *Client) writeTo(connReference *Conn, id string, dstID []byte, payload []byte) (int, error) { + c.mu.Lock() + conn, ok := c.conns[id] + c.mu.Unlock() + if !ok { + return 0, io.EOF + } + + if conn.conn != connReference { + return 0, io.EOF + } + + // todo: use buffer pool instead of create new transport msg. + msg, err := messages.MarshalTransportMsg(dstID, payload) + if err != nil { + log.Errorf("failed to marshal transport message: %s", err) + return 0, err + } + + // the write always return with 0 length because the underling does not support the size feedback. + _, err = c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to write transport message: %s", err) + } + return len(payload), err +} + +func (c *Client) listenForStopEvents(hc *healthcheck.Receiver, conn net.Conn, internalStopFlag *internalStopFlag) { + for { + select { + case _, ok := <-hc.OnTimeout: + if !ok { + return + } + c.log.Errorf("health check timeout") + internalStopFlag.set() + _ = conn.Close() // ignore the err because the readLoop will handle it + return + case <-c.parentCtx.Done(): + err := c.close(true) + if err != nil { + log.Errorf("failed to teardown connection: %s", err) + } + return + } + } +} + +func (c *Client) closeAllConns() { + for _, container := range c.conns { + container.close() + } + c.conns = make(map[string]*connContainer) +} + +func (c *Client) closeConn(connReference *Conn, id string) error { + c.mu.Lock() + defer c.mu.Unlock() + + container, ok := c.conns[id] + if !ok { + return fmt.Errorf("connection already closed") + } + + if container.conn != connReference { + return fmt.Errorf("conn reference mismatch") + } + container.close() + delete(c.conns, id) + + return nil +} + +func (c *Client) close(gracefullyExit bool) error { + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + + c.mu.Lock() + var err error + if !c.serviceIsRunning { + c.mu.Unlock() + return nil + } + + c.serviceIsRunning = false + c.closeAllConns() + if gracefullyExit { + c.writeCloseMsg() + err = c.relayConn.Close() + } + c.mu.Unlock() + + c.wgReadLoop.Wait() + c.log.Infof("relay connection closed with: %s", c.connectionURL) + return err +} + +func (c *Client) notifyDisconnected() { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + + if c.onDisconnectListener == nil { + return + } + go c.onDisconnectListener() +} + +func (c *Client) writeCloseMsg() { + msg := messages.MarshalCloseMsg() + _, err := c.relayConn.Write(msg) + if err != nil { + c.log.Errorf("failed to send close message: %s", err) + } +} + +func (c *Client) readWithTimeout(buf []byte) (int, error) { + ctx, cancel := context.WithTimeout(c.parentCtx, serverResponseTimeout) + defer cancel() + + readDone := make(chan struct{}) + var ( + n int + err error + ) + + go func() { + n, err = c.relayConn.Read(buf) + close(readDone) + }() + + select { + case <-ctx.Done(): + return 0, fmt.Errorf("read operation timed out") + case <-readDone: + return n, err + } +} diff --git a/relay/client/client_test.go b/relay/client/client_test.go new file mode 100644 index 000000000..5ac64f23e --- /dev/null +++ b/relay/client/client_test.go @@ -0,0 +1,631 @@ +package client + +import ( + "context" + "net" + "os" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/util" + + "github.com/netbirdio/netbird/relay/server" +) + +var ( + av = &auth.AllowAllAuth{} + hmacTokenStore = &hmac.TokenStore{} + serverListenAddr = "127.0.0.1:1234" + serverURL = "rel://127.0.0.1:1234" +) + +func TestMain(m *testing.M) { + _ = util.InitLog("error", "console") + code := m.Run() + os.Exit(code) +} + +func TestClient(t *testing.T) { + ctx := context.Background() + + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + listenCfg := server.ListenerConfig{Address: serverListenAddr} + err := srv.Listen(listenCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + t.Log("alice connecting to server") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientAlice.Close() + + t.Log("placeholder connecting to server") + clientPlaceHolder := NewClient(ctx, serverURL, hmacTokenStore, "clientPlaceHolder") + err = clientPlaceHolder.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientPlaceHolder.Close() + + t.Log("Bob connecting to server") + clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") + err = clientBob.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientBob.Close() + + t.Log("Alice open connection to Bob") + connAliceToBob, err := clientAlice.OpenConn("bob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + t.Log("Bob open connection to Alice") + connBobToAlice, err := clientBob.OpenConn("alice") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + log.Debugf("alice sent message to bob") + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + log.Debugf("on new message from alice to bob") + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestRegistration(t *testing.T) { + ctx := context.Background() + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + _ = srv.Shutdown(ctx) + t.Fatalf("failed to connect to server: %s", err) + } + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } + err = srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } +} + +func TestRegistrationTimeout(t *testing.T) { + ctx := context.Background() + fakeUDPListener, err := net.ListenUDP("udp", &net.UDPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + }) + if err != nil { + t.Fatalf("failed to bind UDP server: %s", err) + } + defer func(fakeUDPListener *net.UDPConn) { + _ = fakeUDPListener.Close() + }(fakeUDPListener) + + fakeTCPListener, err := net.ListenTCP("tcp", &net.TCPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + }) + if err != nil { + t.Fatalf("failed to bind TCP server: %s", err) + } + defer func(fakeTCPListener *net.TCPListener) { + _ = fakeTCPListener.Close() + }(fakeTCPListener) + + clientAlice := NewClient(ctx, "127.0.0.1:1234", hmacTokenStore, "alice") + err = clientAlice.Connect() + if err == nil { + t.Errorf("failed to connect to server: %s", err) + } + log.Debugf("%s", err) + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } +} + +func TestEcho(t *testing.T) { + ctx := context.Background() + idAlice := "alice" + idBob := "bob" + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice) + err = clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err := clientAlice.Close() + if err != nil { + t.Errorf("failed to close Alice client: %s", err) + } + }() + + clientBob := NewClient(ctx, serverURL, hmacTokenStore, idBob) + err = clientBob.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err := clientBob.Close() + if err != nil { + t.Errorf("failed to close Bob client: %s", err) + } + }() + + connAliceToBob, err := clientAlice.OpenConn(idBob) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + connBobToAlice, err := clientBob.OpenConn(idAlice) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + _, err = connBobToAlice.Write(buf[:n]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + n, err = connAliceToBob.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestBindToUnavailabePeer(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + _, err = clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } +} + +func TestBindReconnect(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + _, err = clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") + err = clientBob.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + chBob, err := clientBob.OpenConn("alice") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing client Alice") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + + clientAlice = NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + chAlice, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + testString := "hello alice, I am bob" + _, err = chBob.Write([]byte(testString)) + if err != nil { + t.Errorf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := chAlice.Read(buf) + if err != nil { + t.Errorf("failed to read from channel: %s", err) + } + + if testString != string(buf[:n]) { + t.Errorf("expected %s, got %s", testString, string(buf[:n])) + } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } +} + +func TestCloseConn(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing connection") + err = conn.Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + _, err = conn.Write([]byte("hello")) + if err == nil { + t.Errorf("unexpected writing from closed connection") + } +} + +func TestCloseRelayConn(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") + err = clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + _ = clientAlice.relayConn.Close() + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + _, err = clientAlice.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } +} + +func TestCloseByServer(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv1, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + + go func() { + err := srv1.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) + err = relayClient.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + + disconnected := make(chan struct{}) + relayClient.SetOnDisconnectListener(func() { + log.Infof("client disconnected") + close(disconnected) + }) + + err = srv1.Shutdown(ctx) + if err != nil { + t.Fatalf("failed to close server: %s", err) + } + + select { + case <-disconnected: + case <-time.After(3 * time.Second): + log.Fatalf("timeout waiting for client to disconnect") + } + + _, err = relayClient.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } +} + +func TestCloseByClient(t *testing.T) { + ctx := context.Background() + + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) + err = relayClient.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + + err = relayClient.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + + _, err = relayClient.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } + + err = srv.Shutdown(ctx) + if err != nil { + t.Fatalf("failed to close server: %s", err) + } +} + +func waitForServerToStart(errChan chan error) error { + select { + case err := <-errChan: + if err != nil { + return err + } + case <-time.After(300 * time.Millisecond): + return nil + } + return nil +} diff --git a/relay/client/conn.go b/relay/client/conn.go new file mode 100644 index 000000000..b4ff903e8 --- /dev/null +++ b/relay/client/conn.go @@ -0,0 +1,76 @@ +package client + +import ( + "io" + "net" + "time" +) + +// Conn represent a connection to a relayed remote peer. +type Conn struct { + client *Client + dstID []byte + dstStringID string + messageChan chan Msg + instanceURL *RelayAddr +} + +// NewConn creates a new connection to a relayed remote peer. +// client: the client instance, it used to send messages to the destination peer +// dstID: the destination peer ID +// dstStringID: the destination peer ID in string format +// messageChan: the channel where the messages will be received +// instanceURL: the relay instance URL, it used to get the proper server instance address for the remote peer +func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg, instanceURL *RelayAddr) *Conn { + c := &Conn{ + client: client, + dstID: dstID, + dstStringID: dstStringID, + messageChan: messageChan, + instanceURL: instanceURL, + } + + return c +} + +func (c *Conn) Write(p []byte) (n int, err error) { + return c.client.writeTo(c, c.dstStringID, c.dstID, p) +} + +func (c *Conn) Read(b []byte) (n int, err error) { + msg, ok := <-c.messageChan + if !ok { + return 0, io.EOF + } + + n = copy(b, msg.Payload) + msg.Free() + return n, nil +} + +func (c *Conn) Close() error { + return c.client.closeConn(c, c.dstStringID) +} + +func (c *Conn) LocalAddr() net.Addr { + return c.client.relayConn.LocalAddr() +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.instanceURL +} + +func (c *Conn) SetDeadline(t time.Time) error { + //TODO implement me + panic("SetDeadline is not implemented") +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + //TODO implement me + panic("SetReadDeadline is not implemented") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + //TODO implement me + panic("SetReadDeadline is not implemented") +} diff --git a/relay/client/dialer/ws/addr.go b/relay/client/dialer/ws/addr.go new file mode 100644 index 000000000..43f5dd6af --- /dev/null +++ b/relay/client/dialer/ws/addr.go @@ -0,0 +1,13 @@ +package ws + +type WebsocketAddr struct { + addr string +} + +func (a WebsocketAddr) Network() string { + return "websocket" +} + +func (a WebsocketAddr) String() string { + return a.addr +} diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go new file mode 100644 index 000000000..e7f771b8d --- /dev/null +++ b/relay/client/dialer/ws/conn.go @@ -0,0 +1,66 @@ +package ws + +import ( + "context" + "fmt" + "net" + "time" + + "nhooyr.io/websocket" +) + +type Conn struct { + ctx context.Context + *websocket.Conn + remoteAddr WebsocketAddr +} + +func NewConn(wsConn *websocket.Conn, serverAddress string) net.Conn { + return &Conn{ + ctx: context.Background(), + Conn: wsConn, + remoteAddr: WebsocketAddr{serverAddress}, + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, ioReader, err := c.Conn.Reader(c.ctx) + if err != nil { + return 0, err + } + + if t != websocket.MessageBinary { + return 0, fmt.Errorf("unexpected message type") + } + + return ioReader.Read(b) +} + +func (c *Conn) Write(b []byte) (n int, err error) { + err = c.Conn.Write(c.ctx, websocket.MessageBinary, b) + return 0, err +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.remoteAddr +} + +func (c *Conn) LocalAddr() net.Addr { + return WebsocketAddr{addr: "unknown"} +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + return fmt.Errorf("SetReadDeadline is not implemented") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + return fmt.Errorf("SetWriteDeadline is not implemented") +} + +func (c *Conn) SetDeadline(t time.Time) error { + return fmt.Errorf("SetDeadline is not implemented") +} + +func (c *Conn) Close() error { + return c.Conn.CloseNow() +} diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go new file mode 100644 index 000000000..bccd6464a --- /dev/null +++ b/relay/client/dialer/ws/ws.go @@ -0,0 +1,59 @@ +package ws + +import ( + "context" + "fmt" + "net" + "net/http" + "strings" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" + + nbnet "github.com/netbirdio/netbird/util/net" +) + +func Dial(address string) (net.Conn, error) { + wsURL, err := prepareURL(address) + if err != nil { + return nil, err + } + + opts := &websocket.DialOptions{ + HTTPClient: httpClientNbDialer(), + } + + wsConn, resp, err := websocket.Dial(context.Background(), wsURL, opts) + if err != nil { + log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err) + return nil, err + } + if resp.Body != nil { + _ = resp.Body.Close() + } + + conn := NewConn(wsConn, address) + return conn, nil +} + +func prepareURL(address string) (string, error) { + if !strings.HasPrefix(address, "rel") { + return "", fmt.Errorf("unsupported scheme: %s", address) + } + + return strings.Replace(address, "rel", "ws", 1), nil +} + +func httpClientNbDialer() *http.Client { + customDialer := nbnet.NewDialer() + + customTransport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return customDialer.DialContext(ctx, network, addr) + }, + } + + return &http.Client{ + Transport: customTransport, + } +} diff --git a/relay/client/doc.go b/relay/client/doc.go new file mode 100644 index 000000000..1339251d9 --- /dev/null +++ b/relay/client/doc.go @@ -0,0 +1,12 @@ +/* +Package client contains the implementation of the Relay client. + +The Relay client is responsible for establishing a connection with the Relay server and sending and receiving messages, +Keep persistent connection with the Relay server and handle the connection issues. +It uses the WebSocket protocol for communication and optionally supports TLS (Transport Layer Security). + +If a peer wants to communicate with a peer on a different relay server, the manager will establish a new connection to +the relay server. The connection with these relay servers will be closed if there is no active connection. The peers +negotiate the common relay instance via signaling service. +*/ +package client diff --git a/relay/client/guard.go b/relay/client/guard.go new file mode 100644 index 000000000..f826cf1b6 --- /dev/null +++ b/relay/client/guard.go @@ -0,0 +1,48 @@ +package client + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" +) + +var ( + reconnectingTimeout = 5 * time.Second +) + +// Guard manage the reconnection tries to the Relay server in case of disconnection event. +type Guard struct { + ctx context.Context + relayClient *Client +} + +// NewGuard creates a new guard for the relay client. +func NewGuard(context context.Context, relayClient *Client) *Guard { + g := &Guard{ + ctx: context, + relayClient: relayClient, + } + return g +} + +// OnDisconnected is called when the relay client is disconnected from the relay server. It will trigger the reconnection +// todo prevent multiple reconnection instances. In the current usage it should not happen, but it is better to prevent +func (g *Guard) OnDisconnected() { + ticker := time.NewTicker(reconnectingTimeout) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + err := g.relayClient.Connect() + if err != nil { + log.Errorf("failed to reconnect to relay server: %s", err) + continue + } + return + case <-g.ctx.Done(): + return + } + } +} diff --git a/relay/client/manager.go b/relay/client/manager.go new file mode 100644 index 000000000..4867dd04d --- /dev/null +++ b/relay/client/manager.go @@ -0,0 +1,314 @@ +package client + +import ( + "container/list" + "context" + "fmt" + "net" + "reflect" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + relayAuth "github.com/netbirdio/netbird/relay/auth/hmac" +) + +var ( + relayCleanupInterval = 60 * time.Second + + errRelayClientNotConnected = fmt.Errorf("relay client not connected") +) + +// RelayTrack hold the relay clients for the foreign relay servers. +// With the mutex can ensure we can open new connection in case the relay connection has been established with +// the relay server. +type RelayTrack struct { + sync.RWMutex + relayClient *Client +} + +func NewRelayTrack() *RelayTrack { + return &RelayTrack{} +} + +type OnServerCloseListener func() + +// ManagerService is the interface for the relay manager. +type ManagerService interface { + Serve() error + OpenConn(serverAddress, peerKey string) (net.Conn, error) + AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error + RelayInstanceAddress() (string, error) + ServerURL() string + HasRelayAddress() bool + UpdateToken(token *relayAuth.Token) +} + +// Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL +// and automatically reconnect to them in case disconnection. +// The manager also manage temporary relay connection. If a client wants to communicate with a client on a +// different relay server, the manager will establish a new connection to the relay server. The connection with these +// relay servers will be closed if there is no active connection. Periodically the manager will check if there is any +// unused relay connection and close it. +type Manager struct { + ctx context.Context + serverURL string + peerID string + tokenStore *relayAuth.TokenStore + + relayClient *Client + reconnectGuard *Guard + + relayClients map[string]*RelayTrack + relayClientsMutex sync.RWMutex + + onDisconnectedListeners map[string]*list.List + listenerLock sync.Mutex +} + +// NewManager creates a new manager instance. +// The serverURL address can be empty. In this case, the manager will not serve. +func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { + return &Manager{ + ctx: ctx, + serverURL: serverURL, + peerID: peerID, + tokenStore: &relayAuth.TokenStore{}, + relayClients: make(map[string]*RelayTrack), + onDisconnectedListeners: make(map[string]*list.List), + } +} + +// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop for +// the unused relay connections. The manager will automatically reconnect to the relay server in case of disconnection. +func (m *Manager) Serve() error { + if m.relayClient != nil { + return fmt.Errorf("manager already serving") + } + + m.relayClient = NewClient(m.ctx, m.serverURL, m.tokenStore, m.peerID) + err := m.relayClient.Connect() + if err != nil { + log.Errorf("failed to connect to relay server: %s", err) + return err + } + + m.reconnectGuard = NewGuard(m.ctx, m.relayClient) + m.relayClient.SetOnDisconnectListener(func() { + m.onServerDisconnected(m.serverURL) + }) + m.startCleanupLoop() + + return nil +} + +// OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be +// established via the relay server. If the peer is on a different relay server, the manager will establish a new +// connection to the relay server. It returns back with a net.Conn what represent the remote peer connection. +func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { + if m.relayClient == nil { + return nil, errRelayClientNotConnected + } + + foreign, err := m.isForeignServer(serverAddress) + if err != nil { + return nil, err + } + + var ( + netConn net.Conn + ) + if !foreign { + log.Debugf("open peer connection via permanent server: %s", peerKey) + netConn, err = m.relayClient.OpenConn(peerKey) + } else { + log.Debugf("open peer connection via foreign server: %s", serverAddress) + netConn, err = m.openConnVia(serverAddress, peerKey) + } + if err != nil { + return nil, err + } + + return netConn, err +} + +// AddCloseListener adds a listener to the given server instance address. The listener will be called if the connection +// closed. +func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error { + foreign, err := m.isForeignServer(serverAddress) + if err != nil { + return err + } + + var listenerAddr string + if foreign { + listenerAddr = serverAddress + } else { + listenerAddr = m.serverURL + } + m.addListener(listenerAddr, onClosedListener) + return nil +} + +// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is +// lost. This address will be sent to the target peer to choose the common relay server for the communication. +func (m *Manager) RelayInstanceAddress() (string, error) { + if m.relayClient == nil { + return "", errRelayClientNotConnected + } + return m.relayClient.ServerInstanceURL() +} + +// ServerURL returns the address of the permanent relay server. +func (m *Manager) ServerURL() string { + return m.serverURL +} + +// HasRelayAddress returns true if the manager is serving. With this method can check if the peer can communicate with +// Relay service. +func (m *Manager) HasRelayAddress() bool { + return m.serverURL != "" +} + +// UpdateToken updates the token in the token store. +func (m *Manager) UpdateToken(token *relayAuth.Token) { + m.tokenStore.UpdateToken(token) +} + +func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { + // check if already has a connection to the desired relay server + m.relayClientsMutex.RLock() + rt, ok := m.relayClients[serverAddress] + if ok { + rt.RLock() + m.relayClientsMutex.RUnlock() + defer rt.RUnlock() + return rt.relayClient.OpenConn(peerKey) + } + m.relayClientsMutex.RUnlock() + + // if not, establish a new connection but check it again (because changed the lock type) before starting the + // connection + m.relayClientsMutex.Lock() + rt, ok = m.relayClients[serverAddress] + if ok { + rt.RLock() + m.relayClientsMutex.Unlock() + defer rt.RUnlock() + return rt.relayClient.OpenConn(peerKey) + } + + // create a new relay client and store it in the relayClients map + rt = NewRelayTrack() + rt.Lock() + m.relayClients[serverAddress] = rt + m.relayClientsMutex.Unlock() + + relayClient := NewClient(m.ctx, serverAddress, m.tokenStore, m.peerID) + err := relayClient.Connect() + if err != nil { + rt.Unlock() + m.relayClientsMutex.Lock() + delete(m.relayClients, serverAddress) + m.relayClientsMutex.Unlock() + return nil, err + } + // if connection closed then delete the relay client from the list + relayClient.SetOnDisconnectListener(func() { + m.onServerDisconnected(serverAddress) + }) + rt.relayClient = relayClient + rt.Unlock() + + conn, err := relayClient.OpenConn(peerKey) + if err != nil { + return nil, err + } + return conn, nil +} + +func (m *Manager) onServerDisconnected(serverAddress string) { + if serverAddress == m.serverURL { + go m.reconnectGuard.OnDisconnected() + } + + m.notifyOnDisconnectListeners(serverAddress) +} + +func (m *Manager) isForeignServer(address string) (bool, error) { + rAddr, err := m.relayClient.ServerInstanceURL() + if err != nil { + return false, fmt.Errorf("relay client not connected") + } + return rAddr != address, nil +} + +func (m *Manager) startCleanupLoop() { + if m.ctx.Err() != nil { + return + } + + ticker := time.NewTicker(relayCleanupInterval) + go func() { + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.cleanUpUnusedRelays() + } + } + }() +} + +func (m *Manager) cleanUpUnusedRelays() { + m.relayClientsMutex.Lock() + defer m.relayClientsMutex.Unlock() + + for addr, rt := range m.relayClients { + rt.Lock() + if rt.relayClient.HasConns() { + rt.Unlock() + continue + } + rt.relayClient.SetOnDisconnectListener(nil) + go func() { + _ = rt.relayClient.Close() + }() + log.Debugf("clean up unused relay server connection: %s", addr) + delete(m.relayClients, addr) + rt.Unlock() + } +} + +func (m *Manager) addListener(serverAddress string, onClosedListener OnServerCloseListener) { + m.listenerLock.Lock() + defer m.listenerLock.Unlock() + l, ok := m.onDisconnectedListeners[serverAddress] + if !ok { + l = list.New() + } + for e := l.Front(); e != nil; e = e.Next() { + if reflect.ValueOf(e.Value).Pointer() == reflect.ValueOf(onClosedListener).Pointer() { + return + } + } + l.PushBack(onClosedListener) + m.onDisconnectedListeners[serverAddress] = l +} + +func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { + m.listenerLock.Lock() + defer m.listenerLock.Unlock() + + l, ok := m.onDisconnectedListeners[serverAddress] + if !ok { + return + } + for e := l.Front(); e != nil; e = e.Next() { + go e.Value.(OnServerCloseListener)() + } + delete(m.onDisconnectedListeners, serverAddress) +} diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go new file mode 100644 index 000000000..ce94d62fe --- /dev/null +++ b/relay/client/manager_test.go @@ -0,0 +1,432 @@ +package client + +import ( + "context" + "testing" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/server" +) + +func TestEmptyURL(t *testing.T) { + mgr := NewManager(context.Background(), "", "alice") + err := mgr.Serve() + if err == nil { + t.Errorf("expected error, got nil") + } +} + +func TestForeignConn(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + srvCfg2 := server.ListenerConfig{ + Address: "localhost:2234", + } + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan2 := make(chan error, 1) + go func() { + err := srv2.Listen(srvCfg2) + if err != nil { + errChan2 <- err + } + }() + + defer func() { + err := srv2.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + idBob := "bob" + log.Debugf("connect by bob") + clientBob := NewManager(mCtx, toURL(srvCfg2), idBob) + err = clientBob.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + bobsSrvAddr, err := clientBob.RelayInstanceAddress() + if err != nil { + t.Fatalf("failed to get relay address: %s", err) + } + connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + _, err = connBobToAlice.Write(buf[:n]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + n, err = connAliceToBob.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestForeginConnClose(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + srvCfg2 := server.ListenerConfig{ + Address: "localhost:2234", + } + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan2 := make(chan error, 1) + go func() { + err := srv2.Listen(srvCfg2) + if err != nil { + errChan2 <- err + } + }() + + defer func() { + err := srv2.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = mgr.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } +} + +func TestForeginAutoClose(t *testing.T) { + ctx := context.Background() + relayCleanupInterval = 1 * time.Second + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + t.Log("binding server 1.") + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + t.Logf("closing server 1.") + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + t.Logf("server 1. closed") + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + srvCfg2 := server.ListenerConfig{ + Address: "localhost:2234", + } + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan2 := make(chan error, 1) + go func() { + t.Log("binding server 2.") + err := srv2.Listen(srvCfg2) + if err != nil { + errChan2 <- err + } + }() + defer func() { + t.Logf("closing server 2.") + err := srv2.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + t.Logf("server 2 closed.") + }() + + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + t.Log("connect to server 1.") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = mgr.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + t.Log("open connection to another peer") + conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + t.Log("close conn") + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } + + t.Logf("waiting for relay cleanup: %s", relayCleanupInterval+1*time.Second) + time.Sleep(relayCleanupInterval + 1*time.Second) + if len(mgr.relayClients) != 0 { + t.Errorf("expected 0, got %d", len(mgr.relayClients)) + } + + t.Logf("closing manager") +} + +func TestAutoReconnect(t *testing.T) { + ctx := context.Background() + reconnectingTimeout = 2 * time.Second + + srvCfg := server.ListenerConfig{ + Address: "localhost:1234", + } + srv, err := server.NewServer(otel.Meter(""), srvCfg.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv.Listen(srvCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg), "alice") + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + ra, err := clientAlice.RelayInstanceAddress() + if err != nil { + t.Errorf("failed to get relay address: %s", err) + } + conn, err := clientAlice.OpenConn(ra, "bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + t.Log("closing client relay connection") + // todo figure out moc server + _ = clientAlice.relayClient.relayConn.Close() + t.Log("start test reading") + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + log.Infof("waiting for reconnection") + time.Sleep(reconnectingTimeout + 1*time.Second) + + log.Infof("reopent the connection") + _, err = clientAlice.OpenConn(ra, "bob") + if err != nil { + t.Errorf("failed to open channel: %s", err) + } +} + +func TestNotifierDoubleAdd(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + conn1, err := clientAlice.OpenConn(clientAlice.ServerURL(), "idBob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + fnCloseListener := OnServerCloseListener(func() { + log.Infof("close listener") + }) + + err = clientAlice.AddCloseListener(clientAlice.ServerURL(), fnCloseListener) + if err != nil { + t.Fatalf("failed to add close listener: %s", err) + } + + err = clientAlice.AddCloseListener(clientAlice.ServerURL(), fnCloseListener) + if err != nil { + t.Fatalf("failed to add close listener: %s", err) + } + + err = conn1.Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + +} + +func toURL(address server.ListenerConfig) string { + return "rel://" + address.Address +} diff --git a/relay/cmd/env.go b/relay/cmd/env.go new file mode 100644 index 000000000..85d3e922b --- /dev/null +++ b/relay/cmd/env.go @@ -0,0 +1,35 @@ +package main + +import ( + "os" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// setFlagsFromEnvVars reads and updates flag values from environment variables with prefix NB_ +func setFlagsFromEnvVars(cmd *cobra.Command) { + flags := cmd.PersistentFlags() + flags.VisitAll(func(f *pflag.Flag) { + newEnvVar := flagNameToEnvVar(f.Name, "NB_") + value, present := os.LookupEnv(newEnvVar) + if !present { + return + } + + err := flags.Set(f.Name, value) + if err != nil { + log.Infof("unable to configure flag %s using variable %s, err: %v", f.Name, newEnvVar, err) + } + }) +} + +// flagNameToEnvVar converts flag name to environment var name adding a prefix, +// replacing dashes and making all uppercase (e.g. setup-keys is converted to NB_SETUP_KEYS according to the input prefix) +func flagNameToEnvVar(cmdFlag string, prefix string) string { + parsed := strings.ReplaceAll(cmdFlag, "-", "_") + upper := strings.ToUpper(parsed) + return prefix + upper +} diff --git a/relay/cmd/main.go b/relay/cmd/main.go new file mode 100644 index 000000000..b61f61d1a --- /dev/null +++ b/relay/cmd/main.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/hashicorp/go-multierror" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/netbirdio/netbird/encryption" + auth "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/signal/metrics" + "github.com/netbirdio/netbird/util" +) + +const ( + metricsPort = 9090 +) + +type Config struct { + ListenAddress string + // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection + // it is a domain:port or ip:port + ExposedAddress string + LetsencryptEmail string + LetsencryptDataDir string + LetsencryptDomains []string + // in case of using Route 53 for DNS challenge the credentials should be provided in the environment variables or + // in the AWS credentials file + LetsencryptAWSRoute53 bool + TlsCertFile string + TlsKeyFile string + AuthSecret string + LogLevel string + LogFile string +} + +func (c Config) Validate() error { + if c.ExposedAddress == "" { + return fmt.Errorf("exposed address is required") + } + if c.AuthSecret == "" { + return fmt.Errorf("auth secret is required") + } + return nil +} + +func (c Config) HasCertConfig() bool { + return c.TlsCertFile != "" && c.TlsKeyFile != "" +} + +func (c Config) HasLetsEncrypt() bool { + return c.LetsencryptDataDir != "" && c.LetsencryptDomains != nil && len(c.LetsencryptDomains) > 0 +} + +var ( + cobraConfig *Config + rootCmd = &cobra.Command{ + Use: "relay", + Short: "Relay service", + Long: "Relay service for Netbird agents", + RunE: execute, + SilenceUsage: true, + SilenceErrors: true, + } +) + +func init() { + _ = util.InitLog("trace", "console") + cobraConfig = &Config{} + rootCmd.PersistentFlags().StringVarP(&cobraConfig.ListenAddress, "listen-address", "l", ":443", "listen address") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.ExposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.LetsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") + rootCmd.PersistentFlags().StringSliceVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration") + rootCmd.PersistentFlags().BoolVar(&cobraConfig.LetsencryptAWSRoute53, "letsencrypt-aws-route53", false, "use AWS Route 53 for Let's Encrypt DNS challenge") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsCertFile, "tls-cert-file", "c", "", "") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsKeyFile, "tls-key-file", "k", "", "") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "auth secret") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LogLevel, "log-level", "info", "log level") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LogFile, "log-file", "console", "log file") + + setFlagsFromEnvVars(rootCmd) +} + +func waitForExitSignal() { + osSigs := make(chan os.Signal, 1) + signal.Notify(osSigs, syscall.SIGINT, syscall.SIGTERM) + <-osSigs +} + +func execute(cmd *cobra.Command, args []string) error { + err := cobraConfig.Validate() + if err != nil { + return fmt.Errorf("invalid config: %s", err) + } + + err = util.InitLog(cobraConfig.LogLevel, cobraConfig.LogFile) + if err != nil { + return fmt.Errorf("failed to initialize log: %s", err) + } + + metricsServer, err := metrics.NewServer(metricsPort, "") + if err != nil { + return fmt.Errorf("setup metrics: %v", err) + } + + go func() { + log.Infof("running metrics server: %s%s", metricsServer.Addr, metricsServer.Endpoint) + if err := metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Failed to start metrics server: %v", err) + } + }() + + srvListenerCfg := server.ListenerConfig{ + Address: cobraConfig.ListenAddress, + } + + tlsConfig, tlsSupport, err := handleTLSConfig(cobraConfig) + if err != nil { + return fmt.Errorf("failed to setup TLS config: %s", err) + } + srvListenerCfg.TLSConfig = tlsConfig + + authenticator := auth.NewTimedHMACValidator(cobraConfig.AuthSecret, 24*time.Hour) + srv, err := server.NewServer(metricsServer.Meter, cobraConfig.ExposedAddress, tlsSupport, authenticator) + if err != nil { + return fmt.Errorf("failed to create relay server: %v", err) + } + log.Infof("server will be available on: %s", srv.InstanceURL()) + go func() { + if err := srv.Listen(srvListenerCfg); err != nil { + log.Fatalf("failed to bind server: %s", err) + } + }() + + // it will block until exit signal + waitForExitSignal() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var shutDownErrors error + if err := srv.Shutdown(ctx); err != nil { + shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close server: %s", err)) + } + + log.Infof("shutting down metrics server") + if err := metricsServer.Shutdown(ctx); err != nil { + shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err)) + } + return shutDownErrors +} + +func handleTLSConfig(cfg *Config) (*tls.Config, bool, error) { + if cfg.LetsencryptAWSRoute53 { + log.Debugf("using Let's Encrypt DNS resolver with Route 53 support") + r53 := encryption.Route53TLS{ + DataDir: cfg.LetsencryptDataDir, + Email: cfg.LetsencryptEmail, + Domains: cfg.LetsencryptDomains, + } + tlsCfg, err := r53.GetCertificate() + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + + if cfg.HasLetsEncrypt() { + log.Infof("setting up TLS with Let's Encrypt.") + tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...) + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + + if cfg.HasCertConfig() { + log.Debugf("using file based TLS config") + tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile) + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + return nil, false, nil +} + +func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) { + certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...) + if err != nil { + return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err) + } + return certManager.TLSConfig(), nil +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Fatalf("%v", err) + } +} diff --git a/relay/doc.go b/relay/doc.go new file mode 100644 index 000000000..d0306204a --- /dev/null +++ b/relay/doc.go @@ -0,0 +1,14 @@ +//Package relay +/* +The `relay` package contains the implementation of the Relay server and client. The Relay server can be used to relay +messages between peers on a single network channel. In this implementation the transport layer is the WebSocket +protocol. + +Between the server and client communication has been design a custom protocol and message format. These messages are +transported over the WebSocket connection. Optionally the server can use TLS to secure the communication. + +The service can support multiple Relay server instances. For this purpose the peers must know the server instance URL. +This URL will be sent to the target peer to choose the common Relay server for the communication via Signal service. + +*/ +package relay diff --git a/relay/healthcheck/doc.go b/relay/healthcheck/doc.go new file mode 100644 index 000000000..da9689c6b --- /dev/null +++ b/relay/healthcheck/doc.go @@ -0,0 +1,17 @@ +/* +The `healthcheck` package is responsible for managing the health checks between the client and the relay server. It +ensures that the connection between the client and the server are alive and functioning properly. + +The `Sender` struct is responsible for sending health check signals to the receiver. The receiver listens for these +signals and sends a new signal back to the sender to acknowledge that the signal has been received. If the sender does +not receive an acknowledgment signal within a certain time frame, it will send a timeout signal via timeout channel +and stop working. + +The `Receiver` struct is responsible for receiving the health check signals from the sender. If the receiver does not +receive a signal within a certain time frame, it will send a timeout signal via the OnTimeout channel and stop working. + +In the Relay usage the signal is sent to the peer in message type Healthcheck. In case of timeout the connection is +closed and the peer is removed from the relay. +*/ + +package healthcheck diff --git a/relay/healthcheck/receiver.go b/relay/healthcheck/receiver.go new file mode 100644 index 000000000..e1ef17e0e --- /dev/null +++ b/relay/healthcheck/receiver.go @@ -0,0 +1,83 @@ +package healthcheck + +import ( + "context" + "time" +) + +var ( + heartbeatTimeout = healthCheckInterval + 3*time.Second +) + +// Receiver is a healthcheck receiver +// It will listen for heartbeat and check if the heartbeat is not received in a certain time +// If the heartbeat is not received in a certain time, it will send a timeout signal and stop to work +// It will also stop if the context is canceled +// The heartbeat timeout is a bit longer than the sender's healthcheck interval +type Receiver struct { + OnTimeout chan struct{} + + ctx context.Context + ctxCancel context.CancelFunc + heartbeat chan struct{} + alive bool +} + +// NewReceiver creates a new healthcheck receiver and start the timer in the background +func NewReceiver() *Receiver { + ctx, ctxCancel := context.WithCancel(context.Background()) + + r := &Receiver{ + OnTimeout: make(chan struct{}, 1), + ctx: ctx, + ctxCancel: ctxCancel, + heartbeat: make(chan struct{}, 1), + } + + go r.waitForHealthcheck() + return r +} + +// Heartbeat acknowledge the heartbeat has been received +func (r *Receiver) Heartbeat() { + select { + case r.heartbeat <- struct{}{}: + default: + } +} + +// Stop check the timeout and do not send new notifications +func (r *Receiver) Stop() { + r.ctxCancel() +} + +func (r *Receiver) waitForHealthcheck() { + ticker := time.NewTicker(heartbeatTimeout) + defer ticker.Stop() + defer r.ctxCancel() + defer close(r.OnTimeout) + + for { + select { + case <-r.heartbeat: + r.alive = true + case <-ticker.C: + if r.alive { + r.alive = false + continue + } + + r.notifyTimeout() + return + case <-r.ctx.Done(): + return + } + } +} + +func (r *Receiver) notifyTimeout() { + select { + case r.OnTimeout <- struct{}{}: + default: + } +} diff --git a/relay/healthcheck/receiver_test.go b/relay/healthcheck/receiver_test.go new file mode 100644 index 000000000..4b4123416 --- /dev/null +++ b/relay/healthcheck/receiver_test.go @@ -0,0 +1,42 @@ +package healthcheck + +import ( + "testing" + "time" +) + +func TestNewReceiver(t *testing.T) { + heartbeatTimeout = 5 * time.Second + r := NewReceiver() + + select { + case <-r.OnTimeout: + t.Error("unexpected timeout") + case <-time.After(1 * time.Second): + + } +} + +func TestNewReceiverNotReceive(t *testing.T) { + heartbeatTimeout = 1 * time.Second + r := NewReceiver() + + select { + case <-r.OnTimeout: + case <-time.After(2 * time.Second): + t.Error("timeout not received") + } +} + +func TestNewReceiverAck(t *testing.T) { + heartbeatTimeout = 2 * time.Second + r := NewReceiver() + + r.Heartbeat() + + select { + case <-r.OnTimeout: + t.Error("unexpected timeout") + case <-time.After(3 * time.Second): + } +} diff --git a/relay/healthcheck/sender.go b/relay/healthcheck/sender.go new file mode 100644 index 000000000..c5d02a4bb --- /dev/null +++ b/relay/healthcheck/sender.go @@ -0,0 +1,71 @@ +package healthcheck + +import ( + "context" + "time" +) + +var ( + healthCheckInterval = 25 * time.Second + healthCheckTimeout = 5 * time.Second +) + +// Sender is a healthcheck sender +// It will send healthcheck signal to the receiver +// If the receiver does not receive the signal in a certain time, it will send a timeout signal and stop to work +// It will also stop if the context is canceled +type Sender struct { + // HealthCheck is a channel to send health check signal to the peer + HealthCheck chan struct{} + // Timeout is a channel to the health check signal is not received in a certain time + Timeout chan struct{} + + ctx context.Context + ack chan struct{} +} + +// NewSender creates a new healthcheck sender +func NewSender(ctx context.Context) *Sender { + hc := &Sender{ + HealthCheck: make(chan struct{}, 1), + Timeout: make(chan struct{}, 1), + ctx: ctx, + ack: make(chan struct{}, 1), + } + + go hc.healthCheck() + return hc +} + +// OnHCResponse sends an acknowledgment signal to the sender +func (hc *Sender) OnHCResponse() { + select { + case hc.ack <- struct{}{}: + default: + } +} + +func (hc *Sender) healthCheck() { + ticker := time.NewTicker(healthCheckInterval) + defer ticker.Stop() + + timeoutTimer := time.NewTimer(healthCheckInterval + healthCheckTimeout) + defer timeoutTimer.Stop() + + defer close(hc.HealthCheck) + defer close(hc.Timeout) + + for { + select { + case <-ticker.C: + hc.HealthCheck <- struct{}{} + case <-timeoutTimer.C: + hc.Timeout <- struct{}{} + return + case <-hc.ack: + timeoutTimer.Stop() + case <-hc.ctx.Done(): + return + } + } +} diff --git a/relay/healthcheck/sender_test.go b/relay/healthcheck/sender_test.go new file mode 100644 index 000000000..5b6db25f6 --- /dev/null +++ b/relay/healthcheck/sender_test.go @@ -0,0 +1,66 @@ +package healthcheck + +import ( + "context" + "os" + "testing" + "time" +) + +func TestMain(m *testing.M) { + // override the health check interval to speed up the test + healthCheckInterval = 1 * time.Second + healthCheckTimeout = 100 * time.Millisecond + code := m.Run() + os.Exit(code) +} + +func TestNewHealthPeriod(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender(ctx) + + iterations := 0 + for i := 0; i < 3; i++ { + select { + case <-hc.HealthCheck: + iterations++ + hc.OnHCResponse() + case <-hc.Timeout: + t.Fatalf("health check is timed out") + case <-time.After(healthCheckInterval + 100*time.Millisecond): + t.Fatalf("health check not received") + } + } +} + +func TestNewHealthFailed(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender(ctx) + + select { + case <-hc.Timeout: + case <-time.After(healthCheckInterval + healthCheckTimeout + 100*time.Millisecond): + t.Fatalf("health check is not timed out") + } +} + +func TestNewHealthcheckStop(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + hc := NewSender(ctx) + + time.Sleep(300 * time.Millisecond) + cancel() + + select { + case <-hc.HealthCheck: + t.Fatalf("health check on received") + case <-hc.Timeout: + t.Fatalf("health check timedout") + case <-ctx.Done(): + // expected + case <-time.After(1 * time.Second): + t.Fatalf("is not exited") + } +} diff --git a/relay/messages/doc.go b/relay/messages/doc.go new file mode 100644 index 000000000..4c719df3a --- /dev/null +++ b/relay/messages/doc.go @@ -0,0 +1,5 @@ +/* +Package messages provides the message types that are used to communicate between the relay and the client. +This package is used to determine the type of message that is being sent and received between the relay and the client. +*/ +package messages diff --git a/relay/messages/id.go b/relay/messages/id.go new file mode 100644 index 000000000..e2162cd3b --- /dev/null +++ b/relay/messages/id.go @@ -0,0 +1,31 @@ +package messages + +import ( + "crypto/sha256" + "encoding/base64" + "fmt" +) + +const ( + prefixLength = 4 + IDSize = prefixLength + sha256.Size +) + +var ( + prefix = []byte("sha-") // 4 bytes +) + +// HashID generates a sha256 hash from the peerID and returns the hash and the human-readable string +func HashID(peerID string) ([]byte, string) { + idHash := sha256.Sum256([]byte(peerID)) + idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:]) + var prefixedHash []byte + prefixedHash = append(prefixedHash, prefix...) + prefixedHash = append(prefixedHash, idHash[:]...) + return prefixedHash, idHashString +} + +// HashIDToString converts a hash to a human-readable string +func HashIDToString(idHash []byte) string { + return fmt.Sprintf("%s%s", idHash[:prefixLength], base64.StdEncoding.EncodeToString(idHash[prefixLength:])) +} diff --git a/relay/messages/id_test.go b/relay/messages/id_test.go new file mode 100644 index 000000000..271a8f90d --- /dev/null +++ b/relay/messages/id_test.go @@ -0,0 +1,13 @@ +package messages + +import ( + "testing" +) + +func TestHashID(t *testing.T) { + hashedID, hashedStringId := HashID("alice") + enc := HashIDToString(hashedID) + if enc != hashedStringId { + t.Errorf("expected %s, got %s", hashedStringId, enc) + } +} diff --git a/relay/messages/message.go b/relay/messages/message.go new file mode 100644 index 000000000..387d87a94 --- /dev/null +++ b/relay/messages/message.go @@ -0,0 +1,221 @@ +package messages + +import ( + "bytes" + "encoding/gob" + "fmt" + + log "github.com/sirupsen/logrus" +) + +const ( + MsgTypeHello MsgType = 0 + MsgTypeHelloResponse MsgType = 1 + MsgTypeTransport MsgType = 2 + MsgTypeClose MsgType = 3 + MsgTypeHealthCheck MsgType = 4 + + sizeOfMsgType = 1 + sizeOfMagicBye = 4 + headerSizeTransport = sizeOfMsgType + IDSize // 1 byte for msg type, IDSize for peerID + headerSizeHello = sizeOfMsgType + sizeOfMagicBye + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID + + MaxHandshakeSize = 8192 +) + +var ( + ErrInvalidMessageLength = fmt.Errorf("invalid message length") + + magicHeader = []byte{0x21, 0x12, 0xA4, 0x42} + + healthCheckMsg = []byte{byte(MsgTypeHealthCheck)} +) + +type MsgType byte + +func (m MsgType) String() string { + switch m { + case MsgTypeHello: + return "hello" + case MsgTypeHelloResponse: + return "hello response" + case MsgTypeTransport: + return "transport" + case MsgTypeClose: + return "close" + case MsgTypeHealthCheck: + return "health check" + default: + return "unknown" + } +} + +type HelloResponse struct { + InstanceAddress string +} + +// DetermineClientMsgType determines the message type from the first byte of the message +func DetermineClientMsgType(msg []byte) (MsgType, error) { + msgType := MsgType(msg[0]) + switch msgType { + case MsgTypeHello: + return msgType, nil + case MsgTypeTransport: + return msgType, nil + case MsgTypeClose: + return msgType, nil + case MsgTypeHealthCheck: + return msgType, nil + default: + return 0, fmt.Errorf("invalid msg type, len: %d", len(msg)) + } +} + +// DetermineServerMsgType determines the message type from the first byte of the message +func DetermineServerMsgType(msg []byte) (MsgType, error) { + msgType := MsgType(msg[0]) + switch msgType { + case MsgTypeHelloResponse: + return msgType, nil + case MsgTypeTransport: + return msgType, nil + case MsgTypeClose: + return msgType, nil + case MsgTypeHealthCheck: + return msgType, nil + default: + return 0, fmt.Errorf("invalid msg type (len: %d)", len(msg)) + } +} + +// MarshalHelloMsg initial hello message +// The Hello message is the first message sent by a client after establishing a connection with the Relay server. This +// message is used to authenticate the client with the server. The authentication is done using an HMAC method. +// The protocol does not limit to use HMAC, it can be any other method. If the authentication failed the server will +// close the network connection without any response. +func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { + if len(peerID) != IDSize { + return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) + } + + // 5 = 1 byte for msg type, 4 byte for magic header + msg := make([]byte, 5, headerSizeHello+len(additions)) + msg[0] = byte(MsgTypeHello) + copy(msg[1:5], magicHeader) + msg = append(msg, peerID...) + msg = append(msg, additions...) + return msg, nil +} + +// UnmarshalHelloMsg extracts the peerID and the additional data from the hello message. The Additional data is used to +// authenticate the client with the server. +func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { + if len(msg) < headerSizeHello { + return nil, nil, fmt.Errorf("invalid 'hello' message") + } + if !bytes.Equal(msg[1:5], magicHeader) { + return nil, nil, fmt.Errorf("invalid magic header") + } + return msg[5 : 5+IDSize], msg[headerSizeHello:], nil +} + +// MarshalHelloResponse creates a response message to the hello message. +// In case of success connection the server response with a Hello Response message. This message contains the server's +// instance URL. This URL will be used by choose the common Relay server in case if the peers are in different Relay +// servers. +func MarshalHelloResponse(DomainAddress string) ([]byte, error) { + payload := HelloResponse{ + InstanceAddress: DomainAddress, + } + + buf := new(bytes.Buffer) + enc := gob.NewEncoder(buf) + + err := enc.Encode(payload) + if err != nil { + log.Errorf("failed to gob encode hello response: %s", err) + return nil, err + } + + msg := make([]byte, 1, 1+buf.Len()) + msg[0] = byte(MsgTypeHelloResponse) + msg = append(msg, buf.Bytes()...) + return msg, nil +} + +// UnmarshalHelloResponse extracts the instance address from the hello response message +func UnmarshalHelloResponse(msg []byte) (string, error) { + if len(msg) < 2 { + return "", fmt.Errorf("invalid 'hello response' message") + } + payload := HelloResponse{} + buf := bytes.NewBuffer(msg[1:]) + dec := gob.NewDecoder(buf) + + err := dec.Decode(&payload) + if err != nil { + log.Errorf("failed to gob decode hello response: %s", err) + return "", err + } + return payload.InstanceAddress, nil +} + +// MarshalCloseMsg creates a close message. +// The close message is used to close the connection gracefully between the client and the server. The server and the +// client can send this message. After receiving this message, the server or client will close the connection. +func MarshalCloseMsg() []byte { + msg := make([]byte, 1) + msg[0] = byte(MsgTypeClose) + return msg +} + +// MarshalTransportMsg creates a transport message. +// The transport message is used to exchange data between peers. The message contains the data to be exchanged and the +// destination peer hashed ID. +func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) { + if len(peerID) != IDSize { + return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) + } + + msg := make([]byte, headerSizeTransport, headerSizeTransport+len(payload)) + msg[0] = byte(MsgTypeTransport) + copy(msg[1:], peerID) + msg = append(msg, payload...) + return msg, nil +} + +// UnmarshalTransportMsg extracts the peerID and the payload from the transport message. +func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) { + if len(buf) < headerSizeTransport { + return nil, nil, ErrInvalidMessageLength + } + + return buf[1:headerSizeTransport], buf[headerSizeTransport:], nil +} + +// UnmarshalTransportID extracts the peerID from the transport message. +func UnmarshalTransportID(buf []byte) ([]byte, error) { + if len(buf) < headerSizeTransport { + log.Debugf("invalid message length: %d, expected: %d, %x", len(buf), headerSizeTransport, buf) + return nil, ErrInvalidMessageLength + } + return buf[1:headerSizeTransport], nil +} + +// UpdateTransportMsg updates the peerID in the transport message. +// With this function the server can reuse the given byte slice to update the peerID in the transport message. So do +// need to allocate a new byte slice. +func UpdateTransportMsg(msg []byte, peerID []byte) error { + if len(msg) < 1+len(peerID) { + return ErrInvalidMessageLength + } + copy(msg[1:], peerID) + return nil +} + +// MarshalHealthcheck creates a health check message. +// Health check message is sent by the server periodically. The client will respond with a health check response +// message. If the client does not respond to the health check message, the server will close the connection. +func MarshalHealthcheck() []byte { + return healthCheckMsg +} diff --git a/relay/messages/message_test.go b/relay/messages/message_test.go new file mode 100644 index 000000000..b0546a77c --- /dev/null +++ b/relay/messages/message_test.go @@ -0,0 +1,43 @@ +package messages + +import ( + "testing" +) + +func TestMarshalHelloMsg(t *testing.T) { + peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") + bHello, err := MarshalHelloMsg(peerID, nil) + if err != nil { + t.Fatalf("error: %v", err) + } + + receivedPeerID, _, err := UnmarshalHelloMsg(bHello) + if err != nil { + t.Fatalf("error: %v", err) + } + if string(receivedPeerID) != string(peerID) { + t.Errorf("expected %s, got %s", peerID, receivedPeerID) + } +} + +func TestMarshalTransportMsg(t *testing.T) { + peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") + payload := []byte("payload") + msg, err := MarshalTransportMsg(peerID, payload) + if err != nil { + t.Fatalf("error: %v", err) + } + + id, respPayload, err := UnmarshalTransportMsg(msg) + if err != nil { + t.Fatalf("error: %v", err) + } + + if string(id) != string(peerID) { + t.Errorf("expected %s, got %s", peerID, id) + } + + if string(respPayload) != string(payload) { + t.Errorf("expected %s, got %s", payload, respPayload) + } +} diff --git a/relay/metrics/realy.go b/relay/metrics/realy.go new file mode 100644 index 000000000..80e12ee6b --- /dev/null +++ b/relay/metrics/realy.go @@ -0,0 +1,136 @@ +package metrics + +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" +) + +const ( + idleTimeout = 30 * time.Second +) + +type Metrics struct { + metric.Meter + + TransferBytesSent metric.Int64Counter + TransferBytesRecv metric.Int64Counter + + peers metric.Int64UpDownCounter + peerActivityChan chan string + peerLastActive map[string]time.Time + mutexActivity sync.Mutex + ctx context.Context +} + +func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) { + bytesSent, err := meter.Int64Counter("relay_transfer_sent_bytes_total") + if err != nil { + return nil, err + } + + bytesRecv, err := meter.Int64Counter("relay_transfer_received_bytes_total") + if err != nil { + return nil, err + } + + peers, err := meter.Int64UpDownCounter("relay_peers") + if err != nil { + return nil, err + } + + peersActive, err := meter.Int64ObservableGauge("relay_peers_active") + if err != nil { + return nil, err + } + + peersIdle, err := meter.Int64ObservableGauge("relay_peers_idle") + if err != nil { + return nil, err + } + + m := &Metrics{ + Meter: meter, + TransferBytesSent: bytesSent, + TransferBytesRecv: bytesRecv, + peers: peers, + + ctx: ctx, + peerActivityChan: make(chan string, 10), + peerLastActive: make(map[string]time.Time), + } + + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + active, idle := m.calculateActiveIdleConnections() + o.ObserveInt64(peersActive, active) + o.ObserveInt64(peersIdle, idle) + return nil + }, + peersActive, peersIdle, + ) + if err != nil { + return nil, err + } + + go m.readPeerActivity() + return m, nil +} + +// PeerConnected increments the number of connected peers and increments number of idle connections +func (m *Metrics) PeerConnected(id string) { + m.peers.Add(m.ctx, 1) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + m.peerLastActive[id] = time.Time{} +} + +// PeerDisconnected decrements the number of connected peers and decrements number of idle or active connections +func (m *Metrics) PeerDisconnected(id string) { + m.peers.Add(m.ctx, -1) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + delete(m.peerLastActive, id) +} + +// PeerActivity increases the active connections +func (m *Metrics) PeerActivity(peerID string) { + select { + case m.peerActivityChan <- peerID: + default: + log.Errorf("peer activity channel is full, dropping activity metrics for peer %s", peerID) + } +} + +func (m *Metrics) calculateActiveIdleConnections() (int64, int64) { + active, idle := int64(0), int64(0) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + for _, lastActive := range m.peerLastActive { + if time.Since(lastActive) > idleTimeout { + idle++ + } else { + active++ + } + } + return active, idle +} + +func (m *Metrics) readPeerActivity() { + for { + select { + case peerID := <-m.peerActivityChan: + m.mutexActivity.Lock() + m.peerLastActive[peerID] = time.Now() + m.mutexActivity.Unlock() + case <-m.ctx.Done(): + return + } + } +} diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go new file mode 100644 index 000000000..535c8bcd9 --- /dev/null +++ b/relay/server/listener/listener.go @@ -0,0 +1,11 @@ +package listener + +import ( + "context" + "net" +) + +type Listener interface { + Listen(func(conn net.Conn)) error + Shutdown(ctx context.Context) error +} diff --git a/relay/server/listener/ws/conn.go b/relay/server/listener/ws/conn.go new file mode 100644 index 000000000..c248963b9 --- /dev/null +++ b/relay/server/listener/ws/conn.go @@ -0,0 +1,114 @@ +package ws + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" +) + +const ( + writeTimeout = 10 * time.Second +) + +type Conn struct { + *websocket.Conn + lAddr *net.TCPAddr + rAddr *net.TCPAddr + + closed bool + closedMu sync.Mutex + ctx context.Context +} + +func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn { + return &Conn{ + Conn: wsConn, + lAddr: lAddr, + rAddr: rAddr, + ctx: context.Background(), + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, r, err := c.Reader(c.ctx) + if err != nil { + return 0, c.ioErrHandling(err) + } + + if t != websocket.MessageBinary { + log.Errorf("unexpected message type: %d", t) + return 0, fmt.Errorf("unexpected message type") + } + + n, err = r.Read(b) + if err != nil { + return 0, c.ioErrHandling(err) + } + return n, err +} + +// Write writes a binary message with the given payload. +// It does not block until fill the internal buffer. +// If the buffer filled up, wait until the buffer is drained or timeout. +func (c *Conn) Write(b []byte) (int, error) { + ctx, ctxCancel := context.WithTimeout(c.ctx, writeTimeout) + defer ctxCancel() + + err := c.Conn.Write(ctx, websocket.MessageBinary, b) + return len(b), err +} + +func (c *Conn) LocalAddr() net.Addr { + return c.lAddr +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.rAddr +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + return fmt.Errorf("SetReadDeadline is not implemented") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + return fmt.Errorf("SetWriteDeadline is not implemented") +} + +func (c *Conn) SetDeadline(t time.Time) error { + return fmt.Errorf("SetDeadline is not implemented") +} + +func (c *Conn) Close() error { + c.closedMu.Lock() + c.closed = true + c.closedMu.Unlock() + return c.Conn.CloseNow() +} + +func (c *Conn) isClosed() bool { + c.closedMu.Lock() + defer c.closedMu.Unlock() + return c.closed +} + +func (c *Conn) ioErrHandling(err error) error { + if c.isClosed() { + return io.EOF + } + + var wErr *websocket.CloseError + if !errors.As(err, &wErr) { + return err + } + if wErr.Code == websocket.StatusNormalClosure { + return io.EOF + } + return err +} diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go new file mode 100644 index 000000000..2034b709a --- /dev/null +++ b/relay/server/listener/ws/listener.go @@ -0,0 +1,83 @@ +package ws + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" +) + +type Listener struct { + // Address is the address to listen on. + Address string + // TLSConfig is the TLS configuration for the server. + TLSConfig *tls.Config + + server *http.Server + acceptFn func(conn net.Conn) +} + +func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { + l.acceptFn = acceptFn + mux := http.NewServeMux() + mux.HandleFunc("/", l.onAccept) + + l.server = &http.Server{ + Addr: l.Address, + Handler: mux, + TLSConfig: l.TLSConfig, + } + + log.Infof("WS server is listening on address: %s", l.Address) + var err error + if l.TLSConfig != nil { + err = l.server.ListenAndServeTLS("", "") + } else { + err = l.server.ListenAndServe() + } + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func (l *Listener) Shutdown(ctx context.Context) error { + if l.server == nil { + return nil + } + + log.Infof("stop WS listener") + if err := l.server.Shutdown(ctx); err != nil { + return fmt.Errorf("server shutdown failed: %v", err) + } + log.Infof("WS listener stopped") + return nil +} + +func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { + wsConn, err := websocket.Accept(w, r, nil) + if err != nil { + log.Errorf("failed to accept ws connection from %s: %s", r.RemoteAddr, err) + return + } + + rAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr) + if err != nil { + _ = wsConn.Close(websocket.StatusInternalError, "internal error") + return + } + + lAddr, err := net.ResolveTCPAddr("tcp", l.server.Addr) + if err != nil { + _ = wsConn.Close(websocket.StatusInternalError, "internal error") + return + } + + conn := NewConn(wsConn, lAddr, rAddr) + l.acceptFn(conn) +} diff --git a/relay/server/peer.go b/relay/server/peer.go new file mode 100644 index 000000000..c09dc8c9f --- /dev/null +++ b/relay/server/peer.go @@ -0,0 +1,180 @@ +package server + +import ( + "context" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/healthcheck" + "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/metrics" +) + +const ( + bufferSize = 8820 +) + +// Peer represents a peer connection +type Peer struct { + metrics *metrics.Metrics + log *log.Entry + idS string + idB []byte + conn net.Conn + connMu sync.RWMutex + store *Store +} + +// NewPeer creates a new Peer instance and prepare custom logging +func NewPeer(metrics *metrics.Metrics, id []byte, conn net.Conn, store *Store) *Peer { + stringID := messages.HashIDToString(id) + return &Peer{ + metrics: metrics, + log: log.WithField("peer_id", stringID), + idS: stringID, + idB: id, + conn: conn, + store: store, + } +} + +// Work reads data from the connection +// It manages the protocol (healthcheck, transport, close). Read the message and determine the message type and handle +// the message accordingly. +func (p *Peer) Work() { + ctx, cancel := context.WithCancel(context.Background()) + hc := healthcheck.NewSender(ctx) + go p.healthcheck(ctx, hc) + defer cancel() + + buf := make([]byte, bufferSize) + for { + n, err := p.conn.Read(buf) + if err != nil { + if err != io.EOF { + p.log.Errorf("failed to read message: %s", err) + } + return + } + + msg := buf[:n] + + msgType, err := messages.DetermineClientMsgType(msg) + if err != nil { + p.log.Errorf("failed to determine message type: %s", err) + return + } + switch msgType { + case messages.MsgTypeHealthCheck: + hc.OnHCResponse() + case messages.MsgTypeTransport: + p.metrics.TransferBytesRecv.Add(ctx, int64(n)) + p.metrics.PeerActivity(p.String()) + p.handleTransportMsg(msg) + case messages.MsgTypeClose: + p.log.Infof("peer exited gracefully") + _ = p.conn.Close() + return + } + } +} + +// Write writes data to the connection +func (p *Peer) Write(b []byte) (int, error) { + p.connMu.RLock() + defer p.connMu.RUnlock() + return p.conn.Write(b) +} + +// CloseGracefully closes the connection with the peer gracefully. Send a close message to the client and close the +// connection. +func (p *Peer) CloseGracefully(ctx context.Context) { + p.connMu.Lock() + _, err := p.writeWithTimeout(ctx, messages.MarshalCloseMsg()) + if err != nil { + p.log.Errorf("failed to send close message to peer: %s", p.String()) + } + + err = p.conn.Close() + if err != nil { + p.log.Errorf("failed to close connection to peer: %s", err) + } + + defer p.connMu.Unlock() +} + +// String returns the peer ID +func (p *Peer) String() string { + return p.idS +} + +func (p *Peer) writeWithTimeout(ctx context.Context, buf []byte) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + writeDone := make(chan struct{}) + var ( + n int + err error + ) + + go func() { + _, err = p.conn.Write(buf) + close(writeDone) + }() + + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-writeDone: + return n, err + } +} + +func (p *Peer) healthcheck(ctx context.Context, hc *healthcheck.Sender) { + for { + select { + case <-hc.HealthCheck: + _, err := p.Write(messages.MarshalHealthcheck()) + if err != nil { + p.log.Errorf("failed to send healthcheck message: %s", err) + return + } + case <-hc.Timeout: + p.log.Errorf("peer healthcheck timeout") + _ = p.conn.Close() + return + case <-ctx.Done(): + return + } + } +} + +func (p *Peer) handleTransportMsg(msg []byte) { + peerID, err := messages.UnmarshalTransportID(msg) + if err != nil { + p.log.Errorf("failed to unmarshal transport message: %s", err) + return + } + stringPeerID := messages.HashIDToString(peerID) + dp, ok := p.store.Peer(stringPeerID) + if !ok { + p.log.Errorf("peer not found: %s", stringPeerID) + return + } + err = messages.UpdateTransportMsg(msg, p.idB) + if err != nil { + p.log.Errorf("failed to update transport message: %s", err) + return + } + n, err := dp.Write(msg) + if err != nil { + p.log.Errorf("failed to write transport message to: %s", dp.String()) + return + } + p.metrics.TransferBytesSent.Add(context.Background(), int64(n)) +} diff --git a/relay/server/relay.go b/relay/server/relay.go new file mode 100644 index 000000000..b47b29cfc --- /dev/null +++ b/relay/server/relay.go @@ -0,0 +1,163 @@ +package server + +import ( + "context" + "fmt" + "net" + "net/url" + "sync" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/metrics" +) + +// Relay represents the relay server +type Relay struct { + metrics *metrics.Metrics + metricsCancel context.CancelFunc + validator auth.Validator + + store *Store + instanceURL string + + closed bool + closeMu sync.RWMutex +} + +// NewRelay creates a new Relay instance +// +// Parameters: +// meter: An instance of metric.Meter from the go.opentelemetry.io/otel/metric package. It is used to create and manage +// metrics for the relay server. +// exposedAddress: A string representing the address that the relay server is exposed on. The client will use this +// address as the relay server's instance URL. +// tlsSupport: A boolean indicating whether the relay server supports TLS (Transport Layer Security) or not. The +// instance URL depends on this value. +// validator: An instance of auth.Validator from the auth package. It is used to validate the authentication of the +// peers. +// +// Returns: +// A pointer to a Relay instance and an error. If the Relay instance is successfully created, the error is nil. +// Otherwise, the error contains the details of what went wrong. +func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, validator auth.Validator) (*Relay, error) { + ctx, metricsCancel := context.WithCancel(context.Background()) + m, err := metrics.NewMetrics(ctx, meter) + if err != nil { + metricsCancel() + return nil, fmt.Errorf("creating app metrics: %v", err) + } + + r := &Relay{ + metrics: m, + metricsCancel: metricsCancel, + validator: validator, + store: NewStore(), + } + + if tlsSupport { + r.instanceURL = fmt.Sprintf("rels://%s", exposedAddress) + } else { + r.instanceURL = fmt.Sprintf("rel://%s", exposedAddress) + } + _, err = url.ParseRequestURI(r.instanceURL) + if err != nil { + return nil, fmt.Errorf("invalid exposed address: %v", err) + } + + return r, nil +} + +// Accept start to handle a new peer connection +func (r *Relay) Accept(conn net.Conn) { + r.closeMu.RLock() + defer r.closeMu.RUnlock() + if r.closed { + return + } + + peerID, err := r.handshake(conn) + if err != nil { + log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err) + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr) + } + return + } + + peer := NewPeer(r.metrics, peerID, conn, r.store) + peer.log.Infof("peer connected from: %s", conn.RemoteAddr()) + r.store.AddPeer(peer) + r.metrics.PeerConnected(peer.String()) + go func() { + peer.Work() + r.store.DeletePeer(peer) + peer.log.Debugf("relay connection closed") + r.metrics.PeerDisconnected(peer.String()) + }() +} + +// Shutdown closes the relay server +// It closes the connection with all peers in gracefully and stops accepting new connections. +func (r *Relay) Shutdown(ctx context.Context) { + log.Infof("close connection with all peers") + r.closeMu.Lock() + wg := sync.WaitGroup{} + peers := r.store.Peers() + for _, peer := range peers { + wg.Add(1) + go func(p *Peer) { + p.CloseGracefully(ctx) + wg.Done() + }(peer) + } + wg.Wait() + r.metricsCancel() + r.closeMu.Unlock() +} + +// InstanceURL returns the instance URL of the relay server +func (r *Relay) InstanceURL() string { + return r.instanceURL +} + +func (r *Relay) handshake(conn net.Conn) ([]byte, error) { + buf := make([]byte, messages.MaxHandshakeSize) + n, err := conn.Read(buf) + if err != nil { + log.Debugf("failed to read message from: %s, %s", conn.RemoteAddr(), err) + return nil, err + } + msgType, err := messages.DetermineClientMsgType(buf[:n]) + if err != nil { + return nil, err + } + + if msgType != messages.MsgTypeHello { + tErr := fmt.Errorf("invalid message type") + log.Debugf("failed to handshake with: %s, %s", conn.RemoteAddr(), tErr) + return nil, tErr + } + + peerID, authPayload, err := messages.UnmarshalHelloMsg(buf[:n]) + if err != nil { + log.Debugf("failed to handshake with: %s, %s", conn.RemoteAddr(), err) + return nil, err + } + + if err := r.validator.Validate(authPayload); err != nil { + log.Debugf("failed to authenticate connection with: %s, %s", conn.RemoteAddr(), err) + return nil, err + } + + msg, _ := messages.MarshalHelloResponse(r.instanceURL) + _, err = conn.Write(msg) + if err != nil { + return nil, err + } + return peerID, nil +} diff --git a/relay/server/server.go b/relay/server/server.go new file mode 100644 index 000000000..0036e2390 --- /dev/null +++ b/relay/server/server.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "crypto/tls" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/server/listener" + "github.com/netbirdio/netbird/relay/server/listener/ws" +) + +// ListenerConfig is the configuration for the listener. +// Address: the address to bind the listener to. It could be an address behind a reverse proxy. +// TLSConfig: the TLS configuration for the listener. +type ListenerConfig struct { + Address string + TLSConfig *tls.Config +} + +// Server is the main entry point for the relay server. +// It is the gate between the WebSocket listener and the Relay server logic. +// In a new HTTP connection, the server will accept the connection and pass it to the Relay server via the Accept method. +type Server struct { + relay *Relay + wSListener listener.Listener +} + +// NewServer creates a new relay server instance. +// meter: the OpenTelemetry meter +// exposedAddress: this address will be used as the instance URL. It should be a domain:port format. +// tlsSupport: if true, the server will support TLS +// authValidator: the auth validator to use for the server +func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authValidator auth.Validator) (*Server, error) { + relay, err := NewRelay(meter, exposedAddress, tlsSupport, authValidator) + if err != nil { + return nil, err + } + return &Server{ + relay: relay, + }, nil +} + +// Listen starts the relay server. +func (r *Server) Listen(cfg ListenerConfig) error { + r.wSListener = &ws.Listener{ + Address: cfg.Address, + TLSConfig: cfg.TLSConfig, + } + + wslErr := r.wSListener.Listen(r.relay.Accept) + if wslErr != nil { + log.Errorf("failed to bind ws server: %s", wslErr) + } + + return wslErr +} + +// Shutdown stops the relay server. If there are active connections, they will be closed gracefully. In case of a context, +// the connections will be forcefully closed. +func (r *Server) Shutdown(ctx context.Context) (err error) { + // stop service new connections + if r.wSListener != nil { + err = r.wSListener.Shutdown(ctx) + } + + r.relay.Shutdown(ctx) + return +} + +// InstanceURL returns the instance URL of the relay server. +func (r *Server) InstanceURL() string { + return r.relay.instanceURL +} diff --git a/relay/server/store.go b/relay/server/store.go new file mode 100644 index 000000000..96879dae1 --- /dev/null +++ b/relay/server/store.go @@ -0,0 +1,64 @@ +package server + +import ( + "sync" +) + +// Store is a thread-safe store of peers +// It is used to store the peers that are connected to the relay server +type Store struct { + peers map[string]*Peer // consider to use [32]byte as key. The Peer(id string) would be faster + peersLock sync.RWMutex +} + +// NewStore creates a new Store instance +func NewStore() *Store { + return &Store{ + peers: make(map[string]*Peer), + } +} + +// AddPeer adds a peer to the store +// todo: consider to close peer conn if the peer already exists +func (s *Store) AddPeer(peer *Peer) { + s.peersLock.Lock() + defer s.peersLock.Unlock() + s.peers[peer.String()] = peer +} + +// DeletePeer deletes a peer from the store +func (s *Store) DeletePeer(peer *Peer) { + s.peersLock.Lock() + defer s.peersLock.Unlock() + + dp, ok := s.peers[peer.String()] + if !ok { + return + } + if dp != peer { + return + } + + delete(s.peers, peer.String()) +} + +// Peer returns a peer by its ID +func (s *Store) Peer(id string) (*Peer, bool) { + s.peersLock.RLock() + defer s.peersLock.RUnlock() + + p, ok := s.peers[id] + return p, ok +} + +// Peers returns all the peers in the store +func (s *Store) Peers() []*Peer { + s.peersLock.RLock() + defer s.peersLock.RUnlock() + + peers := make([]*Peer, 0, len(s.peers)) + for _, p := range s.peers { + peers = append(peers, p) + } + return peers +} diff --git a/relay/server/store_test.go b/relay/server/store_test.go new file mode 100644 index 000000000..4a30bc131 --- /dev/null +++ b/relay/server/store_test.go @@ -0,0 +1,40 @@ +package server + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/metrics" +) + +func TestStore_DeletePeer(t *testing.T) { + s := NewStore() + + m, _ := metrics.NewMetrics(context.Background(), otel.Meter("")) + + p := NewPeer(m, []byte("peer_one"), nil, nil) + s.AddPeer(p) + s.DeletePeer(p) + if _, ok := s.Peer(p.String()); ok { + t.Errorf("peer was not deleted") + } +} + +func TestStore_DeleteDeprecatedPeer(t *testing.T) { + s := NewStore() + + m, _ := metrics.NewMetrics(context.Background(), otel.Meter("")) + + p1 := NewPeer(m, []byte("peer_id"), nil, nil) + p2 := NewPeer(m, []byte("peer_id"), nil, nil) + + s.AddPeer(p1) + s.AddPeer(p2) + s.DeletePeer(p1) + + if _, ok := s.Peer(p2.String()); !ok { + t.Errorf("second peer was deleted") + } +} diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go new file mode 100644 index 000000000..75560705c --- /dev/null +++ b/relay/test/benchmark_test.go @@ -0,0 +1,386 @@ +package test + +import ( + "context" + "crypto/rand" + "fmt" + "net" + "os" + "sync" + "testing" + "time" + + "github.com/pion/logging" + "github.com/pion/turn/v3" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client" + "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/util" +) + +var ( + av = &auth.AllowAllAuth{} + hmacTokenStore = &hmac.TokenStore{} + pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + dataSize = 1024 * 1024 * 10 +) + +func TestMain(m *testing.M) { + _ = util.InitLog("error", "console") + code := m.Run() + os.Exit(code) +} + +func TestRelayDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + transfer(t, testData, peerPairs) + }) + } +} + +// TestTurnDataTransfer run turn server: +// docker run --rm --name coturn -d --network=host coturn/coturn --user test:test +func TestTurnDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + runTurnTest(t, testData, peerPairs) + }) + } +} + +func transfer(t *testing.T, testData []byte, peerPairs int) { + t.Helper() + ctx := context.Background() + port := 35000 + peerPairs + serverAddress := fmt.Sprintf("127.0.0.1:%d", port) + serverConnURL := fmt.Sprintf("rel://%s", serverAddress) + + srv, err := server.NewServer(otel.Meter(""), serverConnURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + listenCfg := server.ListenerConfig{Address: serverAddress} + err := srv.Listen(listenCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Shutdown(ctx) + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientsSender := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + clientsReceiver := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsReceiver); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + + conn, err = clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + + var transferDuration []time.Duration + wg := sync.WaitGroup{} + var writeErr error + var readErr error + for i := 0; i < len(connsSender); i++ { + wg.Add(2) + start := time.Now() + go func(i int) { + defer wg.Done() + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr = connsSender[i].Write(testData[j:end]) + if writeErr != nil { + return + } + } + + }(i) + + go func(i int, start time.Time) { + defer wg.Done() + buf := make([]byte, 8192) + rcv := 0 + var n int + for receivedSize := 0; receivedSize < len(testData); { + + n, readErr = connsReceiver[i].Read(buf) + if readErr != nil { + return + } + + receivedSize += n + rcv += n + } + transferDuration = append(transferDuration, time.Since(start)) + }(i, start) + } + + wg.Wait() + + if writeErr != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + if readErr != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + // calculate the megabytes per second from the average transferDuration against the dataSize + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) + + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + err = connsReceiver[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + } +} + +func runTurnTest(t *testing.T, testData []byte, maxPairs int) { + t.Helper() + var transferDuration []time.Duration + var wg sync.WaitGroup + + for i := 0; i < maxPairs; i++ { + wg.Add(1) + go func() { + defer wg.Done() + d := runTurnDataTransfer(t, testData) + transferDuration = append(transferDuration, d) + }() + + } + wg.Wait() + + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) +} + +func runTurnDataTransfer(t *testing.T, testData []byte) time.Duration { + t.Helper() + testDataLen := len(testData) + relayAddress := "192.168.0.10:3478" + conn, err := net.Dial("tcp", relayAddress) + if err != nil { + t.Fatal(err) + } + defer func(conn net.Conn) { + _ = conn.Close() + }(conn) + + turnClient, err := getTurnClient(t, relayAddress, conn) + if err != nil { + t.Fatal(err) + } + defer turnClient.Close() + + relayConn, err := turnClient.Allocate() + if err != nil { + t.Fatal(err) + } + defer func(relayConn net.PacketConn) { + _ = relayConn.Close() + }(relayConn) + + receiverConn, err := net.Dial("udp", relayConn.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer func(receiverConn net.Conn) { + _ = receiverConn.Close() + }(receiverConn) + + var ( + tb int + start time.Time + timerInit bool + readDone = make(chan struct{}) + ack = make([]byte, 1) + ) + go func() { + defer func() { + readDone <- struct{}{} + }() + buff := make([]byte, 8192) + for { + n, e := receiverConn.Read(buff) + if e != nil { + return + } + if !timerInit { + start = time.Now() + timerInit = true + } + tb += n + _, _ = receiverConn.Write(ack) + + if tb >= testDataLen { + return + } + } + }() + + pieceSize := 1024 + ackBuff := make([]byte, 1) + pipelineSize := 10 + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, err := relayConn.WriteTo(testData[j:end], receiverConn.LocalAddr()) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + if pipelineSize == 0 { + _, _, _ = relayConn.ReadFrom(ackBuff) + } else { + pipelineSize-- + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + select { + case <-readDone: + if tb != testDataLen { + t.Fatalf("failed to read all data: %d/%d", tb, testDataLen) + } + case <-ctx.Done(): + t.Fatal("timeout") + } + return time.Since(start) +} + +func getTurnClient(t *testing.T, address string, conn net.Conn) (*turn.Client, error) { + t.Helper() + // Dial TURN Server + addrStr := fmt.Sprintf("%s:%d", address, 443) + + fac := logging.NewDefaultLoggerFactory() + //fac.DefaultLogLevel = logging.LogLevelTrace + + // Start a new TURN Client and wrap our net.Conn in a STUNConn + // This allows us to simulate datagram based communication over a net.Conn + cfg := &turn.ClientConfig{ + TURNServerAddr: address, + Conn: turn.NewSTUNConn(conn), + Username: "test", + Password: "test", + LoggerFactory: fac, + } + + client, err := turn.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err) + } + + // Start listening on the conn provided. + err = client.Listen() + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err) + } + + return client, nil +} + +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} + +func waitForServerToStart(errChan chan error) error { + select { + case err := <-errChan: + if err != nil { + return err + } + case <-time.After(300 * time.Millisecond): + return nil + } + return nil +} diff --git a/relay/testec2/main.go b/relay/testec2/main.go new file mode 100644 index 000000000..0c8099a5e --- /dev/null +++ b/relay/testec2/main.go @@ -0,0 +1,258 @@ +//go:build linux || darwin + +package main + +import ( + "crypto/rand" + "flag" + "fmt" + "net" + "os" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/util" +) + +const ( + errMsgFailedReadTCP = "failed to read from tcp: %s" +) + +var ( + dataSize = 1024 * 1024 * 50 // 50MB + pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + signalListenAddress = ":8081" + + relaySrvAddress string + turnSrvAddress string + signalURL string + udpListener string // used for TURN test +) + +type testResult struct { + numOfPairs int + duration time.Duration + speed float64 +} + +func (tr testResult) Speed() string { + speed := tr.speed + var unit string + + switch { + case speed < 1024: + unit = "B/s" + case speed < 1048576: + speed /= 1024 + unit = "KB/s" + case speed < 1073741824: + speed /= 1048576 + unit = "MB/s" + default: + speed /= 1073741824 + unit = "GB/s" + } + + return fmt.Sprintf("%.2f %s", speed, unit) +} + +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} + +func avg(transferDuration []time.Duration) (time.Duration, float64) { + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + bps := float64(dataSize) / avgDuration.Seconds() + return avgDuration, bps +} + +func RelayReceiverMain() []testResult { + testResults := make([]testResult, 0, len(pairs)) + for _, p := range pairs { + tr := testResult{numOfPairs: p} + td := relayReceive(relaySrvAddress, p) + tr.duration, tr.speed = avg(td) + + testResults = append(testResults, tr) + } + + return testResults +} + +func RelaySenderMain() { + log.Infof("starting sender") + log.Infof("starting seed phase") + + testData, err := seedRandomData(dataSize) + if err != nil { + log.Fatalf("failed to seed random data: %s", err) + } + + log.Infof("data size: %d", len(testData)) + + for n, p := range pairs { + log.Infof("running test with %d pairs", p) + relayTransfer(relaySrvAddress, testData, p) + + // grant time to prepare new receivers + if n < len(pairs)-1 { + time.Sleep(3 * time.Second) + } + } + +} + +// TRUNSenderMain is the sender +// - allocate turn clients +// - send relayed addresses to signal server in batch +// - wait for signal server to send back addresses in a map +// - send test data to each address in parallel +func TRUNSenderMain() { + log.Infof("starting TURN sender test") + + log.Infof("starting seed random data: %d", dataSize) + testData, err := seedRandomData(dataSize) + if err != nil { + log.Fatalf("failed to seed random data: %s", err) + } + + ss := SignalClient{signalURL} + + for _, p := range pairs { + log.Infof("running test with %d pairs", p) + turnSender := &TurnSender{} + + createTurnConns(p, turnSender) + + log.Infof("send addresses via signal server: %d", len(turnSender.addresses)) + clientAddresses, err := ss.SendAddress(turnSender.addresses) + if err != nil { + log.Fatalf("failed to send address: %s", err) + } + log.Infof("received addresses: %v", clientAddresses.Address) + + createSenderDevices(turnSender, clientAddresses) + + log.Infof("waiting for tcpListeners to be ready") + time.Sleep(2 * time.Second) + + tcpConns := make([]net.Conn, 0, len(turnSender.devices)) + for i := range turnSender.devices { + addr := fmt.Sprintf("10.0.%d.2:9999", i) + log.Infof("dialing: %s", addr) + tcpConn, err := net.Dial("tcp", addr) + if err != nil { + log.Fatalf("failed to dial tcp: %s", err) + } + tcpConns = append(tcpConns, tcpConn) + } + + log.Infof("start test data transfer for %d pairs", p) + testDataLen := len(testData) + wg := sync.WaitGroup{} + wg.Add(len(tcpConns)) + for i, tcpConn := range tcpConns { + log.Infof("sending test data to device: %d", i) + go runTurnWriting(tcpConn, testData, testDataLen, &wg) + } + wg.Wait() + + for _, d := range turnSender.devices { + _ = d.Close() + } + + log.Infof("test finished with %d pairs", p) + } +} + +func TURNReaderMain() []testResult { + log.Infof("starting TURN receiver test") + si := NewSignalService() + go func() { + log.Infof("starting signal server") + err := si.Listen(signalListenAddress) + if err != nil { + log.Errorf("failed to listen: %s", err) + } + }() + + testResults := make([]testResult, 0, len(pairs)) + for range pairs { + addresses := <-si.AddressesChan + instanceNumber := len(addresses) + log.Infof("received addresses: %d", instanceNumber) + + turnReceiver := &TurnReceiver{} + err := createDevices(addresses, turnReceiver) + if err != nil { + log.Fatalf("%s", err) + } + + // send client addresses back via signal server + si.ClientAddressChan <- turnReceiver.clientAddresses + + durations := make(chan time.Duration, instanceNumber) + for _, device := range turnReceiver.devices { + go runTurnReading(device, durations) + } + + durationsList := make([]time.Duration, 0, instanceNumber) + for d := range durations { + durationsList = append(durationsList, d) + if len(durationsList) == instanceNumber { + close(durations) + } + } + + avgDuration, avgSpeed := avg(durationsList) + ts := testResult{ + numOfPairs: len(durationsList), + duration: avgDuration, + speed: avgSpeed, + } + testResults = append(testResults, ts) + + for _, d := range turnReceiver.devices { + _ = d.Close() + } + } + return testResults +} + +func main() { + var mode string + + _ = util.InitLog("debug", "console") + flag.StringVar(&mode, "mode", "sender", "sender or receiver mode") + flag.Parse() + + relaySrvAddress = os.Getenv("TEST_RELAY_SERVER") // rel://ip:port + turnSrvAddress = os.Getenv("TEST_TURN_SERVER") // ip:3478 + signalURL = os.Getenv("TEST_SIGNAL_URL") // http://receiver_ip:8081 + udpListener = os.Getenv("TEST_UDP_LISTENER") // IP:0 + + if mode == "receiver" { + relayResult := RelayReceiverMain() + turnResults := TURNReaderMain() + for i := 0; i < len(turnResults); i++ { + log.Infof("pairs: %d,\tRelay speed:\t%s,\trelay duration:\t%s", relayResult[i].numOfPairs, relayResult[i].Speed(), relayResult[i].duration) + log.Infof("pairs: %d,\tTURN speed:\t%s,\tturn duration:\t%s", turnResults[i].numOfPairs, turnResults[i].Speed(), turnResults[i].duration) + } + } else { + RelaySenderMain() + // grant time for receiver to start + time.Sleep(3 * time.Second) + TRUNSenderMain() + } +} diff --git a/relay/testec2/relay.go b/relay/testec2/relay.go new file mode 100644 index 000000000..93d084387 --- /dev/null +++ b/relay/testec2/relay.go @@ -0,0 +1,176 @@ +//go:build linux || darwin + +package main + +import ( + "context" + "fmt" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client" +) + +var ( + hmacTokenStore = &hmac.TokenStore{} +) + +func relayTransfer(serverConnURL string, testData []byte, peerPairs int) { + connsSender := prepareConnsSender(serverConnURL, peerPairs) + defer func() { + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + log.Errorf("failed to close connection: %s", err) + } + } + }() + + wg := sync.WaitGroup{} + wg.Add(len(connsSender)) + for _, conn := range connsSender { + go func(conn net.Conn) { + defer wg.Done() + runWriter(conn, testData) + }(conn) + } + wg.Wait() +} + +func runWriter(conn net.Conn, testData []byte) { + si := NewStartInidication(time.Now(), len(testData)) + _, err := conn.Write(si) + if err != nil { + log.Errorf("failed to write to channel: %s", err) + return + } + log.Infof("sent start indication") + + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr := conn.Write(testData[j:end]) + if writeErr != nil { + log.Errorf("failed to write to channel: %s", writeErr) + return + } + } +} + +func prepareConnsSender(serverConnURL string, peerPairs int) []net.Conn { + ctx := context.Background() + clientsSender := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + if err := c.Connect(); err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + log.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + } + return connsSender +} + +func relayReceive(serverConnURL string, peerPairs int) []time.Duration { + connsReceiver := prepareConnsReceiver(serverConnURL, peerPairs) + defer func() { + for i := 0; i < len(connsReceiver); i++ { + if err := connsReceiver[i].Close(); err != nil { + log.Errorf("failed to close connection: %s", err) + } + } + }() + + durations := make(chan time.Duration, len(connsReceiver)) + wg := sync.WaitGroup{} + for _, conn := range connsReceiver { + wg.Add(1) + go func(conn net.Conn) { + defer wg.Done() + duration := runReader(conn) + durations <- duration + }(conn) + } + wg.Wait() + + durationsList := make([]time.Duration, 0, len(connsReceiver)) + for d := range durations { + durationsList = append(durationsList, d) + if len(durationsList) == len(connsReceiver) { + close(durations) + } + } + + return durationsList +} + +func runReader(conn net.Conn) time.Duration { + buf := make([]byte, 8192) + + n, readErr := conn.Read(buf) + if readErr != nil { + log.Errorf("failed to read from channel: %s", readErr) + return 0 + } + + si := DecodeStartIndication(buf[:n]) + log.Infof("received start indication: %v", si) + + receivedSize, err := conn.Read(buf) + if err != nil { + log.Fatalf("failed to read from relay: %s", err) + } + now := time.Now() + + rcv := 0 + for receivedSize < si.TransferSize { + n, readErr = conn.Read(buf) + if readErr != nil { + log.Errorf("failed to read from channel: %s", readErr) + return 0 + } + + receivedSize += n + rcv += n + } + return time.Since(now) +} + +func prepareConnsReceiver(serverConnURL string, peerPairs int) []net.Conn { + clientsReceiver := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsReceiver); i++ { + c := client.NewClient(context.Background(), serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsReceiver); i++ { + conn, err := clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + log.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + return connsReceiver +} diff --git a/relay/testec2/signal.go b/relay/testec2/signal.go new file mode 100644 index 000000000..fe93a2fe2 --- /dev/null +++ b/relay/testec2/signal.go @@ -0,0 +1,91 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "encoding/json" + "net/http" + + log "github.com/sirupsen/logrus" +) + +type PeerAddr struct { + Address []string +} + +type ClientPeerAddr struct { + Address map[string]string +} + +type Signal struct { + AddressesChan chan []string + ClientAddressChan chan map[string]string +} + +func NewSignalService() *Signal { + return &Signal{ + AddressesChan: make(chan []string), + ClientAddressChan: make(chan map[string]string), + } +} + +func (rs *Signal) Listen(listenAddr string) error { + http.HandleFunc("/", rs.onNewAddresses) + return http.ListenAndServe(listenAddr, nil) +} + +func (rs *Signal) onNewAddresses(w http.ResponseWriter, r *http.Request) { + var msg PeerAddr + err := json.NewDecoder(r.Body).Decode(&msg) + if err != nil { + log.Errorf("Error decoding message: %v", err) + } + + log.Infof("received addresses: %d", len(msg.Address)) + rs.AddressesChan <- msg.Address + clientAddresses := <-rs.ClientAddressChan + + respMsg := ClientPeerAddr{ + Address: clientAddresses, + } + data, err := json.Marshal(respMsg) + if err != nil { + log.Errorf("Error marshalling message: %v", err) + return + } + + _, err = w.Write(data) + if err != nil { + log.Errorf("Error writing response: %v", err) + } +} + +type SignalClient struct { + SignalURL string +} + +func (ss SignalClient) SendAddress(addresses []string) (*ClientPeerAddr, error) { + msg := PeerAddr{ + Address: addresses, + } + data, err := json.Marshal(msg) + if err != nil { + return nil, err + } + + response, err := http.Post(ss.SignalURL, "application/json", bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + defer response.Body.Close() + + log.Debugf("wait for signal response") + var respPeerAddress ClientPeerAddr + err = json.NewDecoder(response.Body).Decode(&respPeerAddress) + if err != nil { + return nil, err + } + return &respPeerAddress, nil +} diff --git a/relay/testec2/start_msg.go b/relay/testec2/start_msg.go new file mode 100644 index 000000000..19b65380b --- /dev/null +++ b/relay/testec2/start_msg.go @@ -0,0 +1,39 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "encoding/gob" + "time" + + log "github.com/sirupsen/logrus" +) + +type StartIndication struct { + Started time.Time + TransferSize int +} + +func NewStartInidication(started time.Time, transferSize int) []byte { + si := StartIndication{ + Started: started, + TransferSize: transferSize, + } + + var data bytes.Buffer + err := gob.NewEncoder(&data).Encode(si) + if err != nil { + log.Fatal("encode error:", err) + } + return data.Bytes() +} + +func DecodeStartIndication(data []byte) StartIndication { + var si StartIndication + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&si) + if err != nil { + log.Fatal("decode error:", err) + } + return si +} diff --git a/relay/testec2/tun/proxy.go b/relay/testec2/tun/proxy.go new file mode 100644 index 000000000..7d84bece7 --- /dev/null +++ b/relay/testec2/tun/proxy.go @@ -0,0 +1,72 @@ +//go:build linux || darwin + +package tun + +import ( + "net" + "sync/atomic" + + log "github.com/sirupsen/logrus" +) + +type Proxy struct { + Device *Device + PConn net.PacketConn + DstAddr net.Addr + shutdownFlag atomic.Bool +} + +func (p *Proxy) Start() { + go p.readFromDevice() + go p.readFromConn() +} + +func (p *Proxy) Close() { + p.shutdownFlag.Store(true) +} + +func (p *Proxy) readFromDevice() { + buf := make([]byte, 1500) + for { + n, err := p.Device.Read(buf) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to read from device: %s", err) + return + } + + _, err = p.PConn.WriteTo(buf[:n], p.DstAddr) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to write to conn: %s", err) + return + } + } +} + +func (p *Proxy) readFromConn() { + buf := make([]byte, 1500) + for { + n, _, err := p.PConn.ReadFrom(buf) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to read from conn: %s", err) + return + } + + _, err = p.Device.Write(buf[:n]) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to write to device: %s", err) + return + } + } +} diff --git a/relay/testec2/tun/tun.go b/relay/testec2/tun/tun.go new file mode 100644 index 000000000..5580785ce --- /dev/null +++ b/relay/testec2/tun/tun.go @@ -0,0 +1,110 @@ +//go:build linux || darwin + +package tun + +import ( + "net" + + log "github.com/sirupsen/logrus" + "github.com/songgao/water" + "github.com/vishvananda/netlink" +) + +type Device struct { + Name string + IP string + PConn net.PacketConn + DstAddr net.Addr + + iFace *water.Interface + proxy *Proxy +} + +func (d *Device) Up() error { + cfg := water.Config{ + DeviceType: water.TUN, + PlatformSpecificParams: water.PlatformSpecificParams{ + Name: d.Name, + }, + } + iFace, err := water.New(cfg) + if err != nil { + return err + } + d.iFace = iFace + + err = d.assignIP() + if err != nil { + return err + } + + err = d.bringUp() + if err != nil { + return err + } + + d.proxy = &Proxy{ + Device: d, + PConn: d.PConn, + DstAddr: d.DstAddr, + } + d.proxy.Start() + return nil +} + +func (d *Device) Close() error { + if d.proxy != nil { + d.proxy.Close() + } + if d.iFace != nil { + return d.iFace.Close() + } + return nil +} + +func (d *Device) Read(b []byte) (int, error) { + return d.iFace.Read(b) +} + +func (d *Device) Write(b []byte) (int, error) { + return d.iFace.Write(b) +} + +func (d *Device) assignIP() error { + iface, err := netlink.LinkByName(d.Name) + if err != nil { + log.Errorf("failed to get TUN device: %v", err) + return err + } + + ip := net.IPNet{ + IP: net.ParseIP(d.IP), + Mask: net.CIDRMask(24, 32), + } + + addr := &netlink.Addr{ + IPNet: &ip, + } + err = netlink.AddrAdd(iface, addr) + if err != nil { + log.Errorf("failed to add IP address: %v", err) + return err + } + return nil +} + +func (d *Device) bringUp() error { + iface, err := netlink.LinkByName(d.Name) + if err != nil { + log.Errorf("failed to get device: %v", err) + return err + } + + // Bring the interface up + err = netlink.LinkSetUp(iface) + if err != nil { + log.Errorf("failed to set device up: %v", err) + return err + } + return nil +} diff --git a/relay/testec2/turn.go b/relay/testec2/turn.go new file mode 100644 index 000000000..8beb40423 --- /dev/null +++ b/relay/testec2/turn.go @@ -0,0 +1,181 @@ +//go:build linux || darwin + +package main + +import ( + "fmt" + "net" + "sync" + "time" + + "github.com/netbirdio/netbird/relay/testec2/tun" + + log "github.com/sirupsen/logrus" +) + +type TurnReceiver struct { + conns []*net.UDPConn + clientAddresses map[string]string + devices []*tun.Device +} + +type TurnSender struct { + turnConns map[string]*TurnConn + addresses []string + devices []*tun.Device +} + +func runTurnWriting(tcpConn net.Conn, testData []byte, testDataLen int, wg *sync.WaitGroup) { + defer wg.Done() + defer tcpConn.Close() + + log.Infof("start to sending test data: %s", tcpConn.RemoteAddr()) + + si := NewStartInidication(time.Now(), testDataLen) + _, err := tcpConn.Write(si) + if err != nil { + log.Errorf("failed to write to tcp: %s", err) + return + } + + pieceSize := 1024 + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr := tcpConn.Write(testData[j:end]) + if writeErr != nil { + log.Errorf("failed to write to tcp conn: %s", writeErr) + return + } + } + + // grant time to flush out packages + time.Sleep(3 * time.Second) +} + +func createSenderDevices(sender *TurnSender, clientAddresses *ClientPeerAddr) { + var i int + devices := make([]*tun.Device, 0, len(clientAddresses.Address)) + for k, v := range clientAddresses.Address { + tc, ok := sender.turnConns[k] + if !ok { + log.Fatalf("failed to find turn conn: %s", k) + } + + addr, err := net.ResolveUDPAddr("udp", v) + if err != nil { + log.Fatalf("failed to resolve udp address: %s", err) + } + device := &tun.Device{ + Name: fmt.Sprintf("mtun-sender-%d", i), + IP: fmt.Sprintf("10.0.%d.1", i), + PConn: tc.relayConn, + DstAddr: addr, + } + + err = device.Up() + if err != nil { + log.Fatalf("failed to bring up device: %s", err) + } + + devices = append(devices, device) + i++ + } + sender.devices = devices +} + +func createTurnConns(p int, sender *TurnSender) { + turnConns := make(map[string]*TurnConn) + addresses := make([]string, 0, len(pairs)) + for i := 0; i < p; i++ { + tc := AllocateTurnClient(turnSrvAddress) + log.Infof("allocated turn client: %s", tc.Address().String()) + turnConns[tc.Address().String()] = tc + addresses = append(addresses, tc.Address().String()) + } + + sender.turnConns = turnConns + sender.addresses = addresses +} + +func runTurnReading(d *tun.Device, durations chan time.Duration) { + tcpListener, err := net.Listen("tcp", d.IP+":9999") + if err != nil { + log.Fatalf("failed to listen on tcp: %s", err) + } + log := log.WithField("device", tcpListener.Addr()) + + tcpConn, err := tcpListener.Accept() + if err != nil { + _ = tcpListener.Close() + log.Fatalf("failed to accept connection: %s", err) + } + log.Infof("remote peer connected") + + buf := make([]byte, 103) + n, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + + si := DecodeStartIndication(buf[:n]) + log.Infof("received start indication: %v, %d", si, n) + + buf = make([]byte, 8192) + i, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + now := time.Now() + for i < si.TransferSize { + n, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + i += n + } + durations <- time.Since(now) +} + +func createDevices(addresses []string, receiver *TurnReceiver) error { + receiver.conns = make([]*net.UDPConn, 0, len(addresses)) + receiver.clientAddresses = make(map[string]string, len(addresses)) + receiver.devices = make([]*tun.Device, 0, len(addresses)) + for i, addr := range addresses { + localAddr, err := net.ResolveUDPAddr("udp", udpListener) + if err != nil { + return fmt.Errorf("failed to resolve UDP address: %s", err) + } + + conn, err := net.ListenUDP("udp", localAddr) + if err != nil { + return fmt.Errorf("failed to create UDP connection: %s", err) + } + + receiver.conns = append(receiver.conns, conn) + receiver.clientAddresses[addr] = conn.LocalAddr().String() + + dstAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return fmt.Errorf("failed to resolve address: %s", err) + } + + device := &tun.Device{ + Name: fmt.Sprintf("mtun-%d", i), + IP: fmt.Sprintf("10.0.%d.2", i), + PConn: conn, + DstAddr: dstAddr, + } + + if err = device.Up(); err != nil { + return fmt.Errorf("failed to bring up device: %s, %s", device.Name, err) + } + receiver.devices = append(receiver.devices, device) + } + return nil +} diff --git a/relay/testec2/turn_allocator.go b/relay/testec2/turn_allocator.go new file mode 100644 index 000000000..fd86208df --- /dev/null +++ b/relay/testec2/turn_allocator.go @@ -0,0 +1,83 @@ +//go:build linux || darwin + +package main + +import ( + "fmt" + "net" + + "github.com/pion/logging" + "github.com/pion/turn/v3" + log "github.com/sirupsen/logrus" +) + +type TurnConn struct { + conn net.Conn + turnClient *turn.Client + relayConn net.PacketConn +} + +func (tc *TurnConn) Address() net.Addr { + return tc.relayConn.LocalAddr() +} + +func (tc *TurnConn) Close() { + _ = tc.relayConn.Close() + tc.turnClient.Close() + _ = tc.conn.Close() +} + +func AllocateTurnClient(serverAddr string) *TurnConn { + conn, err := net.Dial("tcp", serverAddr) + if err != nil { + log.Fatal(err) + } + + turnClient, err := getTurnClient(serverAddr, conn) + if err != nil { + log.Fatal(err) + } + + relayConn, err := turnClient.Allocate() + if err != nil { + log.Fatal(err) + } + + return &TurnConn{ + conn: conn, + turnClient: turnClient, + relayConn: relayConn, + } +} + +func getTurnClient(address string, conn net.Conn) (*turn.Client, error) { + // Dial TURN Server + addrStr := fmt.Sprintf("%s:%d", address, 443) + + fac := logging.NewDefaultLoggerFactory() + //fac.DefaultLogLevel = logging.LogLevelTrace + + // Start a new TURN Client and wrap our net.Conn in a STUNConn + // This allows us to simulate datagram based communication over a net.Conn + cfg := &turn.ClientConfig{ + TURNServerAddr: address, + Conn: turn.NewSTUNConn(conn), + Username: "test", + Password: "test", + LoggerFactory: fac, + } + + client, err := turn.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err) + } + + // Start listening on the conn provided. + err = client.Listen() + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err) + } + + return client, nil +} diff --git a/signal/client/client.go b/signal/client/client.go index 9d99b3677..ced3fb7d0 100644 --- a/signal/client/client.go +++ b/signal/client/client.go @@ -51,11 +51,10 @@ func UnMarshalCredential(msg *proto.Message) (*Credential, error) { } // MarshalCredential marshal a Credential instance and returns a Message object -func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type, - rosenpassPubKey []byte, rosenpassAddr string) (*proto.Message, error) { +func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey string, credential *Credential, t proto.Body_Type, rosenpassPubKey []byte, rosenpassAddr string, relaySrvAddress string) (*proto.Message, error) { return &proto.Message{ Key: myKey.PublicKey().String(), - RemoteKey: remoteKey.String(), + RemoteKey: remoteKey, Body: &proto.Body{ Type: t, Payload: fmt.Sprintf("%s:%s", credential.UFrag, credential.Pwd), @@ -65,6 +64,7 @@ func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, cre RosenpassPubKey: rosenpassPubKey, RosenpassServerAddr: rosenpassAddr, }, + RelayServerAddress: relaySrvAddress, }, }, nil } diff --git a/signal/proto/signalexchange.pb.go b/signal/proto/signalexchange.pb.go index 782c45da1..30f704c6f 100644 --- a/signal/proto/signalexchange.pb.go +++ b/signal/proto/signalexchange.pb.go @@ -1,15 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.12 // source: signalexchange.proto package proto import ( - _ "github.com/golang/protobuf/protoc-gen-go/descriptor" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/descriptorpb" reflect "reflect" sync "sync" ) @@ -225,6 +225,8 @@ type Body struct { FeaturesSupported []uint32 `protobuf:"varint,6,rep,packed,name=featuresSupported,proto3" json:"featuresSupported,omitempty"` // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig *RosenpassConfig `protobuf:"bytes,7,opt,name=rosenpassConfig,proto3" json:"rosenpassConfig,omitempty"` + // relayServerAddress is an IP:port of the relay server + RelayServerAddress string `protobuf:"bytes,8,opt,name=relayServerAddress,proto3" json:"relayServerAddress,omitempty"` } func (x *Body) Reset() { @@ -308,6 +310,13 @@ func (x *Body) GetRosenpassConfig() *RosenpassConfig { return nil } +func (x *Body) GetRelayServerAddress() string { + if x != nil { + return x.RelayServerAddress + } + return "" +} + // Mode indicates a connection mode type Mode struct { state protoimpl.MessageState @@ -431,7 +440,7 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x52, 0x09, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xf6, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xa6, 0x03, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, @@ -451,7 +460,10 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x52, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x12, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x22, 0x2e, diff --git a/signal/proto/signalexchange.proto b/signal/proto/signalexchange.proto index a8c4c309c..4431edd7c 100644 --- a/signal/proto/signalexchange.proto +++ b/signal/proto/signalexchange.proto @@ -60,6 +60,9 @@ message Body { // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig rosenpassConfig = 7; + + // relayServerAddress is url of the relay server + string relayServerAddress = 8; } // Mode indicates a connection mode