From 57a89cf0cc2c2c54ff65af6a269e2f6181974a71 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 17 May 2024 17:43:28 +0200 Subject: [PATCH 001/199] Add initial relay code --- go.mod | 1 + go.sum | 2 + relay/client/client.go | 224 +++++++++++++++++++ relay/client/conn.go | 59 +++++ relay/client/dialer/tcp/tcp.go | 7 + relay/client/dialer/ws/client_conn.go | 56 +++++ relay/client/dialer/ws/ws.go | 18 ++ relay/messages/message.go | 134 +++++++++++ relay/server/listener/listener.go | 8 + relay/server/listener/tcp/listener.go | 80 +++++++ relay/server/listener/ws/listener.go | 82 +++++++ relay/server/listener/ws/server_conn.go | 52 +++++ relay/server/peer.go | 113 ++++++++++ relay/server/server.go | 149 +++++++++++++ relay/server/store.go | 48 ++++ relay/test/client_test.go | 285 ++++++++++++++++++++++++ 16 files changed, 1318 insertions(+) create mode 100644 relay/client/client.go create mode 100644 relay/client/conn.go create mode 100644 relay/client/dialer/tcp/tcp.go create mode 100644 relay/client/dialer/ws/client_conn.go create mode 100644 relay/client/dialer/ws/ws.go create mode 100644 relay/messages/message.go create mode 100644 relay/server/listener/listener.go create mode 100644 relay/server/listener/tcp/listener.go create mode 100644 relay/server/listener/ws/listener.go create mode 100644 relay/server/listener/ws/server_conn.go create mode 100644 relay/server/peer.go create mode 100644 relay/server/server.go create mode 100644 relay/server/store.go create mode 100644 relay/test/client_test.go diff --git a/go.mod b/go.mod index f75ddcb6f..cf9308a20 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/google/martian/v3 v3.0.0 github.com/google/nftables v0.0.0-20220808154552-2eca00135732 github.com/gopacket/gopacket v1.1.1 + github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 diff --git a/go.sum b/go.sum index 49314e729..1c5dd13c1 100644 --- a/go.sum +++ b/go.sum @@ -317,6 +317,8 @@ github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0 h1:fWY+zXdWhvWnd github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357 h1:Fkzd8ktnpOR9h47SXHe2AYPwelXLH2GjGsjlAloiWfo= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= diff --git a/relay/client/client.go b/relay/client/client.go new file mode 100644 index 000000000..110c75ec2 --- /dev/null +++ b/relay/client/client.go @@ -0,0 +1,224 @@ +package client + +import ( + "context" + "fmt" + "io" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/client/dialer/ws" + "github.com/netbirdio/netbird/relay/messages" +) + +const ( + bufferSize = 65535 // optimise the buffer size +) + +type connContainer struct { + conn *Conn + messages chan []byte +} + +// Client Todo: +// - handle automatic reconnection +type Client struct { + serverAddress string + peerID string + + channelsPending map[string]chan net.Conn // todo: protect map with mutex + channels map[uint16]*connContainer + msgPool sync.Pool + + relayConn net.Conn + relayConnState bool +} + +func NewClient(serverAddress, peerID string) *Client { + return &Client{ + serverAddress: serverAddress, + peerID: peerID, + channelsPending: make(map[string]chan net.Conn), + channels: make(map[uint16]*connContainer), + msgPool: sync.Pool{ + New: func() any { + return make([]byte, bufferSize) + }, + }, + } +} + +func (c *Client) Connect() error { + conn, err := ws.Dial(c.serverAddress) + if err != nil { + return err + } + c.relayConn = conn + + err = c.handShake() + if err != nil { + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection: %s", cErr) + } + c.relayConn = nil + return err + } + + c.relayConnState = true + go c.readLoop() + return nil +} + +func (c *Client) BindChannel(remotePeerID string) (net.Conn, error) { + if c.relayConn == nil { + return nil, fmt.Errorf("client not connected") + } + + bindSuccessChan := make(chan net.Conn, 1) + c.channelsPending[remotePeerID] = bindSuccessChan + msg := messages.MarshalBindNewChannelMsg(remotePeerID) + _, err := c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to write out bind message: %s", err) + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + select { + case <-ctx.Done(): + return nil, fmt.Errorf("bind timeout") + case c := <-bindSuccessChan: + return c, nil + } +} + +func (c *Client) Close() error { + for _, conn := range c.channels { + close(conn.messages) + } + c.channels = make(map[uint16]*connContainer) + c.relayConnState = false + err := c.relayConn.Close() + return err +} + +func (c *Client) handShake() error { + msg, err := messages.MarshalHelloMsg(c.peerID) + if err != nil { + return err + } + _, err = c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to send hello message: %s", err) + return err + } + return nil +} + +func (c *Client) readLoop() { + log := log.WithField("client_id", c.peerID) + var errExit error + var n int + for { + buf := c.msgPool.Get().([]byte) + n, errExit = c.relayConn.Read(buf) + if errExit != nil { + break + } + + msgType, err := messages.DetermineServerMsgType(buf[:n]) + if err != nil { + log.Errorf("failed to determine message type: %s", err) + c.msgPool.Put(buf) + continue + } + + switch msgType { + case messages.MsgTypeBindResponse: + channelId, peerId, err := messages.UnmarshalBindResponseMsg(buf[:n]) + if err != nil { + log.Errorf("failed to parse bind response message: %v", err) + } else { + c.handleBindResponse(channelId, peerId) + } + c.msgPool.Put(buf) + continue + case messages.MsgTypeTransport: + channelId, payload, err := messages.UnmarshalTransportMsg(buf[:n]) + if err != nil { + log.Errorf("failed to parse transport message: %v", err) + c.msgPool.Put(buf) + continue + } + c.handleTransport(channelId, payload) + } + } + + if c.relayConnState { + log.Errorf("failed to read message from relay server: %s", errExit) + _ = c.relayConn.Close() + } +} + +func (c *Client) handleBindResponse(channelId uint16, peerId string) { + bindSuccessChan, ok := c.channelsPending[peerId] + if !ok { + log.Errorf("unexpected bind response from: %s", peerId) + return + } + delete(c.channelsPending, peerId) + + messageBuffer := make(chan []byte, 10) + conn := NewConn(c, channelId, c.generateConnReaderFN(messageBuffer)) + + c.channels[channelId] = &connContainer{ + conn, + messageBuffer, + } + log.Debugf("bind success for '%s': %d", peerId, channelId) + + bindSuccessChan <- conn +} + +func (c *Client) handleTransport(channelId uint16, payload []byte) { + container, ok := c.channels[channelId] + if !ok { + log.Errorf("c.channels: %v", c.peerID) + log.Errorf("unexpected transport message for channel: %d", channelId) + return + } + + select { + case container.messages <- payload: + default: + log.Errorf("dropping message for channel: %d", channelId) + } +} + +func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { + msg := messages.MarshalTransportMsg(channelID, payload) + n, err := c.relayConn.Write(msg) + if err != nil { + log.Errorf("failed to write transport message: %s", err) + } + return n, err +} + +func (c *Client) generateConnReaderFN(messageBufferChan chan []byte) func(b []byte) (n int, err error) { + return func(b []byte) (n int, err error) { + select { + case msg, ok := <-messageBufferChan: + if !ok { + return 0, io.EOF + } + n = copy(b, msg) + c.msgPool.Put(msg) + } + return n, nil + } +} diff --git a/relay/client/conn.go b/relay/client/conn.go new file mode 100644 index 000000000..a5635e020 --- /dev/null +++ b/relay/client/conn.go @@ -0,0 +1,59 @@ +package client + +import ( + "net" + "time" +) + +type Conn struct { + client *Client + channelID uint16 + readerFn func(b []byte) (n int, err error) +} + +func NewConn(client *Client, channelID uint16, readerFn func(b []byte) (n int, err error)) *Conn { + c := &Conn{ + client: client, + channelID: channelID, + readerFn: readerFn, + } + + return c +} + +func (c *Conn) Write(p []byte) (n int, err error) { + return c.client.writeTo(c.channelID, p) +} + +func (c *Conn) Read(b []byte) (n int, err error) { + return c.readerFn(b) +} + +func (c *Conn) Close() error { + return nil +} + +func (c *Conn) LocalAddr() net.Addr { + //TODO implement me + panic("implement me") +} + +func (c *Conn) RemoteAddr() net.Addr { + //TODO implement me + panic("implement me") +} + +func (c *Conn) SetDeadline(t time.Time) error { + //TODO implement me + panic("implement me") +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + //TODO implement me + panic("implement me") +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + //TODO implement me + panic("implement me") +} diff --git a/relay/client/dialer/tcp/tcp.go b/relay/client/dialer/tcp/tcp.go new file mode 100644 index 000000000..47b0a31d5 --- /dev/null +++ b/relay/client/dialer/tcp/tcp.go @@ -0,0 +1,7 @@ +package tcp + +import "net" + +func Dial(address string) (net.Conn, error) { + return net.Dial("tcp", address) +} diff --git a/relay/client/dialer/ws/client_conn.go b/relay/client/dialer/ws/client_conn.go new file mode 100644 index 000000000..3298bd228 --- /dev/null +++ b/relay/client/dialer/ws/client_conn.go @@ -0,0 +1,56 @@ +package ws + +import ( + "fmt" + "net" + "time" + + "github.com/gorilla/websocket" +) + +type Conn struct { + *websocket.Conn +} + +func NewConn(wsConn *websocket.Conn) net.Conn { + return &Conn{ + wsConn, + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, r, err := c.NextReader() + if err != nil { + return 0, err + } + + if t != websocket.BinaryMessage { + return 0, fmt.Errorf("unexpected message type") + } + + return r.Read(b) +} + +func (c *Conn) Write(b []byte) (int, error) { + err := c.WriteMessage(websocket.BinaryMessage, b) + return len(b), err +} + +func (c *Conn) SetDeadline(t time.Time) error { + errR := c.SetReadDeadline(t) + errW := c.SetWriteDeadline(t) + + if errR != nil { + return errR + } + + if errW != nil { + return errW + } + return nil +} + +func (c *Conn) Close() error { + _ = c.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second*5)) + return c.Conn.Close() +} diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go new file mode 100644 index 000000000..3ab3d38a8 --- /dev/null +++ b/relay/client/dialer/ws/ws.go @@ -0,0 +1,18 @@ +package ws + +import ( + "fmt" + "net" + + "github.com/gorilla/websocket" +) + +func Dial(address string) (net.Conn, error) { + addr := fmt.Sprintf("ws://" + address) + wsConn, _, err := websocket.DefaultDialer.Dial(addr, nil) + if err != nil { + return nil, err + } + conn := NewConn(wsConn) + return conn, nil +} diff --git a/relay/messages/message.go b/relay/messages/message.go new file mode 100644 index 000000000..ee4947afe --- /dev/null +++ b/relay/messages/message.go @@ -0,0 +1,134 @@ +package messages + +import ( + "fmt" +) + +const ( + MsgTypeHello MsgType = 0 + MsgTypeBindNewChannel MsgType = 1 + MsgTypeBindResponse MsgType = 2 + MsgTypeTransport MsgType = 3 +) + +var ( + ErrInvalidMessageLength = fmt.Errorf("invalid message length") +) + +type MsgType byte + +func DetermineClientMsgType(msg []byte) (MsgType, error) { + // todo: validate magic byte + msgType := MsgType(msg[0]) + switch msgType { + case MsgTypeHello: + return msgType, nil + case MsgTypeBindNewChannel: + return msgType, nil + case MsgTypeTransport: + return msgType, nil + default: + return 0, fmt.Errorf("invalid msg type: %s", msg) + } +} + +func DetermineServerMsgType(msg []byte) (MsgType, error) { + // todo: validate magic byte + msgType := MsgType(msg[0]) + switch msgType { + case MsgTypeBindResponse: + return msgType, nil + case MsgTypeTransport: + return msgType, nil + default: + return 0, fmt.Errorf("invalid msg type: %s", msg) + } +} + +// MarshalHelloMsg initial hello message +func MarshalHelloMsg(peerID string) ([]byte, error) { + if len(peerID) == 0 { + return nil, fmt.Errorf("invalid peer id") + } + msg := make([]byte, 1, 1+len(peerID)) + msg[0] = byte(MsgTypeHello) + msg = append(msg, []byte(peerID)...) + return msg, nil +} + +func UnmarshalHelloMsg(msg []byte) (string, error) { + if len(msg) < 2 { + return "", fmt.Errorf("invalid 'hello' messge") + } + return string(msg[1:]), nil +} + +// Bind new channel + +func MarshalBindNewChannelMsg(destinationPeerId string) []byte { + msg := make([]byte, 1, 1+len(destinationPeerId)) + msg[0] = byte(MsgTypeBindNewChannel) + msg = append(msg, []byte(destinationPeerId)...) + return msg +} + +func UnmarshalBindNewChannel(msg []byte) (string, error) { + if len(msg) < 2 { + return "", fmt.Errorf("invalid 'bind new channel' messge") + } + return string(msg[1:]), nil +} + +// Bind response + +func MarshalBindResponseMsg(channelId uint16, id string) []byte { + data := []byte(id) + msg := make([]byte, 3, 3+len(data)) + msg[0] = byte(MsgTypeBindResponse) + msg[1], msg[2] = uint8(channelId>>8), uint8(channelId&0xff) + msg = append(msg, data...) + return msg +} + +func UnmarshalBindResponseMsg(buf []byte) (uint16, string, error) { + if len(buf) < 3 { + return 0, "", ErrInvalidMessageLength + } + channelId := uint16(buf[1])<<8 | uint16(buf[2]) + peerID := string(buf[3:]) + return channelId, peerID, nil +} + +// Transport message + +func MarshalTransportMsg(channelId uint16, payload []byte) []byte { + msg := make([]byte, 3, 3+len(payload)) + msg[0] = byte(MsgTypeTransport) + msg[1], msg[2] = uint8(channelId>>8), uint8(channelId&0xff) + msg = append(msg, payload...) + return msg +} + +func UnmarshalTransportMsg(buf []byte) (uint16, []byte, error) { + if len(buf) < 3 { + return 0, nil, ErrInvalidMessageLength + } + channelId := uint16(buf[1])<<8 | uint16(buf[2]) + return channelId, buf[3:], nil +} + +func UnmarshalTransportID(buf []byte) (uint16, error) { + if len(buf) < 3 { + return 0, ErrInvalidMessageLength + } + channelId := uint16(buf[1])<<8 | uint16(buf[2]) + return channelId, nil +} + +func UpdateTransportMsg(msg []byte, channelId uint16) error { + if len(msg) < 3 { + return ErrInvalidMessageLength + } + msg[1], msg[2] = uint8(channelId>>8), uint8(channelId&0xff) + return nil +} diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go new file mode 100644 index 000000000..66e6d357e --- /dev/null +++ b/relay/server/listener/listener.go @@ -0,0 +1,8 @@ +package listener + +import "net" + +type Listener interface { + Listen(func(conn net.Conn)) error + Close() error +} diff --git a/relay/server/listener/tcp/listener.go b/relay/server/listener/tcp/listener.go new file mode 100644 index 000000000..bd96e3b30 --- /dev/null +++ b/relay/server/listener/tcp/listener.go @@ -0,0 +1,80 @@ +package tcp + +import ( + "net" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/server/listener" +) + +// Listener +// Is it just demo code. It does not work in real life environment because the TCP is a streaming protocol, adn +// it does not handle framing. +type Listener struct { + address string + + onAcceptFn func(conn net.Conn) + wg sync.WaitGroup + quit chan struct{} + listener net.Listener + lock sync.Mutex +} + +func NewListener(address string) listener.Listener { + return &Listener{ + address: address, + } +} + +func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { + l.lock.Lock() + + l.onAcceptFn = onAcceptFn + l.quit = make(chan struct{}) + + li, err := net.Listen("tcp", l.address) + if err != nil { + log.Errorf("failed to listen on address: %s, %s", l.address, err) + l.lock.Unlock() + return err + } + log.Debugf("TCP server is listening on address: %s", l.address) + l.listener = li + l.wg.Add(1) + go l.acceptLoop() + + l.lock.Unlock() + <-l.quit + return nil +} + +// Close todo: prevent multiple call (do not close two times the channel) +func (l *Listener) Close() error { + l.lock.Lock() + defer l.lock.Unlock() + + close(l.quit) + err := l.listener.Close() + l.wg.Wait() + return err +} + +func (l *Listener) acceptLoop() { + defer l.wg.Done() + + for { + conn, err := l.listener.Accept() + if err != nil { + select { + case <-l.quit: + return + default: + log.Errorf("failed to accept connection: %s", err) + continue + } + } + go l.onAcceptFn(conn) + } +} diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go new file mode 100644 index 000000000..cee57e348 --- /dev/null +++ b/relay/server/listener/ws/listener.go @@ -0,0 +1,82 @@ +package ws + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "sync" + "time" + + "github.com/gorilla/websocket" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/server/listener" +) + +var ( + upgrader = websocket.Upgrader{} // use default options +) + +type Listener struct { + address string + + wg sync.WaitGroup + server *http.Server + acceptFn func(conn net.Conn) +} + +func NewListener(address string) listener.Listener { + return &Listener{ + address: address, + } +} + +// Listen todo: prevent multiple call +func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { + l.acceptFn = acceptFn + http.HandleFunc("/", l.onAccept) + + l.server = &http.Server{ + Addr: l.address, + } + + log.Debugf("WS server is listening on address: %s", l.address) + err := l.server.ListenAndServe() + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func (l *Listener) Close() error { + if l.server == nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + log.Debugf("closing WS server") + if err := l.server.Shutdown(ctx); err != nil { + return fmt.Errorf("server shutdown failed: %v", err) + } + + l.wg.Wait() + return nil +} + +func (l *Listener) onAccept(writer http.ResponseWriter, request *http.Request) { + l.wg.Add(1) + defer l.wg.Done() + + wsConn, err := upgrader.Upgrade(writer, request, nil) + if err != nil { + log.Errorf("failed to upgrade connection: %s", err) + return + } + conn := NewConn(wsConn) + l.acceptFn(conn) + return +} diff --git a/relay/server/listener/ws/server_conn.go b/relay/server/listener/ws/server_conn.go new file mode 100644 index 000000000..5c7be69fd --- /dev/null +++ b/relay/server/listener/ws/server_conn.go @@ -0,0 +1,52 @@ +package ws + +import ( + "fmt" + "time" + + "github.com/gorilla/websocket" + log "github.com/sirupsen/logrus" +) + +type Conn struct { + *websocket.Conn +} + +func NewConn(wsConn *websocket.Conn) *Conn { + return &Conn{ + wsConn, + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, r, err := c.NextReader() + if err != nil { + return 0, err + } + + if t != websocket.BinaryMessage { + log.Errorf("unexpected message type: %d", t) + return 0, fmt.Errorf("unexpected message type") + } + + return r.Read(b) +} + +func (c *Conn) Write(b []byte) (int, error) { + err := c.WriteMessage(websocket.BinaryMessage, b) + return len(b), err +} + +func (c *Conn) SetDeadline(t time.Time) error { + errR := c.SetReadDeadline(t) + errW := c.SetWriteDeadline(t) + + if errR != nil { + return errR + } + + if errW != nil { + return errW + } + return nil +} diff --git a/relay/server/peer.go b/relay/server/peer.go new file mode 100644 index 000000000..2e40cbb12 --- /dev/null +++ b/relay/server/peer.go @@ -0,0 +1,113 @@ +package server + +import ( + "fmt" + "net" + "sync" + + log "github.com/sirupsen/logrus" +) + +type Participant struct { + ChannelID uint16 + ChannelIDForeign uint16 + ConnForeign net.Conn + Peer *Peer +} + +type Peer struct { + Log *log.Entry + id string + conn net.Conn + + pendingParticipantByChannelID map[uint16]*Participant + participantByID map[uint16]*Participant // used for package transfer + participantByPeerID map[string]*Participant // used for channel linking + + lastId uint16 + lastIdLock sync.Mutex +} + +func NewPeer(id string, conn net.Conn) *Peer { + return &Peer{ + Log: log.WithField("peer_id", id), + id: id, + conn: conn, + pendingParticipantByChannelID: make(map[uint16]*Participant), + participantByID: make(map[uint16]*Participant), + participantByPeerID: make(map[string]*Participant), + } +} +func (p *Peer) BindChannel(remotePeerId string) uint16 { + ch, ok := p.participantByPeerID[remotePeerId] + if ok { + return ch.ChannelID + } + + channelID := p.newChannelID() + channel := &Participant{ + ChannelID: channelID, + } + p.pendingParticipantByChannelID[channelID] = channel + p.participantByPeerID[remotePeerId] = channel + return channelID +} + +func (p *Peer) UnBindChannel(remotePeerId string) { + pa, ok := p.participantByPeerID[remotePeerId] + if !ok { + return + } + + p.Log.Debugf("unbind channel with '%s': %d", remotePeerId, pa.ChannelID) + p.pendingParticipantByChannelID[pa.ChannelID] = pa + delete(p.participantByID, pa.ChannelID) +} + +func (p *Peer) AddParticipant(peer *Peer, remoteChannelID uint16) (uint16, bool) { + participant, ok := p.participantByPeerID[peer.ID()] + if !ok { + return 0, false + } + participant.ChannelIDForeign = remoteChannelID + participant.ConnForeign = peer.conn + participant.Peer = peer + + delete(p.pendingParticipantByChannelID, participant.ChannelID) + p.participantByID[participant.ChannelID] = participant + return participant.ChannelID, true +} + +func (p *Peer) DeleteParticipants() { + for _, participant := range p.participantByID { + participant.Peer.UnBindChannel(p.id) + } +} + +func (p *Peer) ConnByChannelID(dstID uint16) (uint16, net.Conn, error) { + ch, ok := p.participantByID[dstID] + if !ok { + return 0, nil, fmt.Errorf("destination channel not found") + } + + return ch.ChannelIDForeign, ch.ConnForeign, nil +} + +func (p *Peer) ID() string { + return p.id +} + +func (p *Peer) newChannelID() uint16 { + p.lastIdLock.Lock() + defer p.lastIdLock.Unlock() + for { + p.lastId++ + if _, ok := p.pendingParticipantByChannelID[p.lastId]; ok { + continue + } + if _, ok := p.participantByID[p.lastId]; ok { + continue + } + return p.lastId + } +} diff --git a/relay/server/server.go b/relay/server/server.go new file mode 100644 index 000000000..7c465132f --- /dev/null +++ b/relay/server/server.go @@ -0,0 +1,149 @@ +package server + +import ( + "fmt" + "io" + "net" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/server/listener" + "github.com/netbirdio/netbird/relay/server/listener/ws" +) + +// Server +// todo: +// authentication: provide JWT token via RPC call. The MGM server can forward the token to the agents. +// connection timeout handling +// implement HA (High Availability) mode +type Server struct { + store *Store + + listener listener.Listener +} + +func NewServer() *Server { + return &Server{ + store: NewStore(), + } +} + +func (r *Server) Listen(address string) error { + r.listener = ws.NewListener(address) + return r.listener.Listen(r.accept) +} + +func (r *Server) Close() error { + if r.listener == nil { + return nil + } + return r.listener.Close() +} + +func (r *Server) accept(conn net.Conn) { + peer, err := handShake(conn) + if err != nil { + log.Errorf("failed to handshake wiht %s: %s", conn.RemoteAddr(), err) + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr) + } + return + } + peer.Log.Debugf("on new connection: %s", conn.RemoteAddr()) + + r.store.AddPeer(peer) + defer func() { + peer.Log.Debugf("teardown connection") + r.store.DeletePeer(peer) + }() + + buf := make([]byte, 65535) // todo: optimize buffer size + for { + n, err := conn.Read(buf) + if err != nil { + if err != io.EOF { + peer.Log.Errorf("failed to read message: %s", err) + } + return + } + + msgType, err := messages.DetermineClientMsgType(buf[:n]) + if err != nil { + log.Errorf("failed to determine message type: %s", err) + return + } + switch msgType { + case messages.MsgTypeBindNewChannel: + dstPeerId, err := messages.UnmarshalBindNewChannel(buf[:n]) + if err != nil { + log.Errorf("failed to unmarshal bind new channel message: %s", err) + continue + } + + channelID := r.store.Link(peer, dstPeerId) + + msg := messages.MarshalBindResponseMsg(channelID, dstPeerId) + _, err = conn.Write(msg) + if err != nil { + peer.Log.Errorf("failed to response to bind request: %s", err) + continue + } + peer.Log.Debugf("bind new channel with '%s', channelID: %d", dstPeerId, channelID) + case messages.MsgTypeTransport: + msg := buf[:n] + channelId, err := messages.UnmarshalTransportID(msg) + if err != nil { + peer.Log.Errorf("failed to unmarshal transport message: %s", err) + continue + } + + foreignChannelID, remoteConn, err := peer.ConnByChannelID(channelId) + if err != nil { + peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) + continue + } + + err = transportTo(remoteConn, foreignChannelID, msg) + if err != nil { + peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) + continue + } + } + } +} + +func transportTo(conn net.Conn, channelID uint16, msg []byte) error { + err := messages.UpdateTransportMsg(msg, channelID) + if err != nil { + return err + } + _, err = conn.Write(msg) + return err +} + +func handShake(conn net.Conn) (*Peer, error) { + buf := make([]byte, 65535) // todo: reduce the buffer size + n, err := conn.Read(buf) + if err != nil { + log.Errorf("failed to read message: %s", err) + return nil, err + } + msgType, err := messages.DetermineClientMsgType(buf[:n]) + if err != nil { + return nil, err + } + if msgType != messages.MsgTypeHello { + tErr := fmt.Errorf("invalid message type") + log.Errorf("failed to handshake: %s", tErr) + return nil, tErr + } + peerId, err := messages.UnmarshalHelloMsg(buf[:n]) + if err != nil { + log.Errorf("failed to handshake: %s", err) + return nil, err + } + p := NewPeer(peerId, conn) + return p, nil +} diff --git a/relay/server/store.go b/relay/server/store.go new file mode 100644 index 000000000..fdbc118e4 --- /dev/null +++ b/relay/server/store.go @@ -0,0 +1,48 @@ +package server + +import ( + "sync" +) + +type Store struct { + peers map[string]*Peer // Key is the id (public key or sha-256) of the peer + peersLock sync.Mutex +} + +func NewStore() *Store { + return &Store{ + peers: make(map[string]*Peer), + } +} + +func (s *Store) AddPeer(peer *Peer) { + s.peersLock.Lock() + defer s.peersLock.Unlock() + s.peers[peer.ID()] = peer +} + +func (s *Store) Link(peer *Peer, peerForeignID string) uint16 { + s.peersLock.Lock() + defer s.peersLock.Unlock() + + channelId := peer.BindChannel(peerForeignID) + dstPeer, ok := s.peers[peerForeignID] + if !ok { + return channelId + } + + foreignChannelID, ok := dstPeer.AddParticipant(peer, channelId) + if !ok { + return channelId + } + peer.AddParticipant(dstPeer, foreignChannelID) + return channelId +} + +func (s *Store) DeletePeer(peer *Peer) { + s.peersLock.Lock() + defer s.peersLock.Unlock() + + delete(s.peers, peer.ID()) + peer.DeleteParticipants() +} diff --git a/relay/test/client_test.go b/relay/test/client_test.go new file mode 100644 index 000000000..c6a862ab9 --- /dev/null +++ b/relay/test/client_test.go @@ -0,0 +1,285 @@ +package test + +import ( + "os" + "testing" + + "github.com/netbirdio/netbird/util" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/client" + "github.com/netbirdio/netbird/relay/server" +) + +func TestMain(m *testing.M) { + _ = util.InitLog("trace", "console") + code := m.Run() + os.Exit(code) +} + +func TestClient(t *testing.T) { + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := client.NewClient(addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientAlice.Close() + + clientPlaceHolder := client.NewClient(addr, "clientPlaceHolder") + err = clientPlaceHolder.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientPlaceHolder.Close() + + _, err = clientAlice.BindChannel("clientPlaceHolder") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + clientBob := client.NewClient(addr, "bob") + err = clientBob.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer clientBob.Close() + + connAliceToBob, err := clientAlice.BindChannel("bob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + connBobToAlice, err := clientBob.BindChannel("alice") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + log.Debugf("alice sent message to bob") + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + log.Debugf("on new message from alice to bob") + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestEcho(t *testing.T) { + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := client.NewClient(addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err := clientAlice.Close() + if err != nil { + t.Errorf("failed to close Alice client: %s", err) + } + }() + + clientBob := client.NewClient(addr, "bob") + err = clientBob.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err := clientBob.Close() + if err != nil { + t.Errorf("failed to close Bob client: %s", err) + } + }() + + connAliceToBob, err := clientAlice.BindChannel("bob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + connBobToAlice, err := clientBob.BindChannel("alice") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + _, err = connBobToAlice.Write(buf[:n]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + n, err = connAliceToBob.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} + +func TestBindToUnavailabePeer(t *testing.T) { + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Errorf("failed to bind server: %s", err) + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := client.NewClient(addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + defer func() { + log.Infof("closing client") + err := clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + }() + + _, err = clientAlice.BindChannel("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } +} + +func TestBindReconnect(t *testing.T) { + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Errorf("failed to bind server: %s", err) + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := client.NewClient(addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + _, err = clientAlice.BindChannel("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + clientBob := client.NewClient(addr, "bob") + err = clientBob.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + chBob, err := clientBob.BindChannel("alice") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + + clientAlice = client.NewClient(addr, "alice") + err = clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + chAlice, err := clientAlice.BindChannel("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + testString := "hello alice, I am bob" + _, err = chBob.Write([]byte(testString)) + if err != nil { + t.Errorf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := chAlice.Read(buf) + if err != nil { + t.Errorf("failed to read from channel: %s", err) + } + + if testString != string(buf[:n]) { + t.Errorf("expected %s, got %s", testString, string(buf[:n])) + } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } +} From 49dfbc82d934b0f14a195c5fe83cd60eb94439c2 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 17 May 2024 20:24:06 +0200 Subject: [PATCH 002/199] Add relay cmd --- relay/cmd/main.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 relay/cmd/main.go diff --git a/relay/cmd/main.go b/relay/cmd/main.go new file mode 100644 index 000000000..cfae8232a --- /dev/null +++ b/relay/cmd/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "github.com/netbirdio/netbird/util" + "os" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/server" +) + +func init() { + util.InitLog("trace", "console") +} + +func main() { + + address := "0.0.0.0:1234" + srv := server.NewServer() + err := srv.Listen(address) + if err != nil { + log.Errorf("failed to bind server: %s", err) + os.Exit(1) + } +} From 6ae7a790f2f0abf588625f37e97ccdaf51cd8202 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 17 May 2024 23:29:47 +0200 Subject: [PATCH 003/199] Fix buffer handling --- relay/client/client.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 110c75ec2..a83ea1415 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -149,13 +149,13 @@ func (c *Client) readLoop() { c.msgPool.Put(buf) continue case messages.MsgTypeTransport: - channelId, payload, err := messages.UnmarshalTransportMsg(buf[:n]) + channelId, err := messages.UnmarshalTransportID(buf[:n]) if err != nil { log.Errorf("failed to parse transport message: %v", err) c.msgPool.Put(buf) continue } - c.handleTransport(channelId, payload) + c.handleTransport(channelId, buf[:n]) } } @@ -185,16 +185,15 @@ func (c *Client) handleBindResponse(channelId uint16, peerId string) { bindSuccessChan <- conn } -func (c *Client) handleTransport(channelId uint16, payload []byte) { +func (c *Client) handleTransport(channelId uint16, msg []byte) { container, ok := c.channels[channelId] if !ok { - log.Errorf("c.channels: %v", c.peerID) log.Errorf("unexpected transport message for channel: %d", channelId) return } select { - case container.messages <- payload: + case container.messages <- msg: default: log.Errorf("dropping message for channel: %d", channelId) } @@ -211,13 +210,19 @@ func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { func (c *Client) generateConnReaderFN(messageBufferChan chan []byte) func(b []byte) (n int, err error) { return func(b []byte) (n int, err error) { + defer c.msgPool.Put(b) select { case msg, ok := <-messageBufferChan: if !ok { return 0, io.EOF } - n = copy(b, msg) - c.msgPool.Put(msg) + + payload, err := messages.UnmarshalTransportPayload(msg) + if err != nil { + return 0, err + } + + n = copy(b, payload) } return n, nil } From d4eaec5cbd0161234c3624786fa42b02906590cb Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 17 May 2024 23:41:47 +0200 Subject: [PATCH 004/199] Followup messages modification --- relay/messages/message.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/relay/messages/message.go b/relay/messages/message.go index ee4947afe..a7d5452f0 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -109,12 +109,11 @@ func MarshalTransportMsg(channelId uint16, payload []byte) []byte { return msg } -func UnmarshalTransportMsg(buf []byte) (uint16, []byte, error) { +func UnmarshalTransportPayload(buf []byte) ([]byte, error) { if len(buf) < 3 { - return 0, nil, ErrInvalidMessageLength + return nil, ErrInvalidMessageLength } - channelId := uint16(buf[1])<<8 | uint16(buf[2]) - return channelId, buf[3:], nil + return buf[3:], nil } func UnmarshalTransportID(buf []byte) (uint16, error) { From 9ac5a1ed3ff5be4165265bfbcd6cf28bc0407ca5 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 19 May 2024 12:41:06 +0200 Subject: [PATCH 005/199] Add udp listener and did some change for debug purpose. --- relay/client/client.go | 10 +-- relay/client/conn.go | 6 +- relay/client/dialer/udp/udp.go | 14 ++++ relay/messages/message.go | 17 ++++- relay/server/listener/udp/listener.go | 96 +++++++++++++++++++++++++ relay/server/listener/udp/udp_conn.go | 68 ++++++++++++++++++ relay/server/listener/ws/server_conn.go | 7 +- relay/server/server.go | 30 ++++---- relay/test/client_test.go | 2 +- 9 files changed, 224 insertions(+), 26 deletions(-) create mode 100644 relay/client/dialer/udp/udp.go create mode 100644 relay/server/listener/udp/listener.go create mode 100644 relay/server/listener/udp/udp_conn.go diff --git a/relay/client/client.go b/relay/client/client.go index a83ea1415..94c000123 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -10,12 +10,12 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/client/dialer/ws" + "github.com/netbirdio/netbird/relay/client/dialer/udp" "github.com/netbirdio/netbird/relay/messages" ) const ( - bufferSize = 65535 // optimise the buffer size + bufferSize = 1500 // optimise the buffer size ) type connContainer struct { @@ -52,7 +52,7 @@ func NewClient(serverAddress, peerID string) *Client { } func (c *Client) Connect() error { - conn, err := ws.Dial(c.serverAddress) + conn, err := udp.Dial(c.serverAddress) if err != nil { return err } @@ -128,6 +128,7 @@ func (c *Client) readLoop() { buf := c.msgPool.Get().([]byte) n, errExit = c.relayConn.Read(buf) if errExit != nil { + log.Debugf("failed to read message from relay server: %s", errExit) break } @@ -155,7 +156,8 @@ func (c *Client) readLoop() { c.msgPool.Put(buf) continue } - c.handleTransport(channelId, buf[:n]) + go c.handleTransport(channelId, buf[:n]) + } } diff --git a/relay/client/conn.go b/relay/client/conn.go index a5635e020..d450c3f30 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -34,13 +34,11 @@ func (c *Conn) Close() error { } func (c *Conn) LocalAddr() net.Addr { - //TODO implement me - panic("implement me") + return c.client.relayConn.LocalAddr() } func (c *Conn) RemoteAddr() net.Addr { - //TODO implement me - panic("implement me") + return c.client.relayConn.RemoteAddr() } func (c *Conn) SetDeadline(t time.Time) error { diff --git a/relay/client/dialer/udp/udp.go b/relay/client/dialer/udp/udp.go new file mode 100644 index 000000000..ff0fa9c83 --- /dev/null +++ b/relay/client/dialer/udp/udp.go @@ -0,0 +1,14 @@ +package udp + +import ( + "net" +) + +func Dial(address string) (net.Conn, error) { + udpAddr, err := net.ResolveUDPAddr("udp", address) + if err != nil { + return nil, err + } + + return net.DialUDP("udp", nil, udpAddr) +} diff --git a/relay/messages/message.go b/relay/messages/message.go index a7d5452f0..9d728a498 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -17,6 +17,21 @@ var ( type MsgType byte +func (m MsgType) String() string { + switch m { + case MsgTypeHello: + return "hello" + case MsgTypeBindNewChannel: + return "bind new channel" + case MsgTypeBindResponse: + return "bind response" + case MsgTypeTransport: + return "transport" + default: + return "unknown" + } +} + func DetermineClientMsgType(msg []byte) (MsgType, error) { // todo: validate magic byte msgType := MsgType(msg[0]) @@ -41,7 +56,7 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { case MsgTypeTransport: return msgType, nil default: - return 0, fmt.Errorf("invalid msg type: %s", msg) + return 0, fmt.Errorf("invalid msg type, len: %d", len(msg)) } } diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go new file mode 100644 index 000000000..df7fa4c64 --- /dev/null +++ b/relay/server/listener/udp/listener.go @@ -0,0 +1,96 @@ +package udp + +import ( + "net" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/server/listener" +) + +type Listener struct { + address string + + onAcceptFn func(conn net.Conn) + + conns map[string]*UDPConn + wg sync.WaitGroup + quit chan struct{} + lock sync.Mutex + listener *net.UDPConn +} + +func NewListener(address string) listener.Listener { + return &Listener{ + address: address, + conns: make(map[string]*UDPConn), + } +} + +func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { + l.lock.Lock() + + l.onAcceptFn = onAcceptFn + l.quit = make(chan struct{}) + + addr := &net.UDPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + } + li, err := net.ListenUDP("udp", addr) + if err != nil { + log.Errorf("%s", err) + l.lock.Unlock() + return err + } + log.Debugf("udp server is listening on address: %s", l.address) + l.listener = li + l.wg.Add(1) + go l.readLoop() + + l.lock.Unlock() + <-l.quit + return nil +} + +// Close todo: prevent multiple call (do not close two times the channel) +func (l *Listener) Close() error { + l.lock.Lock() + defer l.lock.Unlock() + + close(l.quit) + err := l.listener.Close() + l.wg.Wait() + return err +} + +func (l *Listener) readLoop() { + defer l.wg.Done() + + for { + buf := make([]byte, 1500) + n, addr, err := l.listener.ReadFromUDP(buf) + if err != nil { + select { + case <-l.quit: + return + default: + log.Errorf("failed to accept connection: %s", err) + continue + } + } + + pConn, ok := l.conns[addr.String()] + if ok { + pConn.onNewMsg(buf[:n]) + continue + } + + pConn = NewConn(l.listener, addr) + l.conns[addr.String()] = pConn + go l.onAcceptFn(pConn) + pConn.onNewMsg(buf[:n]) + + } +} diff --git a/relay/server/listener/udp/udp_conn.go b/relay/server/listener/udp/udp_conn.go new file mode 100644 index 000000000..24b6d4640 --- /dev/null +++ b/relay/server/listener/udp/udp_conn.go @@ -0,0 +1,68 @@ +package udp + +import ( + "io" + "net" + "time" +) + +type UDPConn struct { + *net.UDPConn + addr *net.UDPAddr + msgChannel chan []byte +} + +func NewConn(conn *net.UDPConn, addr *net.UDPAddr) *UDPConn { + return &UDPConn{ + UDPConn: conn, + addr: addr, + msgChannel: make(chan []byte), + } +} + +func (u *UDPConn) Read(b []byte) (n int, err error) { + msg, ok := <-u.msgChannel + if !ok { + return 0, io.EOF + } + + n = copy(b, msg) + return n, nil +} + +func (u *UDPConn) Write(b []byte) (n int, err error) { + return u.UDPConn.WriteTo(b, u.addr) +} + +func (u *UDPConn) Close() error { + //TODO implement me + //panic("implement me") + return nil +} + +func (u *UDPConn) LocalAddr() net.Addr { + return u.UDPConn.LocalAddr() +} + +func (u *UDPConn) RemoteAddr() net.Addr { + return u.addr +} + +func (u *UDPConn) SetDeadline(t time.Time) error { + //TODO implement me + panic("implement me") +} + +func (u *UDPConn) SetReadDeadline(t time.Time) error { + //TODO implement me + panic("implement me") +} + +func (u *UDPConn) SetWriteDeadline(t time.Time) error { + //TODO implement me + panic("implement me") +} + +func (u *UDPConn) onNewMsg(b []byte) { + u.msgChannel <- b +} diff --git a/relay/server/listener/ws/server_conn.go b/relay/server/listener/ws/server_conn.go index 5c7be69fd..de3a16781 100644 --- a/relay/server/listener/ws/server_conn.go +++ b/relay/server/listener/ws/server_conn.go @@ -2,6 +2,7 @@ package ws import ( "fmt" + "sync" "time" "github.com/gorilla/websocket" @@ -10,11 +11,13 @@ import ( type Conn struct { *websocket.Conn + + mu sync.Mutex } func NewConn(wsConn *websocket.Conn) *Conn { return &Conn{ - wsConn, + Conn: wsConn, } } @@ -33,7 +36,9 @@ func (c *Conn) Read(b []byte) (n int, err error) { } func (c *Conn) Write(b []byte) (int, error) { + c.mu.Lock() err := c.WriteMessage(websocket.BinaryMessage, b) + c.mu.Unlock() return len(b), err } diff --git a/relay/server/server.go b/relay/server/server.go index 7c465132f..bcfda16e7 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -9,7 +9,7 @@ import ( "github.com/netbirdio/netbird/relay/messages" "github.com/netbirdio/netbird/relay/server/listener" - "github.com/netbirdio/netbird/relay/server/listener/ws" + "github.com/netbirdio/netbird/relay/server/listener/udp" ) // Server @@ -30,7 +30,7 @@ func NewServer() *Server { } func (r *Server) Listen(address string) error { - r.listener = ws.NewListener(address) + r.listener = udp.NewListener(address) return r.listener.Listen(r.accept) } @@ -51,7 +51,7 @@ func (r *Server) accept(conn net.Conn) { } return } - peer.Log.Debugf("on new connection: %s", conn.RemoteAddr()) + peer.Log.Debugf("peer connected from: %s", conn.RemoteAddr()) r.store.AddPeer(peer) defer func() { @@ -59,8 +59,8 @@ func (r *Server) accept(conn net.Conn) { r.store.DeletePeer(peer) }() - buf := make([]byte, 65535) // todo: optimize buffer size for { + buf := make([]byte, 1500) // todo: optimize buffer size n, err := conn.Read(buf) if err != nil { if err != io.EOF { @@ -98,18 +98,18 @@ func (r *Server) accept(conn net.Conn) { peer.Log.Errorf("failed to unmarshal transport message: %s", err) continue } + go func() { + foreignChannelID, remoteConn, err := peer.ConnByChannelID(channelId) + if err != nil { + peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) + return + } - foreignChannelID, remoteConn, err := peer.ConnByChannelID(channelId) - if err != nil { - peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) - continue - } - - err = transportTo(remoteConn, foreignChannelID, msg) - if err != nil { - peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) - continue - } + err = transportTo(remoteConn, foreignChannelID, msg) + if err != nil { + peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) + } + }() } } } diff --git a/relay/test/client_test.go b/relay/test/client_test.go index c6a862ab9..b98d6690e 100644 --- a/relay/test/client_test.go +++ b/relay/test/client_test.go @@ -173,7 +173,7 @@ func TestBindToUnavailabePeer(t *testing.T) { go func() { err := srv.Listen(addr) if err != nil { - t.Errorf("failed to bind server: %s", err) + t.Fatalf("failed to bind server: %s", err) } }() From 1c9c9ae47eef3f1474f4c434a15a00d697e0d080 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 20 May 2024 11:38:23 +0200 Subject: [PATCH 006/199] Remove sync.pool --- relay/client/client.go | 17 +++-------------- relay/server/server.go | 2 +- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 94c000123..4cb29857d 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -5,12 +5,11 @@ import ( "fmt" "io" "net" - "sync" "time" log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/client/dialer/udp" + "github.com/netbirdio/netbird/relay/client/dialer/ws" "github.com/netbirdio/netbird/relay/messages" ) @@ -31,7 +30,6 @@ type Client struct { channelsPending map[string]chan net.Conn // todo: protect map with mutex channels map[uint16]*connContainer - msgPool sync.Pool relayConn net.Conn relayConnState bool @@ -43,16 +41,11 @@ func NewClient(serverAddress, peerID string) *Client { peerID: peerID, channelsPending: make(map[string]chan net.Conn), channels: make(map[uint16]*connContainer), - msgPool: sync.Pool{ - New: func() any { - return make([]byte, bufferSize) - }, - }, } } func (c *Client) Connect() error { - conn, err := udp.Dial(c.serverAddress) + conn, err := ws.Dial(c.serverAddress) if err != nil { return err } @@ -125,7 +118,7 @@ func (c *Client) readLoop() { var errExit error var n int for { - buf := c.msgPool.Get().([]byte) + buf := make([]byte, bufferSize) // todo optimise buffer size, use pool n, errExit = c.relayConn.Read(buf) if errExit != nil { log.Debugf("failed to read message from relay server: %s", errExit) @@ -135,7 +128,6 @@ func (c *Client) readLoop() { msgType, err := messages.DetermineServerMsgType(buf[:n]) if err != nil { log.Errorf("failed to determine message type: %s", err) - c.msgPool.Put(buf) continue } @@ -147,13 +139,11 @@ func (c *Client) readLoop() { } else { c.handleBindResponse(channelId, peerId) } - c.msgPool.Put(buf) continue case messages.MsgTypeTransport: channelId, err := messages.UnmarshalTransportID(buf[:n]) if err != nil { log.Errorf("failed to parse transport message: %v", err) - c.msgPool.Put(buf) continue } go c.handleTransport(channelId, buf[:n]) @@ -212,7 +202,6 @@ func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { func (c *Client) generateConnReaderFN(messageBufferChan chan []byte) func(b []byte) (n int, err error) { return func(b []byte) (n int, err error) { - defer c.msgPool.Put(b) select { case msg, ok := <-messageBufferChan: if !ok { diff --git a/relay/server/server.go b/relay/server/server.go index bcfda16e7..e26ea3f5c 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -124,7 +124,7 @@ func transportTo(conn net.Conn, channelID uint16, msg []byte) error { } func handShake(conn net.Conn) (*Peer, error) { - buf := make([]byte, 65535) // todo: reduce the buffer size + buf := make([]byte, 1500) n, err := conn.Read(buf) if err != nil { log.Errorf("failed to read message: %s", err) From 13eb457132e9291559b538b1d55429e94e879e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 21 May 2024 15:51:37 +0200 Subject: [PATCH 007/199] Add registration response message to the communication --- relay/client/client.go | 37 +++++++++++++++++++--- relay/messages/message.go | 17 +++++++--- relay/server/server.go | 5 ++- relay/test/client_test.go | 66 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 10 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 4cb29857d..3e7fe185a 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -9,12 +9,13 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/client/dialer/ws" + "github.com/netbirdio/netbird/relay/client/dialer/udp" "github.com/netbirdio/netbird/relay/messages" ) const ( - bufferSize = 1500 // optimise the buffer size + bufferSize = 1500 // optimise the buffer size + serverResponseTimeout = 8 * time.Second ) type connContainer struct { @@ -45,7 +46,7 @@ func NewClient(serverAddress, peerID string) *Client { } func (c *Client) Connect() error { - conn, err := ws.Dial(c.serverAddress) + conn, err := udp.Dial(c.serverAddress) if err != nil { return err } @@ -80,7 +81,7 @@ func (c *Client) BindChannel(remotePeerID string) (net.Conn, error) { return nil, err } - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), serverResponseTimeout) defer cancel() select { case <-ctx.Done(): @@ -91,6 +92,10 @@ func (c *Client) BindChannel(remotePeerID string) (net.Conn, error) { } func (c *Client) Close() error { + if !c.relayConnState { + return nil + } + for _, conn := range c.channels { close(conn.messages) } @@ -110,6 +115,30 @@ func (c *Client) handShake() error { log.Errorf("failed to send hello message: %s", err) return err } + + err = c.relayConn.SetReadDeadline(time.Now().Add(serverResponseTimeout)) + if err != nil { + log.Errorf("failed to set read deadline: %s", err) + return err + } + + buf := make([]byte, 1500) // todo: optimise buffer size + n, err := c.relayConn.Read(buf) + if err != nil { + log.Errorf("failed to read hello response: %s", err) + return err + } + + msgType, err := messages.DetermineServerMsgType(buf[:n]) + if err != nil { + log.Errorf("failed to determine message type: %s", err) + return err + } + + if msgType != messages.MsgTypeHelloResponse { + log.Errorf("unexpected message type: %s", msgType) + return fmt.Errorf("unexpected message type") + } return nil } diff --git a/relay/messages/message.go b/relay/messages/message.go index 9d728a498..02945e2f1 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -6,9 +6,10 @@ import ( const ( MsgTypeHello MsgType = 0 - MsgTypeBindNewChannel MsgType = 1 - MsgTypeBindResponse MsgType = 2 - MsgTypeTransport MsgType = 3 + MsgTypeHelloResponse MsgType = 1 + MsgTypeBindNewChannel MsgType = 2 + MsgTypeBindResponse MsgType = 3 + MsgTypeTransport MsgType = 4 ) var ( @@ -51,12 +52,14 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { // todo: validate magic byte msgType := MsgType(msg[0]) switch msgType { + case MsgTypeHelloResponse: + return msgType, nil case MsgTypeBindResponse: return msgType, nil case MsgTypeTransport: return msgType, nil default: - return 0, fmt.Errorf("invalid msg type, len: %d", len(msg)) + return 0, fmt.Errorf("invalid msg type (len: %d)", len(msg)) } } @@ -78,6 +81,12 @@ func UnmarshalHelloMsg(msg []byte) (string, error) { return string(msg[1:]), nil } +func MarshalHelloResponse() []byte { + msg := make([]byte, 1) + msg[0] = byte(MsgTypeHelloResponse) + return msg +} + // Bind new channel func MarshalBindNewChannelMsg(destinationPeerId string) []byte { diff --git a/relay/server/server.go b/relay/server/server.go index e26ea3f5c..a66341e51 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -145,5 +145,8 @@ func handShake(conn net.Conn) (*Peer, error) { return nil, err } p := NewPeer(peerId, conn) - return p, nil + + msg := messages.MarshalHelloResponse() + _, err = conn.Write(msg) + return p, err } diff --git a/relay/test/client_test.go b/relay/test/client_test.go index b98d6690e..87a07fa94 100644 --- a/relay/test/client_test.go +++ b/relay/test/client_test.go @@ -1,12 +1,14 @@ package test import ( + "net" "os" "testing" - "github.com/netbirdio/netbird/util" log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/util" + "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/relay/server" ) @@ -89,6 +91,68 @@ func TestClient(t *testing.T) { } } +func TestRegistration(t *testing.T) { + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := client.NewClient(addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + defer func() { + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } + }() +} + +func TestRegistrationTimeout(t *testing.T) { + udpListener, err := net.ListenUDP("udp", &net.UDPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + }) + if err != nil { + t.Fatalf("failed to bind UDP server: %s", err) + } + defer udpListener.Close() + + tcpListener, err := net.ListenTCP("tcp", &net.TCPAddr{ + Port: 1234, + IP: net.ParseIP("0.0.0.0"), + }) + if err != nil { + t.Fatalf("failed to bind TCP server: %s", err) + } + defer tcpListener.Close() + + clientAlice := client.NewClient("127.0.0.1:1234", "alice") + err = clientAlice.Connect() + if err == nil { + t.Errorf("failed to connect to server: %s", err) + } + defer func() { + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } + }() +} + func TestEcho(t *testing.T) { addr := "localhost:1234" srv := server.NewServer() From e82c0a55a31e63d673cf4bbbc7750bfd025fac96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 21 May 2024 16:21:29 +0200 Subject: [PATCH 008/199] Set to blocking the message queue --- relay/client/client.go | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 3e7fe185a..b2235ec46 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -146,6 +146,11 @@ func (c *Client) readLoop() { log := log.WithField("client_id", c.peerID) var errExit error var n int + err := c.relayConn.SetReadDeadline(time.Time{}) + if err != nil { + log.Errorf("failed to set read deadline: %s", err) + return + } for { buf := make([]byte, bufferSize) // todo optimise buffer size, use pool n, errExit = c.relayConn.Read(buf) @@ -175,7 +180,13 @@ func (c *Client) readLoop() { log.Errorf("failed to parse transport message: %v", err) continue } - go c.handleTransport(channelId, buf[:n]) + container, ok := c.channels[channelId] + if !ok { + log.Errorf("unexpected transport message for channel: %d", channelId) + return + } + + container.messages <- buf[:n] } } @@ -194,7 +205,7 @@ func (c *Client) handleBindResponse(channelId uint16, peerId string) { } delete(c.channelsPending, peerId) - messageBuffer := make(chan []byte, 10) + messageBuffer := make(chan []byte, 2) conn := NewConn(c, channelId, c.generateConnReaderFN(messageBuffer)) c.channels[channelId] = &connContainer{ @@ -206,20 +217,6 @@ func (c *Client) handleBindResponse(channelId uint16, peerId string) { bindSuccessChan <- conn } -func (c *Client) handleTransport(channelId uint16, msg []byte) { - container, ok := c.channels[channelId] - if !ok { - log.Errorf("unexpected transport message for channel: %d", channelId) - return - } - - select { - case container.messages <- msg: - default: - log.Errorf("dropping message for channel: %d", channelId) - } -} - func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { msg := messages.MarshalTransportMsg(channelID, payload) n, err := c.relayConn.Write(msg) From 0a05f8b4d417254416c9b56d2bdf97eff2e39c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 22 May 2024 00:38:41 +0200 Subject: [PATCH 009/199] Use buffer pool and protect exported functions --- relay/client/client.go | 64 +++++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index b2235ec46..130cb8592 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net" + "sync" "time" log "github.com/sirupsen/logrus" @@ -18,9 +19,14 @@ const ( serverResponseTimeout = 8 * time.Second ) +type bufMsg struct { + bufPtr *[]byte + buf []byte +} + type connContainer struct { conn *Conn - messages chan []byte + messages chan bufMsg } // Client Todo: @@ -31,9 +37,11 @@ type Client struct { channelsPending map[string]chan net.Conn // todo: protect map with mutex channels map[uint16]*connContainer + msgPool sync.Pool relayConn net.Conn relayConnState bool + mu sync.Mutex } func NewClient(serverAddress, peerID string) *Client { @@ -42,10 +50,18 @@ func NewClient(serverAddress, peerID string) *Client { peerID: peerID, channelsPending: make(map[string]chan net.Conn), channels: make(map[uint16]*connContainer), + msgPool: sync.Pool{ + New: func() any { + buf := make([]byte, bufferSize) + return &buf + }, + }, } } func (c *Client) Connect() error { + c.mu.Lock() + defer c.mu.Unlock() conn, err := udp.Dial(c.serverAddress) if err != nil { return err @@ -62,14 +78,23 @@ func (c *Client) Connect() error { return err } + err = c.relayConn.SetReadDeadline(time.Time{}) + if err != nil { + log.Errorf("failed to reset read deadline: %s", err) + return err + } + c.relayConnState = true go c.readLoop() return nil } func (c *Client) BindChannel(remotePeerID string) (net.Conn, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.relayConn == nil { - return nil, fmt.Errorf("client not connected") + return nil, fmt.Errorf("client not connected to the relay server") } bindSuccessChan := make(chan net.Conn, 1) @@ -92,6 +117,9 @@ func (c *Client) BindChannel(remotePeerID string) (net.Conn, error) { } func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + if !c.relayConnState { return nil } @@ -146,22 +174,20 @@ func (c *Client) readLoop() { log := log.WithField("client_id", c.peerID) var errExit error var n int - err := c.relayConn.SetReadDeadline(time.Time{}) - if err != nil { - log.Errorf("failed to set read deadline: %s", err) - return - } for { - buf := make([]byte, bufferSize) // todo optimise buffer size, use pool + bufPtr := c.msgPool.Get().(*[]byte) + buf := *bufPtr n, errExit = c.relayConn.Read(buf) if errExit != nil { log.Debugf("failed to read message from relay server: %s", errExit) + c.freeBuf(bufPtr) break } msgType, err := messages.DetermineServerMsgType(buf[:n]) if err != nil { log.Errorf("failed to determine message type: %s", err) + c.freeBuf(bufPtr) continue } @@ -173,21 +199,26 @@ func (c *Client) readLoop() { } else { c.handleBindResponse(channelId, peerId) } + c.freeBuf(bufPtr) continue case messages.MsgTypeTransport: channelId, err := messages.UnmarshalTransportID(buf[:n]) if err != nil { log.Errorf("failed to parse transport message: %v", err) + c.freeBuf(bufPtr) continue } container, ok := c.channels[channelId] if !ok { log.Errorf("unexpected transport message for channel: %d", channelId) + c.freeBuf(bufPtr) return } - container.messages <- buf[:n] - + container.messages <- bufMsg{ + bufPtr, + buf[:n], + } } } @@ -205,7 +236,7 @@ func (c *Client) handleBindResponse(channelId uint16, peerId string) { } delete(c.channelsPending, peerId) - messageBuffer := make(chan []byte, 2) + messageBuffer := make(chan bufMsg, 2) conn := NewConn(c, channelId, c.generateConnReaderFN(messageBuffer)) c.channels[channelId] = &connContainer{ @@ -226,21 +257,26 @@ func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { return n, err } -func (c *Client) generateConnReaderFN(messageBufferChan chan []byte) func(b []byte) (n int, err error) { +func (c *Client) generateConnReaderFN(messageBufferChan chan bufMsg) func(b []byte) (n int, err error) { return func(b []byte) (n int, err error) { select { - case msg, ok := <-messageBufferChan: + case bufMsg, ok := <-messageBufferChan: if !ok { return 0, io.EOF } - payload, err := messages.UnmarshalTransportPayload(msg) + payload, err := messages.UnmarshalTransportPayload(bufMsg.buf) if err != nil { return 0, err } n = copy(b, payload) + c.freeBuf(bufMsg.bufPtr) } return n, nil } } + +func (c *Client) freeBuf(ptr *[]byte) { + c.msgPool.Put(ptr) +} From 36b2cd16ccc53350fae358b16917218a50a4753f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 23 May 2024 13:24:02 +0200 Subject: [PATCH 010/199] Remove channel binding logic --- relay/client/client.go | 167 ++++++++------------------ relay/client/conn.go | 16 +-- relay/client/manager.go | 43 +++++++ relay/cmd/main.go | 5 +- relay/messages/id.go | 20 +++ relay/messages/message.go | 99 +++++---------- relay/server/listener/udp/listener.go | 36 +++--- relay/server/peer.go | 109 +++-------------- relay/server/server.go | 46 ++----- relay/server/store.go | 35 ++---- relay/test/client_test.go | 27 ++--- 11 files changed, 229 insertions(+), 374 deletions(-) create mode 100644 relay/client/manager.go create mode 100644 relay/messages/id.go diff --git a/relay/client/client.go b/relay/client/client.go index 130cb8592..d574e04a1 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -1,7 +1,6 @@ package client import ( - "context" "fmt" "io" "net" @@ -19,25 +18,21 @@ const ( serverResponseTimeout = 8 * time.Second ) -type bufMsg struct { - bufPtr *[]byte - buf []byte +type Msg struct { + buf []byte } type connContainer struct { conn *Conn - messages chan bufMsg + messages chan Msg } -// Client Todo: -// - handle automatic reconnection type Client struct { + log *log.Entry serverAddress string - peerID string + hashedID []byte - channelsPending map[string]chan net.Conn // todo: protect map with mutex - channels map[uint16]*connContainer - msgPool sync.Pool + conns map[string]*connContainer relayConn net.Conn relayConnState bool @@ -45,17 +40,12 @@ type Client struct { } func NewClient(serverAddress, peerID string) *Client { + hashedID, hashedStringId := messages.HashID(peerID) return &Client{ - serverAddress: serverAddress, - peerID: peerID, - channelsPending: make(map[string]chan net.Conn), - channels: make(map[uint16]*connContainer), - msgPool: sync.Pool{ - New: func() any { - buf := make([]byte, bufferSize) - return &buf - }, - }, + log: log.WithField("client_id", hashedStringId), + serverAddress: serverAddress, + hashedID: hashedID, + conns: make(map[string]*connContainer), } } @@ -89,31 +79,17 @@ func (c *Client) Connect() error { return nil } -func (c *Client) BindChannel(remotePeerID string) (net.Conn, error) { - c.mu.Lock() - defer c.mu.Unlock() +func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { + hashedID, hashedStringID := messages.HashID(dstPeerID) + log.Infof("open connection to peer: %s", hashedStringID) + messageBuffer := make(chan Msg, 2) + conn := NewConn(c, hashedID, c.generateConnReaderFN(messageBuffer)) - if c.relayConn == nil { - return nil, fmt.Errorf("client not connected to the relay server") - } - - bindSuccessChan := make(chan net.Conn, 1) - c.channelsPending[remotePeerID] = bindSuccessChan - msg := messages.MarshalBindNewChannelMsg(remotePeerID) - _, err := c.relayConn.Write(msg) - if err != nil { - log.Errorf("failed to write out bind message: %s", err) - return nil, err - } - - ctx, cancel := context.WithTimeout(context.Background(), serverResponseTimeout) - defer cancel() - select { - case <-ctx.Done(): - return nil, fmt.Errorf("bind timeout") - case c := <-bindSuccessChan: - return c, nil + c.conns[hashedStringID] = &connContainer{ + conn, + messageBuffer, } + return conn, nil } func (c *Client) Close() error { @@ -124,18 +100,15 @@ func (c *Client) Close() error { return nil } - for _, conn := range c.channels { - close(conn.messages) - } - c.channels = make(map[uint16]*connContainer) c.relayConnState = false err := c.relayConn.Close() return err } func (c *Client) handShake() error { - msg, err := messages.MarshalHelloMsg(c.peerID) + msg, err := messages.MarshalHelloMsg(c.hashedID) if err != nil { + log.Errorf("failed to marshal hello message: %s", err) return err } _, err = c.relayConn.Write(msg) @@ -171,85 +144,56 @@ func (c *Client) handShake() error { } func (c *Client) readLoop() { - log := log.WithField("client_id", c.peerID) + defer func() { + c.log.Debugf("exit from read loop") + }() var errExit error var n int for { - bufPtr := c.msgPool.Get().(*[]byte) - buf := *bufPtr + buf := make([]byte, bufferSize) n, errExit = c.relayConn.Read(buf) if errExit != nil { - log.Debugf("failed to read message from relay server: %s", errExit) - c.freeBuf(bufPtr) + if c.relayConnState { + c.log.Debugf("failed to read message from relay server: %s", errExit) + } break } msgType, err := messages.DetermineServerMsgType(buf[:n]) if err != nil { - log.Errorf("failed to determine message type: %s", err) - c.freeBuf(bufPtr) + c.log.Errorf("failed to determine message type: %s", err) continue } switch msgType { - case messages.MsgTypeBindResponse: - channelId, peerId, err := messages.UnmarshalBindResponseMsg(buf[:n]) - if err != nil { - log.Errorf("failed to parse bind response message: %v", err) - } else { - c.handleBindResponse(channelId, peerId) - } - c.freeBuf(bufPtr) - continue case messages.MsgTypeTransport: - channelId, err := messages.UnmarshalTransportID(buf[:n]) + peerID, err := messages.UnmarshalTransportID(buf[:n]) if err != nil { - log.Errorf("failed to parse transport message: %v", err) - c.freeBuf(bufPtr) + c.log.Errorf("failed to parse transport message: %v", err) continue } - container, ok := c.channels[channelId] + stringID := messages.HashIDToString(peerID) + + container, ok := c.conns[stringID] if !ok { - log.Errorf("unexpected transport message for channel: %d", channelId) - c.freeBuf(bufPtr) - return + c.log.Errorf("peer not found: %s", stringID) + continue } - container.messages <- bufMsg{ - bufPtr, + container.messages <- Msg{ buf[:n], } } } if c.relayConnState { - log.Errorf("failed to read message from relay server: %s", errExit) + c.log.Errorf("failed to read message from relay server: %s", errExit) _ = c.relayConn.Close() } } -func (c *Client) handleBindResponse(channelId uint16, peerId string) { - bindSuccessChan, ok := c.channelsPending[peerId] - if !ok { - log.Errorf("unexpected bind response from: %s", peerId) - return - } - delete(c.channelsPending, peerId) - - messageBuffer := make(chan bufMsg, 2) - conn := NewConn(c, channelId, c.generateConnReaderFN(messageBuffer)) - - c.channels[channelId] = &connContainer{ - conn, - messageBuffer, - } - log.Debugf("bind success for '%s': %d", peerId, channelId) - - bindSuccessChan <- conn -} - -func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { - msg := messages.MarshalTransportMsg(channelID, payload) +func (c *Client) writeTo(dstID []byte, payload []byte) (int, error) { + msg := messages.MarshalTransportMsg(dstID, payload) n, err := c.relayConn.Write(msg) if err != nil { log.Errorf("failed to write transport message: %s", err) @@ -257,26 +201,19 @@ func (c *Client) writeTo(channelID uint16, payload []byte) (int, error) { return n, err } -func (c *Client) generateConnReaderFN(messageBufferChan chan bufMsg) func(b []byte) (n int, err error) { +func (c *Client) generateConnReaderFN(msgChannel chan Msg) func(b []byte) (n int, err error) { return func(b []byte) (n int, err error) { - select { - case bufMsg, ok := <-messageBufferChan: - if !ok { - return 0, io.EOF - } - - payload, err := messages.UnmarshalTransportPayload(bufMsg.buf) - if err != nil { - return 0, err - } - - n = copy(b, payload) - c.freeBuf(bufMsg.bufPtr) + msg, ok := <-msgChannel + if !ok { + return 0, io.EOF } + + payload, err := messages.UnmarshalTransportPayload(msg.buf) + if err != nil { + return 0, err + } + + n = copy(b, payload) return n, nil } } - -func (c *Client) freeBuf(ptr *[]byte) { - c.msgPool.Put(ptr) -} diff --git a/relay/client/conn.go b/relay/client/conn.go index d450c3f30..aea07ff9b 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -6,23 +6,23 @@ import ( ) type Conn struct { - client *Client - channelID uint16 - readerFn func(b []byte) (n int, err error) + client *Client + dstID []byte + readerFn func(b []byte) (n int, err error) } -func NewConn(client *Client, channelID uint16, readerFn func(b []byte) (n int, err error)) *Conn { +func NewConn(client *Client, dstID []byte, readerFn func(b []byte) (n int, err error)) *Conn { c := &Conn{ - client: client, - channelID: channelID, - readerFn: readerFn, + client: client, + dstID: dstID, + readerFn: readerFn, } return c } func (c *Conn) Write(p []byte) (n int, err error) { - return c.client.writeTo(c.channelID, p) + return c.client.writeTo(c.dstID, p) } func (c *Conn) Read(b []byte) (n int, err error) { diff --git a/relay/client/manager.go b/relay/client/manager.go new file mode 100644 index 000000000..4d1aeca79 --- /dev/null +++ b/relay/client/manager.go @@ -0,0 +1,43 @@ +package client + +import ( + "context" + "sync" +) + +type Manager struct { + ctx context.Context + ctxCancel context.CancelFunc + srvAddress string + peerID string + + wg sync.WaitGroup + + clients map[string]*Client + clientsMutex sync.RWMutex +} + +func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { + ctx, cancel := context.WithCancel(ctx) + return &Manager{ + ctx: ctx, + ctxCancel: cancel, + srvAddress: serverAddress, + peerID: peerID, + clients: make(map[string]*Client), + } +} + +func (m *Manager) Teardown() { + m.ctxCancel() + m.wg.Wait() +} + +func (m *Manager) newSrvConnection(address string) { + if _, ok := m.clients[address]; ok { + return + } + + // client := NewClient(address, m.peerID) + //err = client.Connect() +} diff --git a/relay/cmd/main.go b/relay/cmd/main.go index cfae8232a..b89ab26ef 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -1,9 +1,10 @@ package main import ( - "github.com/netbirdio/netbird/util" "os" + "github.com/netbirdio/netbird/util" + log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/relay/server" @@ -15,7 +16,7 @@ func init() { func main() { - address := "0.0.0.0:1234" + address := "10.145.236.1:1235" srv := server.NewServer() err := srv.Listen(address) if err != nil { diff --git a/relay/messages/id.go b/relay/messages/id.go new file mode 100644 index 000000000..d37dc37f8 --- /dev/null +++ b/relay/messages/id.go @@ -0,0 +1,20 @@ +package messages + +import ( + "crypto/sha256" + "encoding/base64" +) + +const ( + IDSize = sha256.Size +) + +func HashID(peerID string) ([]byte, string) { + idHash := sha256.Sum256([]byte(peerID)) + idHashString := base64.StdEncoding.EncodeToString(idHash[:]) + return idHash[:], idHashString +} + +func HashIDToString(idHash []byte) string { + return base64.StdEncoding.EncodeToString(idHash[:]) +} diff --git a/relay/messages/message.go b/relay/messages/message.go index 02945e2f1..c71d203b1 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -5,11 +5,9 @@ import ( ) const ( - MsgTypeHello MsgType = 0 - MsgTypeHelloResponse MsgType = 1 - MsgTypeBindNewChannel MsgType = 2 - MsgTypeBindResponse MsgType = 3 - MsgTypeTransport MsgType = 4 + MsgTypeHello MsgType = 0 + MsgTypeHelloResponse MsgType = 1 + MsgTypeTransport MsgType = 2 ) var ( @@ -22,10 +20,8 @@ func (m MsgType) String() string { switch m { case MsgTypeHello: return "hello" - case MsgTypeBindNewChannel: - return "bind new channel" - case MsgTypeBindResponse: - return "bind response" + case MsgTypeHelloResponse: + return "hello response" case MsgTypeTransport: return "transport" default: @@ -39,8 +35,6 @@ func DetermineClientMsgType(msg []byte) (MsgType, error) { switch msgType { case MsgTypeHello: return msgType, nil - case MsgTypeBindNewChannel: - return msgType, nil case MsgTypeTransport: return msgType, nil default: @@ -54,8 +48,6 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { switch msgType { case MsgTypeHelloResponse: return msgType, nil - case MsgTypeBindResponse: - return msgType, nil case MsgTypeTransport: return msgType, nil default: @@ -64,21 +56,21 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { } // MarshalHelloMsg initial hello message -func MarshalHelloMsg(peerID string) ([]byte, error) { - if len(peerID) == 0 { - return nil, fmt.Errorf("invalid peer id") +func MarshalHelloMsg(peerID []byte) ([]byte, error) { + if len(peerID) != IDSize { + return nil, fmt.Errorf("invalid peerID length") } msg := make([]byte, 1, 1+len(peerID)) msg[0] = byte(MsgTypeHello) - msg = append(msg, []byte(peerID)...) + msg = append(msg, peerID...) return msg, nil } -func UnmarshalHelloMsg(msg []byte) (string, error) { +func UnmarshalHelloMsg(msg []byte) ([]byte, error) { if len(msg) < 2 { - return "", fmt.Errorf("invalid 'hello' messge") + return nil, fmt.Errorf("invalid 'hello' messge") } - return string(msg[1:]), nil + return msg[1:], nil } func MarshalHelloResponse() []byte { @@ -87,71 +79,40 @@ func MarshalHelloResponse() []byte { return msg } -// Bind new channel - -func MarshalBindNewChannelMsg(destinationPeerId string) []byte { - msg := make([]byte, 1, 1+len(destinationPeerId)) - msg[0] = byte(MsgTypeBindNewChannel) - msg = append(msg, []byte(destinationPeerId)...) - return msg -} - -func UnmarshalBindNewChannel(msg []byte) (string, error) { - if len(msg) < 2 { - return "", fmt.Errorf("invalid 'bind new channel' messge") - } - return string(msg[1:]), nil -} - -// Bind response - -func MarshalBindResponseMsg(channelId uint16, id string) []byte { - data := []byte(id) - msg := make([]byte, 3, 3+len(data)) - msg[0] = byte(MsgTypeBindResponse) - msg[1], msg[2] = uint8(channelId>>8), uint8(channelId&0xff) - msg = append(msg, data...) - return msg -} - -func UnmarshalBindResponseMsg(buf []byte) (uint16, string, error) { - if len(buf) < 3 { - return 0, "", ErrInvalidMessageLength - } - channelId := uint16(buf[1])<<8 | uint16(buf[2]) - peerID := string(buf[3:]) - return channelId, peerID, nil -} - // Transport message -func MarshalTransportMsg(channelId uint16, payload []byte) []byte { - msg := make([]byte, 3, 3+len(payload)) +func MarshalTransportMsg(peerID []byte, payload []byte) []byte { + if len(peerID) != IDSize { + return nil + } + + msg := make([]byte, 1+IDSize, 1+IDSize+len(payload)) msg[0] = byte(MsgTypeTransport) - msg[1], msg[2] = uint8(channelId>>8), uint8(channelId&0xff) + copy(msg[1:], peerID) msg = append(msg, payload...) return msg } func UnmarshalTransportPayload(buf []byte) ([]byte, error) { - if len(buf) < 3 { + headerSize := 1 + IDSize + if len(buf) < headerSize { return nil, ErrInvalidMessageLength } - return buf[3:], nil + return buf[headerSize:], nil } -func UnmarshalTransportID(buf []byte) (uint16, error) { - if len(buf) < 3 { - return 0, ErrInvalidMessageLength +func UnmarshalTransportID(buf []byte) ([]byte, error) { + headerSize := 1 + IDSize + if len(buf) < headerSize { + return nil, ErrInvalidMessageLength } - channelId := uint16(buf[1])<<8 | uint16(buf[2]) - return channelId, nil + return buf[1:headerSize], nil } -func UpdateTransportMsg(msg []byte, channelId uint16) error { - if len(msg) < 3 { +func UpdateTransportMsg(msg []byte, peerID []byte) error { + if len(msg) < 1+len(peerID) { return ErrInvalidMessageLength } - msg[1], msg[2] = uint8(channelId>>8), uint8(channelId&0xff) + copy(msg[1:], peerID) return nil } diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go index df7fa4c64..400b68a88 100644 --- a/relay/server/listener/udp/listener.go +++ b/relay/server/listener/udp/listener.go @@ -10,15 +10,15 @@ import ( ) type Listener struct { - address string - + address string + conns map[string]*UDPConn onAcceptFn func(conn net.Conn) - conns map[string]*UDPConn - wg sync.WaitGroup - quit chan struct{} - lock sync.Mutex listener *net.UDPConn + + wg sync.WaitGroup + quit chan struct{} + lock sync.Mutex } func NewListener(address string) listener.Listener { @@ -34,17 +34,20 @@ func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { l.onAcceptFn = onAcceptFn l.quit = make(chan struct{}) - addr := &net.UDPAddr{ - Port: 1234, - IP: net.ParseIP("0.0.0.0"), - } - li, err := net.ListenUDP("udp", addr) + addr, err := net.ResolveUDPAddr("udp", l.address) if err != nil { - log.Errorf("%s", err) + log.Errorf("invalid listen address '%s': %s", l.address, err) l.lock.Unlock() return err } - log.Debugf("udp server is listening on address: %s", l.address) + + li, err := net.ListenUDP("udp", addr) + if err != nil { + log.Fatalf("%s", err) + l.lock.Unlock() + return err + } + log.Debugf("udp server is listening on address: %s", addr.String()) l.listener = li l.wg.Add(1) go l.readLoop() @@ -54,14 +57,18 @@ func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { return nil } -// Close todo: prevent multiple call (do not close two times the channel) func (l *Listener) Close() error { l.lock.Lock() defer l.lock.Unlock() + if l.listener == nil { + return nil + } + close(l.quit) err := l.listener.Close() l.wg.Wait() + l.listener = nil return err } @@ -91,6 +98,5 @@ func (l *Listener) readLoop() { l.conns[addr.String()] = pConn go l.onAcceptFn(pConn) pConn.onNewMsg(buf[:n]) - } } diff --git a/relay/server/peer.go b/relay/server/peer.go index 2e40cbb12..d4b98b9b4 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -1,113 +1,34 @@ package server import ( - "fmt" "net" - "sync" log "github.com/sirupsen/logrus" -) -type Participant struct { - ChannelID uint16 - ChannelIDForeign uint16 - ConnForeign net.Conn - Peer *Peer -} + "github.com/netbirdio/netbird/relay/messages" +) type Peer struct { Log *log.Entry - id string + idS string + idB []byte conn net.Conn - - pendingParticipantByChannelID map[uint16]*Participant - participantByID map[uint16]*Participant // used for package transfer - participantByPeerID map[string]*Participant // used for channel linking - - lastId uint16 - lastIdLock sync.Mutex } -func NewPeer(id string, conn net.Conn) *Peer { +func NewPeer(id []byte, conn net.Conn) *Peer { + log.Debugf("new peer: %v", id) + stringID := messages.HashIDToString(id) return &Peer{ - Log: log.WithField("peer_id", id), - id: id, - conn: conn, - pendingParticipantByChannelID: make(map[uint16]*Participant), - participantByID: make(map[uint16]*Participant), - participantByPeerID: make(map[string]*Participant), + Log: log.WithField("peer_id", stringID), + idB: id, + idS: stringID, + conn: conn, } } -func (p *Peer) BindChannel(remotePeerId string) uint16 { - ch, ok := p.participantByPeerID[remotePeerId] - if ok { - return ch.ChannelID - } - - channelID := p.newChannelID() - channel := &Participant{ - ChannelID: channelID, - } - p.pendingParticipantByChannelID[channelID] = channel - p.participantByPeerID[remotePeerId] = channel - return channelID +func (p *Peer) ID() []byte { + return p.idB } -func (p *Peer) UnBindChannel(remotePeerId string) { - pa, ok := p.participantByPeerID[remotePeerId] - if !ok { - return - } - - p.Log.Debugf("unbind channel with '%s': %d", remotePeerId, pa.ChannelID) - p.pendingParticipantByChannelID[pa.ChannelID] = pa - delete(p.participantByID, pa.ChannelID) -} - -func (p *Peer) AddParticipant(peer *Peer, remoteChannelID uint16) (uint16, bool) { - participant, ok := p.participantByPeerID[peer.ID()] - if !ok { - return 0, false - } - participant.ChannelIDForeign = remoteChannelID - participant.ConnForeign = peer.conn - participant.Peer = peer - - delete(p.pendingParticipantByChannelID, participant.ChannelID) - p.participantByID[participant.ChannelID] = participant - return participant.ChannelID, true -} - -func (p *Peer) DeleteParticipants() { - for _, participant := range p.participantByID { - participant.Peer.UnBindChannel(p.id) - } -} - -func (p *Peer) ConnByChannelID(dstID uint16) (uint16, net.Conn, error) { - ch, ok := p.participantByID[dstID] - if !ok { - return 0, nil, fmt.Errorf("destination channel not found") - } - - return ch.ChannelIDForeign, ch.ConnForeign, nil -} - -func (p *Peer) ID() string { - return p.id -} - -func (p *Peer) newChannelID() uint16 { - p.lastIdLock.Lock() - defer p.lastIdLock.Unlock() - for { - p.lastId++ - if _, ok := p.pendingParticipantByChannelID[p.lastId]; ok { - continue - } - if _, ok := p.participantByID[p.lastId]; ok { - continue - } - return p.lastId - } +func (p *Peer) String() string { + return p.idS } diff --git a/relay/server/server.go b/relay/server/server.go index a66341e51..4b05975ec 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -16,7 +16,6 @@ import ( // todo: // authentication: provide JWT token via RPC call. The MGM server can forward the token to the agents. // connection timeout handling -// implement HA (High Availability) mode type Server struct { store *Store @@ -75,54 +74,35 @@ func (r *Server) accept(conn net.Conn) { return } switch msgType { - case messages.MsgTypeBindNewChannel: - dstPeerId, err := messages.UnmarshalBindNewChannel(buf[:n]) - if err != nil { - log.Errorf("failed to unmarshal bind new channel message: %s", err) - continue - } - - channelID := r.store.Link(peer, dstPeerId) - - msg := messages.MarshalBindResponseMsg(channelID, dstPeerId) - _, err = conn.Write(msg) - if err != nil { - peer.Log.Errorf("failed to response to bind request: %s", err) - continue - } - peer.Log.Debugf("bind new channel with '%s', channelID: %d", dstPeerId, channelID) case messages.MsgTypeTransport: msg := buf[:n] - channelId, err := messages.UnmarshalTransportID(msg) + peerID, err := messages.UnmarshalTransportID(msg) if err != nil { peer.Log.Errorf("failed to unmarshal transport message: %s", err) continue } go func() { - foreignChannelID, remoteConn, err := peer.ConnByChannelID(channelId) - if err != nil { - peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) + stringPeerID := messages.HashIDToString(peerID) + dp, ok := r.store.Peer(stringPeerID) + if !ok { + peer.Log.Errorf("peer not found: %s", stringPeerID) return } - - err = transportTo(remoteConn, foreignChannelID, msg) + err := messages.UpdateTransportMsg(msg, peer.ID()) if err != nil { - peer.Log.Errorf("failed to transport message from peer '%s' to '%d': %s", peer.ID(), channelId, err) + peer.Log.Errorf("failed to update transport message: %s", err) + return } + _, err = dp.conn.Write(msg) + if err != nil { + peer.Log.Errorf("failed to write transport message to: %s", dp.String()) + } + return }() } } } -func transportTo(conn net.Conn, channelID uint16, msg []byte) error { - err := messages.UpdateTransportMsg(msg, channelID) - if err != nil { - return err - } - _, err = conn.Write(msg) - return err -} - func handShake(conn net.Conn) (*Peer, error) { buf := make([]byte, 1500) n, err := conn.Read(buf) diff --git a/relay/server/store.go b/relay/server/store.go index fdbc118e4..f785f4d0e 100644 --- a/relay/server/store.go +++ b/relay/server/store.go @@ -5,8 +5,8 @@ import ( ) type Store struct { - peers map[string]*Peer // Key is the id (public key or sha-256) of the peer - peersLock sync.Mutex + peers map[string]*Peer // consider to use [32]byte as key. The Peer(id string) would be faster + peersLock sync.RWMutex } func NewStore() *Store { @@ -18,31 +18,20 @@ func NewStore() *Store { func (s *Store) AddPeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() - s.peers[peer.ID()] = peer -} - -func (s *Store) Link(peer *Peer, peerForeignID string) uint16 { - s.peersLock.Lock() - defer s.peersLock.Unlock() - - channelId := peer.BindChannel(peerForeignID) - dstPeer, ok := s.peers[peerForeignID] - if !ok { - return channelId - } - - foreignChannelID, ok := dstPeer.AddParticipant(peer, channelId) - if !ok { - return channelId - } - peer.AddParticipant(dstPeer, foreignChannelID) - return channelId + s.peers[peer.String()] = peer } func (s *Store) DeletePeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() - delete(s.peers, peer.ID()) - peer.DeleteParticipants() + delete(s.peers, peer.String()) +} + +func (s *Store) Peer(id string) (*Peer, bool) { + s.peersLock.RLock() + defer s.peersLock.RUnlock() + + p, ok := s.peers[id] + return p, ok } diff --git a/relay/test/client_test.go b/relay/test/client_test.go index 87a07fa94..675962eef 100644 --- a/relay/test/client_test.go +++ b/relay/test/client_test.go @@ -50,11 +50,6 @@ func TestClient(t *testing.T) { } defer clientPlaceHolder.Close() - _, err = clientAlice.BindChannel("clientPlaceHolder") - if err != nil { - t.Fatalf("failed to bind channel: %s", err) - } - clientBob := client.NewClient(addr, "bob") err = clientBob.Connect() if err != nil { @@ -62,12 +57,12 @@ func TestClient(t *testing.T) { } defer clientBob.Close() - connAliceToBob, err := clientAlice.BindChannel("bob") + connAliceToBob, err := clientAlice.OpenConn("bob") if err != nil { t.Fatalf("failed to bind channel: %s", err) } - connBobToAlice, err := clientBob.BindChannel("alice") + connBobToAlice, err := clientBob.OpenConn("alice") if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -154,6 +149,8 @@ func TestRegistrationTimeout(t *testing.T) { } func TestEcho(t *testing.T) { + idAlice := "alice" + idBob := "bob" addr := "localhost:1234" srv := server.NewServer() go func() { @@ -170,7 +167,7 @@ func TestEcho(t *testing.T) { } }() - clientAlice := client.NewClient(addr, "alice") + clientAlice := client.NewClient(addr, idAlice) err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -182,7 +179,7 @@ func TestEcho(t *testing.T) { } }() - clientBob := client.NewClient(addr, "bob") + clientBob := client.NewClient(addr, idBob) err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -194,12 +191,12 @@ func TestEcho(t *testing.T) { } }() - connAliceToBob, err := clientAlice.BindChannel("bob") + connAliceToBob, err := clientAlice.OpenConn(idBob) if err != nil { t.Fatalf("failed to bind channel: %s", err) } - connBobToAlice, err := clientBob.BindChannel("alice") + connBobToAlice, err := clientBob.OpenConn(idAlice) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -262,7 +259,7 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() - _, err = clientAlice.BindChannel("bob") + _, err = clientAlice.OpenConn("bob") if err != nil { t.Errorf("failed to bind channel: %s", err) } @@ -292,7 +289,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to connect to server: %s", err) } - _, err = clientAlice.BindChannel("bob") + _, err = clientAlice.OpenConn("bob") if err != nil { t.Errorf("failed to bind channel: %s", err) } @@ -303,7 +300,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to connect to server: %s", err) } - chBob, err := clientBob.BindChannel("alice") + chBob, err := clientBob.OpenConn("alice") if err != nil { t.Errorf("failed to bind channel: %s", err) } @@ -320,7 +317,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to connect to server: %s", err) } - chAlice, err := clientAlice.BindChannel("bob") + chAlice, err := clientAlice.OpenConn("bob") if err != nil { t.Errorf("failed to bind channel: %s", err) } From 173ca25dacaecd0385b35b1d5f9379c7741de4df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Sun, 26 May 2024 22:14:33 +0200 Subject: [PATCH 011/199] Fix in client the close event --- relay/client/client.go | 67 +++++++++++++++--- relay/client/dialer/ws/client_conn.go | 6 +- relay/client/manager.go | 70 ++++++++++++++----- relay/cmd/main.go | 2 + .../listener/udp/{udp_conn.go => conn.go} | 0 relay/server/listener/udp/listener.go | 2 + .../listener/ws/{server_conn.go => conn.go} | 21 +++++- relay/server/listener/ws/listener.go | 3 +- relay/server/peer.go | 1 - relay/server/server.go | 50 ++++++++++--- relay/test/client_test.go | 32 ++++++--- relay/test/manager_test.go | 57 +++++++++++++++ 12 files changed, 257 insertions(+), 54 deletions(-) rename relay/server/listener/udp/{udp_conn.go => conn.go} (100%) rename relay/server/listener/ws/{server_conn.go => conn.go} (71%) create mode 100644 relay/test/manager_test.go diff --git a/relay/client/client.go b/relay/client/client.go index d574e04a1..e7492afee 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -1,6 +1,7 @@ package client import ( + "context" "fmt" "io" "net" @@ -29,20 +30,25 @@ type connContainer struct { type Client struct { log *log.Entry + ctx context.Context + ctxCancel context.CancelFunc serverAddress string hashedID []byte - conns map[string]*connContainer + conns map[string]*connContainer // todo handle it in thread safe way relayConn net.Conn relayConnState bool mu sync.Mutex } -func NewClient(serverAddress, peerID string) *Client { +func NewClient(ctx context.Context, serverAddress, peerID string) *Client { + ctx, ctxCancel := context.WithCancel(ctx) hashedID, hashedStringId := messages.HashID(peerID) return &Client{ log: log.WithField("client_id", hashedStringId), + ctx: ctx, + ctxCancel: ctxCancel, serverAddress: serverAddress, hashedID: hashedID, conns: make(map[string]*connContainer), @@ -51,7 +57,11 @@ func NewClient(serverAddress, peerID string) *Client { func (c *Client) Connect() error { c.mu.Lock() - defer c.mu.Unlock() + if c.relayConnState { + c.mu.Unlock() + return nil + } + conn, err := udp.Dial(c.serverAddress) if err != nil { return err @@ -68,18 +78,39 @@ func (c *Client) Connect() error { return err } - err = c.relayConn.SetReadDeadline(time.Time{}) - if err != nil { - log.Errorf("failed to reset read deadline: %s", err) - return err - } - c.relayConnState = true - go c.readLoop() + c.mu.Unlock() + + go func() { + <-c.ctx.Done() + cErr := c.close() + if cErr != nil { + log.Errorf("failed to close relay connection: %s", cErr) + } + }() + // blocking function + c.readLoop() + + c.mu.Lock() + + // close all Conn types + for _, container := range c.conns { + close(container.messages) + } + c.conns = make(map[string]*connContainer) + + c.mu.Unlock() + return nil } func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { + c.mu.Lock() + defer c.mu.Unlock() + if !c.relayConnState { + return nil, fmt.Errorf("relay connection is not established") + } + hashedID, hashedStringID := messages.HashID(dstPeerID) log.Infof("open connection to peer: %s", hashedStringID) messageBuffer := make(chan Msg, 2) @@ -93,6 +124,11 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { } func (c *Client) Close() error { + c.ctxCancel() + return c.close() +} + +func (c *Client) close() error { c.mu.Lock() defer c.mu.Unlock() @@ -101,11 +137,20 @@ func (c *Client) Close() error { } c.relayConnState = false + err := c.relayConn.Close() + return err } func (c *Client) handShake() error { + defer func() { + err := c.relayConn.SetReadDeadline(time.Time{}) + if err != nil { + log.Errorf("failed to reset read deadline: %s", err) + } + }() + msg, err := messages.MarshalHelloMsg(c.hashedID) if err != nil { log.Errorf("failed to marshal hello message: %s", err) @@ -145,7 +190,7 @@ func (c *Client) handShake() error { func (c *Client) readLoop() { defer func() { - c.log.Debugf("exit from read loop") + c.log.Tracef("exit from read loop") }() var errExit error var n int diff --git a/relay/client/dialer/ws/client_conn.go b/relay/client/dialer/ws/client_conn.go index 3298bd228..72a3fa9b4 100644 --- a/relay/client/dialer/ws/client_conn.go +++ b/relay/client/dialer/ws/client_conn.go @@ -6,6 +6,7 @@ import ( "time" "github.com/gorilla/websocket" + log "github.com/sirupsen/logrus" ) type Conn struct { @@ -51,6 +52,9 @@ func (c *Conn) SetDeadline(t time.Time) error { } func (c *Conn) Close() error { - _ = c.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second*5)) + err := c.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second*5)) + if err != nil { + log.Errorf("failed to close conn?: %s", err) + } return c.Conn.Close() } diff --git a/relay/client/manager.go b/relay/client/manager.go index 4d1aeca79..97793b3ea 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -2,42 +2,74 @@ package client import ( "context" + "net" "sync" + "time" + + log "github.com/sirupsen/logrus" ) type Manager struct { ctx context.Context - ctxCancel context.CancelFunc srvAddress string peerID string - wg sync.WaitGroup + reconnectTime time.Duration - clients map[string]*Client - clientsMutex sync.RWMutex + mu sync.Mutex + client *Client } func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { - ctx, cancel := context.WithCancel(ctx) return &Manager{ - ctx: ctx, - ctxCancel: cancel, - srvAddress: serverAddress, - peerID: peerID, - clients: make(map[string]*Client), + ctx: ctx, + srvAddress: serverAddress, + peerID: peerID, + reconnectTime: 5 * time.Second, } } -func (m *Manager) Teardown() { - m.ctxCancel() - m.wg.Wait() -} - -func (m *Manager) newSrvConnection(address string) { - if _, ok := m.clients[address]; ok { +func (m *Manager) Serve() { + ok := m.mu.TryLock() + if !ok { return } - // client := NewClient(address, m.peerID) - //err = client.Connect() + m.client = NewClient(m.ctx, m.srvAddress, m.peerID) + + go func() { + defer m.mu.Unlock() + + // todo this is not thread safe + for { + select { + case <-m.ctx.Done(): + return + default: + m.connect() + } + + select { + case <-m.ctx.Done(): + return + case <-time.After(2 * time.Second): //timeout + } + } + }() +} + +func (m *Manager) OpenConn(peerKey string) (net.Conn, error) { + // todo m.client nil check + return m.client.OpenConn(peerKey) +} + +// connect is blocking +func (m *Manager) connect() { + err := m.client.Connect() + if err != nil { + if m.ctx.Err() != nil { + return + } + log.Errorf("connection error with '%s': %s", m.srvAddress, err) + } } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index b89ab26ef..80e71486a 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -23,4 +23,6 @@ func main() { log.Errorf("failed to bind server: %s", err) os.Exit(1) } + + select {} } diff --git a/relay/server/listener/udp/udp_conn.go b/relay/server/listener/udp/conn.go similarity index 100% rename from relay/server/listener/udp/udp_conn.go rename to relay/server/listener/udp/conn.go diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go index 400b68a88..ebd1c53f1 100644 --- a/relay/server/listener/udp/listener.go +++ b/relay/server/listener/udp/listener.go @@ -61,6 +61,7 @@ func (l *Listener) Close() error { l.lock.Lock() defer l.lock.Unlock() + log.Infof("closing UDP server") if l.listener == nil { return nil } @@ -95,6 +96,7 @@ func (l *Listener) readLoop() { } pConn = NewConn(l.listener, addr) + log.Infof("new connection from: %s", pConn.RemoteAddr()) l.conns[addr.String()] = pConn go l.onAcceptFn(pConn) pConn.onNewMsg(buf[:n]) diff --git a/relay/server/listener/ws/server_conn.go b/relay/server/listener/ws/conn.go similarity index 71% rename from relay/server/listener/ws/server_conn.go rename to relay/server/listener/ws/conn.go index de3a16781..8734293ac 100644 --- a/relay/server/listener/ws/server_conn.go +++ b/relay/server/listener/ws/conn.go @@ -1,7 +1,9 @@ package ws import ( + "errors" "fmt" + "io" "sync" "time" @@ -24,7 +26,7 @@ func NewConn(wsConn *websocket.Conn) *Conn { func (c *Conn) Read(b []byte) (n int, err error) { t, r, err := c.NextReader() if err != nil { - return 0, err + return 0, ioErrHandling(err) } if t != websocket.BinaryMessage { @@ -32,7 +34,11 @@ func (c *Conn) Read(b []byte) (n int, err error) { return 0, fmt.Errorf("unexpected message type") } - return r.Read(b) + n, err = r.Read(b) + if err != nil { + return 0, ioErrHandling(err) + } + return n, err } func (c *Conn) Write(b []byte) (int, error) { @@ -55,3 +61,14 @@ func (c *Conn) SetDeadline(t time.Time) error { } return nil } + +func ioErrHandling(err error) error { + var wErr *websocket.CloseError + if !errors.As(err, &wErr) { + return err + } + if wErr.Code == websocket.CloseNormalClosure { + return io.EOF + } + return err +} diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index cee57e348..d93dfe0c3 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -42,7 +42,7 @@ func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { Addr: l.address, } - log.Debugf("WS server is listening on address: %s", l.address) + log.Infof("WS server is listening on address: %s", l.address) err := l.server.ListenAndServe() if errors.Is(err, http.ErrServerClosed) { return nil @@ -77,6 +77,7 @@ func (l *Listener) onAccept(writer http.ResponseWriter, request *http.Request) { return } conn := NewConn(wsConn) + log.Infof("new connection from: %s", conn.RemoteAddr()) l.acceptFn(conn) return } diff --git a/relay/server/peer.go b/relay/server/peer.go index d4b98b9b4..7af113079 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -16,7 +16,6 @@ type Peer struct { } func NewPeer(id []byte, conn net.Conn) *Peer { - log.Debugf("new peer: %v", id) stringID := messages.HashIDToString(id) return &Peer{ Log: log.WithField("peer_id", stringID), diff --git a/relay/server/server.go b/relay/server/server.go index 4b05975ec..f71c888e3 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -1,15 +1,18 @@ package server import ( + "errors" "fmt" "io" "net" + "sync" log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/relay/messages" "github.com/netbirdio/netbird/relay/server/listener" "github.com/netbirdio/netbird/relay/server/listener/udp" + "github.com/netbirdio/netbird/relay/server/listener/ws" ) // Server @@ -19,7 +22,8 @@ import ( type Server struct { store *Store - listener listener.Listener + UDPListener listener.Listener + WSListener listener.Listener } func NewServer() *Server { @@ -29,15 +33,45 @@ func NewServer() *Server { } func (r *Server) Listen(address string) error { - r.listener = udp.NewListener(address) - return r.listener.Listen(r.accept) + wg := sync.WaitGroup{} + wg.Add(2) + + r.WSListener = ws.NewListener(address) + var wslErr error + go func() { + defer wg.Done() + wslErr = r.WSListener.Listen(r.accept) + if wslErr != nil { + log.Errorf("failed to bind ws server: %s", wslErr) + } + }() + + r.UDPListener = udp.NewListener(address) + var udpLErr error + go func() { + defer wg.Done() + udpLErr = r.UDPListener.Listen(r.accept) + if udpLErr != nil { + log.Errorf("failed to bind ws server: %s", udpLErr) + } + }() + + err := errors.Join(wslErr, udpLErr) + return err } func (r *Server) Close() error { - if r.listener == nil { - return nil + var wErr error + if r.WSListener != nil { + wErr = r.WSListener.Close() } - return r.listener.Close() + + var uErr error + if r.UDPListener != nil { + uErr = r.UDPListener.Close() + } + err := errors.Join(wErr, uErr) + return err } func (r *Server) accept(conn net.Conn) { @@ -50,12 +84,12 @@ func (r *Server) accept(conn net.Conn) { } return } - peer.Log.Debugf("peer connected from: %s", conn.RemoteAddr()) + peer.Log.Infof("peer connected from: %s", conn.RemoteAddr()) r.store.AddPeer(peer) defer func() { - peer.Log.Debugf("teardown connection") r.store.DeletePeer(peer) + peer.Log.Infof("peer left") }() for { diff --git a/relay/test/client_test.go b/relay/test/client_test.go index 675962eef..1d3748abb 100644 --- a/relay/test/client_test.go +++ b/relay/test/client_test.go @@ -1,6 +1,7 @@ package test import ( + "context" "net" "os" "testing" @@ -20,6 +21,8 @@ func TestMain(m *testing.M) { } func TestClient(t *testing.T) { + ctx := context.Background() + addr := "localhost:1234" srv := server.NewServer() go func() { @@ -36,21 +39,21 @@ func TestClient(t *testing.T) { } }() - clientAlice := client.NewClient(addr, "alice") + clientAlice := client.NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientAlice.Close() - clientPlaceHolder := client.NewClient(addr, "clientPlaceHolder") + clientPlaceHolder := client.NewClient(ctx, addr, "clientPlaceHolder") err = clientPlaceHolder.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientPlaceHolder.Close() - clientBob := client.NewClient(addr, "bob") + clientBob := client.NewClient(ctx, addr, "bob") err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -87,6 +90,7 @@ func TestClient(t *testing.T) { } func TestRegistration(t *testing.T) { + ctx := context.Background() addr := "localhost:1234" srv := server.NewServer() go func() { @@ -103,7 +107,7 @@ func TestRegistration(t *testing.T) { } }() - clientAlice := client.NewClient(addr, "alice") + clientAlice := client.NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -117,6 +121,7 @@ func TestRegistration(t *testing.T) { } func TestRegistrationTimeout(t *testing.T) { + ctx := context.Background() udpListener, err := net.ListenUDP("udp", &net.UDPAddr{ Port: 1234, IP: net.ParseIP("0.0.0.0"), @@ -135,7 +140,7 @@ func TestRegistrationTimeout(t *testing.T) { } defer tcpListener.Close() - clientAlice := client.NewClient("127.0.0.1:1234", "alice") + clientAlice := client.NewClient(ctx, "127.0.0.1:1234", "alice") err = clientAlice.Connect() if err == nil { t.Errorf("failed to connect to server: %s", err) @@ -149,6 +154,7 @@ func TestRegistrationTimeout(t *testing.T) { } func TestEcho(t *testing.T) { + ctx := context.Background() idAlice := "alice" idBob := "bob" addr := "localhost:1234" @@ -167,7 +173,7 @@ func TestEcho(t *testing.T) { } }() - clientAlice := client.NewClient(addr, idAlice) + clientAlice := client.NewClient(ctx, addr, idAlice) err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -179,7 +185,7 @@ func TestEcho(t *testing.T) { } }() - clientBob := client.NewClient(addr, idBob) + clientBob := client.NewClient(ctx, addr, idBob) err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -229,6 +235,8 @@ func TestEcho(t *testing.T) { } func TestBindToUnavailabePeer(t *testing.T) { + ctx := context.Background() + addr := "localhost:1234" srv := server.NewServer() go func() { @@ -246,7 +254,7 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() - clientAlice := client.NewClient(addr, "alice") + clientAlice := client.NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -266,6 +274,8 @@ func TestBindToUnavailabePeer(t *testing.T) { } func TestBindReconnect(t *testing.T) { + ctx := context.Background() + addr := "localhost:1234" srv := server.NewServer() go func() { @@ -283,7 +293,7 @@ func TestBindReconnect(t *testing.T) { } }() - clientAlice := client.NewClient(addr, "alice") + clientAlice := client.NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -294,7 +304,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to bind channel: %s", err) } - clientBob := client.NewClient(addr, "bob") + clientBob := client.NewClient(ctx, addr, "bob") err = clientBob.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -311,7 +321,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to close client: %s", err) } - clientAlice = client.NewClient(addr, "alice") + clientAlice = client.NewClient(ctx, addr, "alice") err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) diff --git a/relay/test/manager_test.go b/relay/test/manager_test.go new file mode 100644 index 000000000..09edeff84 --- /dev/null +++ b/relay/test/manager_test.go @@ -0,0 +1,57 @@ +package test + +import ( + "context" + "testing" + "time" + + "github.com/netbirdio/netbird/relay/client" + "github.com/netbirdio/netbird/relay/server" +) + +func TestManager(t *testing.T) { + addr := "localhost:1239" + + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cm := client.NewManager(ctx, addr, "me") + cm.Serve() + + // wait for the relay handshake to complete + time.Sleep(1 * time.Second) + conn, err := cm.OpenConn("remotepeer") + if err != nil { + t.Errorf("failed to open connection: %s", err) + } + + readCtx, readCancel := context.WithCancel(context.Background()) + defer readCancel() + go func() { + _, _ = conn.Read(make([]byte, 1)) + readCancel() + }() + + cancel() + + select { + case <-time.After(2 * time.Second): + t.Errorf("client peer conn did not close automatically") + case <-readCtx.Done(): + // conn exited well + } +} From b4aa7e50f995abb85e009bda3ade2cf07c9bb4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 27 May 2024 09:42:27 +0200 Subject: [PATCH 012/199] Close sockets on server cmd --- relay/cmd/main.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 80e71486a..ba3ff47f7 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -2,20 +2,26 @@ package main import ( "os" - - "github.com/netbirdio/netbird/util" + "os/signal" + "syscall" log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/util" ) func init() { util.InitLog("trace", "console") } -func main() { +func waitForExitSignal() { + osSigs := make(chan os.Signal, 1) + signal.Notify(osSigs, syscall.SIGINT, syscall.SIGTERM) + _ = <-osSigs +} +func main() { address := "10.145.236.1:1235" srv := server.NewServer() err := srv.Listen(address) @@ -24,5 +30,11 @@ func main() { os.Exit(1) } - select {} + waitForExitSignal() + + err = srv.Close() + if err != nil { + log.Errorf("failed to close server: %s", err) + os.Exit(1) + } } From 645a1f31a7cc1707c4f346a2cb8e134a9d0a527b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 27 May 2024 10:25:08 +0200 Subject: [PATCH 013/199] Fix writing/reading to a closed conn --- relay/client/client.go | 50 ++++++++++++++++++++++++++++----------- relay/client/conn.go | 20 +++++++++------- relay/test/client_test.go | 48 +++++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 23 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index e7492afee..44325e1f9 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -39,6 +39,7 @@ type Client struct { relayConn net.Conn relayConnState bool + wgRelayConn sync.WaitGroup mu sync.Mutex } @@ -81,6 +82,9 @@ func (c *Client) Connect() error { c.relayConnState = true c.mu.Unlock() + c.wgRelayConn.Add(1) + go c.readLoop() + go func() { <-c.ctx.Done() cErr := c.close() @@ -88,18 +92,6 @@ func (c *Client) Connect() error { log.Errorf("failed to close relay connection: %s", cErr) } }() - // blocking function - c.readLoop() - - c.mu.Lock() - - // close all Conn types - for _, container := range c.conns { - close(container.messages) - } - c.conns = make(map[string]*connContainer) - - c.mu.Unlock() return nil } @@ -114,7 +106,7 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { hashedID, hashedStringID := messages.HashID(dstPeerID) log.Infof("open connection to peer: %s", hashedStringID) messageBuffer := make(chan Msg, 2) - conn := NewConn(c, hashedID, c.generateConnReaderFN(messageBuffer)) + conn := NewConn(c, hashedID, hashedStringID, c.generateConnReaderFN(messageBuffer)) c.conns[hashedStringID] = &connContainer{ conn, @@ -140,6 +132,14 @@ func (c *Client) close() error { err := c.relayConn.Close() + c.wgRelayConn.Wait() + + // close all Conn types + for _, container := range c.conns { + close(container.messages) + } + c.conns = make(map[string]*connContainer) + return err } @@ -191,6 +191,7 @@ func (c *Client) handShake() error { func (c *Client) readLoop() { defer func() { c.log.Tracef("exit from read loop") + c.wgRelayConn.Done() }() var errExit error var n int @@ -237,7 +238,14 @@ func (c *Client) readLoop() { } } -func (c *Client) writeTo(dstID []byte, payload []byte) (int, error) { +func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { + c.mu.Lock() + _, ok := c.conns[id] + if !ok { + c.mu.Unlock() + return 0, io.EOF + } + c.mu.Unlock() msg := messages.MarshalTransportMsg(dstID, payload) n, err := c.relayConn.Write(msg) if err != nil { @@ -262,3 +270,17 @@ func (c *Client) generateConnReaderFN(msgChannel chan Msg) func(b []byte) (n int return n, nil } } + +func (c *Client) closeConn(id string) error { + c.mu.Lock() + defer c.mu.Unlock() + + conn, ok := c.conns[id] + if !ok { + return fmt.Errorf("connection already closed") + } + close(conn.messages) + delete(c.conns, id) + + return nil +} diff --git a/relay/client/conn.go b/relay/client/conn.go index aea07ff9b..647b0fae4 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -6,23 +6,25 @@ import ( ) type Conn struct { - client *Client - dstID []byte - readerFn func(b []byte) (n int, err error) + client *Client + dstID []byte + dstStringID string + readerFn func(b []byte) (n int, err error) } -func NewConn(client *Client, dstID []byte, readerFn func(b []byte) (n int, err error)) *Conn { +func NewConn(client *Client, dstID []byte, dstStringID string, readerFn func(b []byte) (n int, err error)) *Conn { c := &Conn{ - client: client, - dstID: dstID, - readerFn: readerFn, + client: client, + dstID: dstID, + dstStringID: dstStringID, + readerFn: readerFn, } return c } func (c *Conn) Write(p []byte) (n int, err error) { - return c.client.writeTo(c.dstID, p) + return c.client.writeTo(c.dstStringID, c.dstID, p) } func (c *Conn) Read(b []byte) (n int, err error) { @@ -30,7 +32,7 @@ func (c *Conn) Read(b []byte) (n int, err error) { } func (c *Conn) Close() error { - return nil + return c.client.closeConn(c.dstStringID) } func (c *Conn) LocalAddr() net.Addr { diff --git a/relay/test/client_test.go b/relay/test/client_test.go index 1d3748abb..f70048d22 100644 --- a/relay/test/client_test.go +++ b/relay/test/client_test.go @@ -354,3 +354,51 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to close client: %s", err) } } + +func TestCloseConn(t *testing.T) { + ctx := context.Background() + + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Errorf("failed to bind server: %s", err) + } + }() + + defer func() { + log.Infof("closing server") + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := client.NewClient(ctx, addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + log.Infof("closing connection") + err = conn.Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + _, err = conn.Write([]byte("hello")) + if err == nil { + t.Errorf("unexpected writing from closed connection") + } +} From 076ce69a2427f39f086b2a9f11a40e2880969557 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 28 May 2024 01:00:25 +0200 Subject: [PATCH 014/199] Add reconnect logic --- relay/client/client.go | 146 ++++++++++++++++++-------- relay/{test => client}/client_test.go | 117 ++++++++++++++++++--- 2 files changed, 206 insertions(+), 57 deletions(-) rename relay/{test => client}/client_test.go (76%) diff --git a/relay/client/client.go b/relay/client/client.go index 44325e1f9..2b722c56e 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + "github.com/netbirdio/netbird/relay/client/dialer/udp" "io" "net" "sync" @@ -10,7 +11,6 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/client/dialer/udp" "github.com/netbirdio/netbird/relay/messages" ) @@ -19,6 +19,10 @@ const ( serverResponseTimeout = 8 * time.Second ) +var ( + reconnectingTimeout = 5 * time.Second +) + type Msg struct { buf []byte } @@ -35,56 +39,51 @@ type Client struct { serverAddress string hashedID []byte - conns map[string]*connContainer // todo handle it in thread safe way + relayConnIsEstablished bool + conns map[string]*connContainer + connsMutext sync.Mutex // protect conns and relayConnIsEstablished bool - relayConn net.Conn - relayConnState bool - wgRelayConn sync.WaitGroup - mu sync.Mutex + relayConn net.Conn + serviceIsRunning bool + wgRelayConn sync.WaitGroup + mu sync.Mutex + onDisconnected chan struct{} } func NewClient(ctx context.Context, serverAddress, peerID string) *Client { ctx, ctxCancel := context.WithCancel(ctx) hashedID, hashedStringId := messages.HashID(peerID) return &Client{ - log: log.WithField("client_id", hashedStringId), - ctx: ctx, - ctxCancel: ctxCancel, - serverAddress: serverAddress, - hashedID: hashedID, - conns: make(map[string]*connContainer), + log: log.WithField("client_id", hashedStringId), + ctx: ctx, + ctxCancel: ctxCancel, + serverAddress: serverAddress, + hashedID: hashedID, + conns: make(map[string]*connContainer), + onDisconnected: make(chan struct{}), } } func (c *Client) Connect() error { c.mu.Lock() - if c.relayConnState { + if c.serviceIsRunning { c.mu.Unlock() return nil } - conn, err := udp.Dial(c.serverAddress) + err := c.connect() if err != nil { - return err - } - c.relayConn = conn - - err = c.handShake() - if err != nil { - cErr := conn.Close() - if cErr != nil { - log.Errorf("failed to close connection: %s", cErr) - } - c.relayConn = nil + c.mu.Unlock() return err } - c.relayConnState = true - c.mu.Unlock() + c.serviceIsRunning = true c.wgRelayConn.Add(1) go c.readLoop() + c.mu.Unlock() + go func() { <-c.ctx.Done() cErr := c.close() @@ -93,13 +92,50 @@ func (c *Client) Connect() error { } }() + go c.reconnectGuard() + return nil } +func (c *Client) reconnectGuard() { + for { + c.wgRelayConn.Wait() + + c.mu.Lock() + if !c.serviceIsRunning { + c.mu.Unlock() + return + } + + log.Infof("reconnecting to relay server") + err := c.connect() + if err != nil { + log.Errorf("failed to reconnect to relay server: %s", err) + c.mu.Unlock() + time.Sleep(reconnectingTimeout) + continue + } + log.Infof("reconnected to relay server") + c.wgRelayConn.Add(1) + go c.readLoop() + + c.mu.Unlock() + + } +} + func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { c.mu.Lock() defer c.mu.Unlock() - if !c.relayConnState { + + c.connsMutext.Lock() + defer c.connsMutext.Unlock() + + if !c.relayConnIsEstablished { + return nil, fmt.Errorf("relay connection is not established") + } + + if !c.serviceIsRunning { return nil, fmt.Errorf("relay connection is not established") } @@ -120,26 +156,41 @@ func (c *Client) Close() error { return c.close() } +func (c *Client) connect() error { + conn, err := udp.Dial(c.serverAddress) + if err != nil { + return err + } + c.relayConn = conn + + err = c.handShake() + if err != nil { + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection: %s", cErr) + } + c.relayConn = nil + return err + } + + c.relayConnIsEstablished = true + return nil +} + func (c *Client) close() error { c.mu.Lock() defer c.mu.Unlock() - if !c.relayConnState { + if !c.serviceIsRunning { return nil } - c.relayConnState = false + c.serviceIsRunning = false err := c.relayConn.Close() c.wgRelayConn.Wait() - // close all Conn types - for _, container := range c.conns { - close(container.messages) - } - c.conns = make(map[string]*connContainer) - return err } @@ -189,17 +240,13 @@ func (c *Client) handShake() error { } func (c *Client) readLoop() { - defer func() { - c.log.Tracef("exit from read loop") - c.wgRelayConn.Done() - }() var errExit error var n int for { buf := make([]byte, bufferSize) n, errExit = c.relayConn.Read(buf) if errExit != nil { - if c.relayConnState { + if c.serviceIsRunning { c.log.Debugf("failed to read message from relay server: %s", errExit) } break @@ -232,10 +279,20 @@ func (c *Client) readLoop() { } } - if c.relayConnState { - c.log.Errorf("failed to read message from relay server: %s", errExit) + if c.serviceIsRunning { _ = c.relayConn.Close() } + + c.connsMutext.Lock() + c.relayConnIsEstablished = false + for _, container := range c.conns { + close(container.messages) + } + c.conns = make(map[string]*connContainer) + c.connsMutext.Unlock() + + c.log.Tracef("exit from read loop") + c.wgRelayConn.Done() } func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { @@ -275,6 +332,9 @@ func (c *Client) closeConn(id string) error { c.mu.Lock() defer c.mu.Unlock() + c.connsMutext.Lock() + defer c.connsMutext.Unlock() + conn, ok := c.conns[id] if !ok { return fmt.Errorf("connection already closed") diff --git a/relay/test/client_test.go b/relay/client/client_test.go similarity index 76% rename from relay/test/client_test.go rename to relay/client/client_test.go index f70048d22..c180b922b 100644 --- a/relay/test/client_test.go +++ b/relay/client/client_test.go @@ -1,16 +1,16 @@ -package test +package client import ( "context" "net" "os" "testing" + "time" log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/util" - "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/relay/server" ) @@ -39,21 +39,21 @@ func TestClient(t *testing.T) { } }() - clientAlice := client.NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientAlice.Close() - clientPlaceHolder := client.NewClient(ctx, addr, "clientPlaceHolder") + clientPlaceHolder := NewClient(ctx, addr, "clientPlaceHolder") err = clientPlaceHolder.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientPlaceHolder.Close() - clientBob := client.NewClient(ctx, addr, "bob") + clientBob := NewClient(ctx, addr, "bob") err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -107,7 +107,7 @@ func TestRegistration(t *testing.T) { } }() - clientAlice := client.NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -140,7 +140,7 @@ func TestRegistrationTimeout(t *testing.T) { } defer tcpListener.Close() - clientAlice := client.NewClient(ctx, "127.0.0.1:1234", "alice") + clientAlice := NewClient(ctx, "127.0.0.1:1234", "alice") err = clientAlice.Connect() if err == nil { t.Errorf("failed to connect to server: %s", err) @@ -173,7 +173,7 @@ func TestEcho(t *testing.T) { } }() - clientAlice := client.NewClient(ctx, addr, idAlice) + clientAlice := NewClient(ctx, addr, idAlice) err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -185,7 +185,7 @@ func TestEcho(t *testing.T) { } }() - clientBob := client.NewClient(ctx, addr, idBob) + clientBob := NewClient(ctx, addr, idBob) err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -254,7 +254,7 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() - clientAlice := client.NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -293,7 +293,7 @@ func TestBindReconnect(t *testing.T) { } }() - clientAlice := client.NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -304,7 +304,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to bind channel: %s", err) } - clientBob := client.NewClient(ctx, addr, "bob") + clientBob := NewClient(ctx, addr, "bob") err = clientBob.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -321,7 +321,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to close client: %s", err) } - clientAlice = client.NewClient(ctx, addr, "alice") + clientAlice = NewClient(ctx, addr, "alice") err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -375,7 +375,7 @@ func TestCloseConn(t *testing.T) { } }() - clientAlice := client.NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -402,3 +402,92 @@ func TestCloseConn(t *testing.T) { t.Errorf("unexpected writing from closed connection") } } + +func TestAutoReconnect(t *testing.T) { + ctx := context.Background() + + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Errorf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := NewClient(ctx, addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + _ = clientAlice.relayConn.Close() + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + log.Infof("waiting for reconnection") + time.Sleep(reconnectingTimeout) + + _, err = clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to open channel: %s", err) + } +} + +func TestCloseRelayConn(t *testing.T) { + ctx := context.Background() + + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Errorf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := NewClient(ctx, addr, "alice") + err := clientAlice.Connect() + if err != nil { + t.Errorf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn("bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + _ = clientAlice.relayConn.Close() + + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + _, err = clientAlice.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } +} From 6d627f192377c2994014fbd464316d753e49ea3e Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 28 May 2024 01:27:53 +0200 Subject: [PATCH 015/199] Code cleaning --- relay/client/client.go | 74 ++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 2b722c56e..69e9b391b 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -39,15 +39,15 @@ type Client struct { serverAddress string hashedID []byte - relayConnIsEstablished bool - conns map[string]*connContainer - connsMutext sync.Mutex // protect conns and relayConnIsEstablished bool + readyToOpenConns bool + conns map[string]*connContainer + connsMutext sync.Mutex // protect conns and readyToOpenConns bool - relayConn net.Conn - serviceIsRunning bool - wgRelayConn sync.WaitGroup - mu sync.Mutex - onDisconnected chan struct{} + relayConn net.Conn + serviceIsRunning bool + serviceIsRunningMutex sync.Mutex + wgReadLoop sync.WaitGroup + onDisconnected chan struct{} } func NewClient(ctx context.Context, serverAddress, peerID string) *Client { @@ -65,24 +65,24 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { } func (c *Client) Connect() error { - c.mu.Lock() + c.serviceIsRunningMutex.Lock() if c.serviceIsRunning { - c.mu.Unlock() + c.serviceIsRunningMutex.Unlock() return nil } err := c.connect() if err != nil { - c.mu.Unlock() + c.serviceIsRunningMutex.Unlock() return err } c.serviceIsRunning = true - c.wgRelayConn.Add(1) + c.wgReadLoop.Add(1) go c.readLoop() - c.mu.Unlock() + c.serviceIsRunningMutex.Unlock() go func() { <-c.ctx.Done() @@ -99,11 +99,11 @@ func (c *Client) Connect() error { func (c *Client) reconnectGuard() { for { - c.wgRelayConn.Wait() + c.wgReadLoop.Wait() - c.mu.Lock() + c.serviceIsRunningMutex.Lock() if !c.serviceIsRunning { - c.mu.Unlock() + c.serviceIsRunningMutex.Unlock() return } @@ -111,31 +111,24 @@ func (c *Client) reconnectGuard() { err := c.connect() if err != nil { log.Errorf("failed to reconnect to relay server: %s", err) - c.mu.Unlock() + c.serviceIsRunningMutex.Unlock() time.Sleep(reconnectingTimeout) continue } log.Infof("reconnected to relay server") - c.wgRelayConn.Add(1) + c.wgReadLoop.Add(1) go c.readLoop() - c.mu.Unlock() + c.serviceIsRunningMutex.Unlock() } } func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { - c.mu.Lock() - defer c.mu.Unlock() - c.connsMutext.Lock() defer c.connsMutext.Unlock() - if !c.relayConnIsEstablished { - return nil, fmt.Errorf("relay connection is not established") - } - - if !c.serviceIsRunning { + if !c.readyToOpenConns { return nil, fmt.Errorf("relay connection is not established") } @@ -152,6 +145,12 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { } func (c *Client) Close() error { + c.serviceIsRunningMutex.Lock() + if !c.serviceIsRunning { + c.serviceIsRunningMutex.Unlock() + return nil + } + c.ctxCancel() return c.close() } @@ -173,13 +172,13 @@ func (c *Client) connect() error { return err } - c.relayConnIsEstablished = true + c.readyToOpenConns = true return nil } func (c *Client) close() error { - c.mu.Lock() - defer c.mu.Unlock() + c.serviceIsRunningMutex.Lock() + defer c.serviceIsRunningMutex.Unlock() if !c.serviceIsRunning { return nil @@ -189,7 +188,7 @@ func (c *Client) close() error { err := c.relayConn.Close() - c.wgRelayConn.Wait() + c.wgReadLoop.Wait() return err } @@ -284,7 +283,7 @@ func (c *Client) readLoop() { } c.connsMutext.Lock() - c.relayConnIsEstablished = false + c.readyToOpenConns = false for _, container := range c.conns { close(container.messages) } @@ -292,17 +291,17 @@ func (c *Client) readLoop() { c.connsMutext.Unlock() c.log.Tracef("exit from read loop") - c.wgRelayConn.Done() + c.wgReadLoop.Done() } func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { - c.mu.Lock() + c.connsMutext.Lock() _, ok := c.conns[id] if !ok { - c.mu.Unlock() + c.connsMutext.Unlock() return 0, io.EOF } - c.mu.Unlock() + c.connsMutext.Unlock() msg := messages.MarshalTransportMsg(dstID, payload) n, err := c.relayConn.Write(msg) if err != nil { @@ -329,9 +328,6 @@ func (c *Client) generateConnReaderFN(msgChannel chan Msg) func(b []byte) (n int } func (c *Client) closeConn(id string) error { - c.mu.Lock() - defer c.mu.Unlock() - c.connsMutext.Lock() defer c.connsMutext.Unlock() From 7cc3964a4dcf9bfb33e48c7a8831c580aa294b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 29 May 2024 16:11:58 +0200 Subject: [PATCH 016/199] Use mux for http server Without it can not start multiple http server instances for unit tests --- relay/server/listener/ws/listener.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index d93dfe0c3..b53e97505 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -36,10 +36,12 @@ func NewListener(address string) listener.Listener { // Listen todo: prevent multiple call func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { l.acceptFn = acceptFn - http.HandleFunc("/", l.onAccept) + mux := http.NewServeMux() + mux.HandleFunc("/", l.onAccept) l.server = &http.Server{ - Addr: l.address, + Addr: l.address, + Handler: mux, } log.Infof("WS server is listening on address: %s", l.address) From 4ff069a102718ce04d6132924172c8e39eafde24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 29 May 2024 16:40:26 +0200 Subject: [PATCH 017/199] Support multiple server --- relay/client/client.go | 92 ++++++++++++++++++++++-------- relay/client/manager.go | 106 ++++++++++++++++++++--------------- relay/client/manager_test.go | 97 ++++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 70 deletions(-) create mode 100644 relay/client/manager_test.go diff --git a/relay/client/client.go b/relay/client/client.go index 69e9b391b..c2f6f9c71 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -3,7 +3,6 @@ package client import ( "context" "fmt" - "github.com/netbirdio/netbird/relay/client/dialer/udp" "io" "net" "sync" @@ -11,6 +10,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/relay/client/dialer/udp" "github.com/netbirdio/netbird/relay/messages" ) @@ -48,6 +48,8 @@ type Client struct { serviceIsRunningMutex sync.Mutex wgReadLoop sync.WaitGroup onDisconnected chan struct{} + + remoteAddr net.Addr } func NewClient(ctx context.Context, serverAddress, peerID string) *Client { @@ -97,31 +99,35 @@ func (c *Client) Connect() error { return nil } -func (c *Client) reconnectGuard() { - for { - c.wgReadLoop.Wait() - - c.serviceIsRunningMutex.Lock() - if !c.serviceIsRunning { - c.serviceIsRunningMutex.Unlock() - return - } - - log.Infof("reconnecting to relay server") - err := c.connect() - if err != nil { - log.Errorf("failed to reconnect to relay server: %s", err) - c.serviceIsRunningMutex.Unlock() - time.Sleep(reconnectingTimeout) - continue - } - log.Infof("reconnected to relay server") - c.wgReadLoop.Add(1) - go c.readLoop() - +func (c *Client) ConnectWithoutReconnect() error { + c.serviceIsRunningMutex.Lock() + if c.serviceIsRunning { c.serviceIsRunningMutex.Unlock() - + return nil } + + err := c.connect() + if err != nil { + c.serviceIsRunningMutex.Unlock() + return err + } + + c.serviceIsRunning = true + + c.wgReadLoop.Add(1) + go c.readLoop() + + c.serviceIsRunningMutex.Unlock() + + go func() { + <-c.ctx.Done() + cErr := c.close() + if cErr != nil { + log.Errorf("failed to close relay connection: %s", cErr) + } + }() + + return nil } func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { @@ -144,6 +150,15 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { return conn, nil } +func (c *Client) RelayRemoteAddress() (net.Addr, error) { + c.serviceIsRunningMutex.Lock() + defer c.serviceIsRunningMutex.Unlock() + if c.remoteAddr == nil { + return nil, fmt.Errorf("relay connection is not established") + } + return c.remoteAddr, nil +} + func (c *Client) Close() error { c.serviceIsRunningMutex.Lock() if !c.serviceIsRunning { @@ -172,6 +187,8 @@ func (c *Client) connect() error { return err } + c.remoteAddr = conn.RemoteAddr() + c.readyToOpenConns = true return nil } @@ -193,6 +210,33 @@ func (c *Client) close() error { return err } +func (c *Client) reconnectGuard() { + for { + c.wgReadLoop.Wait() + + c.serviceIsRunningMutex.Lock() + if !c.serviceIsRunning { + c.serviceIsRunningMutex.Unlock() + return + } + + log.Infof("reconnecting to relay server") + err := c.connect() + if err != nil { + log.Errorf("failed to reconnect to relay server: %s", err) + c.serviceIsRunningMutex.Unlock() + time.Sleep(reconnectingTimeout) + continue + } + log.Infof("reconnected to relay server") + c.wgReadLoop.Add(1) + go c.readLoop() + + c.serviceIsRunningMutex.Unlock() + + } +} + func (c *Client) handShake() error { defer func() { err := c.relayConn.SetReadDeadline(time.Time{}) diff --git a/relay/client/manager.go b/relay/client/manager.go index 97793b3ea..686b4ac4f 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -2,74 +2,88 @@ package client import ( "context" + "fmt" "net" - "sync" - "time" - - log "github.com/sirupsen/logrus" ) +// Manager todo: thread safe type Manager struct { ctx context.Context srvAddress string peerID string - reconnectTime time.Duration + relayClient *Client - mu sync.Mutex - client *Client + relayClients map[string]*Client } func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { return &Manager{ - ctx: ctx, - srvAddress: serverAddress, - peerID: peerID, - reconnectTime: 5 * time.Second, + ctx: ctx, + srvAddress: serverAddress, + peerID: peerID, + relayClients: make(map[string]*Client), } } -func (m *Manager) Serve() { - ok := m.mu.TryLock() - if !ok { - return +func (m *Manager) Serve() error { + m.relayClient = NewClient(m.ctx, m.srvAddress, m.peerID) + err := m.relayClient.Connect() + if err != nil { + return err } + return nil +} - m.client = NewClient(m.ctx, m.srvAddress, m.peerID) - - go func() { - defer m.mu.Unlock() - - // todo this is not thread safe - for { - select { - case <-m.ctx.Done(): - return - default: - m.connect() - } - - select { - case <-m.ctx.Done(): - return - case <-time.After(2 * time.Second): //timeout - } - } - }() +func (m *Manager) RelayAddress() (net.Addr, error) { + if m.relayClient == nil { + return nil, fmt.Errorf("relay client not connected") + } + return m.relayClient.RelayRemoteAddress() } func (m *Manager) OpenConn(peerKey string) (net.Conn, error) { - // todo m.client nil check - return m.client.OpenConn(peerKey) + if m.relayClient == nil { + return nil, fmt.Errorf("relay client not connected") + } + + rAddr, err := m.relayClient.RelayRemoteAddress() + if err != nil { + return nil, fmt.Errorf("relay client not connected") + } + + return m.OpenConnTo(rAddr.String(), peerKey) } -// connect is blocking -func (m *Manager) connect() { - err := m.client.Connect() - if err != nil { - if m.ctx.Err() != nil { - return - } - log.Errorf("connection error with '%s': %s", m.srvAddress, err) +func (m *Manager) OpenConnTo(serverAddress, peerKey string) (net.Conn, error) { + if m.relayClient == nil { + return nil, fmt.Errorf("relay client not connected") } + rAddr, err := m.relayClient.RelayRemoteAddress() + if err != nil { + return nil, fmt.Errorf("relay client not connected") + } + + if rAddr.String() == serverAddress { + return m.relayClient.OpenConn(peerKey) + } + + relayClient, ok := m.relayClients[serverAddress] + if ok { + return relayClient.OpenConn(peerKey) + } + + relayClient = NewClient(m.ctx, serverAddress, m.peerID) + err = relayClient.ConnectWithoutReconnect() + if err != nil { + return nil, err + } + + conn, err := relayClient.OpenConn(peerKey) + if err != nil { + return nil, err + } + + m.relayClients[serverAddress] = relayClient + return conn, nil } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go new file mode 100644 index 000000000..6c8edd534 --- /dev/null +++ b/relay/client/manager_test.go @@ -0,0 +1,97 @@ +package client + +import ( + "context" + "testing" + + "github.com/netbirdio/netbird/relay/server" +) + +func TestNewManager(t *testing.T) { + ctx := context.Background() + idAlice := "alice" + idBob := "bob" + addr1 := "localhost:1234" + srv1 := server.NewServer() + go func() { + err := srv1.Listen(addr1) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv1.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + addr2 := "localhost:2234" + srv2 := server.NewServer() + go func() { + err := srv2.Listen(addr2) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv2.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + clientAlice := NewManager(ctx, addr1, idAlice) + err := clientAlice.Serve() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + + clientBob := NewManager(ctx, addr2, idBob) + err = clientBob.Serve() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + + bobsSrvAddr, err := clientBob.RelayAddress() + if err != nil { + t.Fatalf("failed to get relay address: %s", err) + } + connAliceToBob, err := clientAlice.OpenConnTo(bobsSrvAddr.String(), idBob) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + connBobToAlice, err := clientBob.OpenConn(idAlice) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + payload := "hello bob, I am alice" + _, err = connAliceToBob.Write([]byte(payload)) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + buf := make([]byte, 65535) + n, err := connBobToAlice.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + _, err = connBobToAlice.Write(buf[:n]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + n, err = connAliceToBob.Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + if payload != string(buf[:n]) { + t.Fatalf("expected %s, got %s", payload, string(buf[:n])) + } +} From fd4ad15c835f3d6d1196bf41c90399b33a045177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Sat, 1 Jun 2024 11:25:00 +0200 Subject: [PATCH 018/199] Move reconnection logic to separated struct --- relay/client/client.go | 115 ++++++++++++----------------------- relay/client/guard.go | 33 ++++++++++ relay/client/manager.go | 74 ++++++++++++---------- relay/client/manager_test.go | 8 ++- 4 files changed, 121 insertions(+), 109 deletions(-) create mode 100644 relay/client/guard.go diff --git a/relay/client/client.go b/relay/client/client.go index c2f6f9c71..f57bb2b92 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -19,10 +19,6 @@ const ( serverResponseTimeout = 8 * time.Second ) -var ( - reconnectingTimeout = 5 * time.Second -) - type Msg struct { buf []byte } @@ -47,35 +43,42 @@ type Client struct { serviceIsRunning bool serviceIsRunningMutex sync.Mutex wgReadLoop sync.WaitGroup - onDisconnected chan struct{} remoteAddr net.Addr + + onDisconnectListener func() + listenerMutex sync.Mutex } func NewClient(ctx context.Context, serverAddress, peerID string) *Client { ctx, ctxCancel := context.WithCancel(ctx) hashedID, hashedStringId := messages.HashID(peerID) return &Client{ - log: log.WithField("client_id", hashedStringId), - ctx: ctx, - ctxCancel: ctxCancel, - serverAddress: serverAddress, - hashedID: hashedID, - conns: make(map[string]*connContainer), - onDisconnected: make(chan struct{}), + log: log.WithField("client_id", hashedStringId), + ctx: ctx, + ctxCancel: ctxCancel, + serverAddress: serverAddress, + hashedID: hashedID, + conns: make(map[string]*connContainer), } } +func (c *Client) SetOnDisconnectListener(fn func()) { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + c.onDisconnectListener = fn +} + func (c *Client) Connect() error { c.serviceIsRunningMutex.Lock() + defer c.serviceIsRunningMutex.Unlock() + if c.serviceIsRunning { - c.serviceIsRunningMutex.Unlock() return nil } err := c.connect() if err != nil { - c.serviceIsRunningMutex.Unlock() return err } @@ -84,41 +87,6 @@ func (c *Client) Connect() error { c.wgReadLoop.Add(1) go c.readLoop() - c.serviceIsRunningMutex.Unlock() - - go func() { - <-c.ctx.Done() - cErr := c.close() - if cErr != nil { - log.Errorf("failed to close relay connection: %s", cErr) - } - }() - - go c.reconnectGuard() - - return nil -} - -func (c *Client) ConnectWithoutReconnect() error { - c.serviceIsRunningMutex.Lock() - if c.serviceIsRunning { - c.serviceIsRunningMutex.Unlock() - return nil - } - - err := c.connect() - if err != nil { - c.serviceIsRunningMutex.Unlock() - return err - } - - c.serviceIsRunning = true - - c.wgReadLoop.Add(1) - go c.readLoop() - - c.serviceIsRunningMutex.Unlock() - go func() { <-c.ctx.Done() cErr := c.close() @@ -210,33 +178,6 @@ func (c *Client) close() error { return err } -func (c *Client) reconnectGuard() { - for { - c.wgReadLoop.Wait() - - c.serviceIsRunningMutex.Lock() - if !c.serviceIsRunning { - c.serviceIsRunningMutex.Unlock() - return - } - - log.Infof("reconnecting to relay server") - err := c.connect() - if err != nil { - log.Errorf("failed to reconnect to relay server: %s", err) - c.serviceIsRunningMutex.Unlock() - time.Sleep(reconnectingTimeout) - continue - } - log.Infof("reconnected to relay server") - c.wgReadLoop.Add(1) - go c.readLoop() - - c.serviceIsRunningMutex.Unlock() - - } -} - func (c *Client) handShake() error { defer func() { err := c.relayConn.SetReadDeadline(time.Time{}) @@ -322,6 +263,8 @@ func (c *Client) readLoop() { } } + c.notifyDisconnected() + if c.serviceIsRunning { _ = c.relayConn.Close() } @@ -384,3 +327,23 @@ func (c *Client) closeConn(id string) error { return nil } + +func (c *Client) onDisconnect() { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + + if c.onDisconnectListener == nil { + return + } + c.onDisconnectListener() +} + +func (c *Client) notifyDisconnected() { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + + if c.onDisconnectListener == nil { + return + } + go c.onDisconnectListener() +} diff --git a/relay/client/guard.go b/relay/client/guard.go new file mode 100644 index 000000000..47a6ff722 --- /dev/null +++ b/relay/client/guard.go @@ -0,0 +1,33 @@ +package client + +import ( + "context" + "time" +) + +var ( + reconnectingTimeout = 5 * time.Second +) + +type Guard struct { + ctx context.Context + relayClient *Client +} + +func NewGuard(context context.Context, relayClient *Client) *Guard { + g := &Guard{ + ctx: context, + relayClient: relayClient, + } + + return g +} + +func (g *Guard) OnDisconnected() { + select { + case <-time.After(time.Second): + _ = g.relayClient.Connect() + case <-g.ctx.Done(): + return + } +} diff --git a/relay/client/manager.go b/relay/client/manager.go index 686b4ac4f..e2ced6f5b 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -4,17 +4,19 @@ import ( "context" "fmt" "net" + "sync" ) -// Manager todo: thread safe type Manager struct { ctx context.Context srvAddress string peerID string - relayClient *Client + relayClient *Client + reconnectGuard *Guard - relayClients map[string]*Client + relayClients map[string]*Client + relayClientsMutex sync.Mutex } func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { @@ -28,13 +30,33 @@ func NewManager(ctx context.Context, serverAddress string, peerID string) *Manag func (m *Manager) Serve() error { m.relayClient = NewClient(m.ctx, m.srvAddress, m.peerID) + m.reconnectGuard = NewGuard(m.ctx, m.relayClient) + m.relayClient.SetOnDisconnectListener(m.reconnectGuard.OnDisconnected) err := m.relayClient.Connect() if err != nil { return err } + return nil } +func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { + if m.relayClient == nil { + return nil, fmt.Errorf("relay client not connected") + } + + foreign, err := m.isForeignServer(serverAddress) + if err != nil { + return nil, err + } + + if foreign { + return m.openConnVia(serverAddress, peerKey) + } else { + return m.relayClient.OpenConn(peerKey) + } +} + func (m *Manager) RelayAddress() (net.Addr, error) { if m.relayClient == nil { return nil, fmt.Errorf("relay client not connected") @@ -42,48 +64,38 @@ func (m *Manager) RelayAddress() (net.Addr, error) { return m.relayClient.RelayRemoteAddress() } -func (m *Manager) OpenConn(peerKey string) (net.Conn, error) { - if m.relayClient == nil { - return nil, fmt.Errorf("relay client not connected") - } - - rAddr, err := m.relayClient.RelayRemoteAddress() - if err != nil { - return nil, fmt.Errorf("relay client not connected") - } - - return m.OpenConnTo(rAddr.String(), peerKey) -} - -func (m *Manager) OpenConnTo(serverAddress, peerKey string) (net.Conn, error) { - if m.relayClient == nil { - return nil, fmt.Errorf("relay client not connected") - } - rAddr, err := m.relayClient.RelayRemoteAddress() - if err != nil { - return nil, fmt.Errorf("relay client not connected") - } - - if rAddr.String() == serverAddress { - return m.relayClient.OpenConn(peerKey) - } - +func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { relayClient, ok := m.relayClients[serverAddress] if ok { return relayClient.OpenConn(peerKey) } relayClient = NewClient(m.ctx, serverAddress, m.peerID) - err = relayClient.ConnectWithoutReconnect() + err := relayClient.Connect() if err != nil { return nil, err } - + relayClient.SetOnDisconnectListener(func() { + m.deleteRelayConn(serverAddress) + }) conn, err := relayClient.OpenConn(peerKey) if err != nil { return nil, err } m.relayClients[serverAddress] = relayClient + return conn, nil } + +func (m *Manager) deleteRelayConn(address string) { + delete(m.relayClients, address) +} + +func (m *Manager) isForeignServer(address string) (bool, error) { + rAddr, err := m.relayClient.RelayRemoteAddress() + if err != nil { + return false, fmt.Errorf("relay client not connected") + } + return rAddr.String() != address, nil +} diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 6c8edd534..2c041fff4 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -48,6 +48,10 @@ func TestNewManager(t *testing.T) { if err != nil { t.Fatalf("failed to connect to server: %s", err) } + aliceSrvAddr, err := clientAlice.RelayAddress() + if err != nil { + t.Fatalf("failed to get relay address: %s", err) + } clientBob := NewManager(ctx, addr2, idBob) err = clientBob.Serve() @@ -59,12 +63,12 @@ func TestNewManager(t *testing.T) { if err != nil { t.Fatalf("failed to get relay address: %s", err) } - connAliceToBob, err := clientAlice.OpenConnTo(bobsSrvAddr.String(), idBob) + connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr.String(), idBob) if err != nil { t.Fatalf("failed to bind channel: %s", err) } - connBobToAlice, err := clientBob.OpenConn(idAlice) + connBobToAlice, err := clientBob.OpenConn(aliceSrvAddr.String(), idAlice) if err != nil { t.Fatalf("failed to bind channel: %s", err) } From 3430b81622d00aaf159ba3d2a6c8a5eb08c5b635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Sat, 1 Jun 2024 11:48:15 +0200 Subject: [PATCH 019/199] Add relay server tracking --- relay/client/manager.go | 53 +++++++++++++++++++------ relay/client/manager_test.go | 75 +++++++++++++++++++++++++++++++----- 2 files changed, 107 insertions(+), 21 deletions(-) diff --git a/relay/client/manager.go b/relay/client/manager.go index e2ced6f5b..f8d5a28fc 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -5,8 +5,19 @@ import ( "fmt" "net" "sync" + + log "github.com/sirupsen/logrus" ) +type RelayTrack struct { + sync.RWMutex + relayClient *Client +} + +func NewRelayTrack() *RelayTrack { + return &RelayTrack{} +} + type Manager struct { ctx context.Context srvAddress string @@ -15,8 +26,8 @@ type Manager struct { relayClient *Client reconnectGuard *Guard - relayClients map[string]*Client - relayClientsMutex sync.Mutex + relayClients map[string]*RelayTrack + relayClientsMutex sync.RWMutex } func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { @@ -24,7 +35,7 @@ func NewManager(ctx context.Context, serverAddress string, peerID string) *Manag ctx: ctx, srvAddress: serverAddress, peerID: peerID, - relayClients: make(map[string]*Client), + relayClients: make(map[string]*RelayTrack), } } @@ -50,10 +61,10 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { return nil, err } - if foreign { - return m.openConnVia(serverAddress, peerKey) - } else { + if !foreign { return m.relayClient.OpenConn(peerKey) + } else { + return m.openConnVia(serverAddress, peerKey) } } @@ -65,30 +76,50 @@ func (m *Manager) RelayAddress() (net.Addr, error) { } func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { - relayClient, ok := m.relayClients[serverAddress] + m.relayClientsMutex.RLock() + relayTrack, ok := m.relayClients[serverAddress] if ok { - return relayClient.OpenConn(peerKey) + relayTrack.RLock() + m.relayClientsMutex.RUnlock() + defer relayTrack.RUnlock() + return relayTrack.relayClient.OpenConn(peerKey) } + m.relayClientsMutex.RUnlock() - relayClient = NewClient(m.ctx, serverAddress, m.peerID) + rt := NewRelayTrack() + rt.Lock() + + m.relayClientsMutex.Lock() + m.relayClients[serverAddress] = rt + m.relayClientsMutex.Unlock() + + relayClient := NewClient(m.ctx, serverAddress, m.peerID) err := relayClient.Connect() if err != nil { + rt.Unlock() + m.relayClientsMutex.Lock() + delete(m.relayClients, serverAddress) + m.relayClientsMutex.Unlock() return nil, err } relayClient.SetOnDisconnectListener(func() { m.deleteRelayConn(serverAddress) }) + rt.Unlock() + conn, err := relayClient.OpenConn(peerKey) if err != nil { return nil, err } - m.relayClients[serverAddress] = relayClient - return conn, nil } func (m *Manager) deleteRelayConn(address string) { + log.Infof("deleting relay client for %s", address) + m.relayClientsMutex.Lock() + defer m.relayClientsMutex.Unlock() + delete(m.relayClients, address) } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 2c041fff4..0fbdddd7e 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -4,13 +4,14 @@ import ( "context" "testing" + log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/relay/server" ) -func TestNewManager(t *testing.T) { +func TestForeignConn(t *testing.T) { ctx := context.Background() - idAlice := "alice" - idBob := "bob" + addr1 := "localhost:1234" srv1 := server.NewServer() go func() { @@ -43,22 +44,21 @@ func TestNewManager(t *testing.T) { } }() + idAlice := "alice" + log.Debugf("connect by alice") clientAlice := NewManager(ctx, addr1, idAlice) err := clientAlice.Serve() if err != nil { t.Fatalf("failed to connect to server: %s", err) } - aliceSrvAddr, err := clientAlice.RelayAddress() - if err != nil { - t.Fatalf("failed to get relay address: %s", err) - } + idBob := "bob" + log.Debugf("connect by bob") clientBob := NewManager(ctx, addr2, idBob) err = clientBob.Serve() if err != nil { t.Fatalf("failed to connect to server: %s", err) } - bobsSrvAddr, err := clientBob.RelayAddress() if err != nil { t.Fatalf("failed to get relay address: %s", err) @@ -67,8 +67,7 @@ func TestNewManager(t *testing.T) { if err != nil { t.Fatalf("failed to bind channel: %s", err) } - - connBobToAlice, err := clientBob.OpenConn(aliceSrvAddr.String(), idAlice) + connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr.String(), idAlice) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -99,3 +98,59 @@ func TestNewManager(t *testing.T) { t.Fatalf("expected %s, got %s", payload, string(buf[:n])) } } + +func TestForeginConnClose(t *testing.T) { + ctx := context.Background() + + addr1 := "localhost:1234" + srv1 := server.NewServer() + go func() { + err := srv1.Listen(addr1) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv1.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + addr2 := "localhost:2234" + srv2 := server.NewServer() + go func() { + err := srv2.Listen(addr2) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv2.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + idAlice := "alice" + log.Debugf("connect by alice") + clientAlice := NewManager(ctx, addr1, idAlice) + err := clientAlice.Serve() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + + conn, err := clientAlice.OpenConn(addr2, "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } + + select {} +} From 4ced07dd8de54e876bfc547a0973756f7a09222b Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 3 Jun 2024 00:29:08 +0200 Subject: [PATCH 020/199] Fix close conn threading issue --- relay/client/client.go | 142 +++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index f57bb2b92..21a33119e 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -30,19 +30,18 @@ type connContainer struct { type Client struct { log *log.Entry + parentCtx context.Context ctx context.Context ctxCancel context.CancelFunc serverAddress string hashedID []byte - readyToOpenConns bool + relayConn net.Conn conns map[string]*connContainer - connsMutext sync.Mutex // protect conns and readyToOpenConns bool - - relayConn net.Conn - serviceIsRunning bool - serviceIsRunningMutex sync.Mutex - wgReadLoop sync.WaitGroup + serviceIsRunning bool + mu sync.Mutex + readLoopMutex sync.Mutex + wgReadLoop sync.WaitGroup remoteAddr net.Addr @@ -51,12 +50,11 @@ type Client struct { } func NewClient(ctx context.Context, serverAddress, peerID string) *Client { - ctx, ctxCancel := context.WithCancel(ctx) hashedID, hashedStringId := messages.HashID(peerID) return &Client{ log: log.WithField("client_id", hashedStringId), - ctx: ctx, - ctxCancel: ctxCancel, + parentCtx: ctx, + ctxCancel: func() {}, serverAddress: serverAddress, hashedID: hashedID, conns: make(map[string]*connContainer), @@ -70,39 +68,44 @@ func (c *Client) SetOnDisconnectListener(fn func()) { } func (c *Client) Connect() error { - c.serviceIsRunningMutex.Lock() - defer c.serviceIsRunningMutex.Unlock() + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + + c.mu.Lock() if c.serviceIsRunning { + c.mu.Unlock() return nil } + c.mu.Unlock() err := c.connect() if err != nil { + c.mu.Unlock() return err } c.serviceIsRunning = true - c.wgReadLoop.Add(1) - go c.readLoop() - - go func() { - <-c.ctx.Done() - cErr := c.close() + c.ctx, c.ctxCancel = context.WithCancel(c.parentCtx) + context.AfterFunc(c.ctx, func() { + cErr := c.Close() if cErr != nil { log.Errorf("failed to close relay connection: %s", cErr) } - }() + }) + c.wgReadLoop.Add(1) + go c.readLoop(c.relayConn) return nil } +// todo: what should happen of call with the same peerID? func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { - c.connsMutext.Lock() - defer c.connsMutext.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - if !c.readyToOpenConns { + if !c.serviceIsRunning { return nil, fmt.Errorf("relay connection is not established") } @@ -119,8 +122,8 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { } func (c *Client) RelayRemoteAddress() (net.Addr, error) { - c.serviceIsRunningMutex.Lock() - defer c.serviceIsRunningMutex.Unlock() + c.mu.Lock() + defer c.mu.Unlock() if c.remoteAddr == nil { return nil, fmt.Errorf("relay connection is not established") } @@ -128,14 +131,21 @@ func (c *Client) RelayRemoteAddress() (net.Addr, error) { } func (c *Client) Close() error { - c.serviceIsRunningMutex.Lock() - if !c.serviceIsRunning { - c.serviceIsRunningMutex.Unlock() - return nil - } + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + c.mu.Lock() + var err error + if c.serviceIsRunning { + c.serviceIsRunning = false + err = c.relayConn.Close() + } + c.closeAllConns() + c.mu.Unlock() + + c.wgReadLoop.Wait() c.ctxCancel() - return c.close() + return err } func (c *Client) connect() error { @@ -157,27 +167,9 @@ func (c *Client) connect() error { c.remoteAddr = conn.RemoteAddr() - c.readyToOpenConns = true return nil } -func (c *Client) close() error { - c.serviceIsRunningMutex.Lock() - defer c.serviceIsRunningMutex.Unlock() - - if !c.serviceIsRunning { - return nil - } - - c.serviceIsRunning = false - - err := c.relayConn.Close() - - c.wgReadLoop.Wait() - - return err -} - func (c *Client) handShake() error { defer func() { err := c.relayConn.SetReadDeadline(time.Time{}) @@ -223,16 +215,18 @@ func (c *Client) handShake() error { return nil } -func (c *Client) readLoop() { +func (c *Client) readLoop(relayConn net.Conn) { var errExit error var n int for { buf := make([]byte, bufferSize) - n, errExit = c.relayConn.Read(buf) + n, errExit = relayConn.Read(buf) if errExit != nil { + c.mu.Lock() if c.serviceIsRunning { c.log.Debugf("failed to read message from relay server: %s", errExit) } + c.mu.Unlock() break } @@ -251,44 +245,44 @@ func (c *Client) readLoop() { } stringID := messages.HashIDToString(peerID) + c.mu.Lock() + if !c.serviceIsRunning { + c.mu.Unlock() + break + } container, ok := c.conns[stringID] + c.mu.Unlock() if !ok { c.log.Errorf("peer not found: %s", stringID) continue } - container.messages <- Msg{ - buf[:n], - } + container.messages <- Msg{buf[:n]} } } c.notifyDisconnected() - if c.serviceIsRunning { - _ = c.relayConn.Close() - } - - c.connsMutext.Lock() - c.readyToOpenConns = false - for _, container := range c.conns { - close(container.messages) - } - c.conns = make(map[string]*connContainer) - c.connsMutext.Unlock() - c.log.Tracef("exit from read loop") c.wgReadLoop.Done() + + c.Close() } +// todo check by reference too, the id is not enought because the id come from the outer conn func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { - c.connsMutext.Lock() + c.mu.Lock() + // conn, ok := c.conns[id] _, ok := c.conns[id] + c.mu.Unlock() if !ok { - c.connsMutext.Unlock() return 0, io.EOF } - c.connsMutext.Unlock() + /* + if conn != clientRef { + return 0, io.EOF + } + */ msg := messages.MarshalTransportMsg(dstID, payload) n, err := c.relayConn.Write(msg) if err != nil { @@ -314,9 +308,17 @@ func (c *Client) generateConnReaderFN(msgChannel chan Msg) func(b []byte) (n int } } +func (c *Client) closeAllConns() { + for _, container := range c.conns { + close(container.messages) + } + c.conns = make(map[string]*connContainer) +} + +// todo check by reference too, the id is not enought because the id come from the outer conn func (c *Client) closeConn(id string) error { - c.connsMutext.Lock() - defer c.connsMutext.Unlock() + c.mu.Lock() + defer c.mu.Unlock() conn, ok := c.conns[id] if !ok { From 57ddb5f26278576e64a68c047d17386cf3283332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 3 Jun 2024 11:22:16 +0200 Subject: [PATCH 021/199] Add comment --- relay/client/client.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/relay/client/client.go b/relay/client/client.go index 21a33119e..2d660f1db 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -28,6 +28,10 @@ type connContainer struct { messages chan Msg } +// Client is a client for the relay server. It is responsible for establishing a connection to the relay server and +// managing connections to other peers. All exported functions are safe to call concurrently. After close the connection, +// the client can be reused by calling Connect again. When the client is closed, all connections are closed too. +// While the Connect is in progress, the OpenConn function will block until the connection is established. type Client struct { log *log.Entry parentCtx context.Context @@ -49,6 +53,7 @@ type Client struct { listenerMutex sync.Mutex } +// NewClient creates a new client for the relay server. The client is not connected to the server until the Connect func NewClient(ctx context.Context, serverAddress, peerID string) *Client { hashedID, hashedStringId := messages.HashID(peerID) return &Client{ @@ -61,12 +66,14 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { } } +// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. func (c *Client) SetOnDisconnectListener(fn func()) { c.listenerMutex.Lock() defer c.listenerMutex.Unlock() c.onDisconnectListener = fn } +// Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. func (c *Client) Connect() error { c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() @@ -121,6 +128,7 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { return conn, nil } +// RelayRemoteAddress returns the IP address of the relay server. It could change after the close and reopen the connection. func (c *Client) RelayRemoteAddress() (net.Addr, error) { c.mu.Lock() defer c.mu.Unlock() @@ -130,6 +138,7 @@ func (c *Client) RelayRemoteAddress() (net.Addr, error) { return c.remoteAddr, nil } +// Close closes the connection to the relay server and all connections to other peers. func (c *Client) Close() error { c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() From 9d44a476c63779ffc8ce3bc84db69f34f8cec98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 3 Jun 2024 20:14:39 +0200 Subject: [PATCH 022/199] Fix double unlock in client.go --- relay/client/client.go | 4 +--- relay/messages/message.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 2d660f1db..e59110005 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -79,16 +79,14 @@ func (c *Client) Connect() error { defer c.readLoopMutex.Unlock() c.mu.Lock() + defer c.mu.Unlock() if c.serviceIsRunning { - c.mu.Unlock() return nil } - c.mu.Unlock() err := c.connect() if err != nil { - c.mu.Unlock() return err } diff --git a/relay/messages/message.go b/relay/messages/message.go index c71d203b1..d3a96af69 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -38,7 +38,7 @@ func DetermineClientMsgType(msg []byte) (MsgType, error) { case MsgTypeTransport: return msgType, nil default: - return 0, fmt.Errorf("invalid msg type: %s", msg) + return 0, fmt.Errorf("invalid msg type, len: %d", len(msg)) } } From 2b369cd28f2a2fd8c5d00a06d525bbe9af5e8677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 3 Jun 2024 20:17:43 +0200 Subject: [PATCH 023/199] Add quic transporter --- relay/client/dialer/quic/conn.go | 52 ++++++++++++ relay/client/dialer/quic/quic.go | 32 +++++++ relay/server/listener/quic/conn.go | 36 ++++++++ relay/server/listener/quic/listener.go | 111 +++++++++++++++++++++++++ 4 files changed, 231 insertions(+) create mode 100644 relay/client/dialer/quic/conn.go create mode 100644 relay/client/dialer/quic/quic.go create mode 100644 relay/server/listener/quic/conn.go create mode 100644 relay/server/listener/quic/listener.go diff --git a/relay/client/dialer/quic/conn.go b/relay/client/dialer/quic/conn.go new file mode 100644 index 000000000..2c8b29a3c --- /dev/null +++ b/relay/client/dialer/quic/conn.go @@ -0,0 +1,52 @@ +package quic + +import ( + "net" + + "github.com/quic-go/quic-go" + log "github.com/sirupsen/logrus" +) + +type Conn struct { + quic.Stream + qConn quic.Connection +} + +func NewConn(stream quic.Stream, qConn quic.Connection) net.Conn { + return &Conn{ + Stream: stream, + qConn: qConn, + } +} + +func (q *Conn) Write(b []byte) (n int, err error) { + log.Debugf("writing: %d, %x\n", len(b), b) + n, err = q.Stream.Write(b) + if n != len(b) { + log.Errorf("failed to write out the full message") + } + return +} + +func (q *Conn) Close() error { + err := q.Stream.Close() + if err != nil { + log.Errorf("failed to close stream: %s", err) + return err + } + err = q.qConn.CloseWithError(0, "") + if err != nil { + log.Errorf("failed to close connection: %s", err) + return err + + } + return err +} + +func (c *Conn) LocalAddr() net.Addr { + return c.qConn.LocalAddr() +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.qConn.RemoteAddr() +} diff --git a/relay/client/dialer/quic/quic.go b/relay/client/dialer/quic/quic.go new file mode 100644 index 000000000..4863ed7bd --- /dev/null +++ b/relay/client/dialer/quic/quic.go @@ -0,0 +1,32 @@ +package quic + +import ( + "context" + "crypto/tls" + "net" + + "github.com/quic-go/quic-go" + log "github.com/sirupsen/logrus" +) + +func Dial(address string) (net.Conn, error) { + tlsConf := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"quic-echo-example"}, + } + qConn, err := quic.DialAddr(context.Background(), address, tlsConf, &quic.Config{ + EnableDatagrams: true, + }) + if err != nil { + log.Errorf("dial quic address %s failed: %s", address, err) + return nil, err + } + + stream, err := qConn.OpenStreamSync(context.Background()) + if err != nil { + return nil, err + } + + conn := NewConn(stream, qConn) + return conn, nil +} diff --git a/relay/server/listener/quic/conn.go b/relay/server/listener/quic/conn.go new file mode 100644 index 000000000..12d22bb5e --- /dev/null +++ b/relay/server/listener/quic/conn.go @@ -0,0 +1,36 @@ +package quic + +import ( + "net" + + "github.com/quic-go/quic-go" + log "github.com/sirupsen/logrus" +) + +type QuicConn struct { + quic.Stream + qConn quic.Connection +} + +func NewConn(stream quic.Stream, qConn quic.Connection) net.Conn { + return &QuicConn{ + Stream: stream, + qConn: qConn, + } +} + +func (q QuicConn) Write(b []byte) (n int, err error) { + n, err = q.Stream.Write(b) + if n != len(b) { + log.Errorf("failed to write out the full message") + } + return +} + +func (q QuicConn) LocalAddr() net.Addr { + return q.qConn.LocalAddr() +} + +func (q QuicConn) RemoteAddr() net.Addr { + return q.qConn.RemoteAddr() +} diff --git a/relay/server/listener/quic/listener.go b/relay/server/listener/quic/listener.go new file mode 100644 index 000000000..8244bb2e2 --- /dev/null +++ b/relay/server/listener/quic/listener.go @@ -0,0 +1,111 @@ +package quic + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "sync" + + "github.com/quic-go/quic-go" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/server/listener" +) + +type Listener struct { + address string + onAcceptFn func(conn net.Conn) + + listener *quic.Listener + quit chan struct{} + wg sync.WaitGroup +} + +func NewListener(address string) listener.Listener { + return &Listener{ + address: address, + } +} + +func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { + ql, err := quic.ListenAddr(l.address, generateTLSConfig(), &quic.Config{ + EnableDatagrams: true, + }) + if err != nil { + return err + } + l.listener = ql + l.quit = make(chan struct{}) + + log.Infof("quic server is listening on address: %s", l.address) + l.wg.Add(1) + go l.acceptLoop(onAcceptFn) + + <-l.quit + return nil +} + +func (l *Listener) Close() error { + close(l.quit) + err := l.listener.Close() + l.wg.Wait() + return err +} + +func (l *Listener) acceptLoop(acceptFn func(conn net.Conn)) { + defer l.wg.Done() + + for { + qConn, err := l.listener.Accept(context.Background()) + if err != nil { + select { + case <-l.quit: + return + default: + log.Errorf("failed to accept connection: %s", err) + continue + } + } + + log.Infof("new connection from: %s", qConn.RemoteAddr()) + + stream, err := qConn.AcceptStream(context.Background()) + if err != nil { + log.Errorf("failed to open stream: %s", err) + continue + } + + conn := NewConn(stream, qConn) + + go acceptFn(conn) + } +} + +// Setup a bare-bones TLS config for the server +func generateTLSConfig() *tls.Config { + key, err := rsa.GenerateKey(rand.Reader, 1024) + if err != nil { + panic(err) + } + template := x509.Certificate{SerialNumber: big.NewInt(1)} + certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + panic(err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + panic(err) + } + return &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + NextProtos: []string{"quic-echo-example"}, + } +} From 0556dc186028e3c1488dabd72bd8e166d15279ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 3 Jun 2024 21:36:46 +0200 Subject: [PATCH 024/199] Avoid nil pointer exception in test in case of err --- relay/client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index c180b922b..3aa931d68 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -471,7 +471,7 @@ func TestCloseRelayConn(t *testing.T) { clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { - t.Errorf("failed to connect to server: %s", err) + t.Fatalf("failed to connect to server: %s", err) } conn, err := clientAlice.OpenConn("bob") From 15818b72c6c4c3989b7e9436a7f85621aa549478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 3 Jun 2024 21:38:37 +0200 Subject: [PATCH 025/199] Add alternative ws server implementation --- go.mod | 33 ++++---- go.sum | 68 ++++++++------- relay/client/client.go | 4 +- relay/client/dialer/wsnhooyr/client_conn.go | 79 +++++++++++++++++ relay/client/dialer/wsnhooyr/ws.go | 22 +++++ relay/messages/message.go | 3 + relay/server/listener/wsnhooyr/conn.go | 91 ++++++++++++++++++++ relay/server/listener/wsnhooyr/listener.go | 93 +++++++++++++++++++++ relay/server/server.go | 4 +- 9 files changed, 349 insertions(+), 48 deletions(-) create mode 100644 relay/client/dialer/wsnhooyr/client_conn.go create mode 100644 relay/client/dialer/wsnhooyr/ws.go create mode 100644 relay/server/listener/wsnhooyr/conn.go create mode 100644 relay/server/listener/wsnhooyr/listener.go diff --git a/go.mod b/go.mod index cf9308a20..ae0218c10 100644 --- a/go.mod +++ b/go.mod @@ -14,15 +14,15 @@ require ( github.com/gorilla/mux v1.8.0 github.com/kardianos/service v1.2.1-0.20210728001519-a323c3813bc7 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.18.1 + github.com/onsi/gomega v1.27.6 github.com/pion/ice/v3 v3.0.2 github.com/rs/cors v1.8.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 - golang.org/x/crypto v0.21.0 - golang.org/x/sys v0.18.0 + golang.org/x/crypto v0.23.0 + golang.org/x/sys v0.20.0 golang.zx2c4.com/wireguard v0.0.0-20230704135630-469159ecf7d1 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6 golang.zx2c4.com/wireguard/windows v0.5.3 @@ -44,7 +44,7 @@ require ( github.com/gliderlabs/ssh v0.3.4 github.com/godbus/dbus/v5 v5.1.0 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/gopacket v1.1.19 github.com/google/martian/v3 v3.0.0 github.com/google/nftables v0.0.0-20220808154552-2eca00135732 @@ -70,6 +70,7 @@ require ( github.com/pion/transport/v3 v3.0.1 github.com/pion/turn/v3 v3.0.1 github.com/prometheus/client_golang v1.14.0 + github.com/quic-go/quic-go v0.44.0 github.com/rs/xid v1.3.0 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/stretchr/testify v1.8.4 @@ -83,17 +84,18 @@ require ( go.opentelemetry.io/otel/metric v0.33.0 go.opentelemetry.io/otel/sdk/metric v0.33.0 goauthentik.io/api/v3 v3.2023051.3 - golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 - golang.org/x/net v0.23.0 + golang.org/x/net v0.25.0 golang.org/x/oauth2 v0.8.0 - golang.org/x/sync v0.3.0 - golang.org/x/term v0.18.0 + golang.org/x/sync v0.7.0 + golang.org/x/term v0.20.0 google.golang.org/api v0.126.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.3 gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde + nhooyr.io/websocket v1.8.11 ) require ( @@ -125,15 +127,17 @@ require ( github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f // indirect github.com/go-gl/gl v0.0.0-20210813123233-e4099ee2221f // indirect github.com/go-gl/glfw/v3.3/glfw v0.0.0-20211024062804-40e447a793be // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-stack/stack v1.8.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.0.1 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.10.0 // indirect @@ -157,6 +161,7 @@ require ( github.com/moby/term v0.0.0-20221128092401-c43b287e0e0f // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/onsi/ginkgo/v2 v2.9.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.5 // indirect @@ -179,18 +184,18 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/sdk v1.11.1 // indirect go.opentelemetry.io/otel/trace v1.11.1 // indirect + go.uber.org/mock v0.4.0 // indirect golang.org/x/image v0.10.0 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.21.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 // indirect k8s.io/apimachinery v0.23.16 // indirect ) diff --git a/go.sum b/go.sum index 1c5dd13c1..4b6197e3e 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -214,6 +214,8 @@ github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= @@ -278,8 +280,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= @@ -296,6 +298,7 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= @@ -458,18 +461,16 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= -github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= @@ -540,6 +541,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/quic-go/quic-go v0.44.0 h1:So5wOr7jyO4vzL2sd8/pD9Kesciv91zSk8BoFngItQ0= +github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQlpnPcWmek= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -586,6 +589,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -643,6 +647,8 @@ go.opentelemetry.io/otel/sdk/metric v0.33.0/go.mod h1:xdypMeA21JBOvjjzDUtD0kzIcH go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4= goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -657,8 +663,8 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -669,8 +675,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= -golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -698,8 +704,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -738,7 +744,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -749,8 +754,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -773,8 +778,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -848,8 +853,8 @@ golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -857,8 +862,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -872,13 +877,14 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -928,8 +934,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1095,6 +1101,8 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0= +nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/relay/client/client.go b/relay/client/client.go index e59110005..64b20f9e1 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -10,7 +10,7 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/client/dialer/udp" + "github.com/netbirdio/netbird/relay/client/dialer/ws" "github.com/netbirdio/netbird/relay/messages" ) @@ -156,7 +156,7 @@ func (c *Client) Close() error { } func (c *Client) connect() error { - conn, err := udp.Dial(c.serverAddress) + conn, err := ws.Dial(c.serverAddress) if err != nil { return err } diff --git a/relay/client/dialer/wsnhooyr/client_conn.go b/relay/client/dialer/wsnhooyr/client_conn.go new file mode 100644 index 000000000..bd3db74eb --- /dev/null +++ b/relay/client/dialer/wsnhooyr/client_conn.go @@ -0,0 +1,79 @@ +package wsnhooyr + +import ( + "context" + "fmt" + "net" + "time" + + "nhooyr.io/websocket" +) + +type Conn struct { + *websocket.Conn + ctx context.Context +} + +func NewConn(wsConn *websocket.Conn) net.Conn { + return &Conn{ + Conn: wsConn, + ctx: context.Background(), + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, ioReader, err := c.Conn.Reader(c.ctx) + if err != nil { + return 0, err + } + + if t != websocket.MessageBinary { + return 0, fmt.Errorf("unexpected message type") + } + + return ioReader.Read(b) +} + +func (c *Conn) Write(b []byte) (n int, err error) { + err = c.Conn.Write(c.ctx, websocket.MessageBinary, b) + return len(b), err +} + +func (c *Conn) RemoteAddr() net.Addr { + // todo: implement me + return nil +} + +func (c *Conn) LocalAddr() net.Addr { + // todo: implement me + return nil +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + // todo: implement me + return nil +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + // todo: implement me + return nil +} + +func (c *Conn) SetDeadline(t time.Time) error { + // todo: implement me + errR := c.SetReadDeadline(t) + errW := c.SetWriteDeadline(t) + + if errR != nil { + return errR + } + + if errW != nil { + return errW + } + return nil +} + +func (c *Conn) Close() error { + return c.Conn.Close(websocket.StatusNormalClosure, "") +} diff --git a/relay/client/dialer/wsnhooyr/ws.go b/relay/client/dialer/wsnhooyr/ws.go new file mode 100644 index 000000000..ccec97402 --- /dev/null +++ b/relay/client/dialer/wsnhooyr/ws.go @@ -0,0 +1,22 @@ +package wsnhooyr + +import ( + "context" + "fmt" + "net" + + "nhooyr.io/websocket" +) + +func Dial(address string) (net.Conn, error) { + + addr := fmt.Sprintf("ws://" + address) + wsConn, _, err := websocket.Dial(context.Background(), addr, nil) + if err != nil { + return nil, err + } + + conn := NewConn(wsConn) + + return conn, nil +} diff --git a/relay/messages/message.go b/relay/messages/message.go index d3a96af69..1c34d5034 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -2,6 +2,8 @@ package messages import ( "fmt" + + log "github.com/sirupsen/logrus" ) const ( @@ -104,6 +106,7 @@ func UnmarshalTransportPayload(buf []byte) ([]byte, error) { func UnmarshalTransportID(buf []byte) ([]byte, error) { headerSize := 1 + IDSize if len(buf) < headerSize { + log.Debugf("invalid message length: %d, expected: %d, %x", len(buf), headerSize, buf) return nil, ErrInvalidMessageLength } return buf[1:headerSize], nil diff --git a/relay/server/listener/wsnhooyr/conn.go b/relay/server/listener/wsnhooyr/conn.go new file mode 100644 index 000000000..c3461cb85 --- /dev/null +++ b/relay/server/listener/wsnhooyr/conn.go @@ -0,0 +1,91 @@ +package ws + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "time" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" +) + +type Conn struct { + *websocket.Conn + lAddr *net.TCPAddr + rAddr *net.TCPAddr + + ctx context.Context +} + +func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn { + return &Conn{ + Conn: wsConn, + lAddr: lAddr, + rAddr: rAddr, + ctx: context.Background(), + } +} + +func (c *Conn) Read(b []byte) (n int, err error) { + t, r, err := c.Reader(c.ctx) + if err != nil { + return 0, ioErrHandling(err) + } + + if t != websocket.MessageBinary { + log.Errorf("unexpected message type: %d", t) + return 0, fmt.Errorf("unexpected message type") + } + + n, err = r.Read(b) + if err != nil { + return 0, ioErrHandling(err) + } + return n, err +} + +func (c *Conn) Write(b []byte) (int, error) { + err := c.Conn.Write(c.ctx, websocket.MessageBinary, b) + return len(b), err +} + +func (c *Conn) LocalAddr() net.Addr { + return c.lAddr +} + +func (c *Conn) RemoteAddr() net.Addr { + return c.rAddr +} + +func (c *Conn) SetReadDeadline(t time.Time) error { + // todo: implement me + return nil +} + +func (c *Conn) SetWriteDeadline(t time.Time) error { + // todo: implement me + return nil +} + +func (c *Conn) SetDeadline(t time.Time) error { + // todo: implement me + return nil +} + +func (c *Conn) Close() error { + return c.Conn.Close(websocket.StatusNormalClosure, "") +} + +func ioErrHandling(err error) error { + var wErr *websocket.CloseError + if !errors.As(err, &wErr) { + return err + } + if wErr.Code == websocket.StatusNormalClosure { + return io.EOF + } + return err +} diff --git a/relay/server/listener/wsnhooyr/listener.go b/relay/server/listener/wsnhooyr/listener.go new file mode 100644 index 000000000..88370a1fc --- /dev/null +++ b/relay/server/listener/wsnhooyr/listener.go @@ -0,0 +1,93 @@ +package ws + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "nhooyr.io/websocket" + + "github.com/netbirdio/netbird/relay/server/listener" +) + +type Listener struct { + address string + + wg sync.WaitGroup + server *http.Server + acceptFn func(conn net.Conn) +} + +func NewListener(address string) listener.Listener { + return &Listener{ + address: address, + } +} + +// Listen todo: prevent multiple call +func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { + l.acceptFn = acceptFn + mux := http.NewServeMux() + mux.HandleFunc("/", l.onAccept) + + l.server = &http.Server{ + Addr: l.address, + Handler: mux, + } + + log.Infof("WS server is listening on address: %s", l.address) + err := l.server.ListenAndServe() + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func (l *Listener) Close() error { + if l.server == nil { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + log.Debugf("closing WS server") + if err := l.server.Shutdown(ctx); err != nil { + return fmt.Errorf("server shutdown failed: %v", err) + } + + l.wg.Wait() + return nil +} + +func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { + l.wg.Add(1) + defer l.wg.Done() + + wsConn, err := websocket.Accept(w, r, nil) + if err != nil { + log.Errorf("failed to accept ws connection: %s", err) + return + } + + rAddr, err := net.ResolveTCPAddr("tcp", r.RemoteAddr) + if err != nil { + _ = wsConn.Close(websocket.StatusInternalError, "internal error") + return + } + + lAddr, err := net.ResolveTCPAddr("tcp", l.server.Addr) + if err != nil { + _ = wsConn.Close(websocket.StatusInternalError, "internal error") + return + } + + conn := NewConn(wsConn, lAddr, rAddr) + l.acceptFn(conn) + return +} diff --git a/relay/server/server.go b/relay/server/server.go index f71c888e3..7e729c309 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -12,7 +12,7 @@ import ( "github.com/netbirdio/netbird/relay/messages" "github.com/netbirdio/netbird/relay/server/listener" "github.com/netbirdio/netbird/relay/server/listener/udp" - "github.com/netbirdio/netbird/relay/server/listener/ws" + ws "github.com/netbirdio/netbird/relay/server/listener/wsnhooyr" ) // Server @@ -104,7 +104,7 @@ func (r *Server) accept(conn net.Conn) { msgType, err := messages.DetermineClientMsgType(buf[:n]) if err != nil { - log.Errorf("failed to determine message type: %s", err) + peer.Log.Errorf("failed to determine message type: %s", err) return } switch msgType { From a40d4d2f325b56ba92a99c56b3235ca1b3311774 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 4 Jun 2024 14:40:35 +0200 Subject: [PATCH 026/199] - add comments - avoid double closing messages - add cleanup routine for relay manager --- relay/client/client.go | 35 +++++---- relay/client/manager.go | 100 +++++++++++++++++++++---- relay/client/manager_test.go | 74 +++++++++++++++--- relay/server/listener/wsnhooyr/conn.go | 1 + relay/server/server.go | 1 - 5 files changed, 172 insertions(+), 39 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 64b20f9e1..fa4f0da6a 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -35,7 +35,6 @@ type connContainer struct { type Client struct { log *log.Entry parentCtx context.Context - ctx context.Context ctxCancel context.CancelFunc serverAddress string hashedID []byte @@ -66,13 +65,6 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { } } -// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. -func (c *Client) SetOnDisconnectListener(fn func()) { - c.listenerMutex.Lock() - defer c.listenerMutex.Unlock() - c.onDisconnectListener = fn -} - // Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. func (c *Client) Connect() error { c.readLoopMutex.Lock() @@ -92,8 +84,9 @@ func (c *Client) Connect() error { c.serviceIsRunning = true - c.ctx, c.ctxCancel = context.WithCancel(c.parentCtx) - context.AfterFunc(c.ctx, func() { + var ctx context.Context + ctx, c.ctxCancel = context.WithCancel(c.parentCtx) + context.AfterFunc(ctx, func() { cErr := c.Close() if cErr != nil { log.Errorf("failed to close relay connection: %s", cErr) @@ -136,6 +129,19 @@ func (c *Client) RelayRemoteAddress() (net.Addr, error) { return c.remoteAddr, nil } +// SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. +func (c *Client) SetOnDisconnectListener(fn func()) { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + c.onDisconnectListener = fn +} + +func (c *Client) HasConns() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.conns) > 0 +} + // Close closes the connection to the relay server and all connections to other peers. func (c *Client) Close() error { c.readLoopMutex.Lock() @@ -143,14 +149,17 @@ func (c *Client) Close() error { c.mu.Lock() var err error - if c.serviceIsRunning { - c.serviceIsRunning = false - err = c.relayConn.Close() + if !c.serviceIsRunning { + return nil } + + c.serviceIsRunning = false + err = c.relayConn.Close() c.closeAllConns() c.mu.Unlock() c.wgReadLoop.Wait() + c.log.Infof("relay client ha been closed: %s", c.serverAddress) c.ctxCancel() return err } diff --git a/relay/client/manager.go b/relay/client/manager.go index f8d5a28fc..98ec3a1c8 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -5,10 +5,18 @@ import ( "fmt" "net" "sync" + "time" log "github.com/sirupsen/logrus" ) +var ( + relayCleanupInterval = 60 * time.Second +) + +// RelayTrack hold the relay clients for the foregin relay servers. +// With the mutex can ensure we can open new connection in case the relay connection has been established with +// the relay server. type RelayTrack struct { sync.RWMutex relayClient *Client @@ -18,6 +26,12 @@ func NewRelayTrack() *RelayTrack { return &RelayTrack{} } +// Manager is a manager for the relay client. It establish one persistent connection to the given relay server. In case +// of network error the manager will try to reconnect to the server. +// The manager also manage temproary relay connection. If a client wants to communicate with an another client on a +// different relay server, the manager will establish a new connection to the relay server. The connection with these +// relay servers will be closed if there is no active connection. Periodically the manager will check if there is any +// unused relay connection and close it. type Manager struct { ctx context.Context srvAddress string @@ -39,18 +53,26 @@ func NewManager(ctx context.Context, serverAddress string, peerID string) *Manag } } -func (m *Manager) Serve() error { +// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop. +// todo: consider to return an error if the initial connection to the relay server is not established. +func (m *Manager) Serve() { m.relayClient = NewClient(m.ctx, m.srvAddress, m.peerID) m.reconnectGuard = NewGuard(m.ctx, m.relayClient) m.relayClient.SetOnDisconnectListener(m.reconnectGuard.OnDisconnected) err := m.relayClient.Connect() if err != nil { - return err + log.Errorf("failed to connect to relay server, keep try to reconnect: %s", err) + return } - return nil + m.startCleanupLoop() + + return } +// OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be +// established via the relay server. If the peer is on a different relay server, the manager will establish a new +// connection to the relay server. func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { if m.relayClient == nil { return nil, fmt.Errorf("relay client not connected") @@ -68,6 +90,8 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { } } +// RelayAddress returns the address of the permanent relay server. It could change if the network connection is lost. +// This address will be sent to the target peer to choose the common relay server for the communication. func (m *Manager) RelayAddress() (net.Addr, error) { if m.relayClient == nil { return nil, fmt.Errorf("relay client not connected") @@ -76,20 +100,31 @@ func (m *Manager) RelayAddress() (net.Addr, error) { } func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { + // check if already has a connection to the desired relay server m.relayClientsMutex.RLock() - relayTrack, ok := m.relayClients[serverAddress] + rt, ok := m.relayClients[serverAddress] if ok { - relayTrack.RLock() + rt.RLock() m.relayClientsMutex.RUnlock() - defer relayTrack.RUnlock() - return relayTrack.relayClient.OpenConn(peerKey) + defer rt.RUnlock() + return rt.relayClient.OpenConn(peerKey) } m.relayClientsMutex.RUnlock() - rt := NewRelayTrack() - rt.Lock() - + // if not, establish a new connection but check it again (because changed the lock type) before starting the + // connection m.relayClientsMutex.Lock() + rt, ok = m.relayClients[serverAddress] + if ok { + rt.RLock() + m.relayClientsMutex.Unlock() + defer rt.RUnlock() + return rt.relayClient.OpenConn(peerKey) + } + + // create a new relay client and store it in the relayClients map + rt = NewRelayTrack() + rt.Lock() m.relayClients[serverAddress] = rt m.relayClientsMutex.Unlock() @@ -102,25 +137,25 @@ func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { m.relayClientsMutex.Unlock() return nil, err } + // if connection closed then delete the relay client from the list relayClient.SetOnDisconnectListener(func() { m.deleteRelayConn(serverAddress) }) + rt.relayClient = relayClient rt.Unlock() conn, err := relayClient.OpenConn(peerKey) if err != nil { return nil, err } - return conn, nil } func (m *Manager) deleteRelayConn(address string) { log.Infof("deleting relay client for %s", address) m.relayClientsMutex.Lock() - defer m.relayClientsMutex.Unlock() - delete(m.relayClients, address) + m.relayClientsMutex.Unlock() } func (m *Manager) isForeignServer(address string) (bool, error) { @@ -130,3 +165,42 @@ func (m *Manager) isForeignServer(address string) (bool, error) { } return rAddr.String() != address, nil } + +func (m *Manager) startCleanupLoop() { + if m.ctx.Err() != nil { + return + } + + ticker := time.NewTicker(relayCleanupInterval) + go func() { + defer ticker.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.cleanUpUnusedRelays() + } + } + }() +} + +func (m *Manager) cleanUpUnusedRelays() { + m.relayClientsMutex.Lock() + defer m.relayClientsMutex.Unlock() + + for addr, rt := range m.relayClients { + rt.Lock() + if rt.relayClient.HasConns() { + rt.Unlock() + continue + } + rt.relayClient.SetOnDisconnectListener(nil) + go func() { + _ = rt.relayClient.Close() + }() + log.Debugf("clean up relay client: %s", addr) + delete(m.relayClients, addr) + rt.Unlock() + } +} diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 0fbdddd7e..d0e22dabd 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -3,6 +3,7 @@ package client import ( "context" "testing" + "time" log "github.com/sirupsen/logrus" @@ -47,18 +48,13 @@ func TestForeignConn(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") clientAlice := NewManager(ctx, addr1, idAlice) - err := clientAlice.Serve() - if err != nil { - t.Fatalf("failed to connect to server: %s", err) - } + clientAlice.Serve() idBob := "bob" log.Debugf("connect by bob") clientBob := NewManager(ctx, addr2, idBob) - err = clientBob.Serve() - if err != nil { - t.Fatalf("failed to connect to server: %s", err) - } + clientBob.Serve() + bobsSrvAddr, err := clientBob.RelayAddress() if err != nil { t.Fatalf("failed to get relay address: %s", err) @@ -137,10 +133,7 @@ func TestForeginConnClose(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") clientAlice := NewManager(ctx, addr1, idAlice) - err := clientAlice.Serve() - if err != nil { - t.Fatalf("failed to connect to server: %s", err) - } + clientAlice.Serve() conn, err := clientAlice.OpenConn(addr2, "anotherpeer") if err != nil { @@ -154,3 +147,60 @@ func TestForeginConnClose(t *testing.T) { select {} } + +func TestForeginAutoClose(t *testing.T) { + ctx := context.Background() + + addr1 := "localhost:1234" + srv1 := server.NewServer() + go func() { + err := srv1.Listen(addr1) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv1.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + addr2 := "localhost:2234" + srv2 := server.NewServer() + go func() { + err := srv2.Listen(addr2) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv2.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + idAlice := "alice" + log.Debugf("connect by alice") + mgr := NewManager(ctx, addr1, idAlice) + relayCleanupInterval = 2 * time.Second + mgr.Serve() + + conn, err := mgr.OpenConn(addr2, "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } + + time.Sleep(relayCleanupInterval + 1*time.Second) + if len(mgr.relayClients) != 0 { + t.Errorf("expected 0, got %d", len(mgr.relayClients)) + } +} diff --git a/relay/server/listener/wsnhooyr/conn.go b/relay/server/listener/wsnhooyr/conn.go index c3461cb85..72b6bfecb 100644 --- a/relay/server/listener/wsnhooyr/conn.go +++ b/relay/server/listener/wsnhooyr/conn.go @@ -79,6 +79,7 @@ func (c *Conn) Close() error { return c.Conn.Close(websocket.StatusNormalClosure, "") } +// todo: fix io.EOF handling func ioErrHandling(err error) error { var wErr *websocket.CloseError if !errors.As(err, &wErr) { diff --git a/relay/server/server.go b/relay/server/server.go index 7e729c309..cb9816907 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -18,7 +18,6 @@ import ( // Server // todo: // authentication: provide JWT token via RPC call. The MGM server can forward the token to the agents. -// connection timeout handling type Server struct { store *Store From fed9e587af9c670a27e755116e68abc84eb7ae72 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 5 Jun 2024 19:49:30 +0200 Subject: [PATCH 027/199] Add close message type --- relay/client/client.go | 44 +++-- relay/client/client_test.go | 186 ++++++++++++--------- relay/client/dialer/ws/client_conn.go | 5 - relay/client/dialer/ws/ws.go | 6 +- relay/client/manager.go | 2 + relay/client/manager_test.go | 179 +++++++++++++------- relay/messages/message.go | 15 ++ relay/server/listener/listener.go | 1 + relay/server/listener/udp/listener.go | 7 +- relay/server/listener/ws/listener.go | 9 +- relay/server/listener/wsnhooyr/conn.go | 27 ++- relay/server/listener/wsnhooyr/listener.go | 9 +- relay/server/server.go | 39 ++++- relay/server/store.go | 12 +- 14 files changed, 371 insertions(+), 170 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index fa4f0da6a..aa4fff5d7 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -87,7 +87,7 @@ func (c *Client) Connect() error { var ctx context.Context ctx, c.ctxCancel = context.WithCancel(c.parentCtx) context.AfterFunc(ctx, func() { - cErr := c.Close() + cErr := c.close(false) if cErr != nil { log.Errorf("failed to close relay connection: %s", cErr) } @@ -144,22 +144,30 @@ func (c *Client) HasConns() bool { // Close closes the connection to the relay server and all connections to other peers. func (c *Client) Close() error { + return c.close(false) +} + +func (c *Client) close(byServer bool) error { c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() c.mu.Lock() var err error if !c.serviceIsRunning { + c.mu.Unlock() return nil } c.serviceIsRunning = false - err = c.relayConn.Close() c.closeAllConns() + if !byServer { + c.writeCloseMsg() + err = c.relayConn.Close() + } c.mu.Unlock() c.wgReadLoop.Wait() - c.log.Infof("relay client ha been closed: %s", c.serverAddress) + c.log.Infof("relay connection closed with: %s", c.serverAddress) c.ctxCancel() return err } @@ -232,8 +240,11 @@ func (c *Client) handShake() error { } func (c *Client) readLoop(relayConn net.Conn) { - var errExit error - var n int + var ( + errExit error + n int + closedByServer bool + ) for { buf := make([]byte, bufferSize) n, errExit = relayConn.Read(buf) @@ -243,7 +254,7 @@ func (c *Client) readLoop(relayConn net.Conn) { c.log.Debugf("failed to read message from relay server: %s", errExit) } c.mu.Unlock() - break + goto Exit } msgType, err := messages.DetermineServerMsgType(buf[:n]) @@ -264,7 +275,7 @@ func (c *Client) readLoop(relayConn net.Conn) { c.mu.Lock() if !c.serviceIsRunning { c.mu.Unlock() - break + goto Exit } container, ok := c.conns[stringID] c.mu.Unlock() @@ -273,16 +284,19 @@ func (c *Client) readLoop(relayConn net.Conn) { continue } + // todo review is this can cause panic container.messages <- Msg{buf[:n]} + case messages.MsgClose: + closedByServer = true + log.Debugf("relay connection close by server") + goto Exit } } +Exit: c.notifyDisconnected() - - c.log.Tracef("exit from read loop") c.wgReadLoop.Done() - - c.Close() + _ = c.close(closedByServer) } // todo check by reference too, the id is not enought because the id come from the outer conn @@ -365,3 +379,11 @@ func (c *Client) notifyDisconnected() { } go c.onDisconnectListener() } + +func (c *Client) writeCloseMsg() { + msg := messages.MarshalCloseMsg() + _, err := c.relayConn.Write(msg) + if err != nil { + c.log.Errorf("failed to send close message: %s", err) + } +} diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 3aa931d68..f5d122276 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -100,57 +100,56 @@ func TestRegistration(t *testing.T) { } }() - defer func() { - err := srv.Close() - if err != nil { - t.Errorf("failed to close server: %s", err) - } - }() - clientAlice := NewClient(ctx, addr, "alice") err := clientAlice.Connect() if err != nil { + _ = srv.Close() t.Fatalf("failed to connect to server: %s", err) } - defer func() { - err = clientAlice.Close() - if err != nil { - t.Errorf("failed to close conn: %s", err) - } - }() + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } + err = srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } } func TestRegistrationTimeout(t *testing.T) { ctx := context.Background() - udpListener, err := net.ListenUDP("udp", &net.UDPAddr{ + fakeUDPListener, err := net.ListenUDP("udp", &net.UDPAddr{ Port: 1234, IP: net.ParseIP("0.0.0.0"), }) if err != nil { t.Fatalf("failed to bind UDP server: %s", err) } - defer udpListener.Close() + defer func(fakeUDPListener *net.UDPConn) { + _ = fakeUDPListener.Close() + }(fakeUDPListener) - tcpListener, err := net.ListenTCP("tcp", &net.TCPAddr{ + fakeTCPListener, err := net.ListenTCP("tcp", &net.TCPAddr{ Port: 1234, IP: net.ParseIP("0.0.0.0"), }) if err != nil { t.Fatalf("failed to bind TCP server: %s", err) } - defer tcpListener.Close() + defer func(fakeTCPListener *net.TCPListener) { + _ = fakeTCPListener.Close() + }(fakeTCPListener) clientAlice := NewClient(ctx, "127.0.0.1:1234", "alice") err = clientAlice.Connect() if err == nil { t.Errorf("failed to connect to server: %s", err) } - defer func() { - err = clientAlice.Close() - if err != nil { - t.Errorf("failed to close conn: %s", err) - } - }() + log.Debugf("%s", err) + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close conn: %s", err) + } } func TestEcho(t *testing.T) { @@ -259,18 +258,16 @@ func TestBindToUnavailabePeer(t *testing.T) { if err != nil { t.Errorf("failed to connect to server: %s", err) } - defer func() { - log.Infof("closing client") - err := clientAlice.Close() - if err != nil { - t.Errorf("failed to close client: %s", err) - } - }() - _, err = clientAlice.OpenConn("bob") if err != nil { t.Errorf("failed to bind channel: %s", err) } + + log.Infof("closing client") + err = clientAlice.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } } func TestBindReconnect(t *testing.T) { @@ -315,7 +312,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to bind channel: %s", err) } - log.Infof("closing client") + log.Infof("closing client Alice") err = clientAlice.Close() if err != nil { t.Errorf("failed to close client: %s", err) @@ -403,52 +400,6 @@ func TestCloseConn(t *testing.T) { } } -func TestAutoReconnect(t *testing.T) { - ctx := context.Background() - - addr := "localhost:1234" - srv := server.NewServer() - go func() { - err := srv.Listen(addr) - if err != nil { - t.Errorf("failed to bind server: %s", err) - } - }() - - defer func() { - err := srv.Close() - if err != nil { - log.Errorf("failed to close server: %s", err) - } - }() - - clientAlice := NewClient(ctx, addr, "alice") - err := clientAlice.Connect() - if err != nil { - t.Errorf("failed to connect to server: %s", err) - } - - conn, err := clientAlice.OpenConn("bob") - if err != nil { - t.Errorf("failed to bind channel: %s", err) - } - - _ = clientAlice.relayConn.Close() - - _, err = conn.Read(make([]byte, 1)) - if err == nil { - t.Errorf("unexpected reading from closed connection") - } - - log.Infof("waiting for reconnection") - time.Sleep(reconnectingTimeout) - - _, err = clientAlice.OpenConn("bob") - if err != nil { - t.Errorf("failed to open channel: %s", err) - } -} - func TestCloseRelayConn(t *testing.T) { ctx := context.Background() @@ -491,3 +442,82 @@ func TestCloseRelayConn(t *testing.T) { t.Errorf("unexpected opening connection to closed server") } } + +func TestCloseByServer(t *testing.T) { + ctx := context.Background() + + addr1 := "localhost:1234" + srv1 := server.NewServer() + go func() { + err := srv1.Listen(addr1) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + idAlice := "alice" + log.Debugf("connect by alice") + relayClient := NewClient(ctx, addr1, idAlice) + err := relayClient.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + + disconnected := make(chan struct{}) + relayClient.SetOnDisconnectListener(func() { + log.Infof("client disconnected") + close(disconnected) + }) + + err = srv1.Close() + if err != nil { + t.Fatalf("failed to close server: %s", err) + } + + select { + case <-disconnected: + case <-time.After(3 * time.Second): + log.Fatalf("timeout waiting for client to disconnect") + } + + _, err = relayClient.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } +} + +func TestCloseByClient(t *testing.T) { + ctx := context.Background() + + addr1 := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr1) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + idAlice := "alice" + log.Debugf("connect by alice") + relayClient := NewClient(ctx, addr1, idAlice) + err := relayClient.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + + err = relayClient.Close() + if err != nil { + t.Errorf("failed to close client: %s", err) + } + + _, err = relayClient.OpenConn("bob") + if err == nil { + t.Errorf("unexpected opening connection to closed server") + } + + err = srv.Close() + if err != nil { + t.Fatalf("failed to close server: %s", err) + } +} diff --git a/relay/client/dialer/ws/client_conn.go b/relay/client/dialer/ws/client_conn.go index 72a3fa9b4..698cb9d24 100644 --- a/relay/client/dialer/ws/client_conn.go +++ b/relay/client/dialer/ws/client_conn.go @@ -6,7 +6,6 @@ import ( "time" "github.com/gorilla/websocket" - log "github.com/sirupsen/logrus" ) type Conn struct { @@ -52,9 +51,5 @@ func (c *Conn) SetDeadline(t time.Time) error { } func (c *Conn) Close() error { - err := c.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""), time.Now().Add(time.Second*5)) - if err != nil { - log.Errorf("failed to close conn?: %s", err) - } return c.Conn.Close() } diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index 3ab3d38a8..b526bd315 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -3,13 +3,17 @@ package ws import ( "fmt" "net" + "time" "github.com/gorilla/websocket" ) func Dial(address string) (net.Conn, error) { addr := fmt.Sprintf("ws://" + address) - wsConn, _, err := websocket.DefaultDialer.Dial(addr, nil) + wsDialer := websocket.Dialer{ + HandshakeTimeout: 3 * time.Second, + } + wsConn, _, err := wsDialer.Dial(addr, nil) if err != nil { return nil, err } diff --git a/relay/client/manager.go b/relay/client/manager.go index 98ec3a1c8..911561403 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -84,8 +84,10 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { } if !foreign { + log.Debugf("open connection to permanent server: %s", peerKey) return m.relayClient.OpenConn(peerKey) } else { + log.Debugf("open connection to foreign server: %s", serverAddress) return m.openConnVia(serverAddress, peerKey) } } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index d0e22dabd..f4de9b4de 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -47,12 +47,14 @@ func TestForeignConn(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") - clientAlice := NewManager(ctx, addr1, idAlice) + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, addr1, idAlice) clientAlice.Serve() idBob := "bob" log.Debugf("connect by bob") - clientBob := NewManager(ctx, addr2, idBob) + clientBob := NewManager(mCtx, addr2, idBob) clientBob.Serve() bobsSrvAddr, err := clientBob.RelayAddress() @@ -132,61 +134,9 @@ func TestForeginConnClose(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") - clientAlice := NewManager(ctx, addr1, idAlice) - clientAlice.Serve() - - conn, err := clientAlice.OpenConn(addr2, "anotherpeer") - if err != nil { - t.Fatalf("failed to bind channel: %s", err) - } - - err = conn.Close() - if err != nil { - t.Fatalf("failed to close connection: %s", err) - } - - select {} -} - -func TestForeginAutoClose(t *testing.T) { - ctx := context.Background() - - addr1 := "localhost:1234" - srv1 := server.NewServer() - go func() { - err := srv1.Listen(addr1) - if err != nil { - t.Fatalf("failed to bind server: %s", err) - } - }() - - defer func() { - err := srv1.Close() - if err != nil { - t.Errorf("failed to close server: %s", err) - } - }() - - addr2 := "localhost:2234" - srv2 := server.NewServer() - go func() { - err := srv2.Listen(addr2) - if err != nil { - t.Fatalf("failed to bind server: %s", err) - } - }() - - defer func() { - err := srv2.Close() - if err != nil { - t.Errorf("failed to close server: %s", err) - } - }() - - idAlice := "alice" - log.Debugf("connect by alice") - mgr := NewManager(ctx, addr1, idAlice) - relayCleanupInterval = 2 * time.Second + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + mgr := NewManager(mCtx, addr1, idAlice) mgr.Serve() conn, err := mgr.OpenConn(addr2, "anotherpeer") @@ -198,9 +148,124 @@ func TestForeginAutoClose(t *testing.T) { if err != nil { t.Fatalf("failed to close connection: %s", err) } +} +func TestForeginAutoClose(t *testing.T) { + ctx := context.Background() + relayCleanupInterval = 1 * time.Second + addr1 := "localhost:1234" + srv1 := server.NewServer() + go func() { + t.Log("binding server 1.") + err := srv1.Listen(addr1) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + + defer func() { + t.Logf("closing server 1.") + err := srv1.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + t.Logf("server 1. closed") + }() + + addr2 := "localhost:2234" + srv2 := server.NewServer() + go func() { + t.Log("binding server 2.") + err := srv2.Listen(addr2) + if err != nil { + t.Fatalf("failed to bind server: %s", err) + } + }() + defer func() { + t.Logf("closing server 2.") + err := srv2.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + t.Logf("server 2 closed.") + }() + + idAlice := "alice" + t.Log("connect to server 1.") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + mgr := NewManager(mCtx, addr1, idAlice) + mgr.Serve() + + t.Log("open connection to another peer") + conn, err := mgr.OpenConn(addr2, "anotherpeer") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + t.Log("close conn") + err = conn.Close() + if err != nil { + t.Fatalf("failed to close connection: %s", err) + } + + t.Logf("waiting for relay cleanup: %s", relayCleanupInterval+1*time.Second) time.Sleep(relayCleanupInterval + 1*time.Second) if len(mgr.relayClients) != 0 { t.Errorf("expected 0, got %d", len(mgr.relayClients)) } + + t.Logf("closing manager") +} + +func TestAutoReconnect(t *testing.T) { + ctx := context.Background() + reconnectingTimeout = 2 * time.Second + + addr := "localhost:1234" + srv := server.NewServer() + go func() { + err := srv.Listen(addr) + if err != nil { + t.Errorf("failed to bind server: %s", err) + } + }() + + defer func() { + err := srv.Close() + if err != nil { + log.Errorf("failed to close server: %s", err) + } + }() + + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, addr, "alice") + clientAlice.Serve() + ra, err := clientAlice.RelayAddress() + if err != nil { + t.Errorf("failed to get relay address: %s", err) + } + conn, err := clientAlice.OpenConn(ra.String(), "bob") + if err != nil { + t.Errorf("failed to bind channel: %s", err) + } + + t.Log("closing client relay connection") + // todo figure out moc server + _ = clientAlice.relayClient.relayConn.Close() + t.Log("start test reading") + _, err = conn.Read(make([]byte, 1)) + if err == nil { + t.Errorf("unexpected reading from closed connection") + } + + log.Infof("waiting for reconnection") + time.Sleep(reconnectingTimeout + 1*time.Second) + + log.Infof("reopent the connection") + _, err = clientAlice.OpenConn(ra.String(), "bob") + if err != nil { + t.Errorf("failed to open channel: %s", err) + } } diff --git a/relay/messages/message.go b/relay/messages/message.go index 1c34d5034..7f73daa17 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -10,6 +10,7 @@ const ( MsgTypeHello MsgType = 0 MsgTypeHelloResponse MsgType = 1 MsgTypeTransport MsgType = 2 + MsgClose MsgType = 3 ) var ( @@ -26,6 +27,8 @@ func (m MsgType) String() string { return "hello response" case MsgTypeTransport: return "transport" + case MsgClose: + return "close" default: return "unknown" } @@ -39,6 +42,8 @@ func DetermineClientMsgType(msg []byte) (MsgType, error) { return msgType, nil case MsgTypeTransport: return msgType, nil + case MsgClose: + return msgType, nil default: return 0, fmt.Errorf("invalid msg type, len: %d", len(msg)) } @@ -52,6 +57,8 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { return msgType, nil case MsgTypeTransport: return msgType, nil + case MsgClose: + return msgType, nil default: return 0, fmt.Errorf("invalid msg type (len: %d)", len(msg)) } @@ -81,6 +88,14 @@ func MarshalHelloResponse() []byte { return msg } +// Close message + +func MarshalCloseMsg() []byte { + msg := make([]byte, 1) + msg[0] = byte(MsgClose) + return msg +} + // Transport message func MarshalTransportMsg(peerID []byte, payload []byte) []byte { diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go index 66e6d357e..3336e0ad8 100644 --- a/relay/server/listener/listener.go +++ b/relay/server/listener/listener.go @@ -5,4 +5,5 @@ import "net" type Listener interface { Listen(func(conn net.Conn)) error Close() error + WaitForExitAcceptedConns() } diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go index ebd1c53f1..3c2dfc070 100644 --- a/relay/server/listener/udp/listener.go +++ b/relay/server/listener/udp/listener.go @@ -21,6 +21,11 @@ type Listener struct { lock sync.Mutex } +func (l *Listener) WaitForExitAcceptedConns() { + l.wg.Wait() + return +} + func NewListener(address string) listener.Listener { return &Listener{ address: address, @@ -61,11 +66,11 @@ func (l *Listener) Close() error { l.lock.Lock() defer l.lock.Unlock() - log.Infof("closing UDP server") if l.listener == nil { return nil } + log.Infof("closing UDP listener") close(l.quit) err := l.listener.Close() l.wg.Wait() diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index b53e97505..632de153f 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -33,8 +33,11 @@ func NewListener(address string) listener.Listener { } } -// Listen todo: prevent multiple call func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { + if l.server != nil { + return errors.New("server is already running") + } + l.acceptFn = acceptFn mux := http.NewServeMux() mux.HandleFunc("/", l.onAccept) @@ -69,6 +72,10 @@ func (l *Listener) Close() error { return nil } +func (l *Listener) WaitForExitAcceptedConns() { + l.wg.Wait() +} + func (l *Listener) onAccept(writer http.ResponseWriter, request *http.Request) { l.wg.Add(1) defer l.wg.Done() diff --git a/relay/server/listener/wsnhooyr/conn.go b/relay/server/listener/wsnhooyr/conn.go index 72b6bfecb..b52e5d082 100644 --- a/relay/server/listener/wsnhooyr/conn.go +++ b/relay/server/listener/wsnhooyr/conn.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "sync" "time" log "github.com/sirupsen/logrus" @@ -17,7 +18,9 @@ type Conn struct { lAddr *net.TCPAddr rAddr *net.TCPAddr - ctx context.Context + closed bool + closedMu sync.Mutex + ctx context.Context } func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn { @@ -32,7 +35,7 @@ func NewConn(wsConn *websocket.Conn, lAddr, rAddr *net.TCPAddr) *Conn { func (c *Conn) Read(b []byte) (n int, err error) { t, r, err := c.Reader(c.ctx) if err != nil { - return 0, ioErrHandling(err) + return 0, c.ioErrHandling(err) } if t != websocket.MessageBinary { @@ -42,7 +45,7 @@ func (c *Conn) Read(b []byte) (n int, err error) { n, err = r.Read(b) if err != nil { - return 0, ioErrHandling(err) + return 0, c.ioErrHandling(err) } return n, err } @@ -76,11 +79,23 @@ func (c *Conn) SetDeadline(t time.Time) error { } func (c *Conn) Close() error { - return c.Conn.Close(websocket.StatusNormalClosure, "") + c.closedMu.Lock() + c.closed = true + c.closedMu.Unlock() + return c.Conn.CloseNow() } -// todo: fix io.EOF handling -func ioErrHandling(err error) error { +func (c *Conn) isClosed() bool { + c.closedMu.Lock() + defer c.closedMu.Unlock() + return c.closed +} + +func (c *Conn) ioErrHandling(err error) error { + if c.isClosed() { + return io.EOF + } + var wErr *websocket.CloseError if !errors.As(err, &wErr) { return err diff --git a/relay/server/listener/wsnhooyr/listener.go b/relay/server/listener/wsnhooyr/listener.go index 88370a1fc..e47a60b47 100644 --- a/relay/server/listener/wsnhooyr/listener.go +++ b/relay/server/listener/wsnhooyr/listener.go @@ -56,15 +56,18 @@ func (l *Listener) Close() error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - log.Debugf("closing WS server") + log.Infof("stop WS listener") if err := l.server.Shutdown(ctx); err != nil { return fmt.Errorf("server shutdown failed: %v", err) } - - l.wg.Wait() + log.Infof("WS listener stopped") return nil } +func (l *Listener) WaitForExitAcceptedConns() { + l.wg.Wait() +} + func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { l.wg.Add(1) defer l.wg.Done() diff --git a/relay/server/server.go b/relay/server/server.go index cb9816907..bb9666a53 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -15,11 +15,9 @@ import ( ws "github.com/netbirdio/netbird/relay/server/listener/wsnhooyr" ) -// Server -// todo: -// authentication: provide JWT token via RPC call. The MGM server can forward the token to the agents. type Server struct { - store *Store + store *Store + storeMu sync.RWMutex UDPListener listener.Listener WSListener listener.Listener @@ -27,7 +25,8 @@ type Server struct { func NewServer() *Server { return &Server{ - store: NewStore(), + store: NewStore(), + storeMu: sync.RWMutex{}, } } @@ -69,6 +68,11 @@ func (r *Server) Close() error { if r.UDPListener != nil { uErr = r.UDPListener.Close() } + + r.sendCloseMsgs() + + r.WSListener.WaitForExitAcceptedConns() + err := errors.Join(wErr, uErr) return err } @@ -88,7 +92,7 @@ func (r *Server) accept(conn net.Conn) { r.store.AddPeer(peer) defer func() { r.store.DeletePeer(peer) - peer.Log.Infof("peer left") + peer.Log.Infof("relay connection closed") }() for { @@ -132,10 +136,33 @@ func (r *Server) accept(conn net.Conn) { } return }() + case messages.MsgClose: + peer.Log.Infof("peer disconnected gracefully") + _ = conn.Close() + return } } } +func (r *Server) sendCloseMsgs() { + msg := messages.MarshalCloseMsg() + + r.storeMu.Lock() + log.Debugf("sending close messages to %d peers", len(r.store.peers)) + for _, p := range r.store.peers { + _, err := p.conn.Write(msg) + if err != nil { + log.Errorf("failed to send close message to peer: %s", p.String()) + } + + err = p.conn.Close() + if err != nil { + log.Errorf("failed to close connection to peer: %s", err) + } + } + r.storeMu.Unlock() +} + func handShake(conn net.Conn) (*Peer, error) { buf := make([]byte, 1500) n, err := conn.Read(buf) diff --git a/relay/server/store.go b/relay/server/store.go index f785f4d0e..1f0f08600 100644 --- a/relay/server/store.go +++ b/relay/server/store.go @@ -24,7 +24,6 @@ func (s *Store) AddPeer(peer *Peer) { func (s *Store) DeletePeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() - delete(s.peers, peer.String()) } @@ -35,3 +34,14 @@ func (s *Store) Peer(id string) (*Peer, bool) { p, ok := s.peers[id] return p, ok } + +func (s *Store) Peers() []*Peer { + s.peersLock.RLock() + defer s.peersLock.RUnlock() + + peers := make([]*Peer, 0, len(s.peers)) + for _, p := range s.peers { + peers = append(peers, p) + } + return peers +} From ed8def4d9b982a856f7c6ed6257cc997776fc80e Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 7 Jun 2024 16:07:35 +0200 Subject: [PATCH 028/199] Protect ws writing in Gorilla ws --- relay/client/dialer/ws/client_conn.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/relay/client/dialer/ws/client_conn.go b/relay/client/dialer/ws/client_conn.go index 698cb9d24..ee3283ecd 100644 --- a/relay/client/dialer/ws/client_conn.go +++ b/relay/client/dialer/ws/client_conn.go @@ -3,6 +3,7 @@ package ws import ( "fmt" "net" + "sync" "time" "github.com/gorilla/websocket" @@ -10,11 +11,12 @@ import ( type Conn struct { *websocket.Conn + mu sync.Mutex } func NewConn(wsConn *websocket.Conn) net.Conn { return &Conn{ - wsConn, + Conn: wsConn, } } @@ -32,7 +34,9 @@ func (c *Conn) Read(b []byte) (n int, err error) { } func (c *Conn) Write(b []byte) (int, error) { + c.mu.Lock() err := c.WriteMessage(websocket.BinaryMessage, b) + c.mu.Unlock() return len(b), err } From 8c70b7d7ffed473ec79d90a00838d624da40f611 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 9 Jun 2024 12:41:52 +0200 Subject: [PATCH 029/199] Replace ws lib on client side --- relay/client/client.go | 2 +- relay/client/dialer/wsnhooyr/client_conn.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index aa4fff5d7..0a1066145 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -3,6 +3,7 @@ package client import ( "context" "fmt" + ws "github.com/netbirdio/netbird/relay/client/dialer/wsnhooyr" "io" "net" "sync" @@ -10,7 +11,6 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/client/dialer/ws" "github.com/netbirdio/netbird/relay/messages" ) diff --git a/relay/client/dialer/wsnhooyr/client_conn.go b/relay/client/dialer/wsnhooyr/client_conn.go index bd3db74eb..8f02b6701 100644 --- a/relay/client/dialer/wsnhooyr/client_conn.go +++ b/relay/client/dialer/wsnhooyr/client_conn.go @@ -75,5 +75,5 @@ func (c *Conn) SetDeadline(t time.Time) error { } func (c *Conn) Close() error { - return c.Conn.Close(websocket.StatusNormalClosure, "") + return c.Conn.CloseNow() } From 5e93d117cf81c73f89f0b801d034a9a06f893ca1 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 9 Jun 2024 20:27:40 +0200 Subject: [PATCH 030/199] Use buf pool - eliminate reader function generation - fix write to closed channel panic --- relay/client/client.go | 97 +++++++++++++++++++++++++-------------- relay/client/conn.go | 16 +++++-- relay/messages/message.go | 7 +-- 3 files changed, 78 insertions(+), 42 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 0a1066145..f508fb8fc 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -3,7 +3,6 @@ package client import ( "context" "fmt" - ws "github.com/netbirdio/netbird/relay/client/dialer/wsnhooyr" "io" "net" "sync" @@ -11,6 +10,7 @@ import ( log "github.com/sirupsen/logrus" + ws "github.com/netbirdio/netbird/relay/client/dialer/wsnhooyr" "github.com/netbirdio/netbird/relay/messages" ) @@ -19,13 +19,49 @@ const ( serverResponseTimeout = 8 * time.Second ) +// Msg carry the payload from the server to the client. With this sturct, the net.Conn can free the buffer. type Msg struct { - buf []byte + Payload []byte + + bufPool *sync.Pool + bufPtr *[]byte +} + +func (m *Msg) Free() { + m.bufPool.Put(m.bufPtr) } type connContainer struct { - conn *Conn - messages chan Msg + conn *Conn + messages chan Msg + msgChanLock sync.Mutex + closed bool // flag to check if channel is closed +} + +func newConnContainer(conn *Conn, messages chan Msg) *connContainer { + return &connContainer{ + conn: conn, + messages: messages, + } +} + +func (cc *connContainer) writeMsg(msg Msg) { + cc.msgChanLock.Lock() + defer cc.msgChanLock.Unlock() + if cc.closed { + return + } + cc.messages <- msg +} + +func (cc *connContainer) close() { + cc.msgChanLock.Lock() + defer cc.msgChanLock.Unlock() + if cc.closed { + return + } + close(cc.messages) + cc.closed = true } // Client is a client for the relay server. It is responsible for establishing a connection to the relay server and @@ -39,6 +75,8 @@ type Client struct { serverAddress string hashedID []byte + bufPool *sync.Pool + relayConn net.Conn conns map[string]*connContainer serviceIsRunning bool @@ -61,7 +99,13 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { ctxCancel: func() {}, serverAddress: serverAddress, hashedID: hashedID, - conns: make(map[string]*connContainer), + bufPool: &sync.Pool{ + New: func() any { + buf := make([]byte, bufferSize) + return &buf + }, + }, + conns: make(map[string]*connContainer), } } @@ -109,13 +153,10 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { hashedID, hashedStringID := messages.HashID(dstPeerID) log.Infof("open connection to peer: %s", hashedStringID) - messageBuffer := make(chan Msg, 2) - conn := NewConn(c, hashedID, hashedStringID, c.generateConnReaderFN(messageBuffer)) + msgChannel := make(chan Msg, 2) + conn := NewConn(c, hashedID, hashedStringID, msgChannel) - c.conns[hashedStringID] = &connContainer{ - conn, - messageBuffer, - } + c.conns[hashedStringID] = newConnContainer(conn, msgChannel) return conn, nil } @@ -246,7 +287,8 @@ func (c *Client) readLoop(relayConn net.Conn) { closedByServer bool ) for { - buf := make([]byte, bufferSize) + bufPtr := c.bufPool.Get().(*[]byte) + buf := *bufPtr n, errExit = relayConn.Read(buf) if errExit != nil { c.mu.Lock() @@ -265,7 +307,7 @@ func (c *Client) readLoop(relayConn net.Conn) { switch msgType { case messages.MsgTypeTransport: - peerID, err := messages.UnmarshalTransportID(buf[:n]) + peerID, payload, err := messages.UnmarshalTransportMsg(buf[:n]) if err != nil { c.log.Errorf("failed to parse transport message: %v", err) continue @@ -284,8 +326,10 @@ func (c *Client) readLoop(relayConn net.Conn) { continue } - // todo review is this can cause panic - container.messages <- Msg{buf[:n]} + container.writeMsg(Msg{ + bufPool: c.bufPool, + bufPtr: bufPtr, + Payload: payload}) case messages.MsgClose: closedByServer = true log.Debugf("relay connection close by server") @@ -321,26 +365,9 @@ func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { return n, err } -func (c *Client) generateConnReaderFN(msgChannel chan Msg) func(b []byte) (n int, err error) { - return func(b []byte) (n int, err error) { - msg, ok := <-msgChannel - if !ok { - return 0, io.EOF - } - - payload, err := messages.UnmarshalTransportPayload(msg.buf) - if err != nil { - return 0, err - } - - n = copy(b, payload) - return n, nil - } -} - func (c *Client) closeAllConns() { for _, container := range c.conns { - close(container.messages) + container.close() } c.conns = make(map[string]*connContainer) } @@ -350,11 +377,11 @@ func (c *Client) closeConn(id string) error { c.mu.Lock() defer c.mu.Unlock() - conn, ok := c.conns[id] + container, ok := c.conns[id] if !ok { return fmt.Errorf("connection already closed") } - close(conn.messages) + container.close() delete(c.conns, id) return nil diff --git a/relay/client/conn.go b/relay/client/conn.go index 647b0fae4..19f3c8df4 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -1,6 +1,7 @@ package client import ( + "io" "net" "time" ) @@ -9,15 +10,15 @@ type Conn struct { client *Client dstID []byte dstStringID string - readerFn func(b []byte) (n int, err error) + messageChan chan Msg } -func NewConn(client *Client, dstID []byte, dstStringID string, readerFn func(b []byte) (n int, err error)) *Conn { +func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg) *Conn { c := &Conn{ client: client, dstID: dstID, dstStringID: dstStringID, - readerFn: readerFn, + messageChan: messageChan, } return c @@ -28,7 +29,14 @@ func (c *Conn) Write(p []byte) (n int, err error) { } func (c *Conn) Read(b []byte) (n int, err error) { - return c.readerFn(b) + msg, ok := <-c.messageChan + if !ok { + return 0, io.EOF + } + + n = copy(b, msg.Payload) + msg.Free() + return n, nil } func (c *Conn) Close() error { diff --git a/relay/messages/message.go b/relay/messages/message.go index 7f73daa17..b865687ce 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -110,12 +110,13 @@ func MarshalTransportMsg(peerID []byte, payload []byte) []byte { return msg } -func UnmarshalTransportPayload(buf []byte) ([]byte, error) { +func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) { headerSize := 1 + IDSize if len(buf) < headerSize { - return nil, ErrInvalidMessageLength + return nil, nil, ErrInvalidMessageLength } - return buf[headerSize:], nil + + return buf[1:headerSize], buf[headerSize:], nil } func UnmarshalTransportID(buf []byte) ([]byte, error) { From 95040129203a30c10783fcb301664df7a12aa38d Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 9 Jun 2024 21:10:57 +0200 Subject: [PATCH 031/199] Set the proper buffer size in the client code --- relay/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/client/client.go b/relay/client/client.go index f508fb8fc..d433d4443 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -15,7 +15,7 @@ import ( ) const ( - bufferSize = 1500 // optimise the buffer size + bufferSize = 8820 serverResponseTimeout = 8 * time.Second ) From 38f2a59d1b7b9ef8a13a4cb1d405c5edfbb72556 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 12 Jun 2024 10:56:21 +0200 Subject: [PATCH 032/199] Add comment --- relay/client/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/relay/client/client.go b/relay/client/client.go index d433d4443..4e6e63265 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -142,7 +142,10 @@ func (c *Client) Connect() error { return nil } -// todo: what should happen of call with the same peerID? +// OpenConn create a new net.Conn for the destination peer ID. In case if the connection is in progress +// to the relay server, the function will block until the connection is established or timed out. Otherwise, +// it will return immediately. +// todo: what should happen if call with the same peerID with multiple times? func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { c.mu.Lock() defer c.mu.Unlock() From 64f949abbb33193a2d6ae3bbfe9704da61b22f73 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 14 Jun 2024 14:40:31 +0200 Subject: [PATCH 033/199] Integrate relay into peer conn - extend mgm with relay address - extend signaling with remote peer's relay address - start setup relay connection before engine start --- client/internal/connect.go | 9 +- client/internal/engine.go | 27 +- client/internal/peer/conn.go | 692 ++-- client/internal/wgproxy/factory_linux.go | 10 - management/proto/management.pb.go | 3763 ++++++++-------------- management/proto/management.proto | 2 + management/server/config.go | 7 +- management/server/grpcserver.go | 1 + relay/client/manager.go | 17 +- signal/client/client.go | 4 +- signal/proto/signalexchange.pb.go | 20 +- signal/proto/signalexchange.proto | 3 + 12 files changed, 1875 insertions(+), 2680 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index d34d0aab0..44e85be53 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -26,6 +26,7 @@ import ( "github.com/netbirdio/netbird/iface" mgm "github.com/netbirdio/netbird/management/client" mgmProto "github.com/netbirdio/netbird/management/proto" + relayClient "github.com/netbirdio/netbird/relay/client" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/util" "github.com/netbirdio/netbird/version" @@ -244,6 +245,12 @@ func (c *ConnectClient) run( c.statusRecorder.MarkSignalConnected() + relayManager := relayClient.NewManager(engineCtx, loginResp.GetWiretrusteeConfig().GetRelayAddress(), myPrivateKey.PublicKey().String()) + if err = relayManager.Serve(); err != nil { + log.Error(err) + return wrapErr(err) + } + peerConfig := loginResp.GetPeerConfig() engineConfig, err := createEngineConfig(myPrivateKey, c.config, peerConfig) @@ -253,7 +260,7 @@ func (c *ConnectClient) run( } c.engineMutex.Lock() - c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe) + c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe) c.engineMutex.Unlock() err = c.engine.Start() diff --git a/client/internal/engine.go b/client/internal/engine.go index b09235714..a2f395ea0 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -35,6 +35,7 @@ import ( "github.com/netbirdio/netbird/iface/bind" mgm "github.com/netbirdio/netbird/management/client" mgmProto "github.com/netbirdio/netbird/management/proto" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" sProto "github.com/netbirdio/netbird/signal/proto" @@ -154,6 +155,8 @@ type Engine struct { wgProbe *Probe wgConnWorker sync.WaitGroup + + relayManager *relayClient.Manager } // Peer is an instance of the Connection Peer @@ -168,6 +171,7 @@ func NewEngine( clientCancel context.CancelFunc, signalClient signal.Client, mgmClient mgm.Client, + relayManager *relayClient.Manager, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, @@ -177,6 +181,7 @@ func NewEngine( clientCancel, signalClient, mgmClient, + relayManager, config, mobileDep, statusRecorder, @@ -193,6 +198,7 @@ func NewEngineWithProbes( clientCancel context.CancelFunc, signalClient signal.Client, mgmClient mgm.Client, + relayManager *relayClient.Manager, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, @@ -207,6 +213,7 @@ func NewEngineWithProbes( clientCancel: clientCancel, signal: signalClient, mgmClient: mgmClient, + relayManager: relayManager, peerConns: make(map[string]*peer.Conn), syncMsgMux: &sync.Mutex{}, config: config, @@ -493,10 +500,17 @@ func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKe t = sProto.Body_OFFER } - msg, err := signal.MarshalCredential(myKey, offerAnswer.WgListenPort, remoteKey, &signal.Credential{ - UFrag: offerAnswer.IceCredentials.UFrag, - Pwd: offerAnswer.IceCredentials.Pwd, - }, t, offerAnswer.RosenpassPubKey, offerAnswer.RosenpassAddr) + msg, err := signal.MarshalCredential( + myKey, + offerAnswer.WgListenPort, + remoteKey, &signal.Credential{ + UFrag: offerAnswer.IceCredentials.UFrag, + Pwd: offerAnswer.IceCredentials.Pwd, + }, + t, + offerAnswer.RosenpassPubKey, + offerAnswer.RosenpassAddr, + offerAnswer.RelaySrvAddress) if err != nil { return err } @@ -524,6 +538,8 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { return err } + // todo update relay address in the relay manager + // todo update signal } @@ -987,7 +1003,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e RosenpassAddr: e.getRosenpassAddr(), } - peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover) + peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover, e.relayManager) if err != nil { return nil, err } @@ -1082,6 +1098,7 @@ func (e *Engine) receiveSignalEvents() { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), }) case sProto.Body_CANDIDATE: candidate, err := ice.UnmarshalCandidate(msg.GetBody().Payload) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index c64c074a7..b2fb58855 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -19,6 +19,7 @@ import ( "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" "github.com/netbirdio/netbird/iface/bind" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" sProto "github.com/netbirdio/netbird/signal/proto" nbnet "github.com/netbirdio/netbird/util/net" @@ -91,6 +92,9 @@ type OfferAnswer struct { // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message // This value is the local Rosenpass server address when sending the message RosenpassAddr string + + // relay server address + RelaySrvAddress string } // IceCredentials ICE protocol credentials struct @@ -138,6 +142,112 @@ type Conn struct { connID nbnet.ConnectionID beforeAddPeerHooks []BeforeAddPeerHookFunc afterRemovePeerHooks []AfterRemovePeerHookFunc + + relayManager *relayClient.Manager +} + +// NewConn creates a new not opened Conn to the remote peer. +// To establish a connection run Conn.Open +func NewConn(config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { + return &Conn{ + config: config, + mu: sync.Mutex{}, + status: StatusDisconnected, + closeCh: make(chan struct{}), + remoteOffersCh: make(chan OfferAnswer), + remoteAnswerCh: make(chan OfferAnswer), + statusRecorder: statusRecorder, + wgProxyFactory: wgProxyFactory, + adapter: adapter, + iFaceDiscover: iFaceDiscover, + relayManager: relayManager, + }, nil +} + +// Open opens connection to the remote peer starting ICE candidate gathering process. +// Blocks until connection has been closed or connection timeout. +// ConnStatus will be set accordingly +func (conn *Conn) Open(ctx context.Context) error { + log.Debugf("trying to connect to peer %s", conn.config.Key) + + peerState := State{ + PubKey: conn.config.Key, + IP: strings.Split(conn.config.WgConfig.AllowedIps, "/")[0], + ConnStatusUpdate: time.Now(), + ConnStatus: conn.status, + Mux: new(sync.RWMutex), + } + err := conn.statusRecorder.UpdatePeerState(peerState) + if err != nil { + log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + } + + defer func() { + err := conn.cleanup() + if err != nil { + log.Warnf("error while cleaning up peer connection %s: %v", conn.config.Key, err) + return + } + }() + + err = conn.sendOffer() + if err != nil { + return err + } + + log.Debugf("connection offer sent to peer %s, waiting for the confirmation", conn.config.Key) + + // Only continue once we got a connection confirmation from the remote peer. + // The connection timeout could have happened before a confirmation received from the remote. + // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) + remoteOfferAnswer, err := conn.waitForRemoteOfferConfirmation() + if err != nil { + return err + } + + log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", + conn.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) + + // at this point we received offer/answer and we are ready to gather candidates + conn.mu.Lock() + conn.status = StatusConnecting + conn.ctx, conn.notifyDisconnected = context.WithCancel(ctx) + defer conn.notifyDisconnected() + conn.mu.Unlock() + + peerState = State{ + PubKey: conn.config.Key, + ConnStatus: conn.status, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + err = conn.statusRecorder.UpdatePeerState(peerState) + if err != nil { + log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + } + + // in edge case this function can block while the manager set up a new relay server connection + relayOperate := conn.setupRelayConnection(remoteOfferAnswer) + + err = conn.setupICEConnection(remoteOfferAnswer, relayOperate) + if err != nil { + log.Errorf("failed to setup ICE connection: %s", err) + if !relayOperate { + return err + } + } + + // wait until connection disconnected or has been closed externally (upper layer, e.g. engine) + err = conn.waitForDisconnection() + return err +} + +func (conn *Conn) AddBeforeAddPeerHook(hook BeforeAddPeerHookFunc) { + conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook) +} + +func (conn *Conn) AddAfterRemovePeerHook(hook AfterRemovePeerHookFunc) { + conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook) } // GetConf returns the connection config @@ -155,24 +265,126 @@ func (conn *Conn) UpdateStunTurn(turnStun []*stun.URI) { conn.config.StunTurn = turnStun } -// NewConn creates a new not opened Conn to the remote peer. -// To establish a connection run Conn.Open -func NewConn(config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover) (*Conn, error) { - return &Conn{ - config: config, - mu: sync.Mutex{}, - status: StatusDisconnected, - closeCh: make(chan struct{}), - remoteOffersCh: make(chan OfferAnswer), - remoteAnswerCh: make(chan OfferAnswer), - statusRecorder: statusRecorder, - wgProxyFactory: wgProxyFactory, - adapter: adapter, - iFaceDiscover: iFaceDiscover, - }, nil +// SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer +func (conn *Conn) SetSignalOffer(handler func(offer OfferAnswer) error) { + conn.signalOffer = handler } -func (conn *Conn) reCreateAgent() error { +// SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established +func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) { + conn.onConnected = handler +} + +// SetOnDisconnected sets a handler function to be triggered by Conn when a connection to a remote disconnected +func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string)) { + conn.onDisconnected = handler +} + +// SetSignalAnswer sets a handler function to be triggered by Conn when a new connection answer has to be signalled to the remote peer +func (conn *Conn) SetSignalAnswer(handler func(answer OfferAnswer) error) { + conn.signalAnswer = handler +} + +// SetSignalCandidate sets a handler function to be triggered by Conn when a new ICE local connection candidate has to be signalled to the remote peer +func (conn *Conn) SetSignalCandidate(handler func(candidate ice.Candidate) error) { + conn.signalCandidate = handler +} + +// SetSendSignalMessage sets a handler function to be triggered by Conn when there is new message to send via signal +func (conn *Conn) SetSendSignalMessage(handler func(message *sProto.Message) error) { + conn.sendSignalMessage = handler +} + +// Close closes this peer Conn issuing a close event to the Conn closeCh +func (conn *Conn) Close() error { + conn.mu.Lock() + defer conn.mu.Unlock() + select { + case conn.closeCh <- struct{}{}: + return nil + default: + // probably could happen when peer has been added and removed right after not even starting to connect + // todo further investigate + // this really happens due to unordered messages coming from management + // more importantly it causes inconsistency -> 2 Conn objects for the same peer + // e.g. this flow: + // update from management has peers: [1,2,3,4] + // engine creates a Conn for peers: [1,2,3,4] and schedules Open in ~1sec + // before conn.Open() another update from management arrives with peers: [1,2,3] + // engine removes peer 4 and calls conn.Close() which does nothing (this default clause) + // before conn.Open() another update from management arrives with peers: [1,2,3,4,5] + // engine adds a new Conn for 4 and 5 + // therefore peer 4 has 2 Conn objects + log.Warnf("Connection has been already closed or attempted closing not started connection %s", conn.config.Key) + return NewConnectionAlreadyClosed(conn.config.Key) + } +} + +// Status returns current status of the Conn +func (conn *Conn) Status() ConnStatus { + conn.mu.Lock() + defer conn.mu.Unlock() + return conn.status +} + +// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { + log.Debugf("OnRemoteOffer from peer %s on status %s", conn.config.Key, conn.status.String()) + + select { + case conn.remoteOffersCh <- offer: + return true + default: + log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) + // connection might not be ready yet to receive so we ignore the message + return false + } +} + +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { + log.Debugf("OnRemoteAnswer from peer %s on status %s", conn.config.Key, conn.status.String()) + + select { + case conn.remoteAnswerCh <- answer: + return true + default: + // connection might not be ready yet to receive so we ignore the message + log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) + return false + } +} + +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) + go func() { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.agent == nil { + return + } + + if candidateViaRoutes(candidate, haRoutes) { + return + } + + err := conn.agent.AddRemoteCandidate(candidate) + if err != nil { + log.Errorf("error while handling remote candidate from peer %s", conn.config.Key) + return + } + }() +} + +func (conn *Conn) GetKey() string { + return conn.config.Key +} + +func (conn *Conn) reCreateAgent(relaySupport []ice.CandidateType) error { conn.mu.Lock() defer conn.mu.Unlock() @@ -192,7 +404,7 @@ func (conn *Conn) reCreateAgent() error { MulticastDNSMode: ice.MulticastDNSModeDisabled, NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, Urls: conn.config.StunTurn, - CandidateTypes: conn.candidateTypes(), + CandidateTypes: candidateTypes(), FailedTimeout: &failedTimeout, InterfaceFilter: stdnet.InterfaceFilter(conn.config.InterfaceBlackList), UDPMux: conn.config.UDPMux, @@ -242,150 +454,72 @@ func (conn *Conn) reCreateAgent() error { return nil } -func (conn *Conn) candidateTypes() []ice.CandidateType { - if hasICEForceRelayConn() { - return []ice.CandidateType{ice.CandidateTypeRelay} - } - // TODO: remove this once we have refactored userspace proxy into the bind package - if runtime.GOOS == "ios" { - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} - } - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} -} +func (conn *Conn) configureWgConnectionForRelay(remoteConn net.Conn, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) error { + conn.mu.Lock() + defer conn.mu.Unlock() -// Open opens connection to the remote peer starting ICE candidate gathering process. -// Blocks until connection has been closed or connection timeout. -// ConnStatus will be set accordingly -func (conn *Conn) Open(ctx context.Context) error { - log.Debugf("trying to connect to peer %s", conn.config.Key) + conn.wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) + endpoint, err := conn.wgProxy.AddTurnConn(remoteConn) + if err != nil { + return err + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + + conn.connID = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { + log.Errorf("Before add peer hook failed: %v", err) + } + } + + err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) + if err != nil { + if conn.wgProxy != nil { + if err := conn.wgProxy.CloseConn(); err != nil { + log.Warnf("Failed to close relay connection: %v", err) + } + } + // todo: is this nil correct? + return nil + } + + conn.status = StatusConnected peerState := State{ - PubKey: conn.config.Key, - IP: strings.Split(conn.config.WgConfig.AllowedIps, "/")[0], - ConnStatusUpdate: time.Now(), - ConnStatus: conn.status, - Mux: new(sync.RWMutex), - } - err := conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + PubKey: conn.config.Key, + ConnStatus: StatusConnected, + ConnStatusUpdate: time.Now(), + LocalIceCandidateType: "", + RemoteIceCandidateType: "", + LocalIceCandidateEndpoint: "", + RemoteIceCandidateEndpoint: "", + Direct: false, + RosenpassEnabled: isRosenpassEnabled(remoteRosenpassPubKey), + Mux: new(sync.RWMutex), + Relayed: true, } - defer func() { - err := conn.cleanup() - if err != nil { - log.Warnf("error while cleaning up peer connection %s: %v", conn.config.Key, err) - return - } - }() - - err = conn.reCreateAgent() - if err != nil { - return err - } - - err = conn.sendOffer() - if err != nil { - return err - } - - log.Debugf("connection offer sent to peer %s, waiting for the confirmation", conn.config.Key) - - // Only continue once we got a connection confirmation from the remote peer. - // The connection timeout could have happened before a confirmation received from the remote. - // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) - var remoteOfferAnswer OfferAnswer - select { - case remoteOfferAnswer = <-conn.remoteOffersCh: - // received confirmation from the remote peer -> ready to proceed - err = conn.sendAnswer() - if err != nil { - return err - } - case remoteOfferAnswer = <-conn.remoteAnswerCh: - case <-time.After(conn.config.Timeout): - return NewConnectionTimeoutError(conn.config.Key, conn.config.Timeout) - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - } - - log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", - conn.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) - - // at this point we received offer/answer and we are ready to gather candidates - conn.mu.Lock() - conn.status = StatusConnecting - conn.ctx, conn.notifyDisconnected = context.WithCancel(ctx) - defer conn.notifyDisconnected() - conn.mu.Unlock() - - peerState = State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } err = conn.statusRecorder.UpdatePeerState(peerState) if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + log.Warnf("unable to save peer's state, got error: %v", err) } - err = conn.agent.GatherCandidates() + _, ipNet, err := net.ParseCIDR(conn.config.WgConfig.AllowedIps) if err != nil { - return fmt.Errorf("gather candidates: %v", err) + return nil } - // will block until connection succeeded - // but it won't release if ICE Agent went into Disconnected or Failed state, - // so we have to cancel it with the provided context once agent detected a broken connection - isControlling := conn.config.LocalKey > conn.config.Key - var remoteConn *ice.Conn - if isControlling { - remoteConn, err = conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) - } else { - remoteConn, err = conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) - } - if err != nil { - return err + if runtime.GOOS == "ios" { + runtime.GC() } - // dynamically set remote WireGuard port if other side specified a different one from the default one - remoteWgPort := iface.DefaultWgPort - if remoteOfferAnswer.WgListenPort != 0 { - remoteWgPort = remoteOfferAnswer.WgListenPort + if conn.onConnected != nil { + conn.onConnected(conn.config.Key, remoteRosenpassPubKey, ipNet.IP.String(), remoteRosenpassAddr) } - // the ice connection has been established successfully so we are ready to start the proxy - remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort, remoteOfferAnswer.RosenpassPubKey, - remoteOfferAnswer.RosenpassAddr) - if err != nil { - return err - } - - log.Infof("connected to peer %s, endpoint address: %s", conn.config.Key, remoteAddr.String()) - - // wait until connection disconnected or has been closed externally (upper layer, e.g. engine) - select { - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - case <-conn.ctx.Done(): - // disconnected from the remote peer - return NewConnectionDisconnectedError(conn.config.Key) - } -} - -func isRelayCandidate(candidate ice.Candidate) bool { - return candidate.Type() == ice.CandidateTypeRelay -} - -func (conn *Conn) AddBeforeAddPeerHook(hook BeforeAddPeerHookFunc) { - conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook) -} - -func (conn *Conn) AddAfterRemovePeerHook(hook AfterRemovePeerHookFunc) { - conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook) + return nil } // configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected @@ -495,6 +629,17 @@ func (conn *Conn) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { } } +func (conn *Conn) waitForDisconnection() error { + select { + case <-conn.closeCh: + // closed externally + return NewConnectionClosedError(conn.config.Key) + case <-conn.ctx.Done(): + // disconnected from the remote peer + return NewConnectionDisconnectedError(conn.config.Key) + } +} + // cleanup closes all open resources and sets status to StatusDisconnected func (conn *Conn) cleanup() error { log.Debugf("trying to cleanup %s", conn.config.Key) @@ -565,36 +710,6 @@ func (conn *Conn) cleanup() error { return err3 } -// SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer -func (conn *Conn) SetSignalOffer(handler func(offer OfferAnswer) error) { - conn.signalOffer = handler -} - -// SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established -func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) { - conn.onConnected = handler -} - -// SetOnDisconnected sets a handler function to be triggered by Conn when a connection to a remote disconnected -func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string)) { - conn.onDisconnected = handler -} - -// SetSignalAnswer sets a handler function to be triggered by Conn when a new connection answer has to be signalled to the remote peer -func (conn *Conn) SetSignalAnswer(handler func(answer OfferAnswer) error) { - conn.signalAnswer = handler -} - -// SetSignalCandidate sets a handler function to be triggered by Conn when a new ICE local connection candidate has to be signalled to the remote peer -func (conn *Conn) SetSignalCandidate(handler func(candidate ice.Candidate) error) { - conn.signalCandidate = handler -} - -// SetSendSignalMessage sets a handler function to be triggered by Conn when there is new message to send via signal -func (conn *Conn) SetSendSignalMessage(handler func(message *sProto.Message) error) { - conn.sendSignalMessage = handler -} - // onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates // and then signals them to the remote peer func (conn *Conn) onICECandidate(candidate ice.Candidate) { @@ -679,106 +794,20 @@ func (conn *Conn) sendOffer() error { if err != nil { return err } - err = conn.signalOffer(OfferAnswer{ + oa := OfferAnswer{ IceCredentials: IceCredentials{localUFrag, localPwd}, WgListenPort: conn.config.LocalWgPort, Version: version.NetbirdVersion(), RosenpassPubKey: conn.config.RosenpassPubKey, RosenpassAddr: conn.config.RosenpassAddr, - }) - if err != nil { - return err } - return nil -} -// Close closes this peer Conn issuing a close event to the Conn closeCh -func (conn *Conn) Close() error { - conn.mu.Lock() - defer conn.mu.Unlock() - select { - case conn.closeCh <- struct{}{}: - return nil - default: - // probably could happen when peer has been added and removed right after not even starting to connect - // todo further investigate - // this really happens due to unordered messages coming from management - // more importantly it causes inconsistency -> 2 Conn objects for the same peer - // e.g. this flow: - // update from management has peers: [1,2,3,4] - // engine creates a Conn for peers: [1,2,3,4] and schedules Open in ~1sec - // before conn.Open() another update from management arrives with peers: [1,2,3] - // engine removes peer 4 and calls conn.Close() which does nothing (this default clause) - // before conn.Open() another update from management arrives with peers: [1,2,3,4,5] - // engine adds a new Conn for 4 and 5 - // therefore peer 4 has 2 Conn objects - log.Warnf("Connection has been already closed or attempted closing not started connection %s", conn.config.Key) - return NewConnectionAlreadyClosed(conn.config.Key) + relayIPAddress, err := conn.relayManager.RelayAddress() + if err == nil { + oa.RelaySrvAddress = relayIPAddress.String() } -} -// Status returns current status of the Conn -func (conn *Conn) Status() ConnStatus { - conn.mu.Lock() - defer conn.mu.Unlock() - return conn.status -} - -// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { - log.Debugf("OnRemoteOffer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteOffersCh <- offer: - return true - default: - log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - // connection might not be ready yet to receive so we ignore the message - return false - } -} - -// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { - log.Debugf("OnRemoteAnswer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteAnswerCh <- answer: - return true - default: - // connection might not be ready yet to receive so we ignore the message - log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - return false - } -} - -// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. -func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) - go func() { - conn.mu.Lock() - defer conn.mu.Unlock() - - if conn.agent == nil { - return - } - - if candidateViaRoutes(candidate, haRoutes) { - return - } - - err := conn.agent.AddRemoteCandidate(candidate) - if err != nil { - log.Errorf("error while handling remote candidate from peer %s", conn.config.Key) - return - } - }() -} - -func (conn *Conn) GetKey() string { - return conn.config.Key + return conn.signalOffer(oa) } func (conn *Conn) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { @@ -788,6 +817,109 @@ func (conn *Conn) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { return false } +func (conn *Conn) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { + var remoteOfferAnswer OfferAnswer + select { + case remoteOfferAnswer = <-conn.remoteOffersCh: + // received confirmation from the remote peer -> ready to proceed + err := conn.sendAnswer() + if err != nil { + return nil, err + } + case remoteOfferAnswer = <-conn.remoteAnswerCh: + case <-time.After(conn.config.Timeout): + return nil, NewConnectionTimeoutError(conn.config.Key, conn.config.Timeout) + case <-conn.closeCh: + // closed externally + return nil, NewConnectionClosedError(conn.config.Key) + } + + return &remoteOfferAnswer, nil +} + +func (conn *Conn) turnAgentDial(remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { + isControlling := conn.config.LocalKey > conn.config.Key + if isControlling { + return conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } else { + return conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } +} + +func (conn *Conn) setupRelayConnection(remoteOfferAnswer *OfferAnswer) bool { + if !isRelaySupported(remoteOfferAnswer) { + return false + } + + currentRelayAddress, err := conn.relayManager.RelayAddress() + if err != nil { + return false + } + + conn.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) + relayConn, err := conn.relayManager.OpenConn(remoteOfferAnswer.RelaySrvAddress, conn.config.Key) + if err != nil { + return false + } + + err = conn.configureWgConnectionForRelay(relayConn, remoteOfferAnswer.RosenpassPubKey, remoteOfferAnswer.RosenpassAddr) + if err != nil { + log.Errorf("failed to configure WireGuard connection for relay: %s", err) + return false + } + return true +} + +func (conn *Conn) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { + if conn.config.LocalKey > conn.config.Key { + return myRelayAddress + } + return remoteRelayAddress +} + +func (conn *Conn) setupICEConnection(remoteOfferAnswer *OfferAnswer, relayOperate bool) error { + var preferredCandidateTypes []ice.CandidateType + if relayOperate { + preferredCandidateTypes = candidateTypesP2P() + } else { + preferredCandidateTypes = candidateTypes() + } + + err := conn.reCreateAgent(preferredCandidateTypes) + if err != nil { + return err + } + + err = conn.agent.GatherCandidates() + if err != nil { + return fmt.Errorf("gather candidates: %v", err) + } + + // will block until connection succeeded + // but it won't release if ICE Agent went into Disconnected or Failed state, + // so we have to cancel it with the provided context once agent detected a broken connection + remoteConn, err := conn.turnAgentDial(remoteOfferAnswer) + if err != nil { + return err + } + + // dynamically set remote WireGuard port if other side specified a different one from the default one + remoteWgPort := iface.DefaultWgPort + if remoteOfferAnswer.WgListenPort != 0 { + remoteWgPort = remoteOfferAnswer.WgListenPort + } + + // the ice connection has been established successfully so we are ready to start the proxy + remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort, remoteOfferAnswer.RosenpassPubKey, + remoteOfferAnswer.RosenpassAddr) + if err != nil { + return err + } + + log.Infof("connected to peer %s, endpoint address: %s", conn.config.Key, remoteAddr.String()) + return nil +} + func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { relatedAdd := candidate.RelatedAddress() return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ @@ -827,3 +959,31 @@ func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool } return false } + +func candidateTypes() []ice.CandidateType { + if hasICEForceRelayConn() { + return []ice.CandidateType{ice.CandidateTypeRelay} + } + // TODO: remove this once we have refactored userspace proxy into the bind package + if runtime.GOOS == "ios" { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} + } + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} +} + +func candidateTypesP2P() []ice.CandidateType { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} +} + +func isRelayCandidate(candidate ice.Candidate) bool { + return candidate.Type() == ice.CandidateTypeRelay +} + +// todo check my side too +func isRelaySupported(answer *OfferAnswer) bool { + return answer.RelaySrvAddress != "" +} + +func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { + return remoteRosenpassPubKey != nil +} diff --git a/client/internal/wgproxy/factory_linux.go b/client/internal/wgproxy/factory_linux.go index 0262994d7..79f5cd548 100644 --- a/client/internal/wgproxy/factory_linux.go +++ b/client/internal/wgproxy/factory_linux.go @@ -4,20 +4,10 @@ package wgproxy import ( "context" - - log "github.com/sirupsen/logrus" ) func NewFactory(ctx context.Context, wgPort int) *Factory { f := &Factory{wgPort: wgPort} - ebpfProxy := NewWGEBPFProxy(ctx, wgPort) - err := ebpfProxy.listen() - if err != nil { - log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err) - return f - } - - f.ebpfProxy = ebpfProxy return f } diff --git a/management/proto/management.pb.go b/management/proto/management.pb.go index 18077ea89..f5dab4d81 100644 --- a/management/proto/management.pb.go +++ b/management/proto/management.pb.go @@ -1,25 +1,25 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.26.0 -// protoc v4.24.3 // source: management.proto package proto import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" + fmt "fmt" + proto "github.com/golang/protobuf/proto" timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" + math "math" ) -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type HostConfig_Protocol int32 @@ -31,49 +31,28 @@ const ( HostConfig_DTLS HostConfig_Protocol = 4 ) -// Enum value maps for HostConfig_Protocol. -var ( - HostConfig_Protocol_name = map[int32]string{ - 0: "UDP", - 1: "TCP", - 2: "HTTP", - 3: "HTTPS", - 4: "DTLS", - } - HostConfig_Protocol_value = map[string]int32{ - "UDP": 0, - "TCP": 1, - "HTTP": 2, - "HTTPS": 3, - "DTLS": 4, - } -) +var HostConfig_Protocol_name = map[int32]string{ + 0: "UDP", + 1: "TCP", + 2: "HTTP", + 3: "HTTPS", + 4: "DTLS", +} -func (x HostConfig_Protocol) Enum() *HostConfig_Protocol { - p := new(HostConfig_Protocol) - *p = x - return p +var HostConfig_Protocol_value = map[string]int32{ + "UDP": 0, + "TCP": 1, + "HTTP": 2, + "HTTPS": 3, + "DTLS": 4, } func (x HostConfig_Protocol) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) + return proto.EnumName(HostConfig_Protocol_name, int32(x)) } -func (HostConfig_Protocol) Descriptor() protoreflect.EnumDescriptor { - return file_management_proto_enumTypes[0].Descriptor() -} - -func (HostConfig_Protocol) Type() protoreflect.EnumType { - return &file_management_proto_enumTypes[0] -} - -func (x HostConfig_Protocol) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use HostConfig_Protocol.Descriptor instead. func (HostConfig_Protocol) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{11, 0} + return fileDescriptor_edc174f991dc0a25, []int{11, 0} } type DeviceAuthorizationFlowProvider int32 @@ -82,41 +61,20 @@ const ( DeviceAuthorizationFlow_HOSTED DeviceAuthorizationFlowProvider = 0 ) -// Enum value maps for DeviceAuthorizationFlowProvider. -var ( - DeviceAuthorizationFlowProvider_name = map[int32]string{ - 0: "HOSTED", - } - DeviceAuthorizationFlowProvider_value = map[string]int32{ - "HOSTED": 0, - } -) +var DeviceAuthorizationFlowProvider_name = map[int32]string{ + 0: "HOSTED", +} -func (x DeviceAuthorizationFlowProvider) Enum() *DeviceAuthorizationFlowProvider { - p := new(DeviceAuthorizationFlowProvider) - *p = x - return p +var DeviceAuthorizationFlowProvider_value = map[string]int32{ + "HOSTED": 0, } func (x DeviceAuthorizationFlowProvider) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) + return proto.EnumName(DeviceAuthorizationFlowProvider_name, int32(x)) } -func (DeviceAuthorizationFlowProvider) Descriptor() protoreflect.EnumDescriptor { - return file_management_proto_enumTypes[1].Descriptor() -} - -func (DeviceAuthorizationFlowProvider) Type() protoreflect.EnumType { - return &file_management_proto_enumTypes[1] -} - -func (x DeviceAuthorizationFlowProvider) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use DeviceAuthorizationFlowProvider.Descriptor instead. func (DeviceAuthorizationFlowProvider) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{18, 0} + return fileDescriptor_edc174f991dc0a25, []int{18, 0} } type FirewallRuleDirection int32 @@ -126,43 +84,22 @@ const ( FirewallRule_OUT FirewallRuleDirection = 1 ) -// Enum value maps for FirewallRuleDirection. -var ( - FirewallRuleDirection_name = map[int32]string{ - 0: "IN", - 1: "OUT", - } - FirewallRuleDirection_value = map[string]int32{ - "IN": 0, - "OUT": 1, - } -) +var FirewallRuleDirection_name = map[int32]string{ + 0: "IN", + 1: "OUT", +} -func (x FirewallRuleDirection) Enum() *FirewallRuleDirection { - p := new(FirewallRuleDirection) - *p = x - return p +var FirewallRuleDirection_value = map[string]int32{ + "IN": 0, + "OUT": 1, } func (x FirewallRuleDirection) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) + return proto.EnumName(FirewallRuleDirection_name, int32(x)) } -func (FirewallRuleDirection) Descriptor() protoreflect.EnumDescriptor { - return file_management_proto_enumTypes[2].Descriptor() -} - -func (FirewallRuleDirection) Type() protoreflect.EnumType { - return &file_management_proto_enumTypes[2] -} - -func (x FirewallRuleDirection) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use FirewallRuleDirection.Descriptor instead. func (FirewallRuleDirection) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{28, 0} + return fileDescriptor_edc174f991dc0a25, []int{28, 0} } type FirewallRuleAction int32 @@ -172,43 +109,22 @@ const ( FirewallRule_DROP FirewallRuleAction = 1 ) -// Enum value maps for FirewallRuleAction. -var ( - FirewallRuleAction_name = map[int32]string{ - 0: "ACCEPT", - 1: "DROP", - } - FirewallRuleAction_value = map[string]int32{ - "ACCEPT": 0, - "DROP": 1, - } -) +var FirewallRuleAction_name = map[int32]string{ + 0: "ACCEPT", + 1: "DROP", +} -func (x FirewallRuleAction) Enum() *FirewallRuleAction { - p := new(FirewallRuleAction) - *p = x - return p +var FirewallRuleAction_value = map[string]int32{ + "ACCEPT": 0, + "DROP": 1, } func (x FirewallRuleAction) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) + return proto.EnumName(FirewallRuleAction_name, int32(x)) } -func (FirewallRuleAction) Descriptor() protoreflect.EnumDescriptor { - return file_management_proto_enumTypes[3].Descriptor() -} - -func (FirewallRuleAction) Type() protoreflect.EnumType { - return &file_management_proto_enumTypes[3] -} - -func (x FirewallRuleAction) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use FirewallRuleAction.Descriptor instead. func (FirewallRuleAction) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{28, 1} + return fileDescriptor_edc174f991dc0a25, []int{28, 1} } type FirewallRuleProtocol int32 @@ -221,161 +137,121 @@ const ( FirewallRule_ICMP FirewallRuleProtocol = 4 ) -// Enum value maps for FirewallRuleProtocol. -var ( - FirewallRuleProtocol_name = map[int32]string{ - 0: "UNKNOWN", - 1: "ALL", - 2: "TCP", - 3: "UDP", - 4: "ICMP", - } - FirewallRuleProtocol_value = map[string]int32{ - "UNKNOWN": 0, - "ALL": 1, - "TCP": 2, - "UDP": 3, - "ICMP": 4, - } -) +var FirewallRuleProtocol_name = map[int32]string{ + 0: "UNKNOWN", + 1: "ALL", + 2: "TCP", + 3: "UDP", + 4: "ICMP", +} -func (x FirewallRuleProtocol) Enum() *FirewallRuleProtocol { - p := new(FirewallRuleProtocol) - *p = x - return p +var FirewallRuleProtocol_value = map[string]int32{ + "UNKNOWN": 0, + "ALL": 1, + "TCP": 2, + "UDP": 3, + "ICMP": 4, } func (x FirewallRuleProtocol) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) + return proto.EnumName(FirewallRuleProtocol_name, int32(x)) } -func (FirewallRuleProtocol) Descriptor() protoreflect.EnumDescriptor { - return file_management_proto_enumTypes[4].Descriptor() -} - -func (FirewallRuleProtocol) Type() protoreflect.EnumType { - return &file_management_proto_enumTypes[4] -} - -func (x FirewallRuleProtocol) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use FirewallRuleProtocol.Descriptor instead. func (FirewallRuleProtocol) EnumDescriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{28, 2} + return fileDescriptor_edc174f991dc0a25, []int{28, 2} } type EncryptedMessage struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Wireguard public key WgPubKey string `protobuf:"bytes,1,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` // encrypted message Body Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` // Version of the Wiretrustee Management Service protocol - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *EncryptedMessage) Reset() { - *x = EncryptedMessage{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *EncryptedMessage) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*EncryptedMessage) ProtoMessage() {} - -func (x *EncryptedMessage) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use EncryptedMessage.ProtoReflect.Descriptor instead. +func (m *EncryptedMessage) Reset() { *m = EncryptedMessage{} } +func (m *EncryptedMessage) String() string { return proto.CompactTextString(m) } +func (*EncryptedMessage) ProtoMessage() {} func (*EncryptedMessage) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{0} + return fileDescriptor_edc174f991dc0a25, []int{0} } -func (x *EncryptedMessage) GetWgPubKey() string { - if x != nil { - return x.WgPubKey +func (m *EncryptedMessage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_EncryptedMessage.Unmarshal(m, b) +} +func (m *EncryptedMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_EncryptedMessage.Marshal(b, m, deterministic) +} +func (m *EncryptedMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_EncryptedMessage.Merge(m, src) +} +func (m *EncryptedMessage) XXX_Size() int { + return xxx_messageInfo_EncryptedMessage.Size(m) +} +func (m *EncryptedMessage) XXX_DiscardUnknown() { + xxx_messageInfo_EncryptedMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_EncryptedMessage proto.InternalMessageInfo + +func (m *EncryptedMessage) GetWgPubKey() string { + if m != nil { + return m.WgPubKey } return "" } -func (x *EncryptedMessage) GetBody() []byte { - if x != nil { - return x.Body +func (m *EncryptedMessage) GetBody() []byte { + if m != nil { + return m.Body } return nil } -func (x *EncryptedMessage) GetVersion() int32 { - if x != nil { - return x.Version +func (m *EncryptedMessage) GetVersion() int32 { + if m != nil { + return m.Version } return 0 } type SyncRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *SyncRequest) Reset() { - *x = SyncRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SyncRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SyncRequest) ProtoMessage() {} - -func (x *SyncRequest) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SyncRequest.ProtoReflect.Descriptor instead. +func (m *SyncRequest) Reset() { *m = SyncRequest{} } +func (m *SyncRequest) String() string { return proto.CompactTextString(m) } +func (*SyncRequest) ProtoMessage() {} func (*SyncRequest) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{1} + return fileDescriptor_edc174f991dc0a25, []int{1} } +func (m *SyncRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SyncRequest.Unmarshal(m, b) +} +func (m *SyncRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SyncRequest.Marshal(b, m, deterministic) +} +func (m *SyncRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncRequest.Merge(m, src) +} +func (m *SyncRequest) XXX_Size() int { + return xxx_messageInfo_SyncRequest.Size(m) +} +func (m *SyncRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SyncRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncRequest proto.InternalMessageInfo + // SyncResponse represents a state that should be applied to the local peer (e.g. Wiretrustee servers config as well as local peer and remote peers configs) type SyncResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Global config WiretrusteeConfig *WiretrusteeConfig `protobuf:"bytes,1,opt,name=wiretrusteeConfig,proto3" json:"wiretrusteeConfig,omitempty"` // Deprecated. Use NetworkMap.PeerConfig @@ -384,82 +260,74 @@ type SyncResponse struct { RemotePeers []*RemotePeerConfig `protobuf:"bytes,3,rep,name=remotePeers,proto3" json:"remotePeers,omitempty"` // Indicates whether remotePeers array is empty or not to bypass protobuf null and empty array equality. // Deprecated. Use NetworkMap.remotePeersIsEmpty - RemotePeersIsEmpty bool `protobuf:"varint,4,opt,name=remotePeersIsEmpty,proto3" json:"remotePeersIsEmpty,omitempty"` - NetworkMap *NetworkMap `protobuf:"bytes,5,opt,name=NetworkMap,proto3" json:"NetworkMap,omitempty"` + RemotePeersIsEmpty bool `protobuf:"varint,4,opt,name=remotePeersIsEmpty,proto3" json:"remotePeersIsEmpty,omitempty"` + NetworkMap *NetworkMap `protobuf:"bytes,5,opt,name=NetworkMap,proto3" json:"NetworkMap,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *SyncResponse) Reset() { - *x = SyncResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SyncResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SyncResponse) ProtoMessage() {} - -func (x *SyncResponse) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SyncResponse.ProtoReflect.Descriptor instead. +func (m *SyncResponse) Reset() { *m = SyncResponse{} } +func (m *SyncResponse) String() string { return proto.CompactTextString(m) } +func (*SyncResponse) ProtoMessage() {} func (*SyncResponse) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{2} + return fileDescriptor_edc174f991dc0a25, []int{2} } -func (x *SyncResponse) GetWiretrusteeConfig() *WiretrusteeConfig { - if x != nil { - return x.WiretrusteeConfig +func (m *SyncResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SyncResponse.Unmarshal(m, b) +} +func (m *SyncResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SyncResponse.Marshal(b, m, deterministic) +} +func (m *SyncResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncResponse.Merge(m, src) +} +func (m *SyncResponse) XXX_Size() int { + return xxx_messageInfo_SyncResponse.Size(m) +} +func (m *SyncResponse) XXX_DiscardUnknown() { + xxx_messageInfo_SyncResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncResponse proto.InternalMessageInfo + +func (m *SyncResponse) GetWiretrusteeConfig() *WiretrusteeConfig { + if m != nil { + return m.WiretrusteeConfig } return nil } -func (x *SyncResponse) GetPeerConfig() *PeerConfig { - if x != nil { - return x.PeerConfig +func (m *SyncResponse) GetPeerConfig() *PeerConfig { + if m != nil { + return m.PeerConfig } return nil } -func (x *SyncResponse) GetRemotePeers() []*RemotePeerConfig { - if x != nil { - return x.RemotePeers +func (m *SyncResponse) GetRemotePeers() []*RemotePeerConfig { + if m != nil { + return m.RemotePeers } return nil } -func (x *SyncResponse) GetRemotePeersIsEmpty() bool { - if x != nil { - return x.RemotePeersIsEmpty +func (m *SyncResponse) GetRemotePeersIsEmpty() bool { + if m != nil { + return m.RemotePeersIsEmpty } return false } -func (x *SyncResponse) GetNetworkMap() *NetworkMap { - if x != nil { - return x.NetworkMap +func (m *SyncResponse) GetNetworkMap() *NetworkMap { + if m != nil { + return m.NetworkMap } return nil } type LoginRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Pre-authorized setup key (can be empty) SetupKey string `protobuf:"bytes,1,opt,name=setupKey,proto3" json:"setupKey,omitempty"` // Meta data of the peer (e.g. name, os_name, os_version, @@ -467,65 +335,61 @@ type LoginRequest struct { // SSO token (can be empty) JwtToken string `protobuf:"bytes,3,opt,name=jwtToken,proto3" json:"jwtToken,omitempty"` // Can be absent for now. - PeerKeys *PeerKeys `protobuf:"bytes,4,opt,name=peerKeys,proto3" json:"peerKeys,omitempty"` + PeerKeys *PeerKeys `protobuf:"bytes,4,opt,name=peerKeys,proto3" json:"peerKeys,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *LoginRequest) Reset() { - *x = LoginRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *LoginRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*LoginRequest) ProtoMessage() {} - -func (x *LoginRequest) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use LoginRequest.ProtoReflect.Descriptor instead. +func (m *LoginRequest) Reset() { *m = LoginRequest{} } +func (m *LoginRequest) String() string { return proto.CompactTextString(m) } +func (*LoginRequest) ProtoMessage() {} func (*LoginRequest) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{3} + return fileDescriptor_edc174f991dc0a25, []int{3} } -func (x *LoginRequest) GetSetupKey() string { - if x != nil { - return x.SetupKey +func (m *LoginRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoginRequest.Unmarshal(m, b) +} +func (m *LoginRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoginRequest.Marshal(b, m, deterministic) +} +func (m *LoginRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoginRequest.Merge(m, src) +} +func (m *LoginRequest) XXX_Size() int { + return xxx_messageInfo_LoginRequest.Size(m) +} +func (m *LoginRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoginRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_LoginRequest proto.InternalMessageInfo + +func (m *LoginRequest) GetSetupKey() string { + if m != nil { + return m.SetupKey } return "" } -func (x *LoginRequest) GetMeta() *PeerSystemMeta { - if x != nil { - return x.Meta +func (m *LoginRequest) GetMeta() *PeerSystemMeta { + if m != nil { + return m.Meta } return nil } -func (x *LoginRequest) GetJwtToken() string { - if x != nil { - return x.JwtToken +func (m *LoginRequest) GetJwtToken() string { + if m != nil { + return m.JwtToken } return "" } -func (x *LoginRequest) GetPeerKeys() *PeerKeys { - if x != nil { - return x.PeerKeys +func (m *LoginRequest) GetPeerKeys() *PeerKeys { + if m != nil { + return m.PeerKeys } return nil } @@ -533,561 +397,506 @@ func (x *LoginRequest) GetPeerKeys() *PeerKeys { // PeerKeys is additional peer info like SSH pub key and WireGuard public key. // This message is sent on Login or register requests, or when a key rotation has to happen. type PeerKeys struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // sshPubKey represents a public SSH key of the peer. Can be absent. SshPubKey []byte `protobuf:"bytes,1,opt,name=sshPubKey,proto3" json:"sshPubKey,omitempty"` // wgPubKey represents a public WireGuard key of the peer. Can be absent. - WgPubKey []byte `protobuf:"bytes,2,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` + WgPubKey []byte `protobuf:"bytes,2,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *PeerKeys) Reset() { - *x = PeerKeys{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PeerKeys) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PeerKeys) ProtoMessage() {} - -func (x *PeerKeys) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PeerKeys.ProtoReflect.Descriptor instead. +func (m *PeerKeys) Reset() { *m = PeerKeys{} } +func (m *PeerKeys) String() string { return proto.CompactTextString(m) } +func (*PeerKeys) ProtoMessage() {} func (*PeerKeys) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{4} + return fileDescriptor_edc174f991dc0a25, []int{4} } -func (x *PeerKeys) GetSshPubKey() []byte { - if x != nil { - return x.SshPubKey +func (m *PeerKeys) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PeerKeys.Unmarshal(m, b) +} +func (m *PeerKeys) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PeerKeys.Marshal(b, m, deterministic) +} +func (m *PeerKeys) XXX_Merge(src proto.Message) { + xxx_messageInfo_PeerKeys.Merge(m, src) +} +func (m *PeerKeys) XXX_Size() int { + return xxx_messageInfo_PeerKeys.Size(m) +} +func (m *PeerKeys) XXX_DiscardUnknown() { + xxx_messageInfo_PeerKeys.DiscardUnknown(m) +} + +var xxx_messageInfo_PeerKeys proto.InternalMessageInfo + +func (m *PeerKeys) GetSshPubKey() []byte { + if m != nil { + return m.SshPubKey } return nil } -func (x *PeerKeys) GetWgPubKey() []byte { - if x != nil { - return x.WgPubKey +func (m *PeerKeys) GetWgPubKey() []byte { + if m != nil { + return m.WgPubKey } return nil } // Environment is part of the PeerSystemMeta and describes the environment the agent is running in. type Environment struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // cloud is the cloud provider the agent is running in if applicable. Cloud string `protobuf:"bytes,1,opt,name=cloud,proto3" json:"cloud,omitempty"` // platform is the platform the agent is running on if applicable. - Platform string `protobuf:"bytes,2,opt,name=platform,proto3" json:"platform,omitempty"` + Platform string `protobuf:"bytes,2,opt,name=platform,proto3" json:"platform,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *Environment) Reset() { - *x = Environment{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Environment) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Environment) ProtoMessage() {} - -func (x *Environment) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Environment.ProtoReflect.Descriptor instead. +func (m *Environment) Reset() { *m = Environment{} } +func (m *Environment) String() string { return proto.CompactTextString(m) } +func (*Environment) ProtoMessage() {} func (*Environment) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{5} + return fileDescriptor_edc174f991dc0a25, []int{5} } -func (x *Environment) GetCloud() string { - if x != nil { - return x.Cloud +func (m *Environment) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Environment.Unmarshal(m, b) +} +func (m *Environment) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Environment.Marshal(b, m, deterministic) +} +func (m *Environment) XXX_Merge(src proto.Message) { + xxx_messageInfo_Environment.Merge(m, src) +} +func (m *Environment) XXX_Size() int { + return xxx_messageInfo_Environment.Size(m) +} +func (m *Environment) XXX_DiscardUnknown() { + xxx_messageInfo_Environment.DiscardUnknown(m) +} + +var xxx_messageInfo_Environment proto.InternalMessageInfo + +func (m *Environment) GetCloud() string { + if m != nil { + return m.Cloud } return "" } -func (x *Environment) GetPlatform() string { - if x != nil { - return x.Platform +func (m *Environment) GetPlatform() string { + if m != nil { + return m.Platform } return "" } // PeerSystemMeta is machine meta data like OS and version. type PeerSystemMeta struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` - GoOS string `protobuf:"bytes,2,opt,name=goOS,proto3" json:"goOS,omitempty"` - Kernel string `protobuf:"bytes,3,opt,name=kernel,proto3" json:"kernel,omitempty"` - Core string `protobuf:"bytes,4,opt,name=core,proto3" json:"core,omitempty"` - Platform string `protobuf:"bytes,5,opt,name=platform,proto3" json:"platform,omitempty"` - OS string `protobuf:"bytes,6,opt,name=OS,proto3" json:"OS,omitempty"` - WiretrusteeVersion string `protobuf:"bytes,7,opt,name=wiretrusteeVersion,proto3" json:"wiretrusteeVersion,omitempty"` - UiVersion string `protobuf:"bytes,8,opt,name=uiVersion,proto3" json:"uiVersion,omitempty"` - KernelVersion string `protobuf:"bytes,9,opt,name=kernelVersion,proto3" json:"kernelVersion,omitempty"` - OSVersion string `protobuf:"bytes,10,opt,name=OSVersion,proto3" json:"OSVersion,omitempty"` - NetworkAddresses []*NetworkAddress `protobuf:"bytes,11,rep,name=networkAddresses,proto3" json:"networkAddresses,omitempty"` - SysSerialNumber string `protobuf:"bytes,12,opt,name=sysSerialNumber,proto3" json:"sysSerialNumber,omitempty"` - SysProductName string `protobuf:"bytes,13,opt,name=sysProductName,proto3" json:"sysProductName,omitempty"` - SysManufacturer string `protobuf:"bytes,14,opt,name=sysManufacturer,proto3" json:"sysManufacturer,omitempty"` - Environment *Environment `protobuf:"bytes,15,opt,name=environment,proto3" json:"environment,omitempty"` + Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` + GoOS string `protobuf:"bytes,2,opt,name=goOS,proto3" json:"goOS,omitempty"` + Kernel string `protobuf:"bytes,3,opt,name=kernel,proto3" json:"kernel,omitempty"` + Core string `protobuf:"bytes,4,opt,name=core,proto3" json:"core,omitempty"` + Platform string `protobuf:"bytes,5,opt,name=platform,proto3" json:"platform,omitempty"` + OS string `protobuf:"bytes,6,opt,name=OS,proto3" json:"OS,omitempty"` + WiretrusteeVersion string `protobuf:"bytes,7,opt,name=wiretrusteeVersion,proto3" json:"wiretrusteeVersion,omitempty"` + UiVersion string `protobuf:"bytes,8,opt,name=uiVersion,proto3" json:"uiVersion,omitempty"` + KernelVersion string `protobuf:"bytes,9,opt,name=kernelVersion,proto3" json:"kernelVersion,omitempty"` + OSVersion string `protobuf:"bytes,10,opt,name=OSVersion,proto3" json:"OSVersion,omitempty"` + NetworkAddresses []*NetworkAddress `protobuf:"bytes,11,rep,name=networkAddresses,proto3" json:"networkAddresses,omitempty"` + SysSerialNumber string `protobuf:"bytes,12,opt,name=sysSerialNumber,proto3" json:"sysSerialNumber,omitempty"` + SysProductName string `protobuf:"bytes,13,opt,name=sysProductName,proto3" json:"sysProductName,omitempty"` + SysManufacturer string `protobuf:"bytes,14,opt,name=sysManufacturer,proto3" json:"sysManufacturer,omitempty"` + Environment *Environment `protobuf:"bytes,15,opt,name=environment,proto3" json:"environment,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *PeerSystemMeta) Reset() { - *x = PeerSystemMeta{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PeerSystemMeta) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PeerSystemMeta) ProtoMessage() {} - -func (x *PeerSystemMeta) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PeerSystemMeta.ProtoReflect.Descriptor instead. +func (m *PeerSystemMeta) Reset() { *m = PeerSystemMeta{} } +func (m *PeerSystemMeta) String() string { return proto.CompactTextString(m) } +func (*PeerSystemMeta) ProtoMessage() {} func (*PeerSystemMeta) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{6} + return fileDescriptor_edc174f991dc0a25, []int{6} } -func (x *PeerSystemMeta) GetHostname() string { - if x != nil { - return x.Hostname +func (m *PeerSystemMeta) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PeerSystemMeta.Unmarshal(m, b) +} +func (m *PeerSystemMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PeerSystemMeta.Marshal(b, m, deterministic) +} +func (m *PeerSystemMeta) XXX_Merge(src proto.Message) { + xxx_messageInfo_PeerSystemMeta.Merge(m, src) +} +func (m *PeerSystemMeta) XXX_Size() int { + return xxx_messageInfo_PeerSystemMeta.Size(m) +} +func (m *PeerSystemMeta) XXX_DiscardUnknown() { + xxx_messageInfo_PeerSystemMeta.DiscardUnknown(m) +} + +var xxx_messageInfo_PeerSystemMeta proto.InternalMessageInfo + +func (m *PeerSystemMeta) GetHostname() string { + if m != nil { + return m.Hostname } return "" } -func (x *PeerSystemMeta) GetGoOS() string { - if x != nil { - return x.GoOS +func (m *PeerSystemMeta) GetGoOS() string { + if m != nil { + return m.GoOS } return "" } -func (x *PeerSystemMeta) GetKernel() string { - if x != nil { - return x.Kernel +func (m *PeerSystemMeta) GetKernel() string { + if m != nil { + return m.Kernel } return "" } -func (x *PeerSystemMeta) GetCore() string { - if x != nil { - return x.Core +func (m *PeerSystemMeta) GetCore() string { + if m != nil { + return m.Core } return "" } -func (x *PeerSystemMeta) GetPlatform() string { - if x != nil { - return x.Platform +func (m *PeerSystemMeta) GetPlatform() string { + if m != nil { + return m.Platform } return "" } -func (x *PeerSystemMeta) GetOS() string { - if x != nil { - return x.OS +func (m *PeerSystemMeta) GetOS() string { + if m != nil { + return m.OS } return "" } -func (x *PeerSystemMeta) GetWiretrusteeVersion() string { - if x != nil { - return x.WiretrusteeVersion +func (m *PeerSystemMeta) GetWiretrusteeVersion() string { + if m != nil { + return m.WiretrusteeVersion } return "" } -func (x *PeerSystemMeta) GetUiVersion() string { - if x != nil { - return x.UiVersion +func (m *PeerSystemMeta) GetUiVersion() string { + if m != nil { + return m.UiVersion } return "" } -func (x *PeerSystemMeta) GetKernelVersion() string { - if x != nil { - return x.KernelVersion +func (m *PeerSystemMeta) GetKernelVersion() string { + if m != nil { + return m.KernelVersion } return "" } -func (x *PeerSystemMeta) GetOSVersion() string { - if x != nil { - return x.OSVersion +func (m *PeerSystemMeta) GetOSVersion() string { + if m != nil { + return m.OSVersion } return "" } -func (x *PeerSystemMeta) GetNetworkAddresses() []*NetworkAddress { - if x != nil { - return x.NetworkAddresses +func (m *PeerSystemMeta) GetNetworkAddresses() []*NetworkAddress { + if m != nil { + return m.NetworkAddresses } return nil } -func (x *PeerSystemMeta) GetSysSerialNumber() string { - if x != nil { - return x.SysSerialNumber +func (m *PeerSystemMeta) GetSysSerialNumber() string { + if m != nil { + return m.SysSerialNumber } return "" } -func (x *PeerSystemMeta) GetSysProductName() string { - if x != nil { - return x.SysProductName +func (m *PeerSystemMeta) GetSysProductName() string { + if m != nil { + return m.SysProductName } return "" } -func (x *PeerSystemMeta) GetSysManufacturer() string { - if x != nil { - return x.SysManufacturer +func (m *PeerSystemMeta) GetSysManufacturer() string { + if m != nil { + return m.SysManufacturer } return "" } -func (x *PeerSystemMeta) GetEnvironment() *Environment { - if x != nil { - return x.Environment +func (m *PeerSystemMeta) GetEnvironment() *Environment { + if m != nil { + return m.Environment } return nil } type LoginResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Global config WiretrusteeConfig *WiretrusteeConfig `protobuf:"bytes,1,opt,name=wiretrusteeConfig,proto3" json:"wiretrusteeConfig,omitempty"` // Peer local config - PeerConfig *PeerConfig `protobuf:"bytes,2,opt,name=peerConfig,proto3" json:"peerConfig,omitempty"` + PeerConfig *PeerConfig `protobuf:"bytes,2,opt,name=peerConfig,proto3" json:"peerConfig,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *LoginResponse) Reset() { - *x = LoginResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *LoginResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*LoginResponse) ProtoMessage() {} - -func (x *LoginResponse) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use LoginResponse.ProtoReflect.Descriptor instead. +func (m *LoginResponse) Reset() { *m = LoginResponse{} } +func (m *LoginResponse) String() string { return proto.CompactTextString(m) } +func (*LoginResponse) ProtoMessage() {} func (*LoginResponse) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{7} + return fileDescriptor_edc174f991dc0a25, []int{7} } -func (x *LoginResponse) GetWiretrusteeConfig() *WiretrusteeConfig { - if x != nil { - return x.WiretrusteeConfig +func (m *LoginResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoginResponse.Unmarshal(m, b) +} +func (m *LoginResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoginResponse.Marshal(b, m, deterministic) +} +func (m *LoginResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoginResponse.Merge(m, src) +} +func (m *LoginResponse) XXX_Size() int { + return xxx_messageInfo_LoginResponse.Size(m) +} +func (m *LoginResponse) XXX_DiscardUnknown() { + xxx_messageInfo_LoginResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_LoginResponse proto.InternalMessageInfo + +func (m *LoginResponse) GetWiretrusteeConfig() *WiretrusteeConfig { + if m != nil { + return m.WiretrusteeConfig } return nil } -func (x *LoginResponse) GetPeerConfig() *PeerConfig { - if x != nil { - return x.PeerConfig +func (m *LoginResponse) GetPeerConfig() *PeerConfig { + if m != nil { + return m.PeerConfig } return nil } type ServerKeyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Server's Wireguard public key Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // Key expiration timestamp after which the key should be fetched again by the client ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=expiresAt,proto3" json:"expiresAt,omitempty"` // Version of the Wiretrustee Management Service protocol - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *ServerKeyResponse) Reset() { - *x = ServerKeyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ServerKeyResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ServerKeyResponse) ProtoMessage() {} - -func (x *ServerKeyResponse) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ServerKeyResponse.ProtoReflect.Descriptor instead. +func (m *ServerKeyResponse) Reset() { *m = ServerKeyResponse{} } +func (m *ServerKeyResponse) String() string { return proto.CompactTextString(m) } +func (*ServerKeyResponse) ProtoMessage() {} func (*ServerKeyResponse) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{8} + return fileDescriptor_edc174f991dc0a25, []int{8} } -func (x *ServerKeyResponse) GetKey() string { - if x != nil { - return x.Key +func (m *ServerKeyResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerKeyResponse.Unmarshal(m, b) +} +func (m *ServerKeyResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerKeyResponse.Marshal(b, m, deterministic) +} +func (m *ServerKeyResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerKeyResponse.Merge(m, src) +} +func (m *ServerKeyResponse) XXX_Size() int { + return xxx_messageInfo_ServerKeyResponse.Size(m) +} +func (m *ServerKeyResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ServerKeyResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerKeyResponse proto.InternalMessageInfo + +func (m *ServerKeyResponse) GetKey() string { + if m != nil { + return m.Key } return "" } -func (x *ServerKeyResponse) GetExpiresAt() *timestamppb.Timestamp { - if x != nil { - return x.ExpiresAt +func (m *ServerKeyResponse) GetExpiresAt() *timestamppb.Timestamp { + if m != nil { + return m.ExpiresAt } return nil } -func (x *ServerKeyResponse) GetVersion() int32 { - if x != nil { - return x.Version +func (m *ServerKeyResponse) GetVersion() int32 { + if m != nil { + return m.Version } return 0 } type Empty struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *Empty) Reset() { - *x = Empty{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Empty) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Empty) ProtoMessage() {} - -func (x *Empty) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Empty.ProtoReflect.Descriptor instead. +func (m *Empty) Reset() { *m = Empty{} } +func (m *Empty) String() string { return proto.CompactTextString(m) } +func (*Empty) ProtoMessage() {} func (*Empty) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{9} + return fileDescriptor_edc174f991dc0a25, []int{9} } +func (m *Empty) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Empty.Unmarshal(m, b) +} +func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Empty.Marshal(b, m, deterministic) +} +func (m *Empty) XXX_Merge(src proto.Message) { + xxx_messageInfo_Empty.Merge(m, src) +} +func (m *Empty) XXX_Size() int { + return xxx_messageInfo_Empty.Size(m) +} +func (m *Empty) XXX_DiscardUnknown() { + xxx_messageInfo_Empty.DiscardUnknown(m) +} + +var xxx_messageInfo_Empty proto.InternalMessageInfo + // WiretrusteeConfig is a common configuration of any Wiretrustee peer. It contains STUN, TURN, Signal and Management servers configurations type WiretrusteeConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // a list of STUN servers Stuns []*HostConfig `protobuf:"bytes,1,rep,name=stuns,proto3" json:"stuns,omitempty"` // a list of TURN servers Turns []*ProtectedHostConfig `protobuf:"bytes,2,rep,name=turns,proto3" json:"turns,omitempty"` // a Signal server config - Signal *HostConfig `protobuf:"bytes,3,opt,name=signal,proto3" json:"signal,omitempty"` + Signal *HostConfig `protobuf:"bytes,3,opt,name=signal,proto3" json:"signal,omitempty"` + RelayAddress string `protobuf:"bytes,4,opt,name=RelayAddress,proto3" json:"RelayAddress,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *WiretrusteeConfig) Reset() { - *x = WiretrusteeConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *WiretrusteeConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*WiretrusteeConfig) ProtoMessage() {} - -func (x *WiretrusteeConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use WiretrusteeConfig.ProtoReflect.Descriptor instead. +func (m *WiretrusteeConfig) Reset() { *m = WiretrusteeConfig{} } +func (m *WiretrusteeConfig) String() string { return proto.CompactTextString(m) } +func (*WiretrusteeConfig) ProtoMessage() {} func (*WiretrusteeConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{10} + return fileDescriptor_edc174f991dc0a25, []int{10} } -func (x *WiretrusteeConfig) GetStuns() []*HostConfig { - if x != nil { - return x.Stuns +func (m *WiretrusteeConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_WiretrusteeConfig.Unmarshal(m, b) +} +func (m *WiretrusteeConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_WiretrusteeConfig.Marshal(b, m, deterministic) +} +func (m *WiretrusteeConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_WiretrusteeConfig.Merge(m, src) +} +func (m *WiretrusteeConfig) XXX_Size() int { + return xxx_messageInfo_WiretrusteeConfig.Size(m) +} +func (m *WiretrusteeConfig) XXX_DiscardUnknown() { + xxx_messageInfo_WiretrusteeConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_WiretrusteeConfig proto.InternalMessageInfo + +func (m *WiretrusteeConfig) GetStuns() []*HostConfig { + if m != nil { + return m.Stuns } return nil } -func (x *WiretrusteeConfig) GetTurns() []*ProtectedHostConfig { - if x != nil { - return x.Turns +func (m *WiretrusteeConfig) GetTurns() []*ProtectedHostConfig { + if m != nil { + return m.Turns } return nil } -func (x *WiretrusteeConfig) GetSignal() *HostConfig { - if x != nil { - return x.Signal +func (m *WiretrusteeConfig) GetSignal() *HostConfig { + if m != nil { + return m.Signal } return nil } -// HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) -type HostConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // URI of the resource e.g. turns://stun.wiretrustee.com:4430 or signal.wiretrustee.com:10000 - Uri string `protobuf:"bytes,1,opt,name=uri,proto3" json:"uri,omitempty"` - Protocol HostConfig_Protocol `protobuf:"varint,2,opt,name=protocol,proto3,enum=management.HostConfig_Protocol" json:"protocol,omitempty"` -} - -func (x *HostConfig) Reset() { - *x = HostConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *HostConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*HostConfig) ProtoMessage() {} - -func (x *HostConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use HostConfig.ProtoReflect.Descriptor instead. -func (*HostConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{11} -} - -func (x *HostConfig) GetUri() string { - if x != nil { - return x.Uri +func (m *WiretrusteeConfig) GetRelayAddress() string { + if m != nil { + return m.RelayAddress } return "" } -func (x *HostConfig) GetProtocol() HostConfig_Protocol { - if x != nil { - return x.Protocol +// HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) +type HostConfig struct { + // URI of the resource e.g. turns://stun.wiretrustee.com:4430 or signal.wiretrustee.com:10000 + Uri string `protobuf:"bytes,1,opt,name=uri,proto3" json:"uri,omitempty"` + Protocol HostConfig_Protocol `protobuf:"varint,2,opt,name=protocol,proto3,enum=management.HostConfig_Protocol" json:"protocol,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HostConfig) Reset() { *m = HostConfig{} } +func (m *HostConfig) String() string { return proto.CompactTextString(m) } +func (*HostConfig) ProtoMessage() {} +func (*HostConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_edc174f991dc0a25, []int{11} +} + +func (m *HostConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HostConfig.Unmarshal(m, b) +} +func (m *HostConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HostConfig.Marshal(b, m, deterministic) +} +func (m *HostConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_HostConfig.Merge(m, src) +} +func (m *HostConfig) XXX_Size() int { + return xxx_messageInfo_HostConfig.Size(m) +} +func (m *HostConfig) XXX_DiscardUnknown() { + xxx_messageInfo_HostConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_HostConfig proto.InternalMessageInfo + +func (m *HostConfig) GetUri() string { + if m != nil { + return m.Uri + } + return "" +} + +func (m *HostConfig) GetProtocol() HostConfig_Protocol { + if m != nil { + return m.Protocol } return HostConfig_UDP } @@ -1095,64 +904,56 @@ func (x *HostConfig) GetProtocol() HostConfig_Protocol { // ProtectedHostConfig is similar to HostConfig but has additional user and password // Mostly used for TURN servers type ProtectedHostConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - HostConfig *HostConfig `protobuf:"bytes,1,opt,name=hostConfig,proto3" json:"hostConfig,omitempty"` - User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` - Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` + HostConfig *HostConfig `protobuf:"bytes,1,opt,name=hostConfig,proto3" json:"hostConfig,omitempty"` + User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` + Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *ProtectedHostConfig) Reset() { - *x = ProtectedHostConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ProtectedHostConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProtectedHostConfig) ProtoMessage() {} - -func (x *ProtectedHostConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ProtectedHostConfig.ProtoReflect.Descriptor instead. +func (m *ProtectedHostConfig) Reset() { *m = ProtectedHostConfig{} } +func (m *ProtectedHostConfig) String() string { return proto.CompactTextString(m) } +func (*ProtectedHostConfig) ProtoMessage() {} func (*ProtectedHostConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{12} + return fileDescriptor_edc174f991dc0a25, []int{12} } -func (x *ProtectedHostConfig) GetHostConfig() *HostConfig { - if x != nil { - return x.HostConfig +func (m *ProtectedHostConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ProtectedHostConfig.Unmarshal(m, b) +} +func (m *ProtectedHostConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ProtectedHostConfig.Marshal(b, m, deterministic) +} +func (m *ProtectedHostConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProtectedHostConfig.Merge(m, src) +} +func (m *ProtectedHostConfig) XXX_Size() int { + return xxx_messageInfo_ProtectedHostConfig.Size(m) +} +func (m *ProtectedHostConfig) XXX_DiscardUnknown() { + xxx_messageInfo_ProtectedHostConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_ProtectedHostConfig proto.InternalMessageInfo + +func (m *ProtectedHostConfig) GetHostConfig() *HostConfig { + if m != nil { + return m.HostConfig } return nil } -func (x *ProtectedHostConfig) GetUser() string { - if x != nil { - return x.User +func (m *ProtectedHostConfig) GetUser() string { + if m != nil { + return m.User } return "" } -func (x *ProtectedHostConfig) GetPassword() string { - if x != nil { - return x.Password +func (m *ProtectedHostConfig) GetPassword() string { + if m != nil { + return m.Password } return "" } @@ -1160,10 +961,6 @@ func (x *ProtectedHostConfig) GetPassword() string { // PeerConfig represents a configuration of a "our" peer. // The properties are used to configure local Wireguard type PeerConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Peer's virtual IP address within the Wiretrustee VPN (a Wireguard address config) Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` // Wiretrustee DNS server (a Wireguard DNS config) @@ -1171,75 +968,67 @@ type PeerConfig struct { // SSHConfig of the peer. SshConfig *SSHConfig `protobuf:"bytes,3,opt,name=sshConfig,proto3" json:"sshConfig,omitempty"` // Peer fully qualified domain name - Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` + Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *PeerConfig) Reset() { - *x = PeerConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PeerConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PeerConfig) ProtoMessage() {} - -func (x *PeerConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PeerConfig.ProtoReflect.Descriptor instead. +func (m *PeerConfig) Reset() { *m = PeerConfig{} } +func (m *PeerConfig) String() string { return proto.CompactTextString(m) } +func (*PeerConfig) ProtoMessage() {} func (*PeerConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{13} + return fileDescriptor_edc174f991dc0a25, []int{13} } -func (x *PeerConfig) GetAddress() string { - if x != nil { - return x.Address +func (m *PeerConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PeerConfig.Unmarshal(m, b) +} +func (m *PeerConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PeerConfig.Marshal(b, m, deterministic) +} +func (m *PeerConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_PeerConfig.Merge(m, src) +} +func (m *PeerConfig) XXX_Size() int { + return xxx_messageInfo_PeerConfig.Size(m) +} +func (m *PeerConfig) XXX_DiscardUnknown() { + xxx_messageInfo_PeerConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_PeerConfig proto.InternalMessageInfo + +func (m *PeerConfig) GetAddress() string { + if m != nil { + return m.Address } return "" } -func (x *PeerConfig) GetDns() string { - if x != nil { - return x.Dns +func (m *PeerConfig) GetDns() string { + if m != nil { + return m.Dns } return "" } -func (x *PeerConfig) GetSshConfig() *SSHConfig { - if x != nil { - return x.SshConfig +func (m *PeerConfig) GetSshConfig() *SSHConfig { + if m != nil { + return m.SshConfig } return nil } -func (x *PeerConfig) GetFqdn() string { - if x != nil { - return x.Fqdn +func (m *PeerConfig) GetFqdn() string { + if m != nil { + return m.Fqdn } return "" } // NetworkMap represents a network state of the peer with the corresponding configuration parameters to establish peer-to-peer connections type NetworkMap struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // Serial is an ID of the network state to be used by clients to order updates. // The larger the Serial the newer the configuration. // E.g. the client app should keep track of this id locally and discard all the configurations with a lower value @@ -1259,100 +1048,96 @@ type NetworkMap struct { // FirewallRule represents a list of firewall rules to be applied to peer FirewallRules []*FirewallRule `protobuf:"bytes,8,rep,name=FirewallRules,proto3" json:"FirewallRules,omitempty"` // firewallRulesIsEmpty indicates whether FirewallRule array is empty or not to bypass protobuf null and empty array equality. - FirewallRulesIsEmpty bool `protobuf:"varint,9,opt,name=firewallRulesIsEmpty,proto3" json:"firewallRulesIsEmpty,omitempty"` + FirewallRulesIsEmpty bool `protobuf:"varint,9,opt,name=firewallRulesIsEmpty,proto3" json:"firewallRulesIsEmpty,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *NetworkMap) Reset() { - *x = NetworkMap{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NetworkMap) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NetworkMap) ProtoMessage() {} - -func (x *NetworkMap) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NetworkMap.ProtoReflect.Descriptor instead. +func (m *NetworkMap) Reset() { *m = NetworkMap{} } +func (m *NetworkMap) String() string { return proto.CompactTextString(m) } +func (*NetworkMap) ProtoMessage() {} func (*NetworkMap) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{14} + return fileDescriptor_edc174f991dc0a25, []int{14} } -func (x *NetworkMap) GetSerial() uint64 { - if x != nil { - return x.Serial +func (m *NetworkMap) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NetworkMap.Unmarshal(m, b) +} +func (m *NetworkMap) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NetworkMap.Marshal(b, m, deterministic) +} +func (m *NetworkMap) XXX_Merge(src proto.Message) { + xxx_messageInfo_NetworkMap.Merge(m, src) +} +func (m *NetworkMap) XXX_Size() int { + return xxx_messageInfo_NetworkMap.Size(m) +} +func (m *NetworkMap) XXX_DiscardUnknown() { + xxx_messageInfo_NetworkMap.DiscardUnknown(m) +} + +var xxx_messageInfo_NetworkMap proto.InternalMessageInfo + +func (m *NetworkMap) GetSerial() uint64 { + if m != nil { + return m.Serial } return 0 } -func (x *NetworkMap) GetPeerConfig() *PeerConfig { - if x != nil { - return x.PeerConfig +func (m *NetworkMap) GetPeerConfig() *PeerConfig { + if m != nil { + return m.PeerConfig } return nil } -func (x *NetworkMap) GetRemotePeers() []*RemotePeerConfig { - if x != nil { - return x.RemotePeers +func (m *NetworkMap) GetRemotePeers() []*RemotePeerConfig { + if m != nil { + return m.RemotePeers } return nil } -func (x *NetworkMap) GetRemotePeersIsEmpty() bool { - if x != nil { - return x.RemotePeersIsEmpty +func (m *NetworkMap) GetRemotePeersIsEmpty() bool { + if m != nil { + return m.RemotePeersIsEmpty } return false } -func (x *NetworkMap) GetRoutes() []*Route { - if x != nil { - return x.Routes +func (m *NetworkMap) GetRoutes() []*Route { + if m != nil { + return m.Routes } return nil } -func (x *NetworkMap) GetDNSConfig() *DNSConfig { - if x != nil { - return x.DNSConfig +func (m *NetworkMap) GetDNSConfig() *DNSConfig { + if m != nil { + return m.DNSConfig } return nil } -func (x *NetworkMap) GetOfflinePeers() []*RemotePeerConfig { - if x != nil { - return x.OfflinePeers +func (m *NetworkMap) GetOfflinePeers() []*RemotePeerConfig { + if m != nil { + return m.OfflinePeers } return nil } -func (x *NetworkMap) GetFirewallRules() []*FirewallRule { - if x != nil { - return x.FirewallRules +func (m *NetworkMap) GetFirewallRules() []*FirewallRule { + if m != nil { + return m.FirewallRules } return nil } -func (x *NetworkMap) GetFirewallRulesIsEmpty() bool { - if x != nil { - return x.FirewallRulesIsEmpty +func (m *NetworkMap) GetFirewallRulesIsEmpty() bool { + if m != nil { + return m.FirewallRulesIsEmpty } return false } @@ -1360,10 +1145,6 @@ func (x *NetworkMap) GetFirewallRulesIsEmpty() bool { // RemotePeerConfig represents a configuration of a remote peer. // The properties are used to configure WireGuard Peers sections type RemotePeerConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // A WireGuard public key of a remote peer WgPubKey string `protobuf:"bytes,1,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` // WireGuard allowed IPs of a remote peer e.g. [10.30.30.1/32] @@ -1371,321 +1152,275 @@ type RemotePeerConfig struct { // SSHConfig is a SSH config of the remote peer. SSHConfig.sshPubKey should be ignored because peer knows it's SSH key. SshConfig *SSHConfig `protobuf:"bytes,3,opt,name=sshConfig,proto3" json:"sshConfig,omitempty"` // Peer fully qualified domain name - Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` + Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *RemotePeerConfig) Reset() { - *x = RemotePeerConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RemotePeerConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RemotePeerConfig) ProtoMessage() {} - -func (x *RemotePeerConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RemotePeerConfig.ProtoReflect.Descriptor instead. +func (m *RemotePeerConfig) Reset() { *m = RemotePeerConfig{} } +func (m *RemotePeerConfig) String() string { return proto.CompactTextString(m) } +func (*RemotePeerConfig) ProtoMessage() {} func (*RemotePeerConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{15} + return fileDescriptor_edc174f991dc0a25, []int{15} } -func (x *RemotePeerConfig) GetWgPubKey() string { - if x != nil { - return x.WgPubKey +func (m *RemotePeerConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_RemotePeerConfig.Unmarshal(m, b) +} +func (m *RemotePeerConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_RemotePeerConfig.Marshal(b, m, deterministic) +} +func (m *RemotePeerConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_RemotePeerConfig.Merge(m, src) +} +func (m *RemotePeerConfig) XXX_Size() int { + return xxx_messageInfo_RemotePeerConfig.Size(m) +} +func (m *RemotePeerConfig) XXX_DiscardUnknown() { + xxx_messageInfo_RemotePeerConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_RemotePeerConfig proto.InternalMessageInfo + +func (m *RemotePeerConfig) GetWgPubKey() string { + if m != nil { + return m.WgPubKey } return "" } -func (x *RemotePeerConfig) GetAllowedIps() []string { - if x != nil { - return x.AllowedIps +func (m *RemotePeerConfig) GetAllowedIps() []string { + if m != nil { + return m.AllowedIps } return nil } -func (x *RemotePeerConfig) GetSshConfig() *SSHConfig { - if x != nil { - return x.SshConfig +func (m *RemotePeerConfig) GetSshConfig() *SSHConfig { + if m != nil { + return m.SshConfig } return nil } -func (x *RemotePeerConfig) GetFqdn() string { - if x != nil { - return x.Fqdn +func (m *RemotePeerConfig) GetFqdn() string { + if m != nil { + return m.Fqdn } return "" } // SSHConfig represents SSH configurations of a peer. type SSHConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // sshEnabled indicates whether a SSH server is enabled on this peer SshEnabled bool `protobuf:"varint,1,opt,name=sshEnabled,proto3" json:"sshEnabled,omitempty"` // sshPubKey is a SSH public key of a peer to be added to authorized_hosts. // This property should be ignore if SSHConfig comes from PeerConfig. - SshPubKey []byte `protobuf:"bytes,2,opt,name=sshPubKey,proto3" json:"sshPubKey,omitempty"` + SshPubKey []byte `protobuf:"bytes,2,opt,name=sshPubKey,proto3" json:"sshPubKey,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *SSHConfig) Reset() { - *x = SSHConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SSHConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SSHConfig) ProtoMessage() {} - -func (x *SSHConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SSHConfig.ProtoReflect.Descriptor instead. +func (m *SSHConfig) Reset() { *m = SSHConfig{} } +func (m *SSHConfig) String() string { return proto.CompactTextString(m) } +func (*SSHConfig) ProtoMessage() {} func (*SSHConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{16} + return fileDescriptor_edc174f991dc0a25, []int{16} } -func (x *SSHConfig) GetSshEnabled() bool { - if x != nil { - return x.SshEnabled +func (m *SSHConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SSHConfig.Unmarshal(m, b) +} +func (m *SSHConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SSHConfig.Marshal(b, m, deterministic) +} +func (m *SSHConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_SSHConfig.Merge(m, src) +} +func (m *SSHConfig) XXX_Size() int { + return xxx_messageInfo_SSHConfig.Size(m) +} +func (m *SSHConfig) XXX_DiscardUnknown() { + xxx_messageInfo_SSHConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_SSHConfig proto.InternalMessageInfo + +func (m *SSHConfig) GetSshEnabled() bool { + if m != nil { + return m.SshEnabled } return false } -func (x *SSHConfig) GetSshPubKey() []byte { - if x != nil { - return x.SshPubKey +func (m *SSHConfig) GetSshPubKey() []byte { + if m != nil { + return m.SshPubKey } return nil } // DeviceAuthorizationFlowRequest empty struct for future expansion type DeviceAuthorizationFlowRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *DeviceAuthorizationFlowRequest) Reset() { - *x = DeviceAuthorizationFlowRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DeviceAuthorizationFlowRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DeviceAuthorizationFlowRequest) ProtoMessage() {} - -func (x *DeviceAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DeviceAuthorizationFlowRequest.ProtoReflect.Descriptor instead. +func (m *DeviceAuthorizationFlowRequest) Reset() { *m = DeviceAuthorizationFlowRequest{} } +func (m *DeviceAuthorizationFlowRequest) String() string { return proto.CompactTextString(m) } +func (*DeviceAuthorizationFlowRequest) ProtoMessage() {} func (*DeviceAuthorizationFlowRequest) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{17} + return fileDescriptor_edc174f991dc0a25, []int{17} } +func (m *DeviceAuthorizationFlowRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeviceAuthorizationFlowRequest.Unmarshal(m, b) +} +func (m *DeviceAuthorizationFlowRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeviceAuthorizationFlowRequest.Marshal(b, m, deterministic) +} +func (m *DeviceAuthorizationFlowRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeviceAuthorizationFlowRequest.Merge(m, src) +} +func (m *DeviceAuthorizationFlowRequest) XXX_Size() int { + return xxx_messageInfo_DeviceAuthorizationFlowRequest.Size(m) +} +func (m *DeviceAuthorizationFlowRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeviceAuthorizationFlowRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeviceAuthorizationFlowRequest proto.InternalMessageInfo + // DeviceAuthorizationFlow represents Device Authorization Flow information // that can be used by the client to login initiate a Oauth 2.0 device authorization grant flow // see https://datatracker.ietf.org/doc/html/rfc8628 type DeviceAuthorizationFlow struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // An IDP provider , (eg. Auth0) - Provider DeviceAuthorizationFlowProvider `protobuf:"varint,1,opt,name=Provider,proto3,enum=management.DeviceAuthorizationFlowProvider" json:"Provider,omitempty"` - ProviderConfig *ProviderConfig `protobuf:"bytes,2,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` + Provider DeviceAuthorizationFlowProvider `protobuf:"varint,1,opt,name=Provider,proto3,enum=management.DeviceAuthorizationFlowProvider" json:"Provider,omitempty"` + ProviderConfig *ProviderConfig `protobuf:"bytes,2,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *DeviceAuthorizationFlow) Reset() { - *x = DeviceAuthorizationFlow{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DeviceAuthorizationFlow) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DeviceAuthorizationFlow) ProtoMessage() {} - -func (x *DeviceAuthorizationFlow) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DeviceAuthorizationFlow.ProtoReflect.Descriptor instead. +func (m *DeviceAuthorizationFlow) Reset() { *m = DeviceAuthorizationFlow{} } +func (m *DeviceAuthorizationFlow) String() string { return proto.CompactTextString(m) } +func (*DeviceAuthorizationFlow) ProtoMessage() {} func (*DeviceAuthorizationFlow) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{18} + return fileDescriptor_edc174f991dc0a25, []int{18} } -func (x *DeviceAuthorizationFlow) GetProvider() DeviceAuthorizationFlowProvider { - if x != nil { - return x.Provider +func (m *DeviceAuthorizationFlow) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeviceAuthorizationFlow.Unmarshal(m, b) +} +func (m *DeviceAuthorizationFlow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeviceAuthorizationFlow.Marshal(b, m, deterministic) +} +func (m *DeviceAuthorizationFlow) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeviceAuthorizationFlow.Merge(m, src) +} +func (m *DeviceAuthorizationFlow) XXX_Size() int { + return xxx_messageInfo_DeviceAuthorizationFlow.Size(m) +} +func (m *DeviceAuthorizationFlow) XXX_DiscardUnknown() { + xxx_messageInfo_DeviceAuthorizationFlow.DiscardUnknown(m) +} + +var xxx_messageInfo_DeviceAuthorizationFlow proto.InternalMessageInfo + +func (m *DeviceAuthorizationFlow) GetProvider() DeviceAuthorizationFlowProvider { + if m != nil { + return m.Provider } return DeviceAuthorizationFlow_HOSTED } -func (x *DeviceAuthorizationFlow) GetProviderConfig() *ProviderConfig { - if x != nil { - return x.ProviderConfig +func (m *DeviceAuthorizationFlow) GetProviderConfig() *ProviderConfig { + if m != nil { + return m.ProviderConfig } return nil } // PKCEAuthorizationFlowRequest empty struct for future expansion type PKCEAuthorizationFlowRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *PKCEAuthorizationFlowRequest) Reset() { - *x = PKCEAuthorizationFlowRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PKCEAuthorizationFlowRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PKCEAuthorizationFlowRequest) ProtoMessage() {} - -func (x *PKCEAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PKCEAuthorizationFlowRequest.ProtoReflect.Descriptor instead. +func (m *PKCEAuthorizationFlowRequest) Reset() { *m = PKCEAuthorizationFlowRequest{} } +func (m *PKCEAuthorizationFlowRequest) String() string { return proto.CompactTextString(m) } +func (*PKCEAuthorizationFlowRequest) ProtoMessage() {} func (*PKCEAuthorizationFlowRequest) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{19} + return fileDescriptor_edc174f991dc0a25, []int{19} } +func (m *PKCEAuthorizationFlowRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PKCEAuthorizationFlowRequest.Unmarshal(m, b) +} +func (m *PKCEAuthorizationFlowRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PKCEAuthorizationFlowRequest.Marshal(b, m, deterministic) +} +func (m *PKCEAuthorizationFlowRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PKCEAuthorizationFlowRequest.Merge(m, src) +} +func (m *PKCEAuthorizationFlowRequest) XXX_Size() int { + return xxx_messageInfo_PKCEAuthorizationFlowRequest.Size(m) +} +func (m *PKCEAuthorizationFlowRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PKCEAuthorizationFlowRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PKCEAuthorizationFlowRequest proto.InternalMessageInfo + // PKCEAuthorizationFlow represents Authorization Code Flow information // that can be used by the client to login initiate a Oauth 2.0 authorization code grant flow // with Proof Key for Code Exchange (PKCE). See https://datatracker.ietf.org/doc/html/rfc7636 type PKCEAuthorizationFlow struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ProviderConfig *ProviderConfig `protobuf:"bytes,1,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` + ProviderConfig *ProviderConfig `protobuf:"bytes,1,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *PKCEAuthorizationFlow) Reset() { - *x = PKCEAuthorizationFlow{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PKCEAuthorizationFlow) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PKCEAuthorizationFlow) ProtoMessage() {} - -func (x *PKCEAuthorizationFlow) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PKCEAuthorizationFlow.ProtoReflect.Descriptor instead. +func (m *PKCEAuthorizationFlow) Reset() { *m = PKCEAuthorizationFlow{} } +func (m *PKCEAuthorizationFlow) String() string { return proto.CompactTextString(m) } +func (*PKCEAuthorizationFlow) ProtoMessage() {} func (*PKCEAuthorizationFlow) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{20} + return fileDescriptor_edc174f991dc0a25, []int{20} } -func (x *PKCEAuthorizationFlow) GetProviderConfig() *ProviderConfig { - if x != nil { - return x.ProviderConfig +func (m *PKCEAuthorizationFlow) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PKCEAuthorizationFlow.Unmarshal(m, b) +} +func (m *PKCEAuthorizationFlow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PKCEAuthorizationFlow.Marshal(b, m, deterministic) +} +func (m *PKCEAuthorizationFlow) XXX_Merge(src proto.Message) { + xxx_messageInfo_PKCEAuthorizationFlow.Merge(m, src) +} +func (m *PKCEAuthorizationFlow) XXX_Size() int { + return xxx_messageInfo_PKCEAuthorizationFlow.Size(m) +} +func (m *PKCEAuthorizationFlow) XXX_DiscardUnknown() { + xxx_messageInfo_PKCEAuthorizationFlow.DiscardUnknown(m) +} + +var xxx_messageInfo_PKCEAuthorizationFlow proto.InternalMessageInfo + +func (m *PKCEAuthorizationFlow) GetProviderConfig() *ProviderConfig { + if m != nil { + return m.ProviderConfig } return nil } // ProviderConfig has all attributes needed to initiate a device/pkce authorization flow type ProviderConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - // An IDP application client id ClientID string `protobuf:"bytes,1,opt,name=ClientID,proto3" json:"ClientID,omitempty"` // An IDP application client secret @@ -1706,1511 +1441,773 @@ type ProviderConfig struct { // AuthorizationEndpoint is the endpoint of an IDP manager where clients can obtain authorization code. AuthorizationEndpoint string `protobuf:"bytes,9,opt,name=AuthorizationEndpoint,proto3" json:"AuthorizationEndpoint,omitempty"` // RedirectURLs handles authorization code from IDP manager - RedirectURLs []string `protobuf:"bytes,10,rep,name=RedirectURLs,proto3" json:"RedirectURLs,omitempty"` + RedirectURLs []string `protobuf:"bytes,10,rep,name=RedirectURLs,proto3" json:"RedirectURLs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *ProviderConfig) Reset() { - *x = ProviderConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ProviderConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ProviderConfig) ProtoMessage() {} - -func (x *ProviderConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ProviderConfig.ProtoReflect.Descriptor instead. +func (m *ProviderConfig) Reset() { *m = ProviderConfig{} } +func (m *ProviderConfig) String() string { return proto.CompactTextString(m) } +func (*ProviderConfig) ProtoMessage() {} func (*ProviderConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{21} + return fileDescriptor_edc174f991dc0a25, []int{21} } -func (x *ProviderConfig) GetClientID() string { - if x != nil { - return x.ClientID +func (m *ProviderConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ProviderConfig.Unmarshal(m, b) +} +func (m *ProviderConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ProviderConfig.Marshal(b, m, deterministic) +} +func (m *ProviderConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_ProviderConfig.Merge(m, src) +} +func (m *ProviderConfig) XXX_Size() int { + return xxx_messageInfo_ProviderConfig.Size(m) +} +func (m *ProviderConfig) XXX_DiscardUnknown() { + xxx_messageInfo_ProviderConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_ProviderConfig proto.InternalMessageInfo + +func (m *ProviderConfig) GetClientID() string { + if m != nil { + return m.ClientID } return "" } -func (x *ProviderConfig) GetClientSecret() string { - if x != nil { - return x.ClientSecret +func (m *ProviderConfig) GetClientSecret() string { + if m != nil { + return m.ClientSecret } return "" } -func (x *ProviderConfig) GetDomain() string { - if x != nil { - return x.Domain +func (m *ProviderConfig) GetDomain() string { + if m != nil { + return m.Domain } return "" } -func (x *ProviderConfig) GetAudience() string { - if x != nil { - return x.Audience +func (m *ProviderConfig) GetAudience() string { + if m != nil { + return m.Audience } return "" } -func (x *ProviderConfig) GetDeviceAuthEndpoint() string { - if x != nil { - return x.DeviceAuthEndpoint +func (m *ProviderConfig) GetDeviceAuthEndpoint() string { + if m != nil { + return m.DeviceAuthEndpoint } return "" } -func (x *ProviderConfig) GetTokenEndpoint() string { - if x != nil { - return x.TokenEndpoint +func (m *ProviderConfig) GetTokenEndpoint() string { + if m != nil { + return m.TokenEndpoint } return "" } -func (x *ProviderConfig) GetScope() string { - if x != nil { - return x.Scope +func (m *ProviderConfig) GetScope() string { + if m != nil { + return m.Scope } return "" } -func (x *ProviderConfig) GetUseIDToken() bool { - if x != nil { - return x.UseIDToken +func (m *ProviderConfig) GetUseIDToken() bool { + if m != nil { + return m.UseIDToken } return false } -func (x *ProviderConfig) GetAuthorizationEndpoint() string { - if x != nil { - return x.AuthorizationEndpoint +func (m *ProviderConfig) GetAuthorizationEndpoint() string { + if m != nil { + return m.AuthorizationEndpoint } return "" } -func (x *ProviderConfig) GetRedirectURLs() []string { - if x != nil { - return x.RedirectURLs +func (m *ProviderConfig) GetRedirectURLs() []string { + if m != nil { + return m.RedirectURLs } return nil } // Route represents a route.Route object type Route struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty"` - NetworkType int64 `protobuf:"varint,3,opt,name=NetworkType,proto3" json:"NetworkType,omitempty"` - Peer string `protobuf:"bytes,4,opt,name=Peer,proto3" json:"Peer,omitempty"` - Metric int64 `protobuf:"varint,5,opt,name=Metric,proto3" json:"Metric,omitempty"` - Masquerade bool `protobuf:"varint,6,opt,name=Masquerade,proto3" json:"Masquerade,omitempty"` - NetID string `protobuf:"bytes,7,opt,name=NetID,proto3" json:"NetID,omitempty"` + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty"` + NetworkType int64 `protobuf:"varint,3,opt,name=NetworkType,proto3" json:"NetworkType,omitempty"` + Peer string `protobuf:"bytes,4,opt,name=Peer,proto3" json:"Peer,omitempty"` + Metric int64 `protobuf:"varint,5,opt,name=Metric,proto3" json:"Metric,omitempty"` + Masquerade bool `protobuf:"varint,6,opt,name=Masquerade,proto3" json:"Masquerade,omitempty"` + NetID string `protobuf:"bytes,7,opt,name=NetID,proto3" json:"NetID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *Route) Reset() { - *x = Route{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Route) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Route) ProtoMessage() {} - -func (x *Route) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Route.ProtoReflect.Descriptor instead. +func (m *Route) Reset() { *m = Route{} } +func (m *Route) String() string { return proto.CompactTextString(m) } +func (*Route) ProtoMessage() {} func (*Route) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{22} + return fileDescriptor_edc174f991dc0a25, []int{22} } -func (x *Route) GetID() string { - if x != nil { - return x.ID +func (m *Route) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Route.Unmarshal(m, b) +} +func (m *Route) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Route.Marshal(b, m, deterministic) +} +func (m *Route) XXX_Merge(src proto.Message) { + xxx_messageInfo_Route.Merge(m, src) +} +func (m *Route) XXX_Size() int { + return xxx_messageInfo_Route.Size(m) +} +func (m *Route) XXX_DiscardUnknown() { + xxx_messageInfo_Route.DiscardUnknown(m) +} + +var xxx_messageInfo_Route proto.InternalMessageInfo + +func (m *Route) GetID() string { + if m != nil { + return m.ID } return "" } -func (x *Route) GetNetwork() string { - if x != nil { - return x.Network +func (m *Route) GetNetwork() string { + if m != nil { + return m.Network } return "" } -func (x *Route) GetNetworkType() int64 { - if x != nil { - return x.NetworkType +func (m *Route) GetNetworkType() int64 { + if m != nil { + return m.NetworkType } return 0 } -func (x *Route) GetPeer() string { - if x != nil { - return x.Peer +func (m *Route) GetPeer() string { + if m != nil { + return m.Peer } return "" } -func (x *Route) GetMetric() int64 { - if x != nil { - return x.Metric +func (m *Route) GetMetric() int64 { + if m != nil { + return m.Metric } return 0 } -func (x *Route) GetMasquerade() bool { - if x != nil { - return x.Masquerade +func (m *Route) GetMasquerade() bool { + if m != nil { + return m.Masquerade } return false } -func (x *Route) GetNetID() string { - if x != nil { - return x.NetID +func (m *Route) GetNetID() string { + if m != nil { + return m.NetID } return "" } // DNSConfig represents a dns.Update type DNSConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ServiceEnable bool `protobuf:"varint,1,opt,name=ServiceEnable,proto3" json:"ServiceEnable,omitempty"` - NameServerGroups []*NameServerGroup `protobuf:"bytes,2,rep,name=NameServerGroups,proto3" json:"NameServerGroups,omitempty"` - CustomZones []*CustomZone `protobuf:"bytes,3,rep,name=CustomZones,proto3" json:"CustomZones,omitempty"` + ServiceEnable bool `protobuf:"varint,1,opt,name=ServiceEnable,proto3" json:"ServiceEnable,omitempty"` + NameServerGroups []*NameServerGroup `protobuf:"bytes,2,rep,name=NameServerGroups,proto3" json:"NameServerGroups,omitempty"` + CustomZones []*CustomZone `protobuf:"bytes,3,rep,name=CustomZones,proto3" json:"CustomZones,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *DNSConfig) Reset() { - *x = DNSConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DNSConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DNSConfig) ProtoMessage() {} - -func (x *DNSConfig) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DNSConfig.ProtoReflect.Descriptor instead. +func (m *DNSConfig) Reset() { *m = DNSConfig{} } +func (m *DNSConfig) String() string { return proto.CompactTextString(m) } +func (*DNSConfig) ProtoMessage() {} func (*DNSConfig) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{23} + return fileDescriptor_edc174f991dc0a25, []int{23} } -func (x *DNSConfig) GetServiceEnable() bool { - if x != nil { - return x.ServiceEnable +func (m *DNSConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DNSConfig.Unmarshal(m, b) +} +func (m *DNSConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DNSConfig.Marshal(b, m, deterministic) +} +func (m *DNSConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_DNSConfig.Merge(m, src) +} +func (m *DNSConfig) XXX_Size() int { + return xxx_messageInfo_DNSConfig.Size(m) +} +func (m *DNSConfig) XXX_DiscardUnknown() { + xxx_messageInfo_DNSConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_DNSConfig proto.InternalMessageInfo + +func (m *DNSConfig) GetServiceEnable() bool { + if m != nil { + return m.ServiceEnable } return false } -func (x *DNSConfig) GetNameServerGroups() []*NameServerGroup { - if x != nil { - return x.NameServerGroups +func (m *DNSConfig) GetNameServerGroups() []*NameServerGroup { + if m != nil { + return m.NameServerGroups } return nil } -func (x *DNSConfig) GetCustomZones() []*CustomZone { - if x != nil { - return x.CustomZones +func (m *DNSConfig) GetCustomZones() []*CustomZone { + if m != nil { + return m.CustomZones } return nil } // CustomZone represents a dns.CustomZone type CustomZone struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Domain string `protobuf:"bytes,1,opt,name=Domain,proto3" json:"Domain,omitempty"` - Records []*SimpleRecord `protobuf:"bytes,2,rep,name=Records,proto3" json:"Records,omitempty"` + Domain string `protobuf:"bytes,1,opt,name=Domain,proto3" json:"Domain,omitempty"` + Records []*SimpleRecord `protobuf:"bytes,2,rep,name=Records,proto3" json:"Records,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *CustomZone) Reset() { - *x = CustomZone{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *CustomZone) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*CustomZone) ProtoMessage() {} - -func (x *CustomZone) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use CustomZone.ProtoReflect.Descriptor instead. +func (m *CustomZone) Reset() { *m = CustomZone{} } +func (m *CustomZone) String() string { return proto.CompactTextString(m) } +func (*CustomZone) ProtoMessage() {} func (*CustomZone) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{24} + return fileDescriptor_edc174f991dc0a25, []int{24} } -func (x *CustomZone) GetDomain() string { - if x != nil { - return x.Domain +func (m *CustomZone) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CustomZone.Unmarshal(m, b) +} +func (m *CustomZone) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CustomZone.Marshal(b, m, deterministic) +} +func (m *CustomZone) XXX_Merge(src proto.Message) { + xxx_messageInfo_CustomZone.Merge(m, src) +} +func (m *CustomZone) XXX_Size() int { + return xxx_messageInfo_CustomZone.Size(m) +} +func (m *CustomZone) XXX_DiscardUnknown() { + xxx_messageInfo_CustomZone.DiscardUnknown(m) +} + +var xxx_messageInfo_CustomZone proto.InternalMessageInfo + +func (m *CustomZone) GetDomain() string { + if m != nil { + return m.Domain } return "" } -func (x *CustomZone) GetRecords() []*SimpleRecord { - if x != nil { - return x.Records +func (m *CustomZone) GetRecords() []*SimpleRecord { + if m != nil { + return m.Records } return nil } // SimpleRecord represents a dns.SimpleRecord type SimpleRecord struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Type int64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` - Class string `protobuf:"bytes,3,opt,name=Class,proto3" json:"Class,omitempty"` - TTL int64 `protobuf:"varint,4,opt,name=TTL,proto3" json:"TTL,omitempty"` - RData string `protobuf:"bytes,5,opt,name=RData,proto3" json:"RData,omitempty"` + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type int64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + Class string `protobuf:"bytes,3,opt,name=Class,proto3" json:"Class,omitempty"` + TTL int64 `protobuf:"varint,4,opt,name=TTL,proto3" json:"TTL,omitempty"` + RData string `protobuf:"bytes,5,opt,name=RData,proto3" json:"RData,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *SimpleRecord) Reset() { - *x = SimpleRecord{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SimpleRecord) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SimpleRecord) ProtoMessage() {} - -func (x *SimpleRecord) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SimpleRecord.ProtoReflect.Descriptor instead. +func (m *SimpleRecord) Reset() { *m = SimpleRecord{} } +func (m *SimpleRecord) String() string { return proto.CompactTextString(m) } +func (*SimpleRecord) ProtoMessage() {} func (*SimpleRecord) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{25} + return fileDescriptor_edc174f991dc0a25, []int{25} } -func (x *SimpleRecord) GetName() string { - if x != nil { - return x.Name +func (m *SimpleRecord) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SimpleRecord.Unmarshal(m, b) +} +func (m *SimpleRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SimpleRecord.Marshal(b, m, deterministic) +} +func (m *SimpleRecord) XXX_Merge(src proto.Message) { + xxx_messageInfo_SimpleRecord.Merge(m, src) +} +func (m *SimpleRecord) XXX_Size() int { + return xxx_messageInfo_SimpleRecord.Size(m) +} +func (m *SimpleRecord) XXX_DiscardUnknown() { + xxx_messageInfo_SimpleRecord.DiscardUnknown(m) +} + +var xxx_messageInfo_SimpleRecord proto.InternalMessageInfo + +func (m *SimpleRecord) GetName() string { + if m != nil { + return m.Name } return "" } -func (x *SimpleRecord) GetType() int64 { - if x != nil { - return x.Type +func (m *SimpleRecord) GetType() int64 { + if m != nil { + return m.Type } return 0 } -func (x *SimpleRecord) GetClass() string { - if x != nil { - return x.Class +func (m *SimpleRecord) GetClass() string { + if m != nil { + return m.Class } return "" } -func (x *SimpleRecord) GetTTL() int64 { - if x != nil { - return x.TTL +func (m *SimpleRecord) GetTTL() int64 { + if m != nil { + return m.TTL } return 0 } -func (x *SimpleRecord) GetRData() string { - if x != nil { - return x.RData +func (m *SimpleRecord) GetRData() string { + if m != nil { + return m.RData } return "" } // NameServerGroup represents a dns.NameServerGroup type NameServerGroup struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - NameServers []*NameServer `protobuf:"bytes,1,rep,name=NameServers,proto3" json:"NameServers,omitempty"` Primary bool `protobuf:"varint,2,opt,name=Primary,proto3" json:"Primary,omitempty"` Domains []string `protobuf:"bytes,3,rep,name=Domains,proto3" json:"Domains,omitempty"` SearchDomainsEnabled bool `protobuf:"varint,4,opt,name=SearchDomainsEnabled,proto3" json:"SearchDomainsEnabled,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *NameServerGroup) Reset() { - *x = NameServerGroup{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NameServerGroup) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NameServerGroup) ProtoMessage() {} - -func (x *NameServerGroup) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NameServerGroup.ProtoReflect.Descriptor instead. +func (m *NameServerGroup) Reset() { *m = NameServerGroup{} } +func (m *NameServerGroup) String() string { return proto.CompactTextString(m) } +func (*NameServerGroup) ProtoMessage() {} func (*NameServerGroup) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{26} + return fileDescriptor_edc174f991dc0a25, []int{26} } -func (x *NameServerGroup) GetNameServers() []*NameServer { - if x != nil { - return x.NameServers +func (m *NameServerGroup) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NameServerGroup.Unmarshal(m, b) +} +func (m *NameServerGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NameServerGroup.Marshal(b, m, deterministic) +} +func (m *NameServerGroup) XXX_Merge(src proto.Message) { + xxx_messageInfo_NameServerGroup.Merge(m, src) +} +func (m *NameServerGroup) XXX_Size() int { + return xxx_messageInfo_NameServerGroup.Size(m) +} +func (m *NameServerGroup) XXX_DiscardUnknown() { + xxx_messageInfo_NameServerGroup.DiscardUnknown(m) +} + +var xxx_messageInfo_NameServerGroup proto.InternalMessageInfo + +func (m *NameServerGroup) GetNameServers() []*NameServer { + if m != nil { + return m.NameServers } return nil } -func (x *NameServerGroup) GetPrimary() bool { - if x != nil { - return x.Primary +func (m *NameServerGroup) GetPrimary() bool { + if m != nil { + return m.Primary } return false } -func (x *NameServerGroup) GetDomains() []string { - if x != nil { - return x.Domains +func (m *NameServerGroup) GetDomains() []string { + if m != nil { + return m.Domains } return nil } -func (x *NameServerGroup) GetSearchDomainsEnabled() bool { - if x != nil { - return x.SearchDomainsEnabled +func (m *NameServerGroup) GetSearchDomainsEnabled() bool { + if m != nil { + return m.SearchDomainsEnabled } return false } // NameServer represents a dns.NameServer type NameServer struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - IP string `protobuf:"bytes,1,opt,name=IP,proto3" json:"IP,omitempty"` - NSType int64 `protobuf:"varint,2,opt,name=NSType,proto3" json:"NSType,omitempty"` - Port int64 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` + IP string `protobuf:"bytes,1,opt,name=IP,proto3" json:"IP,omitempty"` + NSType int64 `protobuf:"varint,2,opt,name=NSType,proto3" json:"NSType,omitempty"` + Port int64 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *NameServer) Reset() { - *x = NameServer{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NameServer) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NameServer) ProtoMessage() {} - -func (x *NameServer) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NameServer.ProtoReflect.Descriptor instead. +func (m *NameServer) Reset() { *m = NameServer{} } +func (m *NameServer) String() string { return proto.CompactTextString(m) } +func (*NameServer) ProtoMessage() {} func (*NameServer) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{27} + return fileDescriptor_edc174f991dc0a25, []int{27} } -func (x *NameServer) GetIP() string { - if x != nil { - return x.IP +func (m *NameServer) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NameServer.Unmarshal(m, b) +} +func (m *NameServer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NameServer.Marshal(b, m, deterministic) +} +func (m *NameServer) XXX_Merge(src proto.Message) { + xxx_messageInfo_NameServer.Merge(m, src) +} +func (m *NameServer) XXX_Size() int { + return xxx_messageInfo_NameServer.Size(m) +} +func (m *NameServer) XXX_DiscardUnknown() { + xxx_messageInfo_NameServer.DiscardUnknown(m) +} + +var xxx_messageInfo_NameServer proto.InternalMessageInfo + +func (m *NameServer) GetIP() string { + if m != nil { + return m.IP } return "" } -func (x *NameServer) GetNSType() int64 { - if x != nil { - return x.NSType +func (m *NameServer) GetNSType() int64 { + if m != nil { + return m.NSType } return 0 } -func (x *NameServer) GetPort() int64 { - if x != nil { - return x.Port +func (m *NameServer) GetPort() int64 { + if m != nil { + return m.Port } return 0 } // FirewallRule represents a firewall rule type FirewallRule struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - PeerIP string `protobuf:"bytes,1,opt,name=PeerIP,proto3" json:"PeerIP,omitempty"` - Direction FirewallRuleDirection `protobuf:"varint,2,opt,name=Direction,proto3,enum=management.FirewallRuleDirection" json:"Direction,omitempty"` - Action FirewallRuleAction `protobuf:"varint,3,opt,name=Action,proto3,enum=management.FirewallRuleAction" json:"Action,omitempty"` - Protocol FirewallRuleProtocol `protobuf:"varint,4,opt,name=Protocol,proto3,enum=management.FirewallRuleProtocol" json:"Protocol,omitempty"` - Port string `protobuf:"bytes,5,opt,name=Port,proto3" json:"Port,omitempty"` + PeerIP string `protobuf:"bytes,1,opt,name=PeerIP,proto3" json:"PeerIP,omitempty"` + Direction FirewallRuleDirection `protobuf:"varint,2,opt,name=Direction,proto3,enum=management.FirewallRuleDirection" json:"Direction,omitempty"` + Action FirewallRuleAction `protobuf:"varint,3,opt,name=Action,proto3,enum=management.FirewallRuleAction" json:"Action,omitempty"` + Protocol FirewallRuleProtocol `protobuf:"varint,4,opt,name=Protocol,proto3,enum=management.FirewallRuleProtocol" json:"Protocol,omitempty"` + Port string `protobuf:"bytes,5,opt,name=Port,proto3" json:"Port,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *FirewallRule) Reset() { - *x = FirewallRule{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FirewallRule) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FirewallRule) ProtoMessage() {} - -func (x *FirewallRule) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FirewallRule.ProtoReflect.Descriptor instead. +func (m *FirewallRule) Reset() { *m = FirewallRule{} } +func (m *FirewallRule) String() string { return proto.CompactTextString(m) } +func (*FirewallRule) ProtoMessage() {} func (*FirewallRule) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{28} + return fileDescriptor_edc174f991dc0a25, []int{28} } -func (x *FirewallRule) GetPeerIP() string { - if x != nil { - return x.PeerIP +func (m *FirewallRule) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FirewallRule.Unmarshal(m, b) +} +func (m *FirewallRule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FirewallRule.Marshal(b, m, deterministic) +} +func (m *FirewallRule) XXX_Merge(src proto.Message) { + xxx_messageInfo_FirewallRule.Merge(m, src) +} +func (m *FirewallRule) XXX_Size() int { + return xxx_messageInfo_FirewallRule.Size(m) +} +func (m *FirewallRule) XXX_DiscardUnknown() { + xxx_messageInfo_FirewallRule.DiscardUnknown(m) +} + +var xxx_messageInfo_FirewallRule proto.InternalMessageInfo + +func (m *FirewallRule) GetPeerIP() string { + if m != nil { + return m.PeerIP } return "" } -func (x *FirewallRule) GetDirection() FirewallRuleDirection { - if x != nil { - return x.Direction +func (m *FirewallRule) GetDirection() FirewallRuleDirection { + if m != nil { + return m.Direction } return FirewallRule_IN } -func (x *FirewallRule) GetAction() FirewallRuleAction { - if x != nil { - return x.Action +func (m *FirewallRule) GetAction() FirewallRuleAction { + if m != nil { + return m.Action } return FirewallRule_ACCEPT } -func (x *FirewallRule) GetProtocol() FirewallRuleProtocol { - if x != nil { - return x.Protocol +func (m *FirewallRule) GetProtocol() FirewallRuleProtocol { + if m != nil { + return m.Protocol } return FirewallRule_UNKNOWN } -func (x *FirewallRule) GetPort() string { - if x != nil { - return x.Port +func (m *FirewallRule) GetPort() string { + if m != nil { + return m.Port } return "" } type NetworkAddress struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NetIP string `protobuf:"bytes,1,opt,name=netIP,proto3" json:"netIP,omitempty"` - Mac string `protobuf:"bytes,2,opt,name=mac,proto3" json:"mac,omitempty"` + NetIP string `protobuf:"bytes,1,opt,name=netIP,proto3" json:"netIP,omitempty"` + Mac string `protobuf:"bytes,2,opt,name=mac,proto3" json:"mac,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } -func (x *NetworkAddress) Reset() { - *x = NetworkAddress{} - if protoimpl.UnsafeEnabled { - mi := &file_management_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NetworkAddress) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NetworkAddress) ProtoMessage() {} - -func (x *NetworkAddress) ProtoReflect() protoreflect.Message { - mi := &file_management_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NetworkAddress.ProtoReflect.Descriptor instead. +func (m *NetworkAddress) Reset() { *m = NetworkAddress{} } +func (m *NetworkAddress) String() string { return proto.CompactTextString(m) } +func (*NetworkAddress) ProtoMessage() {} func (*NetworkAddress) Descriptor() ([]byte, []int) { - return file_management_proto_rawDescGZIP(), []int{29} + return fileDescriptor_edc174f991dc0a25, []int{29} } -func (x *NetworkAddress) GetNetIP() string { - if x != nil { - return x.NetIP +func (m *NetworkAddress) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_NetworkAddress.Unmarshal(m, b) +} +func (m *NetworkAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_NetworkAddress.Marshal(b, m, deterministic) +} +func (m *NetworkAddress) XXX_Merge(src proto.Message) { + xxx_messageInfo_NetworkAddress.Merge(m, src) +} +func (m *NetworkAddress) XXX_Size() int { + return xxx_messageInfo_NetworkAddress.Size(m) +} +func (m *NetworkAddress) XXX_DiscardUnknown() { + xxx_messageInfo_NetworkAddress.DiscardUnknown(m) +} + +var xxx_messageInfo_NetworkAddress proto.InternalMessageInfo + +func (m *NetworkAddress) GetNetIP() string { + if m != nil { + return m.NetIP } return "" } -func (x *NetworkAddress) GetMac() string { - if x != nil { - return x.Mac +func (m *NetworkAddress) GetMac() string { + if m != nil { + return m.Mac } return "" } -var File_management_proto protoreflect.FileDescriptor - -var file_management_proto_rawDesc = []byte{ - 0x0a, 0x10, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x12, 0x0a, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x1a, 0x1f, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, - 0x5c, 0x0a, 0x10, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, - 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62, - 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x0d, 0x0a, - 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbb, 0x02, 0x0a, - 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, - 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, - 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, - 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, - 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, - 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x12, 0x36, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x52, 0x0a, - 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x22, 0xa8, 0x01, 0x0a, 0x0c, 0x4c, - 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, - 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, - 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, - 0x61, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x6a, 0x77, 0x74, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6a, 0x77, 0x74, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x30, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x08, 0x70, 0x65, 0x65, - 0x72, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x44, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4b, 0x65, 0x79, - 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, - 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x3f, 0x0a, 0x0b, 0x45, - 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0xa9, 0x04, 0x0a, - 0x0e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, 0x61, 0x12, - 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x67, - 0x6f, 0x4f, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x6f, 0x4f, 0x53, 0x12, - 0x16, 0x0a, 0x06, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x72, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x53, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x02, 0x4f, 0x53, 0x12, 0x2e, 0x0a, 0x12, 0x77, 0x69, 0x72, 0x65, 0x74, - 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x12, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x69, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x69, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, - 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x4f, - 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x4f, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x10, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x0b, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, - 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, - 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x4e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x79, 0x73, 0x53, - 0x65, 0x72, 0x69, 0x61, 0x6c, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x73, - 0x79, 0x73, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0d, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x79, 0x73, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x4d, 0x61, 0x6e, 0x75, 0x66, 0x61, - 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x79, - 0x73, 0x4d, 0x61, 0x6e, 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x12, 0x39, 0x0a, - 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x0f, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, - 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x65, 0x6e, 0x76, - 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x11, 0x77, 0x69, - 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, - 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, - 0x79, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x41, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, - 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x22, 0xa8, 0x01, 0x0a, 0x11, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2c, 0x0a, 0x05, 0x73, 0x74, 0x75, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x05, 0x73, 0x74, 0x75, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, - 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x12, 0x2e, - 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x22, 0x98, - 0x01, 0x0a, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, - 0x3b, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, - 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x3b, 0x0a, 0x08, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, - 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x54, - 0x54, 0x50, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x54, 0x54, 0x50, 0x53, 0x10, 0x03, 0x12, - 0x08, 0x0a, 0x04, 0x44, 0x54, 0x4c, 0x53, 0x10, 0x04, 0x22, 0x7d, 0x0a, 0x13, 0x50, 0x72, 0x6f, - 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x12, 0x36, 0x0a, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x68, 0x6f, - 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, - 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x10, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x64, 0x6e, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, - 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0xe2, 0x03, 0x0a, - 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x53, - 0x65, 0x72, 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, 0x65, 0x72, - 0x69, 0x61, 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, - 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, - 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, - 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x29, 0x0a, 0x06, 0x52, - 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x52, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x40, 0x0a, 0x0c, 0x6f, - 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x0c, 0x6f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x3e, 0x0a, - 0x0d, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x08, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x0d, - 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x32, 0x0a, - 0x14, 0x66, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x66, 0x69, 0x72, - 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x22, 0x97, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, - 0x65, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, - 0x70, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, - 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0x49, 0x0a, 0x09, 0x53, - 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x73, - 0x68, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, - 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, - 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, - 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, - 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, 0x76, - 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x42, - 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0a, - 0x0a, 0x06, 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x4b, - 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, - 0x6c, 0x6f, 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5b, 0x0a, 0x15, 0x50, 0x4b, - 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, - 0x6c, 0x6f, 0x77, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, - 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xea, 0x02, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2e, - 0x0a, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, 0x76, 0x69, - 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x24, - 0x0a, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, - 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x55, 0x73, 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x41, 0x75, - 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x41, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x22, 0x0a, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, 0x4c, 0x73, - 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x55, 0x52, 0x4c, 0x73, 0x22, 0xb5, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, - 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, - 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x4e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, - 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x12, 0x16, - 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, - 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, - 0x72, 0x61, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, 0x73, 0x71, - 0x75, 0x65, 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x22, 0xb4, 0x01, 0x0a, - 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, - 0x12, 0x47, 0x0a, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x38, 0x0a, 0x0b, 0x43, 0x75, 0x73, - 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, - 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x52, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, - 0x6e, 0x65, 0x73, 0x22, 0x58, 0x0a, 0x0a, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x07, 0x52, 0x65, 0x63, - 0x6f, 0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x74, 0x0a, - 0x0c, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, - 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x54, - 0x54, 0x4c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x14, 0x0a, - 0x05, 0x52, 0x44, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x44, - 0x61, 0x74, 0x61, 0x22, 0xb3, 0x01, 0x0a, 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x52, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, - 0x6f, 0x72, 0x74, 0x22, 0xf0, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, - 0x52, 0x75, 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, - 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x22, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, - 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, - 0x0a, 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, - 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, - 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x06, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, - 0x75, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, - 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, - 0x07, 0x0a, 0x03, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x22, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x43, 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, - 0x0a, 0x04, 0x44, 0x52, 0x4f, 0x50, 0x10, 0x01, 0x22, 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, - 0x00, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, - 0x50, 0x10, 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, - 0x49, 0x43, 0x4d, 0x50, 0x10, 0x04, 0x22, 0x38, 0x0a, 0x0e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x49, - 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x12, 0x10, - 0x0a, 0x03, 0x6d, 0x61, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, - 0x32, 0xd1, 0x03, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, - 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, - 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, - 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, - 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, - 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, - 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, - 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, - 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, - 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, - 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, - 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, - 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x50, 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, - 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +func init() { + proto.RegisterEnum("management.HostConfig_Protocol", HostConfig_Protocol_name, HostConfig_Protocol_value) + proto.RegisterEnum("management.DeviceAuthorizationFlowProvider", DeviceAuthorizationFlowProvider_name, DeviceAuthorizationFlowProvider_value) + proto.RegisterEnum("management.FirewallRuleDirection", FirewallRuleDirection_name, FirewallRuleDirection_value) + proto.RegisterEnum("management.FirewallRuleAction", FirewallRuleAction_name, FirewallRuleAction_value) + proto.RegisterEnum("management.FirewallRuleProtocol", FirewallRuleProtocol_name, FirewallRuleProtocol_value) + proto.RegisterType((*EncryptedMessage)(nil), "management.EncryptedMessage") + proto.RegisterType((*SyncRequest)(nil), "management.SyncRequest") + proto.RegisterType((*SyncResponse)(nil), "management.SyncResponse") + proto.RegisterType((*LoginRequest)(nil), "management.LoginRequest") + proto.RegisterType((*PeerKeys)(nil), "management.PeerKeys") + proto.RegisterType((*Environment)(nil), "management.Environment") + proto.RegisterType((*PeerSystemMeta)(nil), "management.PeerSystemMeta") + proto.RegisterType((*LoginResponse)(nil), "management.LoginResponse") + proto.RegisterType((*ServerKeyResponse)(nil), "management.ServerKeyResponse") + proto.RegisterType((*Empty)(nil), "management.Empty") + proto.RegisterType((*WiretrusteeConfig)(nil), "management.WiretrusteeConfig") + proto.RegisterType((*HostConfig)(nil), "management.HostConfig") + proto.RegisterType((*ProtectedHostConfig)(nil), "management.ProtectedHostConfig") + proto.RegisterType((*PeerConfig)(nil), "management.PeerConfig") + proto.RegisterType((*NetworkMap)(nil), "management.NetworkMap") + proto.RegisterType((*RemotePeerConfig)(nil), "management.RemotePeerConfig") + proto.RegisterType((*SSHConfig)(nil), "management.SSHConfig") + proto.RegisterType((*DeviceAuthorizationFlowRequest)(nil), "management.DeviceAuthorizationFlowRequest") + proto.RegisterType((*DeviceAuthorizationFlow)(nil), "management.DeviceAuthorizationFlow") + proto.RegisterType((*PKCEAuthorizationFlowRequest)(nil), "management.PKCEAuthorizationFlowRequest") + proto.RegisterType((*PKCEAuthorizationFlow)(nil), "management.PKCEAuthorizationFlow") + proto.RegisterType((*ProviderConfig)(nil), "management.ProviderConfig") + proto.RegisterType((*Route)(nil), "management.Route") + proto.RegisterType((*DNSConfig)(nil), "management.DNSConfig") + proto.RegisterType((*CustomZone)(nil), "management.CustomZone") + proto.RegisterType((*SimpleRecord)(nil), "management.SimpleRecord") + proto.RegisterType((*NameServerGroup)(nil), "management.NameServerGroup") + proto.RegisterType((*NameServer)(nil), "management.NameServer") + proto.RegisterType((*FirewallRule)(nil), "management.FirewallRule") + proto.RegisterType((*NetworkAddress)(nil), "management.NetworkAddress") } -var ( - file_management_proto_rawDescOnce sync.Once - file_management_proto_rawDescData = file_management_proto_rawDesc -) - -func file_management_proto_rawDescGZIP() []byte { - file_management_proto_rawDescOnce.Do(func() { - file_management_proto_rawDescData = protoimpl.X.CompressGZIP(file_management_proto_rawDescData) - }) - return file_management_proto_rawDescData +func init() { + proto.RegisterFile("management.proto", fileDescriptor_edc174f991dc0a25) } -var file_management_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_management_proto_msgTypes = make([]protoimpl.MessageInfo, 30) -var file_management_proto_goTypes = []interface{}{ - (HostConfig_Protocol)(0), // 0: management.HostConfig.Protocol - (DeviceAuthorizationFlowProvider)(0), // 1: management.DeviceAuthorizationFlow.provider - (FirewallRuleDirection)(0), // 2: management.FirewallRule.direction - (FirewallRuleAction)(0), // 3: management.FirewallRule.action - (FirewallRuleProtocol)(0), // 4: management.FirewallRule.protocol - (*EncryptedMessage)(nil), // 5: management.EncryptedMessage - (*SyncRequest)(nil), // 6: management.SyncRequest - (*SyncResponse)(nil), // 7: management.SyncResponse - (*LoginRequest)(nil), // 8: management.LoginRequest - (*PeerKeys)(nil), // 9: management.PeerKeys - (*Environment)(nil), // 10: management.Environment - (*PeerSystemMeta)(nil), // 11: management.PeerSystemMeta - (*LoginResponse)(nil), // 12: management.LoginResponse - (*ServerKeyResponse)(nil), // 13: management.ServerKeyResponse - (*Empty)(nil), // 14: management.Empty - (*WiretrusteeConfig)(nil), // 15: management.WiretrusteeConfig - (*HostConfig)(nil), // 16: management.HostConfig - (*ProtectedHostConfig)(nil), // 17: management.ProtectedHostConfig - (*PeerConfig)(nil), // 18: management.PeerConfig - (*NetworkMap)(nil), // 19: management.NetworkMap - (*RemotePeerConfig)(nil), // 20: management.RemotePeerConfig - (*SSHConfig)(nil), // 21: management.SSHConfig - (*DeviceAuthorizationFlowRequest)(nil), // 22: management.DeviceAuthorizationFlowRequest - (*DeviceAuthorizationFlow)(nil), // 23: management.DeviceAuthorizationFlow - (*PKCEAuthorizationFlowRequest)(nil), // 24: management.PKCEAuthorizationFlowRequest - (*PKCEAuthorizationFlow)(nil), // 25: management.PKCEAuthorizationFlow - (*ProviderConfig)(nil), // 26: management.ProviderConfig - (*Route)(nil), // 27: management.Route - (*DNSConfig)(nil), // 28: management.DNSConfig - (*CustomZone)(nil), // 29: management.CustomZone - (*SimpleRecord)(nil), // 30: management.SimpleRecord - (*NameServerGroup)(nil), // 31: management.NameServerGroup - (*NameServer)(nil), // 32: management.NameServer - (*FirewallRule)(nil), // 33: management.FirewallRule - (*NetworkAddress)(nil), // 34: management.NetworkAddress - (*timestamppb.Timestamp)(nil), // 35: google.protobuf.Timestamp -} -var file_management_proto_depIdxs = []int32{ - 15, // 0: management.SyncResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig - 18, // 1: management.SyncResponse.peerConfig:type_name -> management.PeerConfig - 20, // 2: management.SyncResponse.remotePeers:type_name -> management.RemotePeerConfig - 19, // 3: management.SyncResponse.NetworkMap:type_name -> management.NetworkMap - 11, // 4: management.LoginRequest.meta:type_name -> management.PeerSystemMeta - 9, // 5: management.LoginRequest.peerKeys:type_name -> management.PeerKeys - 34, // 6: management.PeerSystemMeta.networkAddresses:type_name -> management.NetworkAddress - 10, // 7: management.PeerSystemMeta.environment:type_name -> management.Environment - 15, // 8: management.LoginResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig - 18, // 9: management.LoginResponse.peerConfig:type_name -> management.PeerConfig - 35, // 10: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp - 16, // 11: management.WiretrusteeConfig.stuns:type_name -> management.HostConfig - 17, // 12: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig - 16, // 13: management.WiretrusteeConfig.signal:type_name -> management.HostConfig - 0, // 14: management.HostConfig.protocol:type_name -> management.HostConfig.Protocol - 16, // 15: management.ProtectedHostConfig.hostConfig:type_name -> management.HostConfig - 21, // 16: management.PeerConfig.sshConfig:type_name -> management.SSHConfig - 18, // 17: management.NetworkMap.peerConfig:type_name -> management.PeerConfig - 20, // 18: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig - 27, // 19: management.NetworkMap.Routes:type_name -> management.Route - 28, // 20: management.NetworkMap.DNSConfig:type_name -> management.DNSConfig - 20, // 21: management.NetworkMap.offlinePeers:type_name -> management.RemotePeerConfig - 33, // 22: management.NetworkMap.FirewallRules:type_name -> management.FirewallRule - 21, // 23: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig - 1, // 24: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider - 26, // 25: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig - 26, // 26: management.PKCEAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig - 31, // 27: management.DNSConfig.NameServerGroups:type_name -> management.NameServerGroup - 29, // 28: management.DNSConfig.CustomZones:type_name -> management.CustomZone - 30, // 29: management.CustomZone.Records:type_name -> management.SimpleRecord - 32, // 30: management.NameServerGroup.NameServers:type_name -> management.NameServer - 2, // 31: management.FirewallRule.Direction:type_name -> management.FirewallRule.direction - 3, // 32: management.FirewallRule.Action:type_name -> management.FirewallRule.action - 4, // 33: management.FirewallRule.Protocol:type_name -> management.FirewallRule.protocol - 5, // 34: management.ManagementService.Login:input_type -> management.EncryptedMessage - 5, // 35: management.ManagementService.Sync:input_type -> management.EncryptedMessage - 14, // 36: management.ManagementService.GetServerKey:input_type -> management.Empty - 14, // 37: management.ManagementService.isHealthy:input_type -> management.Empty - 5, // 38: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage - 5, // 39: management.ManagementService.GetPKCEAuthorizationFlow:input_type -> management.EncryptedMessage - 5, // 40: management.ManagementService.Login:output_type -> management.EncryptedMessage - 5, // 41: management.ManagementService.Sync:output_type -> management.EncryptedMessage - 13, // 42: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse - 14, // 43: management.ManagementService.isHealthy:output_type -> management.Empty - 5, // 44: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage - 5, // 45: management.ManagementService.GetPKCEAuthorizationFlow:output_type -> management.EncryptedMessage - 40, // [40:46] is the sub-list for method output_type - 34, // [34:40] is the sub-list for method input_type - 34, // [34:34] is the sub-list for extension type_name - 34, // [34:34] is the sub-list for extension extendee - 0, // [0:34] is the sub-list for field type_name -} - -func init() { file_management_proto_init() } -func file_management_proto_init() { - if File_management_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_management_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EncryptedMessage); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SyncRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SyncResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LoginRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerKeys); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Environment); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerSystemMeta); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LoginResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ServerKeyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Empty); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WiretrusteeConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HostConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProtectedHostConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PeerConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NetworkMap); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RemotePeerConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SSHConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeviceAuthorizationFlowRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeviceAuthorizationFlow); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PKCEAuthorizationFlowRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PKCEAuthorizationFlow); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ProviderConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Route); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DNSConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CustomZone); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SimpleRecord); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NameServerGroup); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NameServer); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FirewallRule); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_management_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NetworkAddress); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_management_proto_rawDesc, - NumEnums: 5, - NumMessages: 30, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_management_proto_goTypes, - DependencyIndexes: file_management_proto_depIdxs, - EnumInfos: file_management_proto_enumTypes, - MessageInfos: file_management_proto_msgTypes, - }.Build() - File_management_proto = out.File - file_management_proto_rawDesc = nil - file_management_proto_goTypes = nil - file_management_proto_depIdxs = nil +var fileDescriptor_edc174f991dc0a25 = []byte{ + // 1923 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x4d, 0x6f, 0x23, 0x49, + 0x19, 0x4e, 0xdb, 0xb1, 0x63, 0xbf, 0x76, 0x32, 0x4e, 0x31, 0x33, 0xdb, 0x0a, 0x61, 0x36, 0xb4, + 0x10, 0x0a, 0xd2, 0x2a, 0xbb, 0xca, 0xf2, 0x31, 0x68, 0x61, 0xd9, 0x4c, 0x9c, 0x49, 0xa2, 0x24, + 0x8e, 0x55, 0xed, 0xb0, 0xa3, 0x81, 0x4b, 0xa7, 0xbb, 0xe2, 0x34, 0xd3, 0xee, 0xf2, 0x54, 0x55, + 0x27, 0x18, 0x89, 0x03, 0xff, 0x01, 0x09, 0x7e, 0x02, 0x9c, 0x81, 0x13, 0x07, 0xfe, 0x00, 0x17, + 0xfe, 0x02, 0x47, 0x4e, 0xfc, 0x04, 0x54, 0x5f, 0xfd, 0x61, 0x3b, 0x23, 0xd0, 0x70, 0x80, 0x53, + 0xea, 0xfd, 0xec, 0xb7, 0x9e, 0x7a, 0xeb, 0xad, 0xc7, 0x81, 0xde, 0x24, 0x48, 0x83, 0x31, 0x99, + 0x90, 0x54, 0xec, 0x4d, 0x19, 0x15, 0x14, 0x41, 0xa1, 0xd9, 0xfa, 0x70, 0x4c, 0xe9, 0x38, 0x21, + 0x1f, 0x2b, 0xcb, 0x75, 0x76, 0xf3, 0xb1, 0x88, 0x27, 0x84, 0x8b, 0x60, 0x32, 0xd5, 0xce, 0xde, + 0x4f, 0xa1, 0x77, 0x94, 0x86, 0x6c, 0x36, 0x15, 0x24, 0xba, 0x20, 0x9c, 0x07, 0x63, 0x82, 0xb6, + 0xa0, 0x75, 0x3f, 0x1e, 0x66, 0xd7, 0x67, 0x64, 0xe6, 0x3a, 0x3b, 0xce, 0x6e, 0x1b, 0xe7, 0x32, + 0x42, 0xb0, 0x7a, 0x4d, 0xa3, 0x99, 0x5b, 0xdb, 0x71, 0x76, 0xbb, 0x58, 0xad, 0x91, 0x0b, 0x6b, + 0x77, 0x84, 0xf1, 0x98, 0xa6, 0x6e, 0x7d, 0xc7, 0xd9, 0x6d, 0x60, 0x2b, 0x7a, 0xeb, 0xd0, 0xf1, + 0x67, 0x69, 0x88, 0xc9, 0xdb, 0x8c, 0x70, 0xe1, 0xfd, 0xb9, 0x06, 0x5d, 0x2d, 0xf3, 0x29, 0x4d, + 0x39, 0x41, 0x67, 0xb0, 0x79, 0x1f, 0x33, 0x22, 0x58, 0xc6, 0x05, 0x21, 0x87, 0x34, 0xbd, 0x89, + 0xc7, 0xea, 0x93, 0x9d, 0xfd, 0xaf, 0xed, 0x95, 0x36, 0xf6, 0xe5, 0xbc, 0x13, 0x5e, 0x8c, 0x43, + 0xdf, 0x05, 0x98, 0x12, 0xc2, 0x4c, 0x96, 0x9a, 0xca, 0xf2, 0xb4, 0x9c, 0x65, 0x98, 0x5b, 0x71, + 0xc9, 0x13, 0x7d, 0x0e, 0x1d, 0x46, 0x26, 0x54, 0x10, 0x69, 0xe7, 0x6e, 0x7d, 0xa7, 0xbe, 0xdb, + 0xd9, 0xdf, 0x2e, 0x07, 0xe2, 0xdc, 0x6c, 0xc2, 0xcb, 0x01, 0x68, 0x0f, 0x50, 0x49, 0x3c, 0xe5, + 0x47, 0x93, 0xa9, 0x98, 0xb9, 0xab, 0x3b, 0xce, 0x6e, 0x0b, 0x2f, 0xb1, 0xc8, 0x3a, 0x07, 0x44, + 0xdc, 0x53, 0xf6, 0xe6, 0x22, 0x98, 0xba, 0x8d, 0xc5, 0x3a, 0x0b, 0x2b, 0x2e, 0x79, 0x7a, 0xbf, + 0x73, 0xa0, 0x7b, 0x4e, 0xc7, 0x71, 0x6a, 0xe0, 0x94, 0xe7, 0xc4, 0x89, 0xc8, 0xa6, 0xa5, 0x73, + 0xb2, 0x32, 0xda, 0x83, 0xd5, 0x09, 0x11, 0x81, 0x81, 0x61, 0x6b, 0x1e, 0x06, 0x7f, 0xc6, 0x05, + 0x99, 0x5c, 0x10, 0x11, 0x60, 0xe5, 0x27, 0x73, 0xfd, 0xec, 0x5e, 0x8c, 0xe8, 0x1b, 0xa2, 0x0f, + 0xb1, 0x8d, 0x73, 0x19, 0x7d, 0x02, 0x2d, 0x09, 0xd7, 0x19, 0x99, 0x71, 0xb5, 0xad, 0xce, 0xfe, + 0xe3, 0xf9, 0x7c, 0xd2, 0x86, 0x73, 0x2f, 0xaf, 0x0f, 0x2d, 0xab, 0x45, 0xdb, 0xd0, 0xe6, 0xfc, + 0xb6, 0xd4, 0x4e, 0x5d, 0x5c, 0x28, 0x2a, 0xbd, 0xa6, 0x7b, 0x2a, 0x97, 0xbd, 0x1f, 0x41, 0xe7, + 0x28, 0xbd, 0x8b, 0x19, 0x4d, 0xe5, 0x77, 0xd0, 0x63, 0x68, 0x84, 0x09, 0xcd, 0x22, 0xb3, 0x57, + 0x2d, 0xc8, 0x04, 0xd3, 0x24, 0x10, 0x37, 0x94, 0x4d, 0x54, 0x82, 0x36, 0xce, 0x65, 0xef, 0xf7, + 0xab, 0xb0, 0x51, 0xdd, 0xad, 0x74, 0xbf, 0xa5, 0x5c, 0xa4, 0xc1, 0x84, 0x58, 0xcc, 0xac, 0x2c, + 0x7b, 0x7b, 0x4c, 0x2f, 0x7d, 0x93, 0x46, 0xad, 0xd1, 0x53, 0x68, 0xbe, 0x21, 0x2c, 0x25, 0x89, + 0x41, 0xc5, 0x48, 0xd2, 0x37, 0xa4, 0x8c, 0x28, 0x3c, 0xda, 0x58, 0xad, 0x2b, 0xa5, 0x34, 0xaa, + 0xa5, 0xa0, 0x0d, 0xa8, 0x5d, 0xfa, 0x6e, 0x53, 0x69, 0x6b, 0x97, 0xbe, 0x6c, 0x9a, 0x52, 0x07, + 0xff, 0xd8, 0x5c, 0x9f, 0x35, 0x65, 0x5f, 0x62, 0x91, 0x28, 0x66, 0xb1, 0x75, 0x6b, 0x29, 0xb7, + 0x42, 0x81, 0xbe, 0x01, 0xeb, 0xba, 0x2e, 0xeb, 0xd1, 0x56, 0x1e, 0x55, 0xa5, 0xcc, 0x71, 0xe9, + 0x5b, 0x0f, 0xd0, 0x39, 0x72, 0x05, 0x7a, 0x09, 0xbd, 0x54, 0x37, 0xdb, 0x41, 0x14, 0x31, 0xc2, + 0x39, 0xe1, 0x6e, 0x47, 0xdd, 0x85, 0xad, 0x25, 0xcd, 0x69, 0x7c, 0xf0, 0x42, 0x0c, 0xda, 0x85, + 0x47, 0x7c, 0xc6, 0x7d, 0xc2, 0xe2, 0x20, 0x19, 0x64, 0x93, 0x6b, 0xc2, 0xdc, 0xae, 0xfa, 0xd6, + 0xbc, 0x1a, 0x7d, 0x13, 0x36, 0xf8, 0x8c, 0x0f, 0x19, 0x8d, 0xb2, 0x50, 0x0c, 0xe4, 0x89, 0xac, + 0x2b, 0xc7, 0x39, 0xad, 0xc9, 0x78, 0x11, 0xa4, 0xd9, 0x4d, 0x10, 0x8a, 0x8c, 0x11, 0xe6, 0x6e, + 0xe4, 0x19, 0xcb, 0x6a, 0xf4, 0x7d, 0xe8, 0x90, 0xa2, 0x63, 0xdc, 0x47, 0xaa, 0x59, 0x3f, 0x28, + 0x97, 0x5f, 0x6a, 0x28, 0x5c, 0xf6, 0xf5, 0x7e, 0xed, 0xc0, 0xba, 0xb9, 0x5d, 0xff, 0x43, 0xc3, + 0xc9, 0x9b, 0xc1, 0xa6, 0x4f, 0xd8, 0x9d, 0xba, 0x4b, 0x79, 0x65, 0x3d, 0xa8, 0xbf, 0xc9, 0xef, + 0xbc, 0x5c, 0xa2, 0xe7, 0xd0, 0x26, 0x3f, 0x9f, 0xc6, 0x8c, 0xf0, 0x03, 0x91, 0xdf, 0x79, 0x3d, + 0xfb, 0xf7, 0xec, 0xec, 0xdf, 0x1b, 0xd9, 0xd9, 0x8f, 0x0b, 0xe7, 0x77, 0x0c, 0xef, 0x35, 0x68, + 0xa8, 0x81, 0xe5, 0xfd, 0xd5, 0x81, 0xcd, 0x85, 0x4d, 0xa2, 0x8f, 0xa0, 0xc1, 0x45, 0x96, 0x72, + 0xd7, 0x51, 0x4d, 0x52, 0xd9, 0xcc, 0x09, 0xe5, 0xc2, 0x6c, 0x46, 0x3b, 0xa1, 0xef, 0x40, 0x43, + 0x64, 0x2c, 0xe5, 0x6e, 0x4d, 0x79, 0x7f, 0x58, 0xd9, 0x3a, 0xa3, 0x82, 0x84, 0x82, 0x44, 0xe5, + 0x30, 0xe5, 0x8d, 0xf6, 0xa0, 0xc9, 0xe3, 0x71, 0x1a, 0xe8, 0xeb, 0xf7, 0xf0, 0x57, 0x8c, 0x17, + 0xf2, 0xa0, 0x8b, 0x49, 0x12, 0xcc, 0x4c, 0x3b, 0x9a, 0xeb, 0x59, 0xd1, 0x79, 0xbf, 0x75, 0x00, + 0x8a, 0x50, 0x09, 0x66, 0xc6, 0x62, 0x0b, 0x66, 0xc6, 0x62, 0xf4, 0x19, 0xb4, 0x14, 0x66, 0x21, + 0x4d, 0x14, 0x96, 0x1b, 0xd5, 0x72, 0x8b, 0x58, 0x55, 0xb9, 0x74, 0xc3, 0x79, 0x80, 0xf7, 0x19, + 0xb4, 0xac, 0x16, 0xad, 0x41, 0xfd, 0xaa, 0x3f, 0xec, 0xad, 0xc8, 0xc5, 0xe8, 0x70, 0xd8, 0x73, + 0x50, 0x0b, 0x56, 0x4f, 0x46, 0xa3, 0x61, 0xaf, 0x86, 0xda, 0xd0, 0x90, 0x2b, 0xbf, 0x57, 0x97, + 0xca, 0xfe, 0xe8, 0xdc, 0xef, 0xad, 0x7a, 0xbf, 0x84, 0xaf, 0x2c, 0x01, 0x43, 0x36, 0xcf, 0x6d, + 0x2e, 0x99, 0x16, 0x7c, 0x08, 0x89, 0x92, 0xa7, 0x1c, 0x52, 0x19, 0x27, 0xcc, 0x0e, 0x34, 0xb9, + 0x56, 0x43, 0x2a, 0xe0, 0xfc, 0x9e, 0xb2, 0xc8, 0x0e, 0x7a, 0x2b, 0x7b, 0xbf, 0x72, 0x00, 0x8a, + 0x3e, 0x94, 0xad, 0x11, 0x18, 0x1c, 0x35, 0x3a, 0x56, 0x94, 0x98, 0x45, 0xea, 0x2c, 0x15, 0x66, + 0x51, 0xca, 0xd1, 0xa7, 0x6a, 0xca, 0x9b, 0x0a, 0xf5, 0x59, 0x3d, 0x29, 0x57, 0xe8, 0xfb, 0x27, + 0xa6, 0xc0, 0xc2, 0x4f, 0xd6, 0x77, 0xf3, 0x36, 0x4a, 0xed, 0x10, 0x95, 0x6b, 0xef, 0xef, 0xf5, + 0xf2, 0xf3, 0x28, 0xe7, 0xaf, 0x9e, 0x19, 0xaa, 0x84, 0x55, 0x6c, 0xa4, 0xff, 0x9b, 0xc7, 0xfe, + 0x5b, 0xd0, 0xc4, 0x34, 0x13, 0x84, 0xbb, 0x0d, 0xf5, 0xa9, 0xcd, 0xca, 0xa7, 0xa4, 0x05, 0x1b, + 0x07, 0x09, 0x61, 0x7f, 0xe0, 0x9b, 0x1d, 0x35, 0x17, 0x21, 0xcc, 0x8d, 0xb8, 0xf0, 0x43, 0x5f, + 0x40, 0x97, 0xde, 0xdc, 0x24, 0x71, 0x6a, 0x36, 0xb4, 0xf6, 0x6f, 0x6c, 0xa8, 0x12, 0x81, 0x3e, + 0x87, 0xf5, 0x97, 0x31, 0x23, 0xf7, 0x41, 0x92, 0xe0, 0x2c, 0x21, 0xdc, 0x6d, 0xa9, 0x14, 0x6e, + 0x39, 0x45, 0xd9, 0x01, 0x57, 0xdd, 0xd1, 0x3e, 0x3c, 0xbe, 0x29, 0x2b, 0x2c, 0x26, 0x6d, 0x85, + 0xc9, 0x52, 0x9b, 0xf7, 0x1b, 0x07, 0x7a, 0xf3, 0x65, 0xbd, 0x93, 0x76, 0x3e, 0x03, 0x08, 0x92, + 0x84, 0xde, 0x93, 0xe8, 0x74, 0xaa, 0x67, 0x48, 0x1b, 0x97, 0x34, 0xff, 0xbd, 0xf6, 0x3b, 0x85, + 0x76, 0xee, 0x2b, 0xbf, 0xca, 0xf9, 0xed, 0x51, 0x1a, 0x5c, 0x27, 0x44, 0xd3, 0x8e, 0x16, 0x2e, + 0x69, 0xaa, 0xd4, 0xa6, 0x36, 0x47, 0x6d, 0xbc, 0x1d, 0x78, 0xd6, 0x27, 0x77, 0x71, 0x48, 0x0e, + 0x32, 0x71, 0x4b, 0x59, 0xfc, 0x8b, 0x40, 0xc4, 0x34, 0x7d, 0x99, 0xd0, 0x7b, 0xcb, 0x87, 0xff, + 0xe2, 0xc0, 0x07, 0x0f, 0xb8, 0xa0, 0x13, 0x35, 0x47, 0xee, 0xe2, 0x88, 0x30, 0xf5, 0xe5, 0x8d, + 0xfd, 0x8f, 0x2a, 0xcd, 0xb0, 0x3c, 0x4c, 0x0e, 0x7b, 0x15, 0x83, 0xf3, 0x68, 0xf4, 0x02, 0x36, + 0xec, 0xba, 0x72, 0x5d, 0xb6, 0xe6, 0x66, 0x70, 0xc9, 0x03, 0xcf, 0x45, 0x78, 0x4f, 0xd5, 0x48, + 0xd4, 0xf9, 0x00, 0x9a, 0x27, 0x97, 0xfe, 0xe8, 0xa8, 0xdf, 0x5b, 0xf1, 0x9e, 0xc1, 0xf6, 0xf0, + 0xec, 0xf0, 0xe8, 0xc1, 0x1d, 0xfe, 0x04, 0x9e, 0x2c, 0xb5, 0x2f, 0x29, 0xca, 0xf9, 0x8f, 0x8b, + 0xfa, 0x47, 0x6d, 0x3e, 0x89, 0xec, 0xa1, 0xc3, 0x24, 0x26, 0xa9, 0x38, 0xed, 0xdb, 0x1e, 0xb2, + 0xb2, 0x7c, 0x1b, 0xf4, 0xda, 0x27, 0x21, 0x23, 0xc2, 0x4c, 0xaf, 0x8a, 0x4e, 0x8e, 0x9b, 0x3e, + 0x9d, 0x04, 0xb1, 0x25, 0xc1, 0x46, 0x92, 0x79, 0x0f, 0xb2, 0x28, 0x26, 0x69, 0x68, 0x29, 0x5f, + 0x2e, 0xcb, 0x91, 0x50, 0x9c, 0xc6, 0x51, 0x1a, 0x4d, 0x69, 0x9c, 0x0a, 0x43, 0x00, 0x97, 0x58, + 0x24, 0x59, 0x53, 0xbc, 0x3a, 0x77, 0xd5, 0xac, 0xb0, 0xaa, 0x94, 0x6c, 0xd7, 0x0f, 0xe9, 0x94, + 0x18, 0x4e, 0xa8, 0x05, 0xd9, 0x91, 0x57, 0x9c, 0x9c, 0xf6, 0x35, 0x51, 0x6f, 0xe9, 0x8e, 0x2c, + 0x34, 0xe8, 0xdb, 0xf0, 0xa4, 0x82, 0x75, 0xfe, 0x0d, 0x4d, 0x08, 0x97, 0x1b, 0xf5, 0xab, 0x19, + 0xc5, 0x8c, 0x84, 0xe2, 0x0a, 0x9f, 0x73, 0x17, 0xd4, 0xfd, 0xaa, 0xe8, 0xbc, 0x3f, 0x39, 0xd0, + 0x50, 0x83, 0x4a, 0x52, 0xd9, 0x1c, 0xdd, 0xda, 0x69, 0x5f, 0x3e, 0x13, 0x66, 0x60, 0x1b, 0x48, + 0xad, 0x88, 0x76, 0xa0, 0x63, 0x96, 0xa3, 0xd9, 0x94, 0x28, 0x48, 0xeb, 0xb8, 0xac, 0x92, 0x57, + 0x50, 0x4e, 0x00, 0x7b, 0x05, 0xe5, 0x5a, 0x9e, 0xc1, 0x05, 0x11, 0x2c, 0x0e, 0x15, 0x86, 0x75, + 0x6c, 0x24, 0xb9, 0xf7, 0x8b, 0x80, 0xbf, 0xcd, 0x08, 0x0b, 0x22, 0xa2, 0x40, 0x6b, 0xe1, 0x92, + 0x46, 0x22, 0x36, 0x20, 0xf2, 0xe0, 0x0d, 0x62, 0x4a, 0xf0, 0xfe, 0xe8, 0x94, 0xc6, 0xaa, 0xc4, + 0x5e, 0xd2, 0xa9, 0x38, 0x24, 0xfa, 0x0e, 0x9b, 0x4b, 0x5d, 0x55, 0xa2, 0x63, 0xe8, 0x49, 0xe2, + 0xa9, 0x89, 0xd7, 0x31, 0xa3, 0xd9, 0xd4, 0xf2, 0x96, 0xaf, 0x56, 0xa8, 0x70, 0xd5, 0x07, 0x2f, + 0x04, 0xa1, 0xe7, 0xd0, 0x39, 0xcc, 0xb8, 0xa0, 0x93, 0xd7, 0x34, 0x25, 0xf6, 0xb5, 0xa9, 0x3c, + 0x53, 0x85, 0x19, 0x97, 0x5d, 0xbd, 0x57, 0x00, 0x85, 0x58, 0x6a, 0x4b, 0xa7, 0xd2, 0x96, 0xfb, + 0xb0, 0x86, 0x49, 0x48, 0x59, 0x64, 0xeb, 0xab, 0x4c, 0x6d, 0x3f, 0x9e, 0x4c, 0x13, 0xa2, 0x1d, + 0xb0, 0x75, 0xf4, 0x04, 0x74, 0xcb, 0x06, 0x79, 0x04, 0x83, 0xe2, 0xd7, 0x90, 0x5a, 0x4b, 0x9d, + 0x3a, 0xb1, 0x9a, 0x3a, 0x00, 0xb5, 0x96, 0xf0, 0x1e, 0x26, 0x01, 0xe7, 0xe6, 0x66, 0x68, 0x41, + 0x32, 0x81, 0xd1, 0xe8, 0x5c, 0x9d, 0x5f, 0x1d, 0xcb, 0xa5, 0xf4, 0xc3, 0xfd, 0x40, 0x04, 0xe6, + 0x06, 0x68, 0xc1, 0xfb, 0x83, 0x03, 0x8f, 0xe6, 0xe0, 0x91, 0xe8, 0x14, 0xaa, 0xa5, 0x3c, 0xb2, + 0x30, 0xe3, 0xb2, 0xab, 0x6c, 0xb9, 0x21, 0x8b, 0x27, 0x01, 0xd3, 0x63, 0xb7, 0x85, 0xad, 0x28, + 0x2d, 0x1a, 0x1b, 0x8d, 0x76, 0x1b, 0x5b, 0x51, 0xbe, 0x53, 0x3e, 0x09, 0x58, 0x78, 0x6b, 0x14, + 0x76, 0xac, 0xeb, 0xb7, 0x7b, 0xa9, 0xcd, 0x3b, 0x01, 0x28, 0x3e, 0xab, 0x1a, 0x7f, 0x98, 0x37, + 0xfe, 0x50, 0x9e, 0xca, 0xc0, 0x2f, 0xe1, 0x64, 0x24, 0xd5, 0xd4, 0x94, 0x09, 0xd3, 0xef, 0x6a, + 0xed, 0xfd, 0xb3, 0x06, 0xdd, 0xf2, 0xbb, 0x29, 0x83, 0x65, 0xb7, 0xe7, 0x09, 0x8d, 0x84, 0xbe, + 0x80, 0x76, 0x5f, 0xdd, 0x3a, 0xc9, 0xc8, 0x35, 0xfb, 0xf4, 0x1e, 0x7a, 0x8a, 0xf7, 0x22, 0xeb, + 0x89, 0x8b, 0x20, 0xf4, 0x3d, 0x68, 0x1e, 0xe8, 0xf0, 0xfa, 0x22, 0x79, 0xad, 0x84, 0x07, 0x3a, + 0xd6, 0xb8, 0xa3, 0x1f, 0x16, 0xd4, 0x55, 0xa1, 0xb2, 0xb1, 0xff, 0xf5, 0x07, 0x43, 0xa7, 0x39, + 0xf3, 0xcd, 0xd9, 0xae, 0xdd, 0x76, 0xc3, 0xdc, 0x65, 0xb9, 0xed, 0x6d, 0x68, 0xe7, 0x35, 0xa2, + 0x26, 0xd4, 0x4e, 0x07, 0x9a, 0x0d, 0x5f, 0x5e, 0x8d, 0x7a, 0x8e, 0xf7, 0x0c, 0x9a, 0xba, 0x04, + 0xf9, 0xa6, 0x1c, 0x1c, 0x1e, 0x1e, 0x0d, 0x47, 0xbd, 0x15, 0x45, 0x87, 0xf1, 0xe5, 0xb0, 0xe7, + 0x78, 0x3f, 0x28, 0x88, 0x38, 0xea, 0xc0, 0xda, 0xd5, 0xe0, 0x6c, 0x70, 0xf9, 0xa5, 0xc9, 0x70, + 0x70, 0x7e, 0xde, 0x73, 0x2c, 0xb1, 0xae, 0x59, 0xaa, 0xad, 0xc8, 0xf4, 0xe9, 0xe1, 0xc5, 0xb0, + 0xb7, 0xea, 0x3d, 0x87, 0x8d, 0xea, 0x8f, 0x55, 0xd9, 0x9a, 0x29, 0x11, 0x39, 0xe4, 0x5a, 0x90, + 0x2d, 0x3c, 0x09, 0x42, 0x4b, 0x66, 0x27, 0x41, 0xb8, 0xff, 0xb7, 0x3a, 0x6c, 0x5e, 0xe4, 0x1b, + 0x37, 0xb3, 0x01, 0x1d, 0x41, 0x43, 0xfd, 0x40, 0x44, 0xdb, 0xd5, 0x1f, 0x94, 0xd5, 0xff, 0x9e, + 0x6d, 0xbd, 0xd3, 0xea, 0xad, 0xa0, 0x97, 0xb0, 0xea, 0xcf, 0xd2, 0xf0, 0xfd, 0xb2, 0x7c, 0xe2, + 0xa0, 0x17, 0xd0, 0x3d, 0x26, 0x22, 0xff, 0x71, 0x88, 0x2a, 0xcc, 0x52, 0xd1, 0xac, 0xad, 0xca, + 0xcf, 0xd4, 0x85, 0x9f, 0x91, 0xde, 0x8a, 0xa4, 0x4d, 0x31, 0x3f, 0x21, 0x41, 0x22, 0x6e, 0x97, + 0x26, 0x58, 0x54, 0x79, 0x2b, 0xe8, 0x35, 0x6c, 0x1d, 0x13, 0xf1, 0x10, 0x6f, 0x79, 0x3f, 0x70, + 0x5e, 0x81, 0x7b, 0x4c, 0xc4, 0x72, 0xca, 0xf0, 0x5e, 0x99, 0x5f, 0xb4, 0x5e, 0x37, 0xcd, 0x7f, + 0x41, 0x9b, 0xea, 0xcf, 0xa7, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0xcb, 0xde, 0x32, 0x7a, 0x3a, + 0x15, 0x00, 0x00, } diff --git a/management/proto/management.proto b/management/proto/management.proto index 2cc0efa22..61304c8c4 100644 --- a/management/proto/management.proto +++ b/management/proto/management.proto @@ -146,6 +146,8 @@ message WiretrusteeConfig { // a Signal server config HostConfig signal = 3; + + string RelayAddress = 4; } // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) diff --git a/management/server/config.go b/management/server/config.go index 3e5ff1eaf..cbc8a4e72 100644 --- a/management/server/config.go +++ b/management/server/config.go @@ -32,9 +32,10 @@ const ( // Config of the Management service type Config struct { - Stuns []*Host - TURNConfig *TURNConfig - Signal *Host + Stuns []*Host + TURNConfig *TURNConfig + RelayAddress string + Signal *Host Datadir string DataStoreEncryptionKey string diff --git a/management/server/grpcserver.go b/management/server/grpcserver.go index 32989df7d..8c8b8354f 100644 --- a/management/server/grpcserver.go +++ b/management/server/grpcserver.go @@ -450,6 +450,7 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot Uri: config.Signal.URI, Protocol: ToResponseProto(config.Signal.Proto), }, + RelayAddress: config.RelayAddress, } } diff --git a/relay/client/manager.go b/relay/client/manager.go index 911561403..0223c98ab 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -55,19 +55,24 @@ func NewManager(ctx context.Context, serverAddress string, peerID string) *Manag // Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop. // todo: consider to return an error if the initial connection to the relay server is not established. -func (m *Manager) Serve() { +func (m *Manager) Serve() error { + if m.relayClient != nil { + return fmt.Errorf("manager already serving") + } + m.relayClient = NewClient(m.ctx, m.srvAddress, m.peerID) - m.reconnectGuard = NewGuard(m.ctx, m.relayClient) - m.relayClient.SetOnDisconnectListener(m.reconnectGuard.OnDisconnected) err := m.relayClient.Connect() if err != nil { - log.Errorf("failed to connect to relay server, keep try to reconnect: %s", err) - return + log.Errorf("failed to connect to relay server: %s", err) + return err } + m.reconnectGuard = NewGuard(m.ctx, m.relayClient) + m.relayClient.SetOnDisconnectListener(m.reconnectGuard.OnDisconnected) + m.startCleanupLoop() - return + return nil } // OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be diff --git a/signal/client/client.go b/signal/client/client.go index 9d99b3677..7c54178e2 100644 --- a/signal/client/client.go +++ b/signal/client/client.go @@ -51,8 +51,7 @@ func UnMarshalCredential(msg *proto.Message) (*Credential, error) { } // MarshalCredential marshal a Credential instance and returns a Message object -func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type, - rosenpassPubKey []byte, rosenpassAddr string) (*proto.Message, error) { +func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type, rosenpassPubKey []byte, rosenpassAddr string, relaySrvAddress string) (*proto.Message, error) { return &proto.Message{ Key: myKey.PublicKey().String(), RemoteKey: remoteKey.String(), @@ -65,6 +64,7 @@ func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, cre RosenpassPubKey: rosenpassPubKey, RosenpassServerAddr: rosenpassAddr, }, + RelayServerAddress: relaySrvAddress, }, }, nil } diff --git a/signal/proto/signalexchange.pb.go b/signal/proto/signalexchange.pb.go index 782c45da1..30f704c6f 100644 --- a/signal/proto/signalexchange.pb.go +++ b/signal/proto/signalexchange.pb.go @@ -1,15 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v3.12.4 +// protoc v3.21.12 // source: signalexchange.proto package proto import ( - _ "github.com/golang/protobuf/protoc-gen-go/descriptor" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/descriptorpb" reflect "reflect" sync "sync" ) @@ -225,6 +225,8 @@ type Body struct { FeaturesSupported []uint32 `protobuf:"varint,6,rep,packed,name=featuresSupported,proto3" json:"featuresSupported,omitempty"` // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig *RosenpassConfig `protobuf:"bytes,7,opt,name=rosenpassConfig,proto3" json:"rosenpassConfig,omitempty"` + // relayServerAddress is an IP:port of the relay server + RelayServerAddress string `protobuf:"bytes,8,opt,name=relayServerAddress,proto3" json:"relayServerAddress,omitempty"` } func (x *Body) Reset() { @@ -308,6 +310,13 @@ func (x *Body) GetRosenpassConfig() *RosenpassConfig { return nil } +func (x *Body) GetRelayServerAddress() string { + if x != nil { + return x.RelayServerAddress + } + return "" +} + // Mode indicates a connection mode type Mode struct { state protoimpl.MessageState @@ -431,7 +440,7 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x52, 0x09, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x52, - 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xf6, 0x02, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, + 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0xa6, 0x03, 0x0a, 0x04, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x6f, 0x64, 0x79, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x18, 0x0a, @@ -451,7 +460,10 @@ var file_signalexchange_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x65, 0x78, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x52, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0f, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x12, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x36, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x46, 0x46, 0x45, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x4e, 0x53, 0x57, 0x45, 0x52, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x44, 0x49, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x4d, 0x4f, 0x44, 0x45, 0x10, 0x04, 0x22, 0x2e, diff --git a/signal/proto/signalexchange.proto b/signal/proto/signalexchange.proto index a8c4c309c..5aa24d893 100644 --- a/signal/proto/signalexchange.proto +++ b/signal/proto/signalexchange.proto @@ -60,6 +60,9 @@ message Body { // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig rosenpassConfig = 7; + + // relayServerAddress is an IP:port of the relay server + string relayServerAddress = 8; } // Mode indicates a connection mode From a7760bf0a7aa18ead65807eca043c2ef1df184ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 14 Jun 2024 15:43:18 +0200 Subject: [PATCH 034/199] Configurable relay address with env variable --- client/internal/connect.go | 23 +++++++++++++++++++---- relay/client/manager.go | 6 ++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index 44e85be53..2c0c2657b 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "os" "runtime" "runtime/debug" "strings" @@ -245,10 +246,13 @@ func (c *ConnectClient) run( c.statusRecorder.MarkSignalConnected() - relayManager := relayClient.NewManager(engineCtx, loginResp.GetWiretrusteeConfig().GetRelayAddress(), myPrivateKey.PublicKey().String()) - if err = relayManager.Serve(); err != nil { - log.Error(err) - return wrapErr(err) + relayAddress := relayAddress(loginResp) + relayManager := relayClient.NewManager(engineCtx, relayAddress, myPrivateKey.PublicKey().String()) + if relayAddress != "" { + if err = relayManager.Serve(); err != nil { + log.Error(err) + return wrapErr(err) + } } peerConfig := loginResp.GetPeerConfig() @@ -304,6 +308,17 @@ func (c *ConnectClient) run( return nil } +func relayAddress(resp *mgmProto.LoginResponse) string { + if envRelay := os.Getenv("NB_RELAY_ADDRESS"); envRelay != "" { + return envRelay + } + + if resp.GetWiretrusteeConfig().GetRelayAddress() != "" { + return resp.GetWiretrusteeConfig().GetRelayAddress() + } + return "" +} + func (c *ConnectClient) Engine() *Engine { var e *Engine c.engineMutex.Lock() diff --git a/relay/client/manager.go b/relay/client/manager.go index 0223c98ab..6b816c4fb 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -12,6 +12,8 @@ import ( var ( relayCleanupInterval = 60 * time.Second + + errRelayClientNotConnected = fmt.Errorf("relay client not connected") ) // RelayTrack hold the relay clients for the foregin relay servers. @@ -80,7 +82,7 @@ func (m *Manager) Serve() error { // connection to the relay server. func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { if m.relayClient == nil { - return nil, fmt.Errorf("relay client not connected") + return nil, errRelayClientNotConnected } foreign, err := m.isForeignServer(serverAddress) @@ -101,7 +103,7 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { // This address will be sent to the target peer to choose the common relay server for the communication. func (m *Manager) RelayAddress() (net.Addr, error) { if m.relayClient == nil { - return nil, fmt.Errorf("relay client not connected") + return nil, errRelayClientNotConnected } return m.relayClient.RelayRemoteAddress() } From e407fe02c5730d09939f838f5ec3ae32b737e65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 17 Jun 2024 17:52:22 +0200 Subject: [PATCH 035/199] Separate lifecircle of handshake, ice, relay connections - fix Stun, Turn address update thread safety issue - move conn worker login into peer package --- client/internal/engine.go | 189 +---- client/internal/peer/conn.go | 1072 ++++++------------------ client/internal/peer/conn_ice.go | 436 ++++++++++ client/internal/peer/conn_relay.go | 115 +++ client/internal/peer/conn_test.go | 10 +- client/internal/peer/handshaker.go | 210 +++++ client/internal/peer/stdnet.go | 4 +- client/internal/peer/stdnet_android.go | 4 +- client/internal/signaler.go | 71 ++ relay/client/manager.go | 4 + signal/client/client.go | 4 +- 11 files changed, 1139 insertions(+), 980 deletions(-) create mode 100644 client/internal/peer/conn_ice.go create mode 100644 client/internal/peer/conn_relay.go create mode 100644 client/internal/peer/handshaker.go create mode 100644 client/internal/signaler.go diff --git a/client/internal/engine.go b/client/internal/engine.go index a2f395ea0..d80d1f235 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -12,6 +12,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" "github.com/pion/ice/v3" @@ -95,7 +96,8 @@ type EngineConfig struct { // Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers. type Engine struct { // signal is a Signal Service client - signal signal.Client + signal signal.Client + signaler *Signaler // mgmClient is a Management Service client mgmClient mgm.Client // peerConns is a map that holds all the peers that are known to this peer @@ -116,7 +118,8 @@ type Engine struct { // STUNs is a list of STUN servers used by ICE STUNs []*stun.URI // TURNs is a list of STUN servers used by ICE - TURNs []*stun.URI + TURNs []*stun.URI + StunTurn atomic.Value // clientRoutes is the most recent list of clientRoutes received from the Management Service clientRoutes route.HAMap @@ -154,8 +157,6 @@ type Engine struct { relayProbe *Probe wgProbe *Probe - wgConnWorker sync.WaitGroup - relayManager *relayClient.Manager } @@ -207,11 +208,11 @@ func NewEngineWithProbes( relayProbe *Probe, wgProbe *Probe, ) *Engine { - return &Engine{ clientCtx: clientCtx, clientCancel: clientCancel, signal: signalClient, + signaler: NewSignaler(signalClient, config.WgPrivateKey), mgmClient: mgmClient, relayManager: relayManager, peerConns: make(map[string]*peer.Conn), @@ -258,7 +259,6 @@ func (e *Engine) Stop() error { time.Sleep(500 * time.Millisecond) e.close() - e.wgConnWorker.Wait() log.Infof("stopped Netbird Engine") return nil } @@ -457,72 +457,11 @@ func (e *Engine) removePeer(peerKey string) error { conn, exists := e.peerConns[peerKey] if exists { delete(e.peerConns, peerKey) - err := conn.Close() - if err != nil { - switch err.(type) { - case *peer.ConnectionAlreadyClosedError: - return nil - default: - return err - } - } + conn.Close() } return nil } -func signalCandidate(candidate ice.Candidate, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client) error { - err := s.Send(&sProto.Message{ - Key: myKey.PublicKey().String(), - RemoteKey: remoteKey.String(), - Body: &sProto.Body{ - Type: sProto.Body_CANDIDATE, - Payload: candidate.Marshal(), - }, - }) - if err != nil { - return err - } - - return nil -} - -func sendSignal(message *sProto.Message, s signal.Client) error { - return s.Send(message) -} - -// SignalOfferAnswer signals either an offer or an answer to remote peer -func SignalOfferAnswer(offerAnswer peer.OfferAnswer, myKey wgtypes.Key, remoteKey wgtypes.Key, s signal.Client, - isAnswer bool) error { - var t sProto.Body_Type - if isAnswer { - t = sProto.Body_ANSWER - } else { - t = sProto.Body_OFFER - } - - msg, err := signal.MarshalCredential( - myKey, - offerAnswer.WgListenPort, - remoteKey, &signal.Credential{ - UFrag: offerAnswer.IceCredentials.UFrag, - Pwd: offerAnswer.IceCredentials.Pwd, - }, - t, - offerAnswer.RosenpassPubKey, - offerAnswer.RosenpassAddr, - offerAnswer.RelaySrvAddress) - if err != nil { - return err - } - - err = s.Send(msg) - if err != nil { - return err - } - - return nil -} - func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { e.syncMsgMux.Lock() defer e.syncMsgMux.Unlock() @@ -538,6 +477,11 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { return err } + var stunTurn []*stun.URI + stunTurn = append(stunTurn, e.STUNs...) + stunTurn = append(stunTurn, e.TURNs...) + e.StunTurn.Store(stunTurn) + // todo update relay address in the relay manager // todo update signal @@ -893,57 +837,10 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { if err != nil { log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err) } - - e.wgConnWorker.Add(1) - go e.connWorker(conn, peerKey) } return nil } -func (e *Engine) connWorker(conn *peer.Conn, peerKey string) { - defer e.wgConnWorker.Done() - for { - - // randomize starting time a bit - min := 500 - max := 2000 - duration := time.Duration(rand.Intn(max-min)+min) * time.Millisecond - select { - case <-e.ctx.Done(): - return - case <-time.After(duration): - } - - // if peer has been removed -> give up - if !e.peerExists(peerKey) { - log.Debugf("peer %s doesn't exist anymore, won't retry connection", peerKey) - return - } - - if !e.signal.Ready() { - log.Infof("signal client isn't ready, skipping connection attempt %s", peerKey) - continue - } - - // we might have received new STUN and TURN servers meanwhile, so update them - e.syncMsgMux.Lock() - conn.UpdateStunTurn(append(e.STUNs, e.TURNs...)) - e.syncMsgMux.Unlock() - - err := conn.Open(e.ctx) - if err != nil { - log.Debugf("connection to peer %s failed: %v", peerKey, err) - var connectionClosedError *peer.ConnectionClosedError - switch { - case errors.As(err, &connectionClosedError): - // conn has been forced to close, so we exit the loop - return - default: - } - } - } -} - func (e *Engine) peerExists(peerKey string) bool { e.syncMsgMux.Lock() defer e.syncMsgMux.Unlock() @@ -953,9 +850,6 @@ func (e *Engine) peerExists(peerKey string) bool { func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) { log.Debugf("creating peer connection %s", pubKey) - var stunTurn []*stun.URI - stunTurn = append(stunTurn, e.STUNs...) - stunTurn = append(stunTurn, e.TURNs...) wgConfig := peer.WgConfig{ RemoteKey: pubKey, @@ -988,52 +882,29 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e // randomize connection timeout timeout := time.Duration(rand.Intn(PeerConnectionTimeoutMax-PeerConnectionTimeoutMin)+PeerConnectionTimeoutMin) * time.Millisecond config := peer.ConnConfig{ - Key: pubKey, - LocalKey: e.config.WgPrivateKey.PublicKey().String(), - StunTurn: stunTurn, - InterfaceBlackList: e.config.IFaceBlackList, - DisableIPv6Discovery: e.config.DisableIPv6Discovery, - Timeout: timeout, - UDPMux: e.udpMux.UDPMuxDefault, - UDPMuxSrflx: e.udpMux, - WgConfig: wgConfig, - LocalWgPort: e.config.WgPort, - NATExternalIPs: e.parseNATExternalIPMappings(), - RosenpassPubKey: e.getRosenpassPubKey(), - RosenpassAddr: e.getRosenpassAddr(), + Key: pubKey, + LocalKey: e.config.WgPrivateKey.PublicKey().String(), + Timeout: timeout, + WgConfig: wgConfig, + LocalWgPort: e.config.WgPort, + RosenpassPubKey: e.getRosenpassPubKey(), + RosenpassAddr: e.getRosenpassAddr(), + ICEConfig: peer.ICEConfig{ + StunTurn: e.StunTurn, + InterfaceBlackList: e.config.IFaceBlackList, + DisableIPv6Discovery: e.config.DisableIPv6Discovery, + UDPMux: e.udpMux.UDPMuxDefault, + UDPMuxSrflx: e.udpMux, + NATExternalIPs: e.parseNATExternalIPMappings(), + }, } - peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover, e.relayManager) + peerConn, err := peer.NewConn(e.ctx, config, e.statusRecorder, e.wgProxyFactory, e.signaler, e.mobileDep.IFaceDiscover, e.relayManager) if err != nil { return nil, err } - wgPubKey, err := wgtypes.ParseKey(pubKey) - if err != nil { - return nil, err - } - - signalOffer := func(offerAnswer peer.OfferAnswer) error { - return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, false) - } - - signalCandidate := func(candidate ice.Candidate) error { - return signalCandidate(candidate, e.config.WgPrivateKey, wgPubKey, e.signal) - } - - signalAnswer := func(offerAnswer peer.OfferAnswer) error { - return SignalOfferAnswer(offerAnswer, e.config.WgPrivateKey, wgPubKey, e.signal, true) - } - - peerConn.SetSignalCandidate(signalCandidate) - peerConn.SetSignalOffer(signalOffer) - peerConn.SetSignalAnswer(signalAnswer) - peerConn.SetSendSignalMessage(func(message *sProto.Message) error { - return sendSignal(message, e.signal) - }) - if e.rpManager != nil { - peerConn.SetOnConnected(e.rpManager.OnConnected) peerConn.SetOnDisconnected(e.rpManager.OnDisconnected) } @@ -1107,7 +978,7 @@ func (e *Engine) receiveSignalEvents() { return err } - conn.OnRemoteCandidate(candidate, e.GetClientRoutes()) + go conn.OnRemoteCandidate(candidate, e.GetClientRoutes()) case sProto.Body_MODE: } @@ -1402,7 +1273,7 @@ func (e *Engine) receiveProbeEvents() { for _, peer := range e.peerConns { key := peer.GetKey() - wgStats, err := peer.GetConf().WgConfig.WgInterface.GetStats(key) + wgStats, err := peer.WgConfig().WgInterface.GetStats(key) if err != nil { log.Debugf("failed to get wg stats for peer %s: %s", key, err) } diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b2fb58855..16daf492e 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -2,37 +2,33 @@ package peer import ( "context" - "fmt" "net" - "net/netip" "runtime" "strings" "sync" "time" "github.com/pion/ice/v3" - "github.com/pion/stun/v2" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" - "github.com/netbirdio/netbird/iface/bind" relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" - sProto "github.com/netbirdio/netbird/signal/proto" nbnet "github.com/netbirdio/netbird/util/net" - "github.com/netbirdio/netbird/version" ) -const ( - iceKeepAliveDefault = 4 * time.Second - iceDisconnectedTimeoutDefault = 6 * time.Second - // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package - iceRelayAcceptanceMinWaitDefault = 2 * time.Second +type ConnPriority int +const ( defaultWgKeepAlive = 25 * time.Second + + connPriorityRelay ConnPriority = 1 + connPriorityICETurn = 1 + connPriorityICEP2P = 2 ) type WgConfig struct { @@ -45,637 +41,151 @@ type WgConfig struct { // ConnConfig is a peer Connection configuration type ConnConfig struct { - // Key is a public key of a remote peer Key string // LocalKey is a public key of a local peer LocalKey string - // StunTurn is a list of STUN and TURN URLs - StunTurn []*stun.URI - - // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering - // (e.g. if eth0 is in the list, host candidate of this interface won't be used) - InterfaceBlackList []string - DisableIPv6Discovery bool - Timeout time.Duration WgConfig WgConfig - UDPMux ice.UDPMux - UDPMuxSrflx ice.UniversalUDPMux - LocalWgPort int - NATExternalIPs []string - // RosenpassPubKey is this peer's Rosenpass public key RosenpassPubKey []byte // RosenpassPubKey is this peer's RosenpassAddr server address (IP:port) RosenpassAddr string -} -// OfferAnswer represents a session establishment offer or answer -type OfferAnswer struct { - IceCredentials IceCredentials - // WgListenPort is a remote WireGuard listen port. - // This field is used when establishing a direct WireGuard connection without any proxy. - // We can set the remote peer's endpoint with this port. - WgListenPort int - - // Version of NetBird Agent - Version string - // RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message - // This value is the local Rosenpass server public key when sending the message - RosenpassPubKey []byte - // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message - // This value is the local Rosenpass server address when sending the message - RosenpassAddr string - - // relay server address - RelaySrvAddress string -} - -// IceCredentials ICE protocol credentials struct -type IceCredentials struct { - UFrag string - Pwd string + // ICEConfig ICE protocol configuration + ICEConfig ICEConfig } type BeforeAddPeerHookFunc func(connID nbnet.ConnectionID, IP net.IP) error type AfterRemovePeerHookFunc func(connID nbnet.ConnectionID) error type Conn struct { - config ConnConfig - mu sync.Mutex - - // signalCandidate is a handler function to signal remote peer about local connection candidate - signalCandidate func(candidate ice.Candidate) error - // signalOffer is a handler function to signal remote peer our connection offer (credentials) - signalOffer func(OfferAnswer) error - signalAnswer func(OfferAnswer) error - sendSignalMessage func(message *sProto.Message) error - onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) - onDisconnected func(remotePeer string, wgIP string) - - // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection - remoteOffersCh chan OfferAnswer - // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection - remoteAnswerCh chan OfferAnswer - closeCh chan struct{} - ctx context.Context - notifyDisconnected context.CancelFunc - - agent *ice.Agent - status ConnStatus - + log *log.Entry + mu sync.Mutex + ctx context.Context + ctxCancel context.CancelFunc + config ConnConfig statusRecorder *Status - wgProxyFactory *wgproxy.Factory wgProxy wgproxy.Proxy + signaler *internal.Signaler + allowedIPsIP string + handshaker *Handshaker + closeCh chan struct{} - adapter iface.TunAdapter - iFaceDiscover stdnet.ExternalIFaceDiscover - sentExtraSrflx bool + onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) + onDisconnected func(remotePeer string, wgIP string) + + status ConnStatus + + connectorICE *ConnectorICE + connectorRelay *ConnectorRelay connID nbnet.ConnectionID beforeAddPeerHooks []BeforeAddPeerHookFunc afterRemovePeerHooks []AfterRemovePeerHookFunc - relayManager *relayClient.Manager + currentConnType ConnPriority } // NewConn creates a new not opened Conn to the remote peer. // To establish a connection run Conn.Open -func NewConn(config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { - return &Conn{ +func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *internal.Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { + _, allowedIPsIP, err := net.ParseCIDR(config.WgConfig.AllowedIps) + if err != nil { + log.Errorf("failed to parse allowedIPS: %v", err) + return nil, err + } + + ctx, ctxCancel := context.WithCancel(engineCtx) + + var conn = &Conn{ + log: log.WithField("peer", config.Key), + ctx: ctx, + ctxCancel: ctxCancel, config: config, - mu: sync.Mutex{}, - status: StatusDisconnected, - closeCh: make(chan struct{}), - remoteOffersCh: make(chan OfferAnswer), - remoteAnswerCh: make(chan OfferAnswer), statusRecorder: statusRecorder, wgProxyFactory: wgProxyFactory, - adapter: adapter, - iFaceDiscover: iFaceDiscover, - relayManager: relayManager, - }, nil + signaler: signaler, + allowedIPsIP: allowedIPsIP.String(), + handshaker: NewHandshaker(ctx, config, signaler), + status: StatusDisconnected, + closeCh: make(chan struct{}), + } + conn.connectorICE = NewConnectorICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.doHandshake) + conn.connectorRelay = NewConnectorRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.doHandshake) + return conn, nil } -// Open opens connection to the remote peer starting ICE candidate gathering process. -// Blocks until connection has been closed or connection timeout. -// ConnStatus will be set accordingly -func (conn *Conn) Open(ctx context.Context) error { - log.Debugf("trying to connect to peer %s", conn.config.Key) +// Open opens connection to the remote peer +// It will try to establish a connection using ICE and in parallel with relay. The higher priority connection type will +// be used. +// todo implement on disconnected event from ICE and relay too. +func (conn *Conn) Open() { + conn.log.Debugf("trying to connect to peer") peerState := State{ PubKey: conn.config.Key, IP: strings.Split(conn.config.WgConfig.AllowedIps, "/")[0], ConnStatusUpdate: time.Now(), - ConnStatus: conn.status, + ConnStatus: StatusDisconnected, Mux: new(sync.RWMutex), } err := conn.statusRecorder.UpdatePeerState(peerState) if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) + conn.log.Warnf("error while updating the state err: %v", err) } - defer func() { - err := conn.cleanup() + /* + peerState = State{ + PubKey: conn.config.Key, + ConnStatus: StatusConnecting, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + err = conn.statusRecorder.UpdatePeerState(peerState) if err != nil { - log.Warnf("error while cleaning up peer connection %s: %v", conn.config.Key, err) - return + log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) } - }() - - err = conn.sendOffer() - if err != nil { - return err + */ + relayIsSupportedLocally := conn.connectorRelay.RelayIsSupported() + if relayIsSupportedLocally { + go conn.connectorRelay.SetupRelayConnection() } - - log.Debugf("connection offer sent to peer %s, waiting for the confirmation", conn.config.Key) - - // Only continue once we got a connection confirmation from the remote peer. - // The connection timeout could have happened before a confirmation received from the remote. - // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) - remoteOfferAnswer, err := conn.waitForRemoteOfferConfirmation() - if err != nil { - return err - } - - log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", - conn.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) - - // at this point we received offer/answer and we are ready to gather candidates - conn.mu.Lock() - conn.status = StatusConnecting - conn.ctx, conn.notifyDisconnected = context.WithCancel(ctx) - defer conn.notifyDisconnected() - conn.mu.Unlock() - - peerState = State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) - } - - // in edge case this function can block while the manager set up a new relay server connection - relayOperate := conn.setupRelayConnection(remoteOfferAnswer) - - err = conn.setupICEConnection(remoteOfferAnswer, relayOperate) - if err != nil { - log.Errorf("failed to setup ICE connection: %s", err) - if !relayOperate { - return err - } - } - - // wait until connection disconnected or has been closed externally (upper layer, e.g. engine) - err = conn.waitForDisconnection() - return err -} - -func (conn *Conn) AddBeforeAddPeerHook(hook BeforeAddPeerHookFunc) { - conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook) -} - -func (conn *Conn) AddAfterRemovePeerHook(hook AfterRemovePeerHookFunc) { - conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook) -} - -// GetConf returns the connection config -func (conn *Conn) GetConf() ConnConfig { - return conn.config -} - -// WgConfig returns the WireGuard config -func (conn *Conn) WgConfig() WgConfig { - return conn.config.WgConfig -} - -// UpdateStunTurn update the turn and stun addresses -func (conn *Conn) UpdateStunTurn(turnStun []*stun.URI) { - conn.config.StunTurn = turnStun -} - -// SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer -func (conn *Conn) SetSignalOffer(handler func(offer OfferAnswer) error) { - conn.signalOffer = handler -} - -// SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established -func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) { - conn.onConnected = handler -} - -// SetOnDisconnected sets a handler function to be triggered by Conn when a connection to a remote disconnected -func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string)) { - conn.onDisconnected = handler -} - -// SetSignalAnswer sets a handler function to be triggered by Conn when a new connection answer has to be signalled to the remote peer -func (conn *Conn) SetSignalAnswer(handler func(answer OfferAnswer) error) { - conn.signalAnswer = handler -} - -// SetSignalCandidate sets a handler function to be triggered by Conn when a new ICE local connection candidate has to be signalled to the remote peer -func (conn *Conn) SetSignalCandidate(handler func(candidate ice.Candidate) error) { - conn.signalCandidate = handler -} - -// SetSendSignalMessage sets a handler function to be triggered by Conn when there is new message to send via signal -func (conn *Conn) SetSendSignalMessage(handler func(message *sProto.Message) error) { - conn.sendSignalMessage = handler + go conn.connectorICE.SetupICEConnection(relayIsSupportedLocally) } // Close closes this peer Conn issuing a close event to the Conn closeCh -func (conn *Conn) Close() error { +func (conn *Conn) Close() { conn.mu.Lock() - defer conn.mu.Unlock() - select { - case conn.closeCh <- struct{}{}: - return nil - default: - // probably could happen when peer has been added and removed right after not even starting to connect - // todo further investigate - // this really happens due to unordered messages coming from management - // more importantly it causes inconsistency -> 2 Conn objects for the same peer - // e.g. this flow: - // update from management has peers: [1,2,3,4] - // engine creates a Conn for peers: [1,2,3,4] and schedules Open in ~1sec - // before conn.Open() another update from management arrives with peers: [1,2,3] - // engine removes peer 4 and calls conn.Close() which does nothing (this default clause) - // before conn.Open() another update from management arrives with peers: [1,2,3,4,5] - // engine adds a new Conn for 4 and 5 - // therefore peer 4 has 2 Conn objects - log.Warnf("Connection has been already closed or attempted closing not started connection %s", conn.config.Key) - return NewConnectionAlreadyClosed(conn.config.Key) - } -} - -// Status returns current status of the Conn -func (conn *Conn) Status() ConnStatus { - conn.mu.Lock() - defer conn.mu.Unlock() - return conn.status -} - -// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { - log.Debugf("OnRemoteOffer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteOffersCh <- offer: - return true - default: - log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - // connection might not be ready yet to receive so we ignore the message - return false - } -} - -// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise -// doesn't block, discards the message if connection wasn't ready -func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { - log.Debugf("OnRemoteAnswer from peer %s on status %s", conn.config.Key, conn.status.String()) - - select { - case conn.remoteAnswerCh <- answer: - return true - default: - // connection might not be ready yet to receive so we ignore the message - log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", conn.config.Key, conn.status.String()) - return false - } -} - -// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. -func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) - go func() { - conn.mu.Lock() - defer conn.mu.Unlock() - - if conn.agent == nil { - return - } - - if candidateViaRoutes(candidate, haRoutes) { - return - } - - err := conn.agent.AddRemoteCandidate(candidate) - if err != nil { - log.Errorf("error while handling remote candidate from peer %s", conn.config.Key) - return - } - }() -} - -func (conn *Conn) GetKey() string { - return conn.config.Key -} - -func (conn *Conn) reCreateAgent(relaySupport []ice.CandidateType) error { - conn.mu.Lock() - defer conn.mu.Unlock() - - failedTimeout := 6 * time.Second - - var err error - transportNet, err := conn.newStdNet() - if err != nil { - log.Errorf("failed to create pion's stdnet: %s", err) - } - - iceKeepAlive := iceKeepAlive() - iceDisconnectedTimeout := iceDisconnectedTimeout() - iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() - - agentConfig := &ice.AgentConfig{ - MulticastDNSMode: ice.MulticastDNSModeDisabled, - NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: conn.config.StunTurn, - CandidateTypes: candidateTypes(), - FailedTimeout: &failedTimeout, - InterfaceFilter: stdnet.InterfaceFilter(conn.config.InterfaceBlackList), - UDPMux: conn.config.UDPMux, - UDPMuxSrflx: conn.config.UDPMuxSrflx, - NAT1To1IPs: conn.config.NATExternalIPs, - Net: transportNet, - DisconnectedTimeout: &iceDisconnectedTimeout, - KeepaliveInterval: &iceKeepAlive, - RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, - } - - if conn.config.DisableIPv6Discovery { - agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} - } - - conn.agent, err = ice.NewAgent(agentConfig) - if err != nil { - return err - } - - err = conn.agent.OnCandidate(conn.onICECandidate) - if err != nil { - return err - } - - err = conn.agent.OnConnectionStateChange(conn.onICEConnectionStateChange) - if err != nil { - return err - } - - err = conn.agent.OnSelectedCandidatePairChange(conn.onICESelectedCandidatePair) - if err != nil { - return err - } - - err = conn.agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { - err := conn.statusRecorder.UpdateLatency(conn.config.Key, p.Latency()) - if err != nil { - log.Debugf("failed to update latency for peer %s: %s", conn.config.Key, err) - return - } - }) - if err != nil { - return fmt.Errorf("failed setting binding response callback: %w", err) - } - - return nil -} - -func (conn *Conn) configureWgConnectionForRelay(remoteConn net.Conn, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) error { - conn.mu.Lock() - defer conn.mu.Unlock() - - conn.wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - endpoint, err := conn.wgProxy.AddTurnConn(remoteConn) - if err != nil { - return err - } - - endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) - log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) - - conn.connID = nbnet.GenerateConnID() - for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { - log.Errorf("Before add peer hook failed: %v", err) - } - } - - err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) - if err != nil { - if conn.wgProxy != nil { - if err := conn.wgProxy.CloseConn(); err != nil { - log.Warnf("Failed to close relay connection: %v", err) - } - } - // todo: is this nil correct? - return nil - } - - conn.status = StatusConnected - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: StatusConnected, - ConnStatusUpdate: time.Now(), - LocalIceCandidateType: "", - RemoteIceCandidateType: "", - LocalIceCandidateEndpoint: "", - RemoteIceCandidateEndpoint: "", - Direct: false, - RosenpassEnabled: isRosenpassEnabled(remoteRosenpassPubKey), - Mux: new(sync.RWMutex), - Relayed: true, - } - - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("unable to save peer's state, got error: %v", err) - } - - _, ipNet, err := net.ParseCIDR(conn.config.WgConfig.AllowedIps) - if err != nil { - return nil - } - - if runtime.GOOS == "ios" { - runtime.GC() - } - - if conn.onConnected != nil { - conn.onConnected(conn.config.Key, remoteRosenpassPubKey, ipNet.IP.String(), remoteRosenpassAddr) - } - - return nil -} - -// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected -func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) (net.Addr, error) { - conn.mu.Lock() - defer conn.mu.Unlock() - - pair, err := conn.agent.GetSelectedCandidatePair() - if err != nil { - return nil, err - } - - var endpoint net.Addr - if isRelayCandidate(pair.Local) { - log.Debugf("setup relay connection") - conn.wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - endpoint, err = conn.wgProxy.AddTurnConn(remoteConn) - if err != nil { - return nil, err - } - } else { - // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port - go conn.punchRemoteWGPort(pair, remoteWgPort) - endpoint = remoteConn.RemoteAddr() - } - - endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) - log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) - - conn.connID = nbnet.GenerateConnID() - for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { - log.Errorf("Before add peer hook failed: %v", err) - } - } - - err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) - if err != nil { - if conn.wgProxy != nil { - if err := conn.wgProxy.CloseConn(); err != nil { - log.Warnf("Failed to close turn connection: %v", err) - } - } - return nil, fmt.Errorf("update peer: %w", err) - } - - conn.status = StatusConnected - rosenpassEnabled := false - if remoteRosenpassPubKey != nil { - rosenpassEnabled = true - } - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: conn.status, - ConnStatusUpdate: time.Now(), - LocalIceCandidateType: pair.Local.Type().String(), - RemoteIceCandidateType: pair.Remote.Type().String(), - LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), - RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), - Direct: !isRelayCandidate(pair.Local), - RosenpassEnabled: rosenpassEnabled, - Mux: new(sync.RWMutex), - } - if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { - peerState.Relayed = true - } - - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("unable to save peer's state, got error: %v", err) - } - - _, ipNet, err := net.ParseCIDR(conn.config.WgConfig.AllowedIps) - if err != nil { - return nil, err - } - - if runtime.GOOS == "ios" { - runtime.GC() - } - - if conn.onConnected != nil { - conn.onConnected(conn.config.Key, remoteRosenpassPubKey, ipNet.IP.String(), remoteRosenpassAddr) - } - - return endpoint, nil -} - -func (conn *Conn) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { - // wait local endpoint configuration - time.Sleep(time.Second) - addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) - if err != nil { - log.Warnf("got an error while resolving the udp address, err: %s", err) - return - } - - mux, ok := conn.config.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) - if !ok { - log.Warn("invalid udp mux conversion") - return - } - _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) - if err != nil { - log.Warnf("got an error while sending the punch packet, err: %s", err) - } -} - -func (conn *Conn) waitForDisconnection() error { - select { - case <-conn.closeCh: - // closed externally - return NewConnectionClosedError(conn.config.Key) - case <-conn.ctx.Done(): - // disconnected from the remote peer - return NewConnectionDisconnectedError(conn.config.Key) - } -} - -// cleanup closes all open resources and sets status to StatusDisconnected -func (conn *Conn) cleanup() error { - log.Debugf("trying to cleanup %s", conn.config.Key) - conn.mu.Lock() - defer conn.mu.Unlock() - - conn.sentExtraSrflx = false - - var err1, err2, err3 error - if conn.agent != nil { - err1 = conn.agent.Close() - if err1 == nil { - conn.agent = nil - } - } + conn.ctxCancel() if conn.wgProxy != nil { - err2 = conn.wgProxy.CloseConn() + err := conn.wgProxy.CloseConn() + if err != nil { + conn.log.Errorf("failed to close wg proxy: %v", err) + } conn.wgProxy = nil } // todo: is it problem if we try to remove a peer what is never existed? - err3 = conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if err != nil { + conn.log.Errorf("failed to remove wg endpoint: %v", err) + } if conn.connID != "" { for _, hook := range conn.afterRemovePeerHooks { if err := hook(conn.connID); err != nil { - log.Errorf("After remove peer hook failed: %v", err) + conn.log.Errorf("After remove peer hook failed: %v", err) } } - } - conn.connID = "" - - if conn.notifyDisconnected != nil { - conn.notifyDisconnected() - conn.notifyDisconnected = nil + conn.connID = "" } if conn.status == StatusConnected && conn.onDisconnected != nil { @@ -690,298 +200,240 @@ func (conn *Conn) cleanup() error { ConnStatusUpdate: time.Now(), Mux: new(sync.RWMutex), } - err := conn.statusRecorder.UpdatePeerState(peerState) + err = conn.statusRecorder.UpdatePeerState(peerState) if err != nil { // pretty common error because by that time Engine can already remove the peer and status won't be available. // todo rethink status updates - log.Debugf("error while updating peer's %s state, err: %v", conn.config.Key, err) + conn.log.Debugf("error while updating peer's state, err: %v", err) } if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { - log.Debugf("failed to reset wireguard stats for peer %s: %s", conn.config.Key, err) + conn.log.Debugf("failed to reset wireguard stats for peer: %s", err) } - log.Debugf("cleaned up connection to peer %s", conn.config.Key) - if err1 != nil { - return err1 - } - if err2 != nil { - return err2 - } - return err3 + conn.mu.Unlock() } -// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates -// and then signals them to the remote peer -func (conn *Conn) onICECandidate(candidate ice.Candidate) { - // nil means candidate gathering has been ended - if candidate == nil { - return - } - - // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored - log.Debugf("discovered local candidate %s", candidate.String()) - go func() { - err := conn.signalCandidate(candidate) - if err != nil { - log.Errorf("failed signaling candidate to the remote peer %s %s", conn.config.Key, err) - } - }() - - if !conn.shouldSendExtraSrflxCandidate(candidate) { - return - } - - // sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port) - // this is useful when network has an existing port forwarding rule for the wireguard port and this peer - extraSrflx, err := extraSrflxCandidate(candidate) - if err != nil { - log.Errorf("failed creating extra server reflexive candidate %s", err) - return - } - conn.sentExtraSrflx = true - - go func() { - err = conn.signalCandidate(extraSrflx) - if err != nil { - log.Errorf("failed signaling the extra server reflexive candidate to the remote peer %s: %s", conn.config.Key, err) - } - }() +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { + conn.log.Debugf("OnRemoteAnswer, status %s", conn.status.String()) + return conn.handshaker.OnRemoteAnswer(answer) } -func (conn *Conn) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { - log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), - conn.config.Key) +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + conn.connectorICE.OnRemoteCandidate(candidate, haRoutes) } -// onICEConnectionStateChange registers callback of an ICE Agent to track connection state -func (conn *Conn) onICEConnectionStateChange(state ice.ConnectionState) { - log.Debugf("peer %s ICE ConnectionState has changed to %s", conn.config.Key, state.String()) - if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { - conn.notifyDisconnected() - } +func (conn *Conn) AddBeforeAddPeerHook(hook BeforeAddPeerHookFunc) { + conn.beforeAddPeerHooks = append(conn.beforeAddPeerHooks, hook) } -func (conn *Conn) sendAnswer() error { +func (conn *Conn) AddAfterRemovePeerHook(hook AfterRemovePeerHookFunc) { + conn.afterRemovePeerHooks = append(conn.afterRemovePeerHooks, hook) +} + +// SetOnConnected sets a handler function to be triggered by Conn when a new connection to a remote peer established +func (conn *Conn) SetOnConnected(handler func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string)) { + conn.onConnected = handler +} + +// SetOnDisconnected sets a handler function to be triggered by Conn when a connection to a remote disconnected +func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string)) { + conn.onDisconnected = handler +} + +func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { + conn.log.Debugf("OnRemoteOffer, on status %s", conn.status.String()) + return conn.handshaker.OnRemoteOffer(offer) +} + +// WgConfig returns the WireGuard config +func (conn *Conn) WgConfig() WgConfig { + return conn.config.WgConfig +} + +// Status returns current status of the Conn +func (conn *Conn) Status() ConnStatus { + conn.mu.Lock() + defer conn.mu.Unlock() + return conn.status +} + +func (conn *Conn) GetKey() string { + return conn.config.Key +} + +func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.mu.Lock() defer conn.mu.Unlock() - localUFrag, localPwd, err := conn.agent.GetLocalUserCredentials() - if err != nil { - return err + if conn.ctx.Err() != nil { + return } - log.Debugf("sending answer to %s", conn.config.Key) - err = conn.signalAnswer(OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, - WgListenPort: conn.config.LocalWgPort, - Version: version.NetbirdVersion(), - RosenpassPubKey: conn.config.RosenpassPubKey, - RosenpassAddr: conn.config.RosenpassAddr, - }) - if err != nil { - return err + if conn.currentConnType > connPriorityRelay { + return } - return nil + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + endpoint, err := wgProxy.AddTurnConn(rci.relayedConn) + if err != nil { + conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err) + return + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + + conn.connID = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) + } + } + + err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) + if err != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close relay connection: %v", err) + } + conn.log.Errorf("Failed to update wg peer configuration: %v", err) + return + } + + if conn.wgProxy != nil { + if err := conn.wgProxy.CloseConn(); err != nil { + conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) + } + } + conn.wgProxy = wgProxy + + conn.currentConnType = connPriorityRelay + + peerState := State{ + Direct: false, + Relayed: true, + } + + conn.updateStatus(peerState, rci.rosenpassPubKey, rci.rosenpassAddr) } -// sendOffer prepares local user credentials and signals them to the remote peer -func (conn *Conn) sendOffer() error { +// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected +func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICEConnInfo) { conn.mu.Lock() defer conn.mu.Unlock() - localUFrag, localPwd, err := conn.agent.GetLocalUserCredentials() - if err != nil { - return err - } - oa := OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, - WgListenPort: conn.config.LocalWgPort, - Version: version.NetbirdVersion(), - RosenpassPubKey: conn.config.RosenpassPubKey, - RosenpassAddr: conn.config.RosenpassAddr, + if conn.ctx.Err() != nil { + return } - relayIPAddress, err := conn.relayManager.RelayAddress() - if err == nil { - oa.RelaySrvAddress = relayIPAddress.String() + if conn.currentConnType > priority { + return } - return conn.signalOffer(oa) -} - -func (conn *Conn) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { - if !conn.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { - return true - } - return false -} - -func (conn *Conn) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { - var remoteOfferAnswer OfferAnswer - select { - case remoteOfferAnswer = <-conn.remoteOffersCh: - // received confirmation from the remote peer -> ready to proceed - err := conn.sendAnswer() + var ( + endpoint net.Addr + wgProxy wgproxy.Proxy + ) + if iceConnInfo.RelayedOnLocal { + conn.log.Debugf("setup ice turn connection") + wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) + ep, err := conn.wgProxy.AddTurnConn(iceConnInfo.RemoteConn) if err != nil { - return nil, err + conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) + return } - case remoteOfferAnswer = <-conn.remoteAnswerCh: - case <-time.After(conn.config.Timeout): - return nil, NewConnectionTimeoutError(conn.config.Key, conn.config.Timeout) - case <-conn.closeCh: - // closed externally - return nil, NewConnectionClosedError(conn.config.Key) - } - - return &remoteOfferAnswer, nil -} - -func (conn *Conn) turnAgentDial(remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { - isControlling := conn.config.LocalKey > conn.config.Key - if isControlling { - return conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + endpoint = ep } else { - return conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) - } -} - -func (conn *Conn) setupRelayConnection(remoteOfferAnswer *OfferAnswer) bool { - if !isRelaySupported(remoteOfferAnswer) { - return false + endpoint = iceConnInfo.RemoteConn.RemoteAddr() } - currentRelayAddress, err := conn.relayManager.RelayAddress() - if err != nil { - return false - } + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) - conn.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) - relayConn, err := conn.relayManager.OpenConn(remoteOfferAnswer.RelaySrvAddress, conn.config.Key) - if err != nil { - return false - } - - err = conn.configureWgConnectionForRelay(relayConn, remoteOfferAnswer.RosenpassPubKey, remoteOfferAnswer.RosenpassAddr) - if err != nil { - log.Errorf("failed to configure WireGuard connection for relay: %s", err) - return false - } - return true -} - -func (conn *Conn) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { - if conn.config.LocalKey > conn.config.Key { - return myRelayAddress - } - return remoteRelayAddress -} - -func (conn *Conn) setupICEConnection(remoteOfferAnswer *OfferAnswer, relayOperate bool) error { - var preferredCandidateTypes []ice.CandidateType - if relayOperate { - preferredCandidateTypes = candidateTypesP2P() - } else { - preferredCandidateTypes = candidateTypes() - } - - err := conn.reCreateAgent(preferredCandidateTypes) - if err != nil { - return err - } - - err = conn.agent.GatherCandidates() - if err != nil { - return fmt.Errorf("gather candidates: %v", err) - } - - // will block until connection succeeded - // but it won't release if ICE Agent went into Disconnected or Failed state, - // so we have to cancel it with the provided context once agent detected a broken connection - remoteConn, err := conn.turnAgentDial(remoteOfferAnswer) - if err != nil { - return err - } - - // dynamically set remote WireGuard port if other side specified a different one from the default one - remoteWgPort := iface.DefaultWgPort - if remoteOfferAnswer.WgListenPort != 0 { - remoteWgPort = remoteOfferAnswer.WgListenPort - } - - // the ice connection has been established successfully so we are ready to start the proxy - remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort, remoteOfferAnswer.RosenpassPubKey, - remoteOfferAnswer.RosenpassAddr) - if err != nil { - return err - } - - log.Infof("connected to peer %s, endpoint address: %s", conn.config.Key, remoteAddr.String()) - return nil -} - -func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { - relatedAdd := candidate.RelatedAddress() - return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ - Network: candidate.NetworkType().String(), - Address: candidate.Address(), - Port: relatedAdd.Port, - Component: candidate.Component(), - RelAddr: relatedAdd.Address, - RelPort: relatedAdd.Port, - }) -} - -func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool { - var routePrefixes []netip.Prefix - for _, routes := range clientRoutes { - if len(routes) > 0 && routes[0] != nil { - routePrefixes = append(routePrefixes, routes[0].Network) + conn.connID = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) } } - addr, err := netip.ParseAddr(candidate.Address()) + err := conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) if err != nil { - log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err) - return false + if wgProxy != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close turn connection: %v", err) + } + } + conn.log.Warnf("Failed to update wg peer configuration: %v", err) + return } - for _, prefix := range routePrefixes { - // default route is - if prefix.Bits() == 0 { - continue - } - - if prefix.Contains(addr) { - log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix) - return true + if conn.wgProxy != nil { + if err := conn.wgProxy.CloseConn(); err != nil { + conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) } } - return false + conn.wgProxy = wgProxy + + conn.currentConnType = priority + + peerState := State{ + LocalIceCandidateType: iceConnInfo.LocalIceCandidateType, + RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, + LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, + RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, + Direct: iceConnInfo.Direct, + Relayed: iceConnInfo.Relayed, + } + + conn.updateStatus(peerState, iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) } -func candidateTypes() []ice.CandidateType { - if hasICEForceRelayConn() { - return []ice.CandidateType{ice.CandidateTypeRelay} +func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { + conn.status = StatusConnected + + peerState.PubKey = conn.config.Key + peerState.ConnStatus = StatusConnected + peerState.ConnStatusUpdate = time.Now() + peerState.RosenpassEnabled = isRosenpassEnabled(remoteRosenpassPubKey) + peerState.Mux = new(sync.RWMutex) + + err := conn.statusRecorder.UpdatePeerState(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's state, got error: %v", err) } - // TODO: remove this once we have refactored userspace proxy into the bind package + if runtime.GOOS == "ios" { - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} + runtime.GC() } - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} + + if conn.onConnected != nil { + conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIPsIP, remoteRosenpassAddr) + } + return } -func candidateTypesP2P() []ice.CandidateType { - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} -} +func (conn *Conn) doHandshake() (*OfferAnswer, error) { + if !conn.signaler.Ready() { + return nil, ErrSignalIsNotReady + } -func isRelayCandidate(candidate ice.Candidate) bool { - return candidate.Type() == ice.CandidateTypeRelay -} + uFreg, pwd, err := conn.connectorICE.GetLocalUserCredentials() + if err != nil { + conn.log.Errorf("failed to get local user credentials: %v", err) + } -// todo check my side too -func isRelaySupported(answer *OfferAnswer) bool { - return answer.RelaySrvAddress != "" + addr, err := conn.connectorRelay.RelayAddress() + if err != nil { + conn.log.Errorf("failed to get local relay address: %v", err) + } + return conn.handshaker.Handshake(HandshakeArgs{ + uFreg, + pwd, + addr.String(), + }) } func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { diff --git a/client/internal/peer/conn_ice.go b/client/internal/peer/conn_ice.go new file mode 100644 index 000000000..e3d33a4d7 --- /dev/null +++ b/client/internal/peer/conn_ice.go @@ -0,0 +1,436 @@ +package peer + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net" + "net/netip" + "runtime" + "sync/atomic" + "time" + + "github.com/pion/ice/v3" + "github.com/pion/stun/v2" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/iface" + "github.com/netbirdio/netbird/iface/bind" + "github.com/netbirdio/netbird/route" +) + +const ( + iceKeepAliveDefault = 4 * time.Second + iceDisconnectedTimeoutDefault = 6 * time.Second + // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package + iceRelayAcceptanceMinWaitDefault = 2 * time.Second +) + +type ICEConfig struct { + // StunTurn is a list of STUN and TURN URLs + StunTurn atomic.Value // []*stun.URI + + // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering + // (e.g. if eth0 is in the list, host candidate of this interface won't be used) + InterfaceBlackList []string + DisableIPv6Discovery bool + + UDPMux ice.UDPMux + UDPMuxSrflx ice.UniversalUDPMux + + NATExternalIPs []string +} + +type OnICEConnReadyCallback func(ConnPriority, ICEConnInfo) + +type ICEConnInfo struct { + RemoteConn net.Conn + RosenpassPubKey []byte + RosenpassAddr string + LocalIceCandidateType string + RemoteIceCandidateType string + RemoteIceCandidateEndpoint string + LocalIceCandidateEndpoint string + Direct bool + Relayed bool + RelayedOnLocal bool +} + +type ConnectorICE struct { + ctx context.Context + log *log.Entry + config ConnConfig + configICE ICEConfig + signaler *internal.Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + statusRecorder *Status + onICEConnReady OnICEConnReadyCallback + doHandshakeFn DoHandshake + + connPriority ConnPriority + + agent *ice.Agent + + StunTurn []*stun.URI + + sentExtraSrflx bool +} + +func NewConnectorICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *internal.Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, doHandshakeFn DoHandshake) *ConnectorICE { + cice := &ConnectorICE{ + ctx: ctx, + log: log, + config: config, + configICE: configICE, + signaler: signaler, + iFaceDiscover: ifaceDiscover, + statusRecorder: statusRecorder, + onICEConnReady: onICEConnReady, + doHandshakeFn: doHandshakeFn, + } + return cice +} + +// SetupICEConnection sets up an ICE connection with the remote peer. +// If the relay mode is supported then try to connect in p2p way only. +// It is trying to reconnection in a loop until the context is canceled. +// In case of success connection it will call the onICEConnReady callback. +func (conn *ConnectorICE) SetupICEConnection(relayMode bool) { + var preferredCandidateTypes []ice.CandidateType + if relayMode { + conn.connPriority = connPriorityICEP2P + preferredCandidateTypes = candidateTypesP2P() + } else { + conn.connPriority = connPriorityICETurn + preferredCandidateTypes = candidateTypes() + } + + for { + if !conn.waitForReconnectTry() { + return + } + + remoteOfferAnswer, err := conn.doHandshakeFn() + if err != nil { + if errors.Is(err, ErrSignalIsNotReady) { + conn.log.Infof("signal client isn't ready, skipping connection attempt") + } + continue + } + + ctx, ctxCancel := context.WithCancel(conn.ctx) + agent, err := conn.reCreateAgent(ctxCancel, preferredCandidateTypes) + if err != nil { + ctxCancel() + continue + } + conn.agent = agent + + err = conn.agent.GatherCandidates() + if err != nil { + ctxCancel() + continue + } + + // will block until connection succeeded + // but it won't release if ICE Agent went into Disconnected or Failed state, + // so we have to cancel it with the provided context once agent detected a broken connection + remoteConn, err := conn.turnAgentDial(remoteOfferAnswer) + if err != nil { + ctxCancel() + continue + } + + pair, err := conn.agent.GetSelectedCandidatePair() + if err != nil { + ctxCancel() + continue + } + + if !isRelayCandidate(pair.Local) { + // dynamically set remote WireGuard port if other side specified a different one from the default one + remoteWgPort := iface.DefaultWgPort + if remoteOfferAnswer.WgListenPort != 0 { + remoteWgPort = remoteOfferAnswer.WgListenPort + } + + // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port + go conn.punchRemoteWGPort(pair, remoteWgPort) + } + + ci := ICEConnInfo{ + RemoteConn: remoteConn, + RosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + RosenpassAddr: remoteOfferAnswer.RosenpassAddr, + LocalIceCandidateType: pair.Local.Type().String(), + RemoteIceCandidateType: pair.Remote.Type().String(), + LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), + RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), + Direct: !isRelayCandidate(pair.Local), + Relayed: isRelayed(pair), + RelayedOnLocal: isRelayCandidate(pair.Local), + } + go conn.onICEConnReady(conn.connPriority, ci) + + <-ctx.Done() + ctxCancel() + _ = conn.agent.Close() + } +} + +// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. +func (conn *ConnectorICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + conn.log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) + if conn.agent == nil { + return + } + + if candidateViaRoutes(candidate, haRoutes) { + return + } + + err := conn.agent.AddRemoteCandidate(candidate) + if err != nil { + conn.log.Errorf("error while handling remote candidate") + return + } +} + +func (conn *ConnectorICE) GetLocalUserCredentials() (frag string, pwd string, err error) { + if conn.agent == nil { + return "", "", errors.New("ICE Agent is not initialized") + } + return conn.agent.GetLocalUserCredentials() +} + +func (conn *ConnectorICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { + failedTimeout := 6 * time.Second + transportNet, err := conn.newStdNet() + if err != nil { + conn.log.Errorf("failed to create pion's stdnet: %s", err) + } + + iceKeepAlive := iceKeepAlive() + iceDisconnectedTimeout := iceDisconnectedTimeout() + iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() + + agentConfig := &ice.AgentConfig{ + MulticastDNSMode: ice.MulticastDNSModeDisabled, + NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, + Urls: conn.configICE.StunTurn.Load().([]*stun.URI), + CandidateTypes: relaySupport, + FailedTimeout: &failedTimeout, + InterfaceFilter: stdnet.InterfaceFilter(conn.configICE.InterfaceBlackList), + UDPMux: conn.configICE.UDPMux, + UDPMuxSrflx: conn.configICE.UDPMuxSrflx, + NAT1To1IPs: conn.configICE.NATExternalIPs, + Net: transportNet, + DisconnectedTimeout: &iceDisconnectedTimeout, + KeepaliveInterval: &iceKeepAlive, + RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, + } + + if conn.configICE.DisableIPv6Discovery { + agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} + } + + conn.sentExtraSrflx = false + agent, err := ice.NewAgent(agentConfig) + if err != nil { + return nil, err + } + + err = agent.OnCandidate(conn.onICECandidate) + if err != nil { + return nil, err + } + + err = agent.OnConnectionStateChange(func(state ice.ConnectionState) { + conn.log.Debugf("ICE ConnectionState has changed to %s", state.String()) + if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { + ctxCancel() + } + }) + if err != nil { + return nil, err + } + + err = agent.OnSelectedCandidatePairChange(conn.onICESelectedCandidatePair) + if err != nil { + return nil, err + } + + err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { + err := conn.statusRecorder.UpdateLatency(conn.config.Key, p.Latency()) + if err != nil { + conn.log.Debugf("failed to update latency for peer: %s", err) + return + } + }) + if err != nil { + return nil, fmt.Errorf("failed setting binding response callback: %w", err) + } + + return agent, nil +} + +func (conn *ConnectorICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { + // wait local endpoint configuration + time.Sleep(time.Second) + addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) + if err != nil { + conn.log.Warnf("got an error while resolving the udp address, err: %s", err) + return + } + + mux, ok := conn.configICE.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) + if !ok { + conn.log.Warn("invalid udp mux conversion") + return + } + _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) + if err != nil { + conn.log.Warnf("got an error while sending the punch packet, err: %s", err) + } +} + +// onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates +// and then signals them to the remote peer +func (conn *ConnectorICE) onICECandidate(candidate ice.Candidate) { + // nil means candidate gathering has been ended + if candidate == nil { + return + } + + // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored + conn.log.Debugf("discovered local candidate %s", candidate.String()) + go func() { + err := conn.signaler.SignalICECandidate(candidate, conn.config.Key) + if err != nil { + conn.log.Errorf("failed signaling candidate to the remote peer %s %s", conn.config.Key, err) + } + }() + + if !conn.shouldSendExtraSrflxCandidate(candidate) { + return + } + + // sends an extra server reflexive candidate to the remote peer with our related port (usually the wireguard port) + // this is useful when network has an existing port forwarding rule for the wireguard port and this peer + extraSrflx, err := extraSrflxCandidate(candidate) + if err != nil { + conn.log.Errorf("failed creating extra server reflexive candidate %s", err) + return + } + conn.sentExtraSrflx = true + + go func() { + err = conn.signaler.SignalICECandidate(extraSrflx, conn.config.Key) + if err != nil { + conn.log.Errorf("failed signaling the extra server reflexive candidate: %s", err) + } + }() +} + +func (conn *ConnectorICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { + conn.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), + conn.config.Key) +} + +func (conn *ConnectorICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { + if !conn.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { + return true + } + return false +} + +func (conn *ConnectorICE) turnAgentDial(remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { + isControlling := conn.config.LocalKey > conn.config.Key + if isControlling { + return conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } else { + return conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + } +} + +// waitForReconnectTry waits for a random duration before trying to reconnect +func (conn *ConnectorICE) waitForReconnectTry() bool { + minWait := 500 + maxWait := 2000 + duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + select { + case <-conn.ctx.Done(): + return false + case <-time.After(duration): + return true + } +} + +func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { + relatedAdd := candidate.RelatedAddress() + return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ + Network: candidate.NetworkType().String(), + Address: candidate.Address(), + Port: relatedAdd.Port, + Component: candidate.Component(), + RelAddr: relatedAdd.Address, + RelPort: relatedAdd.Port, + }) +} + +func candidateViaRoutes(candidate ice.Candidate, clientRoutes route.HAMap) bool { + var routePrefixes []netip.Prefix + for _, routes := range clientRoutes { + if len(routes) > 0 && routes[0] != nil { + routePrefixes = append(routePrefixes, routes[0].Network) + } + } + + addr, err := netip.ParseAddr(candidate.Address()) + if err != nil { + log.Errorf("Failed to parse IP address %s: %v", candidate.Address(), err) + return false + } + + for _, prefix := range routePrefixes { + // default route is + if prefix.Bits() == 0 { + continue + } + + if prefix.Contains(addr) { + log.Debugf("Ignoring candidate [%s], its address is part of routed network %s", candidate.String(), prefix) + return true + } + } + return false +} + +func candidateTypes() []ice.CandidateType { + if hasICEForceRelayConn() { + return []ice.CandidateType{ice.CandidateTypeRelay} + } + // TODO: remove this once we have refactored userspace proxy into the bind package + if runtime.GOOS == "ios" { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} + } + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive, ice.CandidateTypeRelay} +} + +func candidateTypesP2P() []ice.CandidateType { + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} +} + +func isRelayCandidate(candidate ice.Candidate) bool { + return candidate.Type() == ice.CandidateTypeRelay +} + +func isRelayed(pair *ice.CandidatePair) bool { + if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { + return true + } + return false +} diff --git a/client/internal/peer/conn_relay.go b/client/internal/peer/conn_relay.go new file mode 100644 index 000000000..23bc47263 --- /dev/null +++ b/client/internal/peer/conn_relay.go @@ -0,0 +1,115 @@ +package peer + +import ( + "context" + "errors" + "math/rand" + "net" + "time" + + log "github.com/sirupsen/logrus" + + relayClient "github.com/netbirdio/netbird/relay/client" +) + +type OnRelayReadyCallback func(info RelayConnInfo) + +type RelayConnInfo struct { + relayedConn net.Conn + rosenpassPubKey []byte + rosenpassAddr string +} + +type ConnectorRelay struct { + ctx context.Context + log *log.Entry + relayManager *relayClient.Manager + config ConnConfig + onRelayConnReadyFN OnRelayReadyCallback + doHandshakeFn DoHandshake +} + +func NewConnectorRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, onRelayConnReadyFN OnRelayReadyCallback, doHandshakeFn DoHandshake) *ConnectorRelay { + return &ConnectorRelay{ + ctx: ctx, + log: log, + relayManager: relayManager, + config: config, + onRelayConnReadyFN: onRelayConnReadyFN, + doHandshakeFn: doHandshakeFn, + } +} + +// SetupRelayConnection todo: this function is not completed. Make no sense to put it in a for loop because we are not waiting for any event +func (conn *ConnectorRelay) SetupRelayConnection() { + for { + if !conn.waitForReconnectTry() { + return + } + + remoteOfferAnswer, err := conn.doHandshakeFn() + if err != nil { + if errors.Is(err, ErrSignalIsNotReady) { + conn.log.Infof("signal client isn't ready, skipping connection attempt") + } + conn.log.Errorf("failed to do handshake: %v", err) + continue + } + + if !conn.isRelaySupported(remoteOfferAnswer) { + // todo should we retry? + continue + } + + // the relayManager will return with error in case if the connection has lost with relay server + currentRelayAddress, err := conn.relayManager.RelayAddress() + if err != nil { + continue + } + + srv := conn.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) + relayedConn, err := conn.relayManager.OpenConn(srv, conn.config.Key) + if err != nil { + continue + } + + go conn.onRelayConnReadyFN(RelayConnInfo{ + relayedConn: relayedConn, + rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + rosenpassAddr: remoteOfferAnswer.RosenpassAddr, + }) + } +} + +func (conn *ConnectorRelay) RelayAddress() (net.Addr, error) { + return conn.relayManager.RelayAddress() +} + +// todo check my side too +func (conn *ConnectorRelay) isRelaySupported(answer *OfferAnswer) bool { + return answer.RelaySrvAddress != "" +} + +func (conn *ConnectorRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { + if conn.config.LocalKey > conn.config.Key { + return myRelayAddress + } + return remoteRelayAddress +} + +func (conn *ConnectorRelay) RelayIsSupported() bool { + return conn.relayManager.IsSupported() +} + +// waitForReconnectTry waits for a random duration before trying to reconnect +func (conn *ConnectorRelay) waitForReconnectTry() bool { + minWait := 500 + maxWait := 2000 + duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + select { + case <-conn.ctx.Done(): + return false + case <-time.After(duration): + return true + } +} diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index c16134808..10f1ac5ef 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -40,7 +40,7 @@ func TestConn_GetKey(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil) + conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -55,7 +55,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -92,7 +92,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -128,7 +128,7 @@ func TestConn_Status(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -158,7 +158,7 @@ func TestConn_Close(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go new file mode 100644 index 000000000..6562c12e7 --- /dev/null +++ b/client/internal/peer/handshaker.go @@ -0,0 +1,210 @@ +package peer + +import ( + "context" + "errors" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/conn" + + "github.com/netbirdio/netbird/client/internal" + "github.com/netbirdio/netbird/version" +) + +const ( + handshakeCacheTimeout = 3 * time.Second +) + +var ( + ErrSignalIsNotReady = errors.New("signal is not ready") +) + +type DoHandshake func() (*OfferAnswer, error) + +// IceCredentials ICE protocol credentials struct +type IceCredentials struct { + UFrag string + Pwd string +} + +// OfferAnswer represents a session establishment offer or answer +type OfferAnswer struct { + IceCredentials IceCredentials + // WgListenPort is a remote WireGuard listen port. + // This field is used when establishing a direct WireGuard connection without any proxy. + // We can set the remote peer's endpoint with this port. + WgListenPort int + + // Version of NetBird Agent + Version string + // RosenpassPubKey is the Rosenpass public key of the remote peer when receiving this message + // This value is the local Rosenpass server public key when sending the message + RosenpassPubKey []byte + // RosenpassAddr is the Rosenpass server address (IP:port) of the remote peer when receiving this message + // This value is the local Rosenpass server address when sending the message + RosenpassAddr string + + // relay server address + RelaySrvAddress string +} + +type HandshakeArgs struct { + IceUFrag string + IcePwd string + RelayAddr string +} + +type Handshaker struct { + mu sync.Mutex + ctx context.Context + config ConnConfig + signaler *internal.Signaler + + // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection + remoteOffersCh chan OfferAnswer + // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection + remoteAnswerCh chan OfferAnswer + + remoteOfferAnswer *OfferAnswer + remoteOfferAnswerCreated time.Time +} + +func NewHandshaker(ctx context.Context, config ConnConfig, signaler *internal.Signaler) *Handshaker { + return &Handshaker{ + ctx: ctx, + config: config, + signaler: signaler, + remoteOffersCh: make(chan OfferAnswer), + remoteAnswerCh: make(chan OfferAnswer), + } +} + +func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { + h.mu.Lock() + defer h.mu.Unlock() + + cachedOfferAnswer, ok := h.cachedHandshake() + if ok { + return cachedOfferAnswer, nil + } + + err := h.sendOffer(args) + if err != nil { + return nil, err + } + + // Only continue once we got a connection confirmation from the remote peer. + // The connection timeout could have happened before a confirmation received from the remote. + // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) + remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() + if err != nil { + return nil, err + } + h.storeRemoteOfferAnswer(remoteOfferAnswer) + + log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", + h.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) + + return remoteOfferAnswer, nil +} + +// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { + + select { + case h.remoteOffersCh <- offer: + return true + default: + log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", h.config.Key, conn.status.String()) + // connection might not be ready yet to receive so we ignore the message + return false + } +} + +// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise +// doesn't block, discards the message if connection wasn't ready +func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { + select { + case h.remoteAnswerCh <- answer: + return true + default: + // connection might not be ready yet to receive so we ignore the message + log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", h.config.Key, conn.status.String()) + return false + } +} + +// sendOffer prepares local user credentials and signals them to the remote peer +func (h *Handshaker) sendOffer(args HandshakeArgs) error { + offer := OfferAnswer{ + IceCredentials: IceCredentials{args.IceUFrag, args.IcePwd}, + WgListenPort: h.config.LocalWgPort, + Version: version.NetbirdVersion(), + RosenpassPubKey: h.config.RosenpassPubKey, + RosenpassAddr: h.config.RosenpassAddr, + RelaySrvAddress: args.RelayAddr, + } + + return h.signaler.SignalOffer(offer, h.config.Key) +} + +func (h *Handshaker) sendAnswer() error { + localUFrag, localPwd, err := conn.connectorICE.GetLocalUserCredentials() + if err != nil { + return err + } + + log.Debugf("sending answer to %s", h.config.Key) + answer := OfferAnswer{ + IceCredentials: IceCredentials{localUFrag, localPwd}, + WgListenPort: h.config.LocalWgPort, + Version: version.NetbirdVersion(), + RosenpassPubKey: h.config.RosenpassPubKey, + RosenpassAddr: h.config.RosenpassAddr, + } + err = h.signaler.SignalAnswer(answer, h.config.Key) + if err != nil { + return err + } + + return nil +} + +func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { + select { + case remoteOfferAnswer := <-h.remoteOffersCh: + // received confirmation from the remote peer -> ready to proceed + err := h.sendAnswer() + if err != nil { + return nil, err + } + return &remoteOfferAnswer, nil + case remoteOfferAnswer := <-h.remoteAnswerCh: + return &remoteOfferAnswer, nil + case <-time.After(h.config.Timeout): + return nil, NewConnectionTimeoutError(h.config.Key, h.config.Timeout) + case <-h.ctx.Done(): + // closed externally + return nil, NewConnectionClosedError(h.config.Key) + } +} + +func (h *Handshaker) storeRemoteOfferAnswer(answer *OfferAnswer) { + h.remoteOfferAnswer = answer + h.remoteOfferAnswerCreated = time.Now() +} + +func (h *Handshaker) cachedHandshake() (*OfferAnswer, bool) { + if h.remoteOfferAnswer == nil { + return nil, false + } + + if time.Since(h.remoteOfferAnswerCreated) > handshakeCacheTimeout { + return nil, false + } + + return h.remoteOfferAnswer, true +} diff --git a/client/internal/peer/stdnet.go b/client/internal/peer/stdnet.go index 13f5886f5..06b484010 100644 --- a/client/internal/peer/stdnet.go +++ b/client/internal/peer/stdnet.go @@ -6,6 +6,6 @@ import ( "github.com/netbirdio/netbird/client/internal/stdnet" ) -func (conn *Conn) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNet(conn.config.InterfaceBlackList) +func (conn *ConnectorICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNet(conn.configICE.InterfaceBlackList) } diff --git a/client/internal/peer/stdnet_android.go b/client/internal/peer/stdnet_android.go index 8a2454371..dba94375f 100644 --- a/client/internal/peer/stdnet_android.go +++ b/client/internal/peer/stdnet_android.go @@ -2,6 +2,6 @@ package peer import "github.com/netbirdio/netbird/client/internal/stdnet" -func (conn *Conn) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.config.InterfaceBlackList) +func (conn *ConnectorICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.configICE.InterfaceBlackList) } diff --git a/client/internal/signaler.go b/client/internal/signaler.go new file mode 100644 index 000000000..cd466784d --- /dev/null +++ b/client/internal/signaler.go @@ -0,0 +1,71 @@ +package internal + +import ( + "github.com/pion/ice/v3" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/client/internal/peer" + signal "github.com/netbirdio/netbird/signal/client" + sProto "github.com/netbirdio/netbird/signal/proto" +) + +type Signaler struct { + signal signal.Client + wgPrivateKey wgtypes.Key +} + +func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler { + return &Signaler{ + signal: signal, + wgPrivateKey: wgPrivateKey, + } +} + +func (s *Signaler) SignalOffer(offer peer.OfferAnswer, remoteKey string) error { + return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER) +} + +func (s *Signaler) SignalAnswer(offer peer.OfferAnswer, remoteKey string) error { + return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER) +} + +func (s *Signaler) SignalICECandidate(candidate ice.Candidate, remoteKey string) error { + return s.signal.Send(&sProto.Message{ + Key: s.wgPrivateKey.PublicKey().String(), + RemoteKey: remoteKey, + Body: &sProto.Body{ + Type: sProto.Body_CANDIDATE, + Payload: candidate.Marshal(), + }, + }) +} + +func (s *Signaler) Ready() bool { + return s.signal.Ready() +} + +// SignalOfferAnswer signals either an offer or an answer to remote peer +func (s *Signaler) signalOfferAnswer(offerAnswer peer.OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error { + msg, err := signal.MarshalCredential( + s.wgPrivateKey, + offerAnswer.WgListenPort, + remoteKey, + &signal.Credential{ + UFrag: offerAnswer.IceCredentials.UFrag, + Pwd: offerAnswer.IceCredentials.Pwd, + }, + bodyType, + offerAnswer.RosenpassPubKey, + offerAnswer.RosenpassAddr, + offerAnswer.RelaySrvAddress) + if err != nil { + return err + } + + err = s.signal.Send(msg) + if err != nil { + return err + } + + return nil +} diff --git a/relay/client/manager.go b/relay/client/manager.go index 6b816c4fb..97a791936 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -108,6 +108,10 @@ func (m *Manager) RelayAddress() (net.Addr, error) { return m.relayClient.RelayRemoteAddress() } +func (m *Manager) IsSupported() bool { + return m.srvAddress != "" +} + func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { // check if already has a connection to the desired relay server m.relayClientsMutex.RLock() diff --git a/signal/client/client.go b/signal/client/client.go index 7c54178e2..ced3fb7d0 100644 --- a/signal/client/client.go +++ b/signal/client/client.go @@ -51,10 +51,10 @@ func UnMarshalCredential(msg *proto.Message) (*Credential, error) { } // MarshalCredential marshal a Credential instance and returns a Message object -func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey wgtypes.Key, credential *Credential, t proto.Body_Type, rosenpassPubKey []byte, rosenpassAddr string, relaySrvAddress string) (*proto.Message, error) { +func MarshalCredential(myKey wgtypes.Key, myPort int, remoteKey string, credential *Credential, t proto.Body_Type, rosenpassPubKey []byte, rosenpassAddr string, relaySrvAddress string) (*proto.Message, error) { return &proto.Message{ Key: myKey.PublicKey().String(), - RemoteKey: remoteKey.String(), + RemoteKey: remoteKey, Body: &proto.Body{ Type: t, Payload: fmt.Sprintf("%s:%s", credential.UFrag, credential.Pwd), From deb8203f062182cdc515173eca0619ee149a483c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 17 Jun 2024 18:02:52 +0200 Subject: [PATCH 036/199] fix circle import --- client/internal/engine.go | 2 +- client/internal/peer/conn.go | 5 ++--- client/internal/peer/conn_ice.go | 5 ++--- client/internal/peer/handshaker.go | 5 ++--- client/internal/{ => peer}/signaler.go | 9 ++++----- 5 files changed, 11 insertions(+), 15 deletions(-) rename client/internal/{ => peer}/signaler.go (80%) diff --git a/client/internal/engine.go b/client/internal/engine.go index d80d1f235..ee0974a45 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -97,7 +97,7 @@ type EngineConfig struct { type Engine struct { // signal is a Signal Service client signal signal.Client - signaler *Signaler + signaler *peer.Signaler // mgmClient is a Management Service client mgmClient mgm.Client // peerConns is a map that holds all the peers that are known to this peer diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 16daf492e..609e4d739 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -12,7 +12,6 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" @@ -73,7 +72,7 @@ type Conn struct { statusRecorder *Status wgProxyFactory *wgproxy.Factory wgProxy wgproxy.Proxy - signaler *internal.Signaler + signaler *Signaler allowedIPsIP string handshaker *Handshaker closeCh chan struct{} @@ -95,7 +94,7 @@ type Conn struct { // NewConn creates a new not opened Conn to the remote peer. // To establish a connection run Conn.Open -func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *internal.Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { +func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, relayManager *relayClient.Manager) (*Conn, error) { _, allowedIPsIP, err := net.ParseCIDR(config.WgConfig.AllowedIps) if err != nil { log.Errorf("failed to parse allowedIPS: %v", err) diff --git a/client/internal/peer/conn_ice.go b/client/internal/peer/conn_ice.go index e3d33a4d7..baf7be4f8 100644 --- a/client/internal/peer/conn_ice.go +++ b/client/internal/peer/conn_ice.go @@ -15,7 +15,6 @@ import ( "github.com/pion/stun/v2" log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/iface" "github.com/netbirdio/netbird/iface/bind" @@ -64,7 +63,7 @@ type ConnectorICE struct { log *log.Entry config ConnConfig configICE ICEConfig - signaler *internal.Signaler + signaler *Signaler iFaceDiscover stdnet.ExternalIFaceDiscover statusRecorder *Status onICEConnReady OnICEConnReadyCallback @@ -79,7 +78,7 @@ type ConnectorICE struct { sentExtraSrflx bool } -func NewConnectorICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *internal.Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, doHandshakeFn DoHandshake) *ConnectorICE { +func NewConnectorICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, doHandshakeFn DoHandshake) *ConnectorICE { cice := &ConnectorICE{ ctx: ctx, log: log, diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 6562c12e7..f2f2cff59 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -9,7 +9,6 @@ import ( log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/conn" - "github.com/netbirdio/netbird/client/internal" "github.com/netbirdio/netbird/version" ) @@ -60,7 +59,7 @@ type Handshaker struct { mu sync.Mutex ctx context.Context config ConnConfig - signaler *internal.Signaler + signaler *Signaler // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection remoteOffersCh chan OfferAnswer @@ -71,7 +70,7 @@ type Handshaker struct { remoteOfferAnswerCreated time.Time } -func NewHandshaker(ctx context.Context, config ConnConfig, signaler *internal.Signaler) *Handshaker { +func NewHandshaker(ctx context.Context, config ConnConfig, signaler *Signaler) *Handshaker { return &Handshaker{ ctx: ctx, config: config, diff --git a/client/internal/signaler.go b/client/internal/peer/signaler.go similarity index 80% rename from client/internal/signaler.go rename to client/internal/peer/signaler.go index cd466784d..713123e5d 100644 --- a/client/internal/signaler.go +++ b/client/internal/peer/signaler.go @@ -1,10 +1,9 @@ -package internal +package peer import ( "github.com/pion/ice/v3" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - "github.com/netbirdio/netbird/client/internal/peer" signal "github.com/netbirdio/netbird/signal/client" sProto "github.com/netbirdio/netbird/signal/proto" ) @@ -21,11 +20,11 @@ func NewSignaler(signal signal.Client, wgPrivateKey wgtypes.Key) *Signaler { } } -func (s *Signaler) SignalOffer(offer peer.OfferAnswer, remoteKey string) error { +func (s *Signaler) SignalOffer(offer OfferAnswer, remoteKey string) error { return s.signalOfferAnswer(offer, remoteKey, sProto.Body_OFFER) } -func (s *Signaler) SignalAnswer(offer peer.OfferAnswer, remoteKey string) error { +func (s *Signaler) SignalAnswer(offer OfferAnswer, remoteKey string) error { return s.signalOfferAnswer(offer, remoteKey, sProto.Body_ANSWER) } @@ -45,7 +44,7 @@ func (s *Signaler) Ready() bool { } // SignalOfferAnswer signals either an offer or an answer to remote peer -func (s *Signaler) signalOfferAnswer(offerAnswer peer.OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error { +func (s *Signaler) signalOfferAnswer(offerAnswer OfferAnswer, remoteKey string, bodyType sProto.Body_Type) error { msg, err := signal.MarshalCredential( s.wgPrivateKey, offerAnswer.WgListenPort, From 5b86a7f3f2e1a563913dcb9fe4695d96c3c35f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 18 Jun 2024 11:10:17 +0200 Subject: [PATCH 037/199] Fix relay mode evaulation --- client/internal/engine.go | 2 +- client/internal/peer/conn.go | 2 +- client/internal/peer/conn_ice.go | 20 ++++++++++---------- client/internal/peer/conn_relay.go | 6 ++++-- relay/client/manager.go | 2 +- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index ee0974a45..c4adb4be4 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -212,7 +212,7 @@ func NewEngineWithProbes( clientCtx: clientCtx, clientCancel: clientCancel, signal: signalClient, - signaler: NewSignaler(signalClient, config.WgPrivateKey), + signaler: peer.NewSignaler(signalClient, config.WgPrivateKey), mgmClient: mgmClient, relayManager: relayManager, peerConns: make(map[string]*peer.Conn), diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 609e4d739..64d06fe20 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -152,7 +152,7 @@ func (conn *Conn) Open() { log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) } */ - relayIsSupportedLocally := conn.connectorRelay.RelayIsSupported() + relayIsSupportedLocally := conn.connectorRelay.RelayIsSupportedLocally() if relayIsSupportedLocally { go conn.connectorRelay.SetupRelayConnection() } diff --git a/client/internal/peer/conn_ice.go b/client/internal/peer/conn_ice.go index baf7be4f8..35b306068 100644 --- a/client/internal/peer/conn_ice.go +++ b/client/internal/peer/conn_ice.go @@ -97,16 +97,7 @@ func NewConnectorICE(ctx context.Context, log *log.Entry, config ConnConfig, con // If the relay mode is supported then try to connect in p2p way only. // It is trying to reconnection in a loop until the context is canceled. // In case of success connection it will call the onICEConnReady callback. -func (conn *ConnectorICE) SetupICEConnection(relayMode bool) { - var preferredCandidateTypes []ice.CandidateType - if relayMode { - conn.connPriority = connPriorityICEP2P - preferredCandidateTypes = candidateTypesP2P() - } else { - conn.connPriority = connPriorityICETurn - preferredCandidateTypes = candidateTypes() - } - +func (conn *ConnectorICE) SetupICEConnection(hasRelayOnLocally bool) { for { if !conn.waitForReconnectTry() { return @@ -120,6 +111,15 @@ func (conn *ConnectorICE) SetupICEConnection(relayMode bool) { continue } + var preferredCandidateTypes []ice.CandidateType + if hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { + conn.connPriority = connPriorityICEP2P + preferredCandidateTypes = candidateTypesP2P() + } else { + conn.connPriority = connPriorityICETurn + preferredCandidateTypes = candidateTypes() + } + ctx, ctxCancel := context.WithCancel(conn.ctx) agent, err := conn.reCreateAgent(ctxCancel, preferredCandidateTypes) if err != nil { diff --git a/client/internal/peer/conn_relay.go b/client/internal/peer/conn_relay.go index 23bc47263..bbd1af219 100644 --- a/client/internal/peer/conn_relay.go +++ b/client/internal/peer/conn_relay.go @@ -78,6 +78,8 @@ func (conn *ConnectorRelay) SetupRelayConnection() { rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, rosenpassAddr: remoteOfferAnswer.RosenpassAddr, }) + + // todo: waitForDisconnection() } } @@ -97,8 +99,8 @@ func (conn *ConnectorRelay) preferredRelayServer(myRelayAddress, remoteRelayAddr return remoteRelayAddress } -func (conn *ConnectorRelay) RelayIsSupported() bool { - return conn.relayManager.IsSupported() +func (conn *ConnectorRelay) RelayIsSupportedLocally() bool { + return conn.relayManager.HasRelayAddress() } // waitForReconnectTry waits for a random duration before trying to reconnect diff --git a/relay/client/manager.go b/relay/client/manager.go index 97a791936..a57e3ce55 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -108,7 +108,7 @@ func (m *Manager) RelayAddress() (net.Addr, error) { return m.relayClient.RelayRemoteAddress() } -func (m *Manager) IsSupported() bool { +func (m *Manager) HasRelayAddress() bool { return m.srvAddress != "" } From f7d8d03e55f91e4adae7509462f6b4e40981e142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 18 Jun 2024 11:20:01 +0200 Subject: [PATCH 038/199] Fix timers --- client/internal/peer/conn_ice.go | 6 +++++- client/internal/peer/conn_relay.go | 6 +++++- client/internal/peer/handshaker.go | 6 ++++-- relay/client/guard.go | 5 ++++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/client/internal/peer/conn_ice.go b/client/internal/peer/conn_ice.go index 35b306068..2dacd0c29 100644 --- a/client/internal/peer/conn_ice.go +++ b/client/internal/peer/conn_ice.go @@ -360,10 +360,14 @@ func (conn *ConnectorICE) waitForReconnectTry() bool { minWait := 500 maxWait := 2000 duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + + timeout := time.NewTimer(duration) + defer timeout.Stop() + select { case <-conn.ctx.Done(): return false - case <-time.After(duration): + case <-timeout.C: return true } } diff --git a/client/internal/peer/conn_relay.go b/client/internal/peer/conn_relay.go index bbd1af219..ae42a4597 100644 --- a/client/internal/peer/conn_relay.go +++ b/client/internal/peer/conn_relay.go @@ -108,10 +108,14 @@ func (conn *ConnectorRelay) waitForReconnectTry() bool { minWait := 500 maxWait := 2000 duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + + timeout := time.NewTimer(duration) + defer timeout.Stop() + select { case <-conn.ctx.Done(): return false - case <-time.After(duration): + case <-timeout.C: return true } } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index f2f2cff59..e10705f63 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -112,7 +112,6 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise // doesn't block, discards the message if connection wasn't ready func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { - select { case h.remoteOffersCh <- offer: return true @@ -173,6 +172,9 @@ func (h *Handshaker) sendAnswer() error { } func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { + timeout := time.NewTimer(h.config.Timeout) + defer timeout.Stop() + select { case remoteOfferAnswer := <-h.remoteOffersCh: // received confirmation from the remote peer -> ready to proceed @@ -183,7 +185,7 @@ func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { return &remoteOfferAnswer, nil case remoteOfferAnswer := <-h.remoteAnswerCh: return &remoteOfferAnswer, nil - case <-time.After(h.config.Timeout): + case <-timeout.C: return nil, NewConnectionTimeoutError(h.config.Key, h.config.Timeout) case <-h.ctx.Done(): // closed externally diff --git a/relay/client/guard.go b/relay/client/guard.go index 47a6ff722..12fae4c54 100644 --- a/relay/client/guard.go +++ b/relay/client/guard.go @@ -24,8 +24,11 @@ func NewGuard(context context.Context, relayClient *Client) *Guard { } func (g *Guard) OnDisconnected() { + timeout := time.NewTimer(reconnectingTimeout) + defer timeout.Stop() + select { - case <-time.After(time.Second): + case <-timeout.C: _ = g.relayClient.Connect() case <-g.ctx.Done(): return From 63b4041e9c70afd0086469657a8e07cb28d8a41f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 18 Jun 2024 11:22:40 +0200 Subject: [PATCH 039/199] Rename connector to worker --- client/internal/peer/conn.go | 20 +-- client/internal/peer/stdnet.go | 4 +- client/internal/peer/stdnet_android.go | 4 +- .../peer/{conn_ice.go => worker_ice.go} | 132 +++++++++--------- .../peer/{conn_relay.go => worker_relay.go} | 44 +++--- 5 files changed, 102 insertions(+), 102 deletions(-) rename client/internal/peer/{conn_ice.go => worker_ice.go} (68%) rename client/internal/peer/{conn_relay.go => worker_relay.go} (57%) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 64d06fe20..f729bc61a 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -82,8 +82,8 @@ type Conn struct { status ConnStatus - connectorICE *ConnectorICE - connectorRelay *ConnectorRelay + workerICE *WorkerICE + workerRelay *WorkerRelay connID nbnet.ConnectionID beforeAddPeerHooks []BeforeAddPeerHookFunc @@ -116,8 +116,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu status: StatusDisconnected, closeCh: make(chan struct{}), } - conn.connectorICE = NewConnectorICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.doHandshake) - conn.connectorRelay = NewConnectorRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.doHandshake) + conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.doHandshake) + conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.doHandshake) return conn, nil } @@ -152,11 +152,11 @@ func (conn *Conn) Open() { log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) } */ - relayIsSupportedLocally := conn.connectorRelay.RelayIsSupportedLocally() + relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() if relayIsSupportedLocally { - go conn.connectorRelay.SetupRelayConnection() + go conn.workerRelay.SetupRelayConnection() } - go conn.connectorICE.SetupICEConnection(relayIsSupportedLocally) + go conn.workerICE.SetupICEConnection(relayIsSupportedLocally) } // Close closes this peer Conn issuing a close event to the Conn closeCh @@ -221,7 +221,7 @@ func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. func (conn *Conn) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - conn.connectorICE.OnRemoteCandidate(candidate, haRoutes) + conn.workerICE.OnRemoteCandidate(candidate, haRoutes) } func (conn *Conn) AddBeforeAddPeerHook(hook BeforeAddPeerHookFunc) { @@ -419,12 +419,12 @@ func (conn *Conn) doHandshake() (*OfferAnswer, error) { return nil, ErrSignalIsNotReady } - uFreg, pwd, err := conn.connectorICE.GetLocalUserCredentials() + uFreg, pwd, err := conn.workerICE.GetLocalUserCredentials() if err != nil { conn.log.Errorf("failed to get local user credentials: %v", err) } - addr, err := conn.connectorRelay.RelayAddress() + addr, err := conn.workerRelay.RelayAddress() if err != nil { conn.log.Errorf("failed to get local relay address: %v", err) } diff --git a/client/internal/peer/stdnet.go b/client/internal/peer/stdnet.go index 06b484010..7882ff5da 100644 --- a/client/internal/peer/stdnet.go +++ b/client/internal/peer/stdnet.go @@ -6,6 +6,6 @@ import ( "github.com/netbirdio/netbird/client/internal/stdnet" ) -func (conn *ConnectorICE) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNet(conn.configICE.InterfaceBlackList) +func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNet(w.configICE.InterfaceBlackList) } diff --git a/client/internal/peer/stdnet_android.go b/client/internal/peer/stdnet_android.go index dba94375f..27f49be9f 100644 --- a/client/internal/peer/stdnet_android.go +++ b/client/internal/peer/stdnet_android.go @@ -2,6 +2,6 @@ package peer import "github.com/netbirdio/netbird/client/internal/stdnet" -func (conn *ConnectorICE) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNetWithDiscover(conn.iFaceDiscover, conn.configICE.InterfaceBlackList) +func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { + return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.configICE.InterfaceBlackList) } diff --git a/client/internal/peer/conn_ice.go b/client/internal/peer/worker_ice.go similarity index 68% rename from client/internal/peer/conn_ice.go rename to client/internal/peer/worker_ice.go index 2dacd0c29..70668b493 100644 --- a/client/internal/peer/conn_ice.go +++ b/client/internal/peer/worker_ice.go @@ -58,7 +58,7 @@ type ICEConnInfo struct { RelayedOnLocal bool } -type ConnectorICE struct { +type WorkerICE struct { ctx context.Context log *log.Entry config ConnConfig @@ -78,8 +78,8 @@ type ConnectorICE struct { sentExtraSrflx bool } -func NewConnectorICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, doHandshakeFn DoHandshake) *ConnectorICE { - cice := &ConnectorICE{ +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, doHandshakeFn DoHandshake) *WorkerICE { + cice := &WorkerICE{ ctx: ctx, log: log, config: config, @@ -97,38 +97,38 @@ func NewConnectorICE(ctx context.Context, log *log.Entry, config ConnConfig, con // If the relay mode is supported then try to connect in p2p way only. // It is trying to reconnection in a loop until the context is canceled. // In case of success connection it will call the onICEConnReady callback. -func (conn *ConnectorICE) SetupICEConnection(hasRelayOnLocally bool) { +func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { for { - if !conn.waitForReconnectTry() { + if !w.waitForReconnectTry() { return } - remoteOfferAnswer, err := conn.doHandshakeFn() + remoteOfferAnswer, err := w.doHandshakeFn() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { - conn.log.Infof("signal client isn't ready, skipping connection attempt") + w.log.Infof("signal client isn't ready, skipping connection attempt") } continue } var preferredCandidateTypes []ice.CandidateType if hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { - conn.connPriority = connPriorityICEP2P + w.connPriority = connPriorityICEP2P preferredCandidateTypes = candidateTypesP2P() } else { - conn.connPriority = connPriorityICETurn + w.connPriority = connPriorityICETurn preferredCandidateTypes = candidateTypes() } - ctx, ctxCancel := context.WithCancel(conn.ctx) - agent, err := conn.reCreateAgent(ctxCancel, preferredCandidateTypes) + ctx, ctxCancel := context.WithCancel(w.ctx) + agent, err := w.reCreateAgent(ctxCancel, preferredCandidateTypes) if err != nil { ctxCancel() continue } - conn.agent = agent + w.agent = agent - err = conn.agent.GatherCandidates() + err = w.agent.GatherCandidates() if err != nil { ctxCancel() continue @@ -137,13 +137,13 @@ func (conn *ConnectorICE) SetupICEConnection(hasRelayOnLocally bool) { // will block until connection succeeded // but it won't release if ICE Agent went into Disconnected or Failed state, // so we have to cancel it with the provided context once agent detected a broken connection - remoteConn, err := conn.turnAgentDial(remoteOfferAnswer) + remoteConn, err := w.turnAgentDial(remoteOfferAnswer) if err != nil { ctxCancel() continue } - pair, err := conn.agent.GetSelectedCandidatePair() + pair, err := w.agent.GetSelectedCandidatePair() if err != nil { ctxCancel() continue @@ -157,7 +157,7 @@ func (conn *ConnectorICE) SetupICEConnection(hasRelayOnLocally bool) { } // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port - go conn.punchRemoteWGPort(pair, remoteWgPort) + go w.punchRemoteWGPort(pair, remoteWgPort) } ci := ICEConnInfo{ @@ -172,18 +172,18 @@ func (conn *ConnectorICE) SetupICEConnection(hasRelayOnLocally bool) { Relayed: isRelayed(pair), RelayedOnLocal: isRelayCandidate(pair.Local), } - go conn.onICEConnReady(conn.connPriority, ci) + go w.onICEConnReady(w.connPriority, ci) <-ctx.Done() ctxCancel() - _ = conn.agent.Close() + _ = w.agent.Close() } } // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. -func (conn *ConnectorICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - conn.log.Debugf("OnRemoteCandidate from peer %s -> %s", conn.config.Key, candidate.String()) - if conn.agent == nil { +func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String()) + if w.agent == nil { return } @@ -191,25 +191,25 @@ func (conn *ConnectorICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes ro return } - err := conn.agent.AddRemoteCandidate(candidate) + err := w.agent.AddRemoteCandidate(candidate) if err != nil { - conn.log.Errorf("error while handling remote candidate") + w.log.Errorf("error while handling remote candidate") return } } -func (conn *ConnectorICE) GetLocalUserCredentials() (frag string, pwd string, err error) { - if conn.agent == nil { +func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string, err error) { + if w.agent == nil { return "", "", errors.New("ICE Agent is not initialized") } - return conn.agent.GetLocalUserCredentials() + return w.agent.GetLocalUserCredentials() } -func (conn *ConnectorICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { +func (w *WorkerICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { failedTimeout := 6 * time.Second - transportNet, err := conn.newStdNet() + transportNet, err := w.newStdNet() if err != nil { - conn.log.Errorf("failed to create pion's stdnet: %s", err) + w.log.Errorf("failed to create pion's stdnet: %s", err) } iceKeepAlive := iceKeepAlive() @@ -219,36 +219,36 @@ func (conn *ConnectorICE) reCreateAgent(ctxCancel context.CancelFunc, relaySuppo agentConfig := &ice.AgentConfig{ MulticastDNSMode: ice.MulticastDNSModeDisabled, NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: conn.configICE.StunTurn.Load().([]*stun.URI), + Urls: w.configICE.StunTurn.Load().([]*stun.URI), CandidateTypes: relaySupport, FailedTimeout: &failedTimeout, - InterfaceFilter: stdnet.InterfaceFilter(conn.configICE.InterfaceBlackList), - UDPMux: conn.configICE.UDPMux, - UDPMuxSrflx: conn.configICE.UDPMuxSrflx, - NAT1To1IPs: conn.configICE.NATExternalIPs, + InterfaceFilter: stdnet.InterfaceFilter(w.configICE.InterfaceBlackList), + UDPMux: w.configICE.UDPMux, + UDPMuxSrflx: w.configICE.UDPMuxSrflx, + NAT1To1IPs: w.configICE.NATExternalIPs, Net: transportNet, DisconnectedTimeout: &iceDisconnectedTimeout, KeepaliveInterval: &iceKeepAlive, RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, } - if conn.configICE.DisableIPv6Discovery { + if w.configICE.DisableIPv6Discovery { agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} } - conn.sentExtraSrflx = false + w.sentExtraSrflx = false agent, err := ice.NewAgent(agentConfig) if err != nil { return nil, err } - err = agent.OnCandidate(conn.onICECandidate) + err = agent.OnCandidate(w.onICECandidate) if err != nil { return nil, err } err = agent.OnConnectionStateChange(func(state ice.ConnectionState) { - conn.log.Debugf("ICE ConnectionState has changed to %s", state.String()) + w.log.Debugf("ICE ConnectionState has changed to %s", state.String()) if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { ctxCancel() } @@ -257,15 +257,15 @@ func (conn *ConnectorICE) reCreateAgent(ctxCancel context.CancelFunc, relaySuppo return nil, err } - err = agent.OnSelectedCandidatePairChange(conn.onICESelectedCandidatePair) + err = agent.OnSelectedCandidatePairChange(w.onICESelectedCandidatePair) if err != nil { return nil, err } err = agent.OnSuccessfulSelectedPairBindingResponse(func(p *ice.CandidatePair) { - err := conn.statusRecorder.UpdateLatency(conn.config.Key, p.Latency()) + err := w.statusRecorder.UpdateLatency(w.config.Key, p.Latency()) if err != nil { - conn.log.Debugf("failed to update latency for peer: %s", err) + w.log.Debugf("failed to update latency for peer: %s", err) return } }) @@ -276,44 +276,44 @@ func (conn *ConnectorICE) reCreateAgent(ctxCancel context.CancelFunc, relaySuppo return agent, nil } -func (conn *ConnectorICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { +func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { // wait local endpoint configuration time.Sleep(time.Second) addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", pair.Remote.Address(), remoteWgPort)) if err != nil { - conn.log.Warnf("got an error while resolving the udp address, err: %s", err) + w.log.Warnf("got an error while resolving the udp address, err: %s", err) return } - mux, ok := conn.configICE.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) + mux, ok := w.configICE.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) if !ok { - conn.log.Warn("invalid udp mux conversion") + w.log.Warn("invalid udp mux conversion") return } _, err = mux.GetSharedConn().WriteTo([]byte{0x6e, 0x62}, addr) if err != nil { - conn.log.Warnf("got an error while sending the punch packet, err: %s", err) + w.log.Warnf("got an error while sending the punch packet, err: %s", err) } } // onICECandidate is a callback attached to an ICE Agent to receive new local connection candidates // and then signals them to the remote peer -func (conn *ConnectorICE) onICECandidate(candidate ice.Candidate) { +func (w *WorkerICE) onICECandidate(candidate ice.Candidate) { // nil means candidate gathering has been ended if candidate == nil { return } // TODO: reported port is incorrect for CandidateTypeHost, makes understanding ICE use via logs confusing as port is ignored - conn.log.Debugf("discovered local candidate %s", candidate.String()) + w.log.Debugf("discovered local candidate %s", candidate.String()) go func() { - err := conn.signaler.SignalICECandidate(candidate, conn.config.Key) + err := w.signaler.SignalICECandidate(candidate, w.config.Key) if err != nil { - conn.log.Errorf("failed signaling candidate to the remote peer %s %s", conn.config.Key, err) + w.log.Errorf("failed signaling candidate to the remote peer %s %s", w.config.Key, err) } }() - if !conn.shouldSendExtraSrflxCandidate(candidate) { + if !w.shouldSendExtraSrflxCandidate(candidate) { return } @@ -321,42 +321,42 @@ func (conn *ConnectorICE) onICECandidate(candidate ice.Candidate) { // this is useful when network has an existing port forwarding rule for the wireguard port and this peer extraSrflx, err := extraSrflxCandidate(candidate) if err != nil { - conn.log.Errorf("failed creating extra server reflexive candidate %s", err) + w.log.Errorf("failed creating extra server reflexive candidate %s", err) return } - conn.sentExtraSrflx = true + w.sentExtraSrflx = true go func() { - err = conn.signaler.SignalICECandidate(extraSrflx, conn.config.Key) + err = w.signaler.SignalICECandidate(extraSrflx, w.config.Key) if err != nil { - conn.log.Errorf("failed signaling the extra server reflexive candidate: %s", err) + w.log.Errorf("failed signaling the extra server reflexive candidate: %s", err) } }() } -func (conn *ConnectorICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { - conn.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), - conn.config.Key) +func (w *WorkerICE) onICESelectedCandidatePair(c1 ice.Candidate, c2 ice.Candidate) { + w.log.Debugf("selected candidate pair [local <-> remote] -> [%s <-> %s], peer %s", c1.String(), c2.String(), + w.config.Key) } -func (conn *ConnectorICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { - if !conn.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { +func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool { + if !w.sentExtraSrflx && candidate.Type() == ice.CandidateTypeServerReflexive && candidate.Port() != candidate.RelatedAddress().Port { return true } return false } -func (conn *ConnectorICE) turnAgentDial(remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { - isControlling := conn.config.LocalKey > conn.config.Key +func (w *WorkerICE) turnAgentDial(remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { + isControlling := w.config.LocalKey > w.config.Key if isControlling { - return conn.agent.Dial(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + return w.agent.Dial(w.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) } else { - return conn.agent.Accept(conn.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + return w.agent.Accept(w.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) } } // waitForReconnectTry waits for a random duration before trying to reconnect -func (conn *ConnectorICE) waitForReconnectTry() bool { +func (w *WorkerICE) waitForReconnectTry() bool { minWait := 500 maxWait := 2000 duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond @@ -365,7 +365,7 @@ func (conn *ConnectorICE) waitForReconnectTry() bool { defer timeout.Stop() select { - case <-conn.ctx.Done(): + case <-w.ctx.Done(): return false case <-timeout.C: return true diff --git a/client/internal/peer/conn_relay.go b/client/internal/peer/worker_relay.go similarity index 57% rename from client/internal/peer/conn_relay.go rename to client/internal/peer/worker_relay.go index ae42a4597..762c69b3e 100644 --- a/client/internal/peer/conn_relay.go +++ b/client/internal/peer/worker_relay.go @@ -20,7 +20,7 @@ type RelayConnInfo struct { rosenpassAddr string } -type ConnectorRelay struct { +type WorkerRelay struct { ctx context.Context log *log.Entry relayManager *relayClient.Manager @@ -29,8 +29,8 @@ type ConnectorRelay struct { doHandshakeFn DoHandshake } -func NewConnectorRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, onRelayConnReadyFN OnRelayReadyCallback, doHandshakeFn DoHandshake) *ConnectorRelay { - return &ConnectorRelay{ +func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, onRelayConnReadyFN OnRelayReadyCallback, doHandshakeFn DoHandshake) *WorkerRelay { + return &WorkerRelay{ ctx: ctx, log: log, relayManager: relayManager, @@ -41,39 +41,39 @@ func NewConnectorRelay(ctx context.Context, log *log.Entry, relayManager *relayC } // SetupRelayConnection todo: this function is not completed. Make no sense to put it in a for loop because we are not waiting for any event -func (conn *ConnectorRelay) SetupRelayConnection() { +func (w *WorkerRelay) SetupRelayConnection() { for { - if !conn.waitForReconnectTry() { + if !w.waitForReconnectTry() { return } - remoteOfferAnswer, err := conn.doHandshakeFn() + remoteOfferAnswer, err := w.doHandshakeFn() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { - conn.log.Infof("signal client isn't ready, skipping connection attempt") + w.log.Infof("signal client isn't ready, skipping connection attempt") } - conn.log.Errorf("failed to do handshake: %v", err) + w.log.Errorf("failed to do handshake: %v", err) continue } - if !conn.isRelaySupported(remoteOfferAnswer) { + if !w.isRelaySupported(remoteOfferAnswer) { // todo should we retry? continue } // the relayManager will return with error in case if the connection has lost with relay server - currentRelayAddress, err := conn.relayManager.RelayAddress() + currentRelayAddress, err := w.relayManager.RelayAddress() if err != nil { continue } - srv := conn.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) - relayedConn, err := conn.relayManager.OpenConn(srv, conn.config.Key) + srv := w.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) if err != nil { continue } - go conn.onRelayConnReadyFN(RelayConnInfo{ + go w.onRelayConnReadyFN(RelayConnInfo{ relayedConn: relayedConn, rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, rosenpassAddr: remoteOfferAnswer.RosenpassAddr, @@ -83,28 +83,28 @@ func (conn *ConnectorRelay) SetupRelayConnection() { } } -func (conn *ConnectorRelay) RelayAddress() (net.Addr, error) { - return conn.relayManager.RelayAddress() +func (w *WorkerRelay) RelayAddress() (net.Addr, error) { + return w.relayManager.RelayAddress() } // todo check my side too -func (conn *ConnectorRelay) isRelaySupported(answer *OfferAnswer) bool { +func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { return answer.RelaySrvAddress != "" } -func (conn *ConnectorRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { - if conn.config.LocalKey > conn.config.Key { +func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { + if w.config.LocalKey > w.config.Key { return myRelayAddress } return remoteRelayAddress } -func (conn *ConnectorRelay) RelayIsSupportedLocally() bool { - return conn.relayManager.HasRelayAddress() +func (w *WorkerRelay) RelayIsSupportedLocally() bool { + return w.relayManager.HasRelayAddress() } // waitForReconnectTry waits for a random duration before trying to reconnect -func (conn *ConnectorRelay) waitForReconnectTry() bool { +func (w *WorkerRelay) waitForReconnectTry() bool { minWait := 500 maxWait := 2000 duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond @@ -113,7 +113,7 @@ func (conn *ConnectorRelay) waitForReconnectTry() bool { defer timeout.Stop() select { - case <-conn.ctx.Done(): + case <-w.ctx.Done(): return false case <-timeout.C: return true From d8ab3c16326c7b134c86e64b8f97724aaae52d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 18 Jun 2024 11:23:39 +0200 Subject: [PATCH 040/199] Call peer.Open from engine --- client/internal/engine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/internal/engine.go b/client/internal/engine.go index c4adb4be4..4efe74cab 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -826,6 +826,7 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { if err != nil { return fmt.Errorf("create peer connection: %w", err) } + conn.Open() e.peerConns[peerKey] = conn if e.beforePeerHook != nil && e.afterPeerHook != nil { From a5e664d83da7b1e075258e09f6cc7e2e79eee20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 18 Jun 2024 11:27:18 +0200 Subject: [PATCH 041/199] Code cleaning --- client/internal/peer/conn.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index f729bc61a..1b2538187 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -162,6 +162,7 @@ func (conn *Conn) Open() { // Close closes this peer Conn issuing a close event to the Conn closeCh func (conn *Conn) Close() { conn.mu.Lock() + defer conn.mu.Unlock() conn.ctxCancel() if conn.wgProxy != nil { @@ -208,8 +209,6 @@ func (conn *Conn) Close() { if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { conn.log.Debugf("failed to reset wireguard stats for peer: %s", err) } - - conn.mu.Unlock() } // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise From e26e2c3a75a81c3e13a0d4f3ec08abf4d9a801d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 18 Jun 2024 17:40:37 +0200 Subject: [PATCH 042/199] Add conn status handling and protect agent --- client/internal/peer/conn.go | 105 +++++++++++++++++++++------ client/internal/peer/handshaker.go | 4 +- client/internal/peer/status_test.go | 10 +-- client/internal/peer/worker_ice.go | 59 +++++++++------ client/internal/peer/worker_relay.go | 4 +- 5 files changed, 126 insertions(+), 56 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 1b2538187..268071593 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -80,7 +80,8 @@ type Conn struct { onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) onDisconnected func(remotePeer string, wgIP string) - status ConnStatus + statusRelay ConnStatus + statusICE ConnStatus workerICE *WorkerICE workerRelay *WorkerRelay @@ -113,11 +114,12 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu signaler: signaler, allowedIPsIP: allowedIPsIP.String(), handshaker: NewHandshaker(ctx, config, signaler), - status: StatusDisconnected, + statusRelay: StatusDisconnected, + statusICE: StatusDisconnected, closeCh: make(chan struct{}), } - conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.doHandshake) - conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.doHandshake) + conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.onWorkerICEStateChanged, conn.doHandshake) + conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.onWorkerRelayStateChanged, conn.doHandshake) return conn, nil } @@ -140,18 +142,6 @@ func (conn *Conn) Open() { conn.log.Warnf("error while updating the state err: %v", err) } - /* - peerState = State{ - PubKey: conn.config.Key, - ConnStatus: StatusConnecting, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - log.Warnf("error while updating the state of peer %s,err: %v", conn.config.Key, err) - } - */ relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() if relayIsSupportedLocally { go conn.workerRelay.SetupRelayConnection() @@ -188,15 +178,16 @@ func (conn *Conn) Close() { conn.connID = "" } - if conn.status == StatusConnected && conn.onDisconnected != nil { + if conn.evalStatus() == StatusConnected && conn.onDisconnected != nil { conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) } - conn.status = StatusDisconnected + conn.statusRelay = StatusDisconnected + conn.statusICE = StatusDisconnected peerState := State{ PubKey: conn.config.Key, - ConnStatus: conn.status, + ConnStatus: StatusDisconnected, ConnStatusUpdate: time.Now(), Mux: new(sync.RWMutex), } @@ -214,7 +205,7 @@ func (conn *Conn) Close() { // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise // doesn't block, discards the message if connection wasn't ready func (conn *Conn) OnRemoteAnswer(answer OfferAnswer) bool { - conn.log.Debugf("OnRemoteAnswer, status %s", conn.status.String()) + conn.log.Debugf("OnRemoteAnswer, status ICE: %s, status relay: %s", conn.statusICE, conn.statusRelay) return conn.handshaker.OnRemoteAnswer(answer) } @@ -242,7 +233,7 @@ func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string) } func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { - conn.log.Debugf("OnRemoteOffer, on status %s", conn.status.String()) + conn.log.Debugf("OnRemoteOffer, on status ICE: %s, status relay: %s", conn.statusICE, conn.statusRelay) return conn.handshaker.OnRemoteOffer(offer) } @@ -255,13 +246,65 @@ func (conn *Conn) WgConfig() WgConfig { func (conn *Conn) Status() ConnStatus { conn.mu.Lock() defer conn.mu.Unlock() - return conn.status + return conn.evalStatus() } func (conn *Conn) GetKey() string { return conn.config.Key } +func (conn *Conn) onWorkerICEStateChanged(newState ConnStatus) { + conn.mu.Lock() + defer conn.mu.Unlock() + defer func() { + conn.statusICE = newState + }() + + if conn.statusRelay == StatusConnected { + return + } + + if conn.evalStatus() == newState { + return + } + + if newState > conn.statusICE { + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: newState, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + _ = conn.statusRecorder.UpdatePeerState(peerState) + } +} + +func (conn *Conn) onWorkerRelayStateChanged(newState ConnStatus) { + conn.mu.Lock() + defer conn.mu.Unlock() + defer func() { + conn.statusRelay = newState + }() + + if conn.statusICE == StatusConnected { + return + } + + if conn.evalStatus() == newState { + return + } + + if newState > conn.statusRelay { + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: newState, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + _ = conn.statusRecorder.UpdatePeerState(peerState) + } +} + func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.mu.Lock() defer conn.mu.Unlock() @@ -270,6 +313,8 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { return } + conn.statusRelay = stateConnected + if conn.currentConnType > connPriorityRelay { return } @@ -326,6 +371,8 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon return } + conn.statusICE = stateConnected + if conn.currentConnType > priority { return } @@ -390,8 +437,6 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon } func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { - conn.status = StatusConnected - peerState.PubKey = conn.config.Key peerState.ConnStatus = StatusConnected peerState.ConnStatusUpdate = time.Now() @@ -434,6 +479,18 @@ func (conn *Conn) doHandshake() (*OfferAnswer, error) { }) } +func (conn *Conn) evalStatus() ConnStatus { + if conn.statusRelay == StatusConnected || conn.statusICE == StatusConnected { + return StatusConnected + } + + if conn.statusRelay == StatusConnecting || conn.statusICE == StatusConnecting { + return StatusConnecting + } + + return StatusDisconnected +} + func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index e10705f63..2bc42686f 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -116,7 +116,7 @@ func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { case h.remoteOffersCh <- offer: return true default: - log.Debugf("OnRemoteOffer skipping message from peer %s on status %s because is not ready", h.config.Key, conn.status.String()) + log.Debugf("OnRemoteOffer skipping message from peer %s because is not ready", h.config.Key) // connection might not be ready yet to receive so we ignore the message return false } @@ -130,7 +130,7 @@ func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { return true default: // connection might not be ready yet to receive so we ignore the message - log.Debugf("OnRemoteAnswer skipping message from peer %s on status %s because is not ready", h.config.Key, conn.status.String()) + log.Debugf("OnRemoteAnswer skipping message from peer %s because is not ready", h.config.Key) return false } } diff --git a/client/internal/peer/status_test.go b/client/internal/peer/status_test.go index a4a6e6081..1d283433b 100644 --- a/client/internal/peer/status_test.go +++ b/client/internal/peer/status_test.go @@ -2,8 +2,8 @@ package peer import ( "errors" - "testing" "sync" + "testing" "github.com/stretchr/testify/assert" ) @@ -43,7 +43,7 @@ func TestUpdatePeerState(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -64,7 +64,7 @@ func TestStatus_UpdatePeerFQDN(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -83,7 +83,7 @@ func TestGetPeerStateChangeNotifierLogic(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState @@ -108,7 +108,7 @@ func TestRemovePeer(t *testing.T) { status := NewRecorder("https://mgm") peerState := State{ PubKey: key, - Mux: new(sync.RWMutex), + Mux: new(sync.RWMutex), } status.peers[key] = peerState diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 70668b493..33362e86f 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -8,6 +8,7 @@ import ( "net" "net/netip" "runtime" + "sync" "sync/atomic" "time" @@ -59,36 +60,39 @@ type ICEConnInfo struct { } type WorkerICE struct { - ctx context.Context - log *log.Entry - config ConnConfig - configICE ICEConfig - signaler *Signaler - iFaceDiscover stdnet.ExternalIFaceDiscover - statusRecorder *Status - onICEConnReady OnICEConnReadyCallback - doHandshakeFn DoHandshake + ctx context.Context + log *log.Entry + config ConnConfig + configICE ICEConfig + signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + statusRecorder *Status + onICEConnReady OnICEConnReadyCallback + onStatusChanged func(ConnStatus) + doHandshakeFn DoHandshake - connPriority ConnPriority + selectedPriority ConnPriority - agent *ice.Agent + agent *ice.Agent + muxAgent sync.RWMutex StunTurn []*stun.URI sentExtraSrflx bool } -func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, doHandshakeFn DoHandshake) *WorkerICE { +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, onStatusChanged func(ConnStatus), doHandshakeFn DoHandshake) *WorkerICE { cice := &WorkerICE{ - ctx: ctx, - log: log, - config: config, - configICE: configICE, - signaler: signaler, - iFaceDiscover: ifaceDiscover, - statusRecorder: statusRecorder, - onICEConnReady: onICEConnReady, - doHandshakeFn: doHandshakeFn, + ctx: ctx, + log: log, + config: config, + configICE: configICE, + signaler: signaler, + iFaceDiscover: ifaceDiscover, + statusRecorder: statusRecorder, + onICEConnReady: onICEConnReady, + onStatusChanged: onStatusChanged, + doHandshakeFn: doHandshakeFn, } return cice } @@ -103,6 +107,8 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { return } + w.onStatusChanged(StatusConnecting) + remoteOfferAnswer, err := w.doHandshakeFn() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { @@ -113,10 +119,10 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { var preferredCandidateTypes []ice.CandidateType if hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { - w.connPriority = connPriorityICEP2P + w.selectedPriority = connPriorityICEP2P preferredCandidateTypes = candidateTypesP2P() } else { - w.connPriority = connPriorityICETurn + w.selectedPriority = connPriorityICETurn preferredCandidateTypes = candidateTypes() } @@ -126,7 +132,9 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { ctxCancel() continue } + w.muxAgent.Lock() w.agent = agent + w.muxAgent.Unlock() err = w.agent.GatherCandidates() if err != nil { @@ -172,16 +180,19 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { Relayed: isRelayed(pair), RelayedOnLocal: isRelayCandidate(pair.Local), } - go w.onICEConnReady(w.connPriority, ci) + go w.onICEConnReady(w.selectedPriority, ci) <-ctx.Done() ctxCancel() _ = w.agent.Close() + w.onStatusChanged(StatusDisconnected) } } // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { + w.muxAgent.RLocker() + defer w.muxAgent.RUnlock() w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String()) if w.agent == nil { return diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 762c69b3e..078fe84da 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -26,16 +26,18 @@ type WorkerRelay struct { relayManager *relayClient.Manager config ConnConfig onRelayConnReadyFN OnRelayReadyCallback + onStatusChanged func(ConnStatus) doHandshakeFn DoHandshake } -func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, onRelayConnReadyFN OnRelayReadyCallback, doHandshakeFn DoHandshake) *WorkerRelay { +func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, onRelayConnReadyFN OnRelayReadyCallback, onStatusChanged func(ConnStatus), doHandshakeFn DoHandshake) *WorkerRelay { return &WorkerRelay{ ctx: ctx, log: log, relayManager: relayManager, config: config, onRelayConnReadyFN: onRelayConnReadyFN, + onStatusChanged: onStatusChanged, doHandshakeFn: doHandshakeFn, } } From 24f71bc68a24028a36ceb7628e3ae61d7bbde0a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 09:40:43 +0200 Subject: [PATCH 043/199] Fix and extend test --- client/internal/peer/conn.go | 2 - client/internal/peer/conn_test.go | 77 +++++++++++-------------------- 2 files changed, 26 insertions(+), 53 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 268071593..389b9eb64 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -75,7 +75,6 @@ type Conn struct { signaler *Signaler allowedIPsIP string handshaker *Handshaker - closeCh chan struct{} onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) onDisconnected func(remotePeer string, wgIP string) @@ -116,7 +115,6 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu handshaker: NewHandshaker(ctx, config, signaler), statusRelay: StatusDisconnected, statusICE: StatusDisconnected, - closeCh: make(chan struct{}), } conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.onWorkerICEStateChanged, conn.doHandshake) conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.onWorkerRelayStateChanged, conn.doHandshake) diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index 10f1ac5ef..7dbb06b5a 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/magiconair/properties/assert" - "github.com/pion/stun/v2" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" @@ -15,12 +14,13 @@ import ( ) var connConf = ConnConfig{ - Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - StunTurn: []*stun.URI{}, - InterfaceBlackList: nil, - Timeout: time.Second, - LocalWgPort: 51820, + Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + Timeout: time.Second, + LocalWgPort: 51820, + ICEConfig: ICEConfig{ + InterfaceBlackList: nil, + }, } func TestNewConn_interfaceFilter(t *testing.T) { @@ -40,7 +40,7 @@ func TestConn_GetKey(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil, nil) + conn, err := NewConn(context.Background(), connConf, nil, wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -55,7 +55,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -63,7 +63,7 @@ func TestConn_OnRemoteOffer(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) go func() { - <-conn.remoteOffersCh + <-conn.handshaker.remoteOffersCh wg.Done() }() @@ -92,7 +92,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } @@ -100,7 +100,7 @@ func TestConn_OnRemoteAnswer(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) go func() { - <-conn.remoteAnswerCh + <-conn.handshaker.remoteAnswerCh wg.Done() }() @@ -128,58 +128,33 @@ func TestConn_Status(t *testing.T) { defer func() { _ = wgProxyFactory.Free() }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) + conn, err := NewConn(context.Background(), connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) if err != nil { return } tables := []struct { - name string - status ConnStatus - want ConnStatus + name string + statusIce ConnStatus + statusRelay ConnStatus + want ConnStatus }{ - {"StatusConnected", StatusConnected, StatusConnected}, - {"StatusDisconnected", StatusDisconnected, StatusDisconnected}, - {"StatusConnecting", StatusConnecting, StatusConnecting}, + {"StatusConnected", StatusConnected, StatusConnected, StatusConnected}, + {"StatusDisconnected", StatusDisconnected, StatusDisconnected, StatusDisconnected}, + {"StatusConnecting", StatusConnecting, StatusConnecting, StatusConnecting}, + {"StatusConnectingIce", StatusConnecting, StatusDisconnected, StatusConnecting}, + {"StatusConnectingIceAlternative", StatusConnecting, StatusConnected, StatusConnected}, + {"StatusConnectingRelay", StatusDisconnected, StatusConnecting, StatusConnecting}, + {"StatusConnectingRelayAlternative", StatusConnected, StatusConnecting, StatusConnected}, } for _, table := range tables { t.Run(table.name, func(t *testing.T) { - conn.status = table.status + conn.statusICE = table.statusIce + conn.statusRelay = table.statusRelay got := conn.Status() assert.Equal(t, got, table.want, "they should be equal") }) } } - -func TestConn_Close(t *testing.T) { - wgProxyFactory := wgproxy.NewFactory(context.Background(), connConf.LocalWgPort) - defer func() { - _ = wgProxyFactory.Free() - }() - conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, nil) - if err != nil { - return - } - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - <-conn.closeCh - wg.Done() - }() - - go func() { - for { - err := conn.Close() - if err != nil { - continue - } else { - return - } - } - }() - - wg.Wait() -} From 48310ef99c193b945492c639f7b07de068c60862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 09:59:01 +0200 Subject: [PATCH 044/199] Fix engine test --- client/internal/engine_test.go | 23 +++++++++++++++-------- client/internal/peer/handshaker.go | 15 +++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index f5a98cb7f..172216e73 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -35,6 +35,7 @@ import ( mgmtProto "github.com/netbirdio/netbird/management/proto" "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/activity" + relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/signal/proto" @@ -71,13 +72,15 @@ func TestEngine_SSH(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: "utun101", WgAddr: "100.64.0.1/24", WgPrivateKey: key, WgPort: 33100, ServerSSHAllowed: true, - }, MobileDependency{}, peer.NewRecorder("https://mgm")) + }, + MobileDependency{}, peer.NewRecorder("https://mgm")) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, @@ -206,7 +209,8 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: "utun102", WgAddr: "100.64.0.1/24", WgPrivateKey: key, @@ -402,8 +406,8 @@ func TestEngine_Sync(t *testing.T) { } return nil } - - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{SyncFunc: syncFunc}, relayMgr, &EngineConfig{ WgIfaceName: "utun103", WgAddr: "100.64.0.1/24", WgPrivateKey: key, @@ -562,7 +566,8 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) { wgIfaceName := fmt.Sprintf("utun%d", 104+n) wgAddr := fmt.Sprintf("100.66.%d.1/24", n) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, WgPrivateKey: key, @@ -732,7 +737,8 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) { wgIfaceName := fmt.Sprintf("utun%d", 104+n) wgAddr := fmt.Sprintf("100.66.%d.1/24", n) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, &EngineConfig{ + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ WgIfaceName: wgIfaceName, WgAddr: wgAddr, WgPrivateKey: key, @@ -1008,7 +1014,8 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin WgPort: wgPort, } - e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, conf, MobileDependency{}, peer.NewRecorder("https://mgm")), nil + relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) + e, err := NewEngine(ctx, cancel, signalClient, mgmtClient, relayMgr, conf, MobileDependency{}, peer.NewRecorder("https://mgm")), nil e.ctx = ctx return e, err } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 2bc42686f..797bbb82e 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -7,7 +7,6 @@ import ( "time" log "github.com/sirupsen/logrus" - "golang.zx2c4.com/wireguard/conn" "github.com/netbirdio/netbird/version" ) @@ -68,6 +67,8 @@ type Handshaker struct { remoteOfferAnswer *OfferAnswer remoteOfferAnswerCreated time.Time + + handshakeArgs HandshakeArgs } func NewHandshaker(ctx context.Context, config ConnConfig, signaler *Signaler) *Handshaker { @@ -84,6 +85,8 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { h.mu.Lock() defer h.mu.Unlock() + h.handshakeArgs = args + cachedOfferAnswer, ok := h.cachedHandshake() if ok { return cachedOfferAnswer, nil @@ -150,20 +153,16 @@ func (h *Handshaker) sendOffer(args HandshakeArgs) error { } func (h *Handshaker) sendAnswer() error { - localUFrag, localPwd, err := conn.connectorICE.GetLocalUserCredentials() - if err != nil { - return err - } - log.Debugf("sending answer to %s", h.config.Key) answer := OfferAnswer{ - IceCredentials: IceCredentials{localUFrag, localPwd}, + IceCredentials: IceCredentials{h.handshakeArgs.IceUFrag, h.handshakeArgs.IcePwd}, WgListenPort: h.config.LocalWgPort, Version: version.NetbirdVersion(), RosenpassPubKey: h.config.RosenpassPubKey, RosenpassAddr: h.config.RosenpassAddr, + RelaySrvAddress: h.handshakeArgs.RelayAddr, } - err = h.signaler.SignalAnswer(answer, h.config.Key) + err := h.signaler.SignalAnswer(answer, h.config.Key) if err != nil { return err } From 2f32e0d8cf1142769a773ed7883cb6531658717d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 11:28:01 +0200 Subject: [PATCH 045/199] Fix chicken-egg problem in the ice agent creation --- client/internal/peer/conn.go | 18 +++++----- client/internal/peer/worker_ice.go | 58 +++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 18 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 389b9eb64..49fe10abd 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -393,7 +393,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon } endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) - conn.log.Debugf("Conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + conn.log.Debugf("Conn resolved IP is %s for endopint %s", endpoint, endpointUdpAddr.IP) conn.connID = nbnet.GenerateConnID() for _, hook := range conn.beforeAddPeerHooks { @@ -461,20 +461,20 @@ func (conn *Conn) doHandshake() (*OfferAnswer, error) { return nil, ErrSignalIsNotReady } - uFreg, pwd, err := conn.workerICE.GetLocalUserCredentials() + var ( + ha HandshakeArgs + err error + ) + ha.IceUFrag, ha.IcePwd, err = conn.workerICE.GetLocalUserCredentials() if err != nil { conn.log.Errorf("failed to get local user credentials: %v", err) } addr, err := conn.workerRelay.RelayAddress() - if err != nil { - conn.log.Errorf("failed to get local relay address: %v", err) + if err == nil { + ha.RelayAddr = addr.String() } - return conn.handshaker.Handshake(HandshakeArgs{ - uFreg, - pwd, - addr.String(), - }) + return conn.handshaker.Handshake(ha) } func (conn *Conn) evalStatus() ConnStatus { diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 33362e86f..ed30ca205 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -13,6 +13,7 @@ import ( "time" "github.com/pion/ice/v3" + "github.com/pion/randutil" "github.com/pion/stun/v2" log "github.com/sirupsen/logrus" @@ -27,6 +28,14 @@ const ( iceDisconnectedTimeoutDefault = 6 * time.Second // iceRelayAcceptanceMinWaitDefault is the same as in the Pion ICE package iceRelayAcceptanceMinWaitDefault = 2 * time.Second + + lenUFrag = 16 + lenPwd = 32 + runesAlpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +) + +var ( + failedTimeout = 6 * time.Second ) type ICEConfig struct { @@ -74,11 +83,13 @@ type WorkerICE struct { selectedPriority ConnPriority agent *ice.Agent - muxAgent sync.RWMutex + muxAgent sync.Mutex StunTurn []*stun.URI sentExtraSrflx bool + localUfrag string + localPwd string } func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, onStatusChanged func(ConnStatus), doHandshakeFn DoHandshake) *WorkerICE { @@ -127,13 +138,22 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { } ctx, ctxCancel := context.WithCancel(w.ctx) + w.muxAgent.Lock() agent, err := w.reCreateAgent(ctxCancel, preferredCandidateTypes) if err != nil { ctxCancel() + w.muxAgent.Unlock() continue } - w.muxAgent.Lock() w.agent = agent + // generate credentials for the next loop. Important the credentials are generated before handshake, because + // the handshake could provide a cached offer-answer + w.localUfrag, w.localPwd, err = generateICECredentials() + if err != nil { + ctxCancel() + w.muxAgent.Unlock() + continue + } w.muxAgent.Unlock() err = w.agent.GatherCandidates() @@ -191,8 +211,8 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { - w.muxAgent.RLocker() - defer w.muxAgent.RUnlock() + w.muxAgent.Lock() + defer w.muxAgent.Unlock() w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String()) if w.agent == nil { return @@ -210,14 +230,18 @@ func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HA } func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string, err error) { - if w.agent == nil { - return "", "", errors.New("ICE Agent is not initialized") + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + + if w.localUfrag != "" && w.localPwd != "" { + return w.localUfrag, w.localPwd, nil } - return w.agent.GetLocalUserCredentials() + + w.localUfrag, w.localPwd, err = generateICECredentials() + return w.localUfrag, w.localPwd, err } func (w *WorkerICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { - failedTimeout := 6 * time.Second transportNet, err := w.newStdNet() if err != nil { w.log.Errorf("failed to create pion's stdnet: %s", err) @@ -232,15 +256,17 @@ func (w *WorkerICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []i NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, Urls: w.configICE.StunTurn.Load().([]*stun.URI), CandidateTypes: relaySupport, - FailedTimeout: &failedTimeout, InterfaceFilter: stdnet.InterfaceFilter(w.configICE.InterfaceBlackList), UDPMux: w.configICE.UDPMux, UDPMuxSrflx: w.configICE.UDPMuxSrflx, NAT1To1IPs: w.configICE.NATExternalIPs, Net: transportNet, + FailedTimeout: &failedTimeout, DisconnectedTimeout: &iceDisconnectedTimeout, KeepaliveInterval: &iceKeepAlive, RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, + LocalUfrag: w.localUfrag, + LocalPwd: w.localPwd, } if w.configICE.DisableIPv6Discovery { @@ -448,3 +474,17 @@ func isRelayed(pair *ice.CandidatePair) bool { } return false } + +func generateICECredentials() (string, string, error) { + ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha) + if err != nil { + return "", "", err + } + + pwd, err := randutil.GenerateCryptoRandomString(lenPwd, runesAlpha) + if err != nil { + return "", "", err + } + return ufrag, pwd, nil + +} From 4d2a25b72827be2accd6deaec24be59500202d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 11:52:40 +0200 Subject: [PATCH 046/199] Code cleaning --- client/internal/peer/conn.go | 26 ++++++++++++-- client/internal/peer/worker_ice.go | 54 ++++++++++++++-------------- client/internal/peer/worker_relay.go | 44 ++++++++++++----------- 3 files changed, 74 insertions(+), 50 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 49fe10abd..f467d5c0e 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -63,6 +63,15 @@ type ConnConfig struct { type BeforeAddPeerHookFunc func(connID nbnet.ConnectionID, IP net.IP) error type AfterRemovePeerHookFunc func(connID nbnet.ConnectionID) error +type WorkerCallbacks struct { + OnRelayReadyCallback func(info RelayConnInfo) + OnRelayStatusChanged func(ConnStatus) + + OnICEConnReadyCallback func(ConnPriority, ICEConnInfo) + OnICEStatusChanged func(ConnStatus) + DoHandshake func(*OfferAnswer, error) +} + type Conn struct { log *log.Entry mu sync.Mutex @@ -116,8 +125,21 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu statusRelay: StatusDisconnected, statusICE: StatusDisconnected, } - conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, conn.iCEConnectionIsReady, conn.onWorkerICEStateChanged, conn.doHandshake) - conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, conn.relayConnectionIsReady, conn.onWorkerRelayStateChanged, conn.doHandshake) + + rFns := WorkerRelayCallbacks{ + OnConnReady: conn.relayConnectionIsReady, + OnStatusChanged: conn.onWorkerRelayStateChanged, + DoHandshake: conn.doHandshake, + } + + wFns := WorkerICECallbacks{ + OnConnReady: conn.iCEConnectionIsReady, + OnStatusChanged: conn.onWorkerICEStateChanged, + DoHandshake: conn.doHandshake, + } + + conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, rFns) + conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) return conn, nil } diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index ed30ca205..ade33fd1a 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -53,8 +53,6 @@ type ICEConfig struct { NATExternalIPs []string } -type OnICEConnReadyCallback func(ConnPriority, ICEConnInfo) - type ICEConnInfo struct { RemoteConn net.Conn RosenpassPubKey []byte @@ -68,17 +66,21 @@ type ICEConnInfo struct { RelayedOnLocal bool } +type WorkerICECallbacks struct { + OnConnReady func(ConnPriority, ICEConnInfo) + OnStatusChanged func(ConnStatus) + DoHandshake func() (*OfferAnswer, error) +} + type WorkerICE struct { - ctx context.Context - log *log.Entry - config ConnConfig - configICE ICEConfig - signaler *Signaler - iFaceDiscover stdnet.ExternalIFaceDiscover - statusRecorder *Status - onICEConnReady OnICEConnReadyCallback - onStatusChanged func(ConnStatus) - doHandshakeFn DoHandshake + ctx context.Context + log *log.Entry + config ConnConfig + configICE ICEConfig + signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + statusRecorder *Status + conn WorkerICECallbacks selectedPriority ConnPriority @@ -92,18 +94,16 @@ type WorkerICE struct { localPwd string } -func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, onICEConnReady OnICEConnReadyCallback, onStatusChanged func(ConnStatus), doHandshakeFn DoHandshake) *WorkerICE { +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, callBacks WorkerICECallbacks) *WorkerICE { cice := &WorkerICE{ - ctx: ctx, - log: log, - config: config, - configICE: configICE, - signaler: signaler, - iFaceDiscover: ifaceDiscover, - statusRecorder: statusRecorder, - onICEConnReady: onICEConnReady, - onStatusChanged: onStatusChanged, - doHandshakeFn: doHandshakeFn, + ctx: ctx, + log: log, + config: config, + configICE: configICE, + signaler: signaler, + iFaceDiscover: ifaceDiscover, + statusRecorder: statusRecorder, + conn: callBacks, } return cice } @@ -118,9 +118,9 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { return } - w.onStatusChanged(StatusConnecting) + w.conn.OnStatusChanged(StatusConnecting) - remoteOfferAnswer, err := w.doHandshakeFn() + remoteOfferAnswer, err := w.conn.DoHandshake() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { w.log.Infof("signal client isn't ready, skipping connection attempt") @@ -200,12 +200,12 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { Relayed: isRelayed(pair), RelayedOnLocal: isRelayCandidate(pair.Local), } - go w.onICEConnReady(w.selectedPriority, ci) + go w.conn.OnConnReady(w.selectedPriority, ci) <-ctx.Done() ctxCancel() _ = w.agent.Close() - w.onStatusChanged(StatusDisconnected) + w.conn.OnStatusChanged(StatusDisconnected) } } diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 078fe84da..dce9cb423 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -12,33 +12,33 @@ import ( relayClient "github.com/netbirdio/netbird/relay/client" ) -type OnRelayReadyCallback func(info RelayConnInfo) - type RelayConnInfo struct { relayedConn net.Conn rosenpassPubKey []byte rosenpassAddr string } -type WorkerRelay struct { - ctx context.Context - log *log.Entry - relayManager *relayClient.Manager - config ConnConfig - onRelayConnReadyFN OnRelayReadyCallback - onStatusChanged func(ConnStatus) - doHandshakeFn DoHandshake +type WorkerRelayCallbacks struct { + OnConnReady func(RelayConnInfo) + OnStatusChanged func(ConnStatus) + DoHandshake func() (*OfferAnswer, error) } -func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, onRelayConnReadyFN OnRelayReadyCallback, onStatusChanged func(ConnStatus), doHandshakeFn DoHandshake) *WorkerRelay { +type WorkerRelay struct { + ctx context.Context + log *log.Entry + relayManager *relayClient.Manager + config ConnConfig + conn WorkerRelayCallbacks +} + +func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, callbacks WorkerRelayCallbacks) *WorkerRelay { return &WorkerRelay{ - ctx: ctx, - log: log, - relayManager: relayManager, - config: config, - onRelayConnReadyFN: onRelayConnReadyFN, - onStatusChanged: onStatusChanged, - doHandshakeFn: doHandshakeFn, + ctx: ctx, + log: log, + relayManager: relayManager, + config: config, + conn: callbacks, } } @@ -49,7 +49,7 @@ func (w *WorkerRelay) SetupRelayConnection() { return } - remoteOfferAnswer, err := w.doHandshakeFn() + remoteOfferAnswer, err := w.conn.DoHandshake() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { w.log.Infof("signal client isn't ready, skipping connection attempt") @@ -75,7 +75,7 @@ func (w *WorkerRelay) SetupRelayConnection() { continue } - go w.onRelayConnReadyFN(RelayConnInfo{ + go w.conn.OnConnReady(RelayConnInfo{ relayedConn: relayedConn, rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, rosenpassAddr: remoteOfferAnswer.RosenpassAddr, @@ -89,8 +89,10 @@ func (w *WorkerRelay) RelayAddress() (net.Addr, error) { return w.relayManager.RelayAddress() } -// todo check my side too func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { + if !w.relayManager.HasRelayAddress() { + return false + } return answer.RelaySrvAddress != "" } From 11de2ec42e9fbb5e05eb1854cedea58d6782f118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 12:18:58 +0200 Subject: [PATCH 047/199] Fix open connection --- client/internal/engine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 4efe74cab..2438533a1 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -826,7 +826,6 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { if err != nil { return fmt.Errorf("create peer connection: %w", err) } - conn.Open() e.peerConns[peerKey] = conn if e.beforePeerHook != nil && e.afterPeerHook != nil { @@ -838,6 +837,8 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { if err != nil { log.Warnf("error adding peer %s to status recorder, got error: %v", peerKey, err) } + + conn.Open() } return nil } From 0261e15aadd37573c9625abb5171fa95430ba828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 17:40:16 +0200 Subject: [PATCH 048/199] Extend the cmd with argument handling - add cobra to relay server - add logger instance for handshaker --- client/internal/connect.go | 5 ++--- client/internal/peer/conn.go | 23 ++++++++++++++++++----- client/internal/peer/env_config.go | 7 +++++++ client/internal/peer/handshaker.go | 15 +++++++++------ client/internal/peer/worker_ice.go | 1 + client/internal/peer/worker_relay.go | 6 ++++-- relay/client/client.go | 2 ++ relay/cmd/main.go | 28 ++++++++++++++++++++++++---- 8 files changed, 67 insertions(+), 20 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index 2c0c2657b..49544a921 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net" - "os" "runtime" "runtime/debug" "strings" @@ -309,8 +308,8 @@ func (c *ConnectClient) run( } func relayAddress(resp *mgmProto.LoginResponse) string { - if envRelay := os.Getenv("NB_RELAY_ADDRESS"); envRelay != "" { - return envRelay + if ra := peer.ForcedRelayAddress(); ra != "" { + return ra } if resp.GetWiretrusteeConfig().GetRelayAddress() != "" { diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index f467d5c0e..29218c03a 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -111,9 +111,10 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu } ctx, ctxCancel := context.WithCancel(engineCtx) + connLog := log.WithField("peer", config.Key) var conn = &Conn{ - log: log.WithField("peer", config.Key), + log: connLog, ctx: ctx, ctxCancel: ctxCancel, config: config, @@ -121,7 +122,7 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu wgProxyFactory: wgProxyFactory, signaler: signaler, allowedIPsIP: allowedIPsIP.String(), - handshaker: NewHandshaker(ctx, config, signaler), + handshaker: NewHandshaker(ctx, connLog, config, signaler), statusRelay: StatusDisconnected, statusICE: StatusDisconnected, } @@ -138,8 +139,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu DoHandshake: conn.doHandshake, } - conn.workerRelay = NewWorkerRelay(ctx, conn.log, relayManager, config, rFns) - conn.workerICE = NewWorkerICE(ctx, conn.log, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) + conn.workerRelay = NewWorkerRelay(ctx, connLog, relayManager, config, rFns) + conn.workerICE = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) return conn, nil } @@ -148,7 +149,7 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu // be used. // todo implement on disconnected event from ICE and relay too. func (conn *Conn) Open() { - conn.log.Debugf("trying to connect to peer") + conn.log.Debugf("open connection to peer") peerState := State{ PubKey: conn.config.Key, @@ -333,12 +334,18 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { return } + conn.log.Debugf("relay connection is ready") + conn.statusRelay = stateConnected if conn.currentConnType > connPriorityRelay { return } + if conn.currentConnType != 0 { + conn.log.Infof("update connection to Relay type") + } + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) endpoint, err := wgProxy.AddTurnConn(rci.relayedConn) if err != nil { @@ -391,12 +398,18 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon return } + conn.log.Debugf("ICE connection is ready") + conn.statusICE = stateConnected if conn.currentConnType > priority { return } + if conn.currentConnType != 0 { + conn.log.Infof("update connection to ICE type") + } + var ( endpoint net.Addr wgProxy wgproxy.Proxy diff --git a/client/internal/peer/env_config.go b/client/internal/peer/env_config.go index 87b626df7..21269168f 100644 --- a/client/internal/peer/env_config.go +++ b/client/internal/peer/env_config.go @@ -16,6 +16,13 @@ const ( envICEForceRelayConn = "NB_ICE_FORCE_RELAY_CONN" ) +func ForcedRelayAddress() string { + if envRelay := os.Getenv("NB_RELAY_ADDRESS"); envRelay != "" { + return envRelay + } + return "" +} + func iceKeepAlive() time.Duration { keepAliveEnv := os.Getenv(envICEKeepAliveIntervalSec) if keepAliveEnv == "" { diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 797bbb82e..8e5ad7120 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -57,6 +57,7 @@ type HandshakeArgs struct { type Handshaker struct { mu sync.Mutex ctx context.Context + log *log.Entry config ConnConfig signaler *Signaler @@ -71,9 +72,10 @@ type Handshaker struct { handshakeArgs HandshakeArgs } -func NewHandshaker(ctx context.Context, config ConnConfig, signaler *Signaler) *Handshaker { +func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler) *Handshaker { return &Handshaker{ ctx: ctx, + log: log, config: config, signaler: signaler, remoteOffersCh: make(chan OfferAnswer), @@ -92,6 +94,7 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { return cachedOfferAnswer, nil } + h.log.Debugf("send offer") err := h.sendOffer(args) if err != nil { return nil, err @@ -106,8 +109,8 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { } h.storeRemoteOfferAnswer(remoteOfferAnswer) - log.Debugf("received connection confirmation from peer %s running version %s and with remote WireGuard listen port %d", - h.config.Key, remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) + h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", + remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) return remoteOfferAnswer, nil } @@ -119,7 +122,7 @@ func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { case h.remoteOffersCh <- offer: return true default: - log.Debugf("OnRemoteOffer skipping message from peer %s because is not ready", h.config.Key) + h.log.Debugf("OnRemoteOffer skipping message because is not ready") // connection might not be ready yet to receive so we ignore the message return false } @@ -133,7 +136,7 @@ func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { return true default: // connection might not be ready yet to receive so we ignore the message - log.Debugf("OnRemoteAnswer skipping message from peer %s because is not ready", h.config.Key) + h.log.Debugf("OnRemoteAnswer skipping message because is not ready") return false } } @@ -153,7 +156,7 @@ func (h *Handshaker) sendOffer(args HandshakeArgs) error { } func (h *Handshaker) sendAnswer() error { - log.Debugf("sending answer to %s", h.config.Key) + h.log.Debugf("sending answer") answer := OfferAnswer{ IceCredentials: IceCredentials{h.handshakeArgs.IceUFrag, h.handshakeArgs.IcePwd}, WgListenPort: h.config.LocalWgPort, diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index ade33fd1a..5c49b4033 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -215,6 +215,7 @@ func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HA defer w.muxAgent.Unlock() w.log.Debugf("OnRemoteCandidate from peer %s -> %s", w.config.Key, candidate.String()) if w.agent == nil { + w.log.Warnf("ICE Agent is not initialized yet") return } diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index dce9cb423..25eb41738 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -54,12 +54,14 @@ func (w *WorkerRelay) SetupRelayConnection() { if errors.Is(err, ErrSignalIsNotReady) { w.log.Infof("signal client isn't ready, skipping connection attempt") } - w.log.Errorf("failed to do handshake: %v", err) + w.log.Errorf("%s", err) continue } if !w.isRelaySupported(remoteOfferAnswer) { // todo should we retry? + // if the remote peer doesn't support relay make no sense to retry infinity + // but if the remote peer supports relay just the connection is lost we should retry continue } @@ -81,7 +83,7 @@ func (w *WorkerRelay) SetupRelayConnection() { rosenpassAddr: remoteOfferAnswer.RosenpassAddr, }) - // todo: waitForDisconnection() + <-w.ctx.Done() } } diff --git a/relay/client/client.go b/relay/client/client.go index 4e6e63265..d63629030 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -111,6 +111,7 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { // Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. func (c *Client) Connect() error { + log.Infof("connecting to relay server: %s", c.serverAddress) c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() @@ -139,6 +140,7 @@ func (c *Client) Connect() error { c.wgReadLoop.Add(1) go c.readLoop(c.relayConn) + log.Infof("relay connection established with: %s", c.serverAddress) return nil } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index ba3ff47f7..dc5cfe69d 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -6,13 +6,27 @@ import ( "syscall" log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" "github.com/netbirdio/netbird/relay/server" "github.com/netbirdio/netbird/util" ) +var ( + listenAddress string + + rootCmd = &cobra.Command{ + Use: "relay", + Short: "Relay service", + Long: "Relay service for Netbird agents", + Run: execute, + } +) + func init() { - util.InitLog("trace", "console") + _ = util.InitLog("trace", "console") + rootCmd.PersistentFlags().StringVarP(&listenAddress, "listen-address", "l", ":1235", "listen address") + } func waitForExitSignal() { @@ -21,10 +35,9 @@ func waitForExitSignal() { _ = <-osSigs } -func main() { - address := "10.145.236.1:1235" +func execute(cmd *cobra.Command, args []string) { srv := server.NewServer() - err := srv.Listen(address) + err := srv.Listen(listenAddress) if err != nil { log.Errorf("failed to bind server: %s", err) os.Exit(1) @@ -38,3 +51,10 @@ func main() { os.Exit(1) } } + +func main() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} From 81f2330d4968cc43a23d08c5fcbf4ffd31710af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 18:16:23 +0200 Subject: [PATCH 049/199] Fix remote address in ws client --- relay/client/dialer/wsnhooyr/client_conn.go | 13 +++++++------ relay/client/dialer/wsnhooyr/ws.go | 19 ++++++++++++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/relay/client/dialer/wsnhooyr/client_conn.go b/relay/client/dialer/wsnhooyr/client_conn.go index 8f02b6701..0aa995286 100644 --- a/relay/client/dialer/wsnhooyr/client_conn.go +++ b/relay/client/dialer/wsnhooyr/client_conn.go @@ -10,14 +10,16 @@ import ( ) type Conn struct { - *websocket.Conn ctx context.Context + *websocket.Conn + srvAddr *net.TCPAddr } -func NewConn(wsConn *websocket.Conn) net.Conn { +func NewConn(wsConn *websocket.Conn, srvAddr *net.TCPAddr) net.Conn { return &Conn{ - Conn: wsConn, - ctx: context.Background(), + ctx: context.Background(), + Conn: wsConn, + srvAddr: srvAddr, } } @@ -40,8 +42,7 @@ func (c *Conn) Write(b []byte) (n int, err error) { } func (c *Conn) RemoteAddr() net.Addr { - // todo: implement me - return nil + return c.srvAddr } func (c *Conn) LocalAddr() net.Addr { diff --git a/relay/client/dialer/wsnhooyr/ws.go b/relay/client/dialer/wsnhooyr/ws.go index ccec97402..cdb056bce 100644 --- a/relay/client/dialer/wsnhooyr/ws.go +++ b/relay/client/dialer/wsnhooyr/ws.go @@ -5,18 +5,31 @@ import ( "fmt" "net" + log "github.com/sirupsen/logrus" "nhooyr.io/websocket" ) func Dial(address string) (net.Conn, error) { - addr := fmt.Sprintf("ws://" + address) - wsConn, _, err := websocket.Dial(context.Background(), addr, nil) + hostName, _, err := net.SplitHostPort(address) + + addr, err := net.ResolveTCPAddr("tcp", address) + if err != nil { + log.Errorf("failed to resolve address of Relay server: %s", address) + return nil, err + } + + url := fmt.Sprintf("ws://%s:%d"+addr.IP.String(), addr.Port) + opts := &websocket.DialOptions{ + Host: hostName, + } + + wsConn, _, err := websocket.Dial(context.Background(), url, opts) if err != nil { return nil, err } - conn := NewConn(wsConn) + conn := NewConn(wsConn, addr) return conn, nil } From f7e6aa9b8f98a452a135479dcb90ebaed75d44f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 18:16:43 +0200 Subject: [PATCH 050/199] Change logging logic --- client/internal/peer/handshaker.go | 1 - client/internal/peer/worker_ice.go | 2 ++ client/internal/peer/worker_relay.go | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 8e5ad7120..86beb1f1e 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -94,7 +94,6 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { return cachedOfferAnswer, nil } - h.log.Debugf("send offer") err := h.sendOffer(args) if err != nil { return nil, err diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 5c49b4033..601f88934 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -120,6 +120,8 @@ func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { w.conn.OnStatusChanged(StatusConnecting) + w.log.Debugf("trying to establish ICE connection with peer %s", w.config.Key) + remoteOfferAnswer, err := w.conn.DoHandshake() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 25eb41738..a00e8ef90 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -49,6 +49,8 @@ func (w *WorkerRelay) SetupRelayConnection() { return } + w.log.Debugf("trying to establish Relay connection with peer %s", w.config.Key) + remoteOfferAnswer, err := w.conn.DoHandshake() if err != nil { if errors.Is(err, ErrSignalIsNotReady) { @@ -59,6 +61,7 @@ func (w *WorkerRelay) SetupRelayConnection() { } if !w.isRelaySupported(remoteOfferAnswer) { + w.log.Infof("Relay is not supported by remote peer") // todo should we retry? // if the remote peer doesn't support relay make no sense to retry infinity // but if the remote peer supports relay just the connection is lost we should retry @@ -68,12 +71,14 @@ func (w *WorkerRelay) SetupRelayConnection() { // the relayManager will return with error in case if the connection has lost with relay server currentRelayAddress, err := w.relayManager.RelayAddress() if err != nil { + w.log.Infof("local Relay connection is lost, skipping connection attempt") continue } srv := w.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) if err != nil { + w.log.Infof("failed to open relay connection: %s", err) continue } From 4f890ff7127b94a5d83718a6c2a285420ceaf639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 18:17:52 +0200 Subject: [PATCH 051/199] Typo fix --- relay/client/dialer/wsnhooyr/ws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/client/dialer/wsnhooyr/ws.go b/relay/client/dialer/wsnhooyr/ws.go index cdb056bce..be78cf034 100644 --- a/relay/client/dialer/wsnhooyr/ws.go +++ b/relay/client/dialer/wsnhooyr/ws.go @@ -19,7 +19,7 @@ func Dial(address string) (net.Conn, error) { return nil, err } - url := fmt.Sprintf("ws://%s:%d"+addr.IP.String(), addr.Port) + url := fmt.Sprintf("ws://%s:%d", addr.IP.String(), addr.Port) opts := &websocket.DialOptions{ Host: hostName, } From c7db2c05247632f8c02d2d0989d0a895120598b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 19 Jun 2024 18:40:49 +0200 Subject: [PATCH 052/199] Moc signal message support --- client/internal/peer/handshaker.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 86beb1f1e..8441fe64a 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -117,6 +117,10 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise // doesn't block, discards the message if connection wasn't ready func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { + // todo remove this if signaling can support relay + if ForcedRelayAddress() != "" { + offer.RelaySrvAddress = ForcedRelayAddress() + } select { case h.remoteOffersCh <- offer: return true @@ -130,6 +134,10 @@ func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise // doesn't block, discards the message if connection wasn't ready func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { + // todo remove this if signaling can support relay + if ForcedRelayAddress() != "" { + answer.RelaySrvAddress = ForcedRelayAddress() + } select { case h.remoteAnswerCh <- answer: return true From 6801dcb3f67b640f040559f31aa7555a8c9f5841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 20 Jun 2024 18:17:30 +0200 Subject: [PATCH 053/199] Fallback to relay conn --- client/internal/peer/conn.go | 64 ++++++++++++++++++------ client/internal/peer/conn_test.go | 57 +++++++++++++++++++++ client/internal/peer/handshaker.go | 2 + client/internal/peer/worker_ice.go | 1 + client/internal/wgproxy/factory_linux.go | 11 ++++ relay/client/manager.go | 4 +- 6 files changed, 122 insertions(+), 17 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 29218c03a..d4a0dc5bf 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -80,7 +80,8 @@ type Conn struct { config ConnConfig statusRecorder *Status wgProxyFactory *wgproxy.Factory - wgProxy wgproxy.Proxy + wgProxyICE wgproxy.Proxy + wgProxyRelay wgproxy.Proxy signaler *Signaler allowedIPsIP string handshaker *Handshaker @@ -99,6 +100,8 @@ type Conn struct { afterRemovePeerHooks []AfterRemovePeerHookFunc currentConnType ConnPriority + + endpointRelay *net.UDPAddr } // NewConn creates a new not opened Conn to the remote peer. @@ -147,7 +150,6 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu // Open opens connection to the remote peer // It will try to establish a connection using ICE and in parallel with relay. The higher priority connection type will // be used. -// todo implement on disconnected event from ICE and relay too. func (conn *Conn) Open() { conn.log.Debugf("open connection to peer") @@ -176,12 +178,20 @@ func (conn *Conn) Close() { defer conn.mu.Unlock() conn.ctxCancel() - if conn.wgProxy != nil { - err := conn.wgProxy.CloseConn() + if conn.wgProxyRelay != nil { + err := conn.wgProxyRelay.CloseConn() if err != nil { - conn.log.Errorf("failed to close wg proxy: %v", err) + conn.log.Errorf("failed to close wg proxy for relay: %v", err) } - conn.wgProxy = nil + conn.wgProxyRelay = nil + } + + if conn.wgProxyICE != nil { + err := conn.wgProxyICE.CloseConn() + if err != nil { + conn.log.Errorf("failed to close wg proxy for ice: %v", err) + } + conn.wgProxyICE = nil } // todo: is it problem if we try to remove a peer what is never existed? @@ -277,6 +287,7 @@ func (conn *Conn) GetKey() string { func (conn *Conn) onWorkerICEStateChanged(newState ConnStatus) { conn.mu.Lock() defer conn.mu.Unlock() + log.Debugf("ICE connection state changed to %s", newState) defer func() { conn.statusICE = newState }() @@ -289,6 +300,16 @@ func (conn *Conn) onWorkerICEStateChanged(newState ConnStatus) { return } + if conn.endpointRelay != nil { + err := conn.configureWGEndpoint(conn.endpointRelay) + if err != nil { + conn.log.Errorf("failed to switch back to relay conn: %v", err) + } + // todo update status to relay related things + log.Debugf("switched back to relay connection") + return + } + if newState > conn.statusICE { peerState := State{ PubKey: conn.config.Key, @@ -307,6 +328,8 @@ func (conn *Conn) onWorkerRelayStateChanged(newState ConnStatus) { conn.statusRelay = newState }() + conn.log.Debugf("Relay connection state changed to %s", newState) + if conn.statusICE == StatusConnected { return } @@ -363,7 +386,7 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { } } - err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) + err = conn.configureWGEndpoint(endpointUdpAddr) if err != nil { if err := wgProxy.CloseConn(); err != nil { conn.log.Warnf("Failed to close relay connection: %v", err) @@ -371,14 +394,14 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.log.Errorf("Failed to update wg peer configuration: %v", err) return } + conn.endpointRelay = endpointUdpAddr - if conn.wgProxy != nil { - if err := conn.wgProxy.CloseConn(); err != nil { + if conn.wgProxyRelay != nil { + if err := conn.wgProxyRelay.CloseConn(); err != nil { conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) } } - conn.wgProxy = wgProxy - + conn.wgProxyRelay = wgProxy conn.currentConnType = connPriorityRelay peerState := State{ @@ -408,6 +431,8 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon if conn.currentConnType != 0 { conn.log.Infof("update connection to ICE type") + } else { + conn.log.Infof("set ICE to active connection") } var ( @@ -417,7 +442,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon if iceConnInfo.RelayedOnLocal { conn.log.Debugf("setup ice turn connection") wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - ep, err := conn.wgProxy.AddTurnConn(iceConnInfo.RemoteConn) + ep, err := conn.wgProxyICE.AddTurnConn(iceConnInfo.RemoteConn) if err != nil { conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) return @@ -448,12 +473,12 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon return } - if conn.wgProxy != nil { - if err := conn.wgProxy.CloseConn(); err != nil { + if conn.wgProxyICE != nil { + if err := conn.wgProxyICE.CloseConn(); err != nil { conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) } } - conn.wgProxy = wgProxy + conn.wgProxyICE = wgProxy conn.currentConnType = priority @@ -469,6 +494,15 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.updateStatus(peerState, iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) } +func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error { + return conn.config.WgConfig.WgInterface.UpdatePeer( + conn.config.WgConfig.RemoteKey, + conn.config.WgConfig.AllowedIps, + defaultWgKeepAlive, + addr, + conn.config.WgConfig.PreSharedKey, + ) +} func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { peerState.PubKey = conn.config.Key peerState.ConnStatus = StatusConnected diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index 7dbb06b5a..78ccf9724 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -2,15 +2,19 @@ package peer import ( "context" + "os" "sync" "testing" "time" "github.com/magiconair/properties/assert" + log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" + relayClient "github.com/netbirdio/netbird/relay/client" + "github.com/netbirdio/netbird/util" ) var connConf = ConnConfig{ @@ -23,6 +27,12 @@ var connConf = ConnConfig{ }, } +func TestMain(m *testing.M) { + _ = util.InitLog("trace", "console") + code := m.Run() + os.Exit(code) +} + func TestNewConn_interfaceFilter(t *testing.T) { ignore := []string{iface.WgInterfaceDefault, "tun0", "zt", "ZeroTier", "utun", "wg", "ts", "Tailscale", "tailscale"} @@ -158,3 +168,50 @@ func TestConn_Status(t *testing.T) { }) } } + +func TestConn_Switch(t *testing.T) { + ctx := context.Background() + + wgProxyFactory := wgproxy.NewFactory(ctx, connConf.LocalWgPort) + connConfAlice := ConnConfig{ + Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + Timeout: time.Second, + LocalWgPort: 51820, + ICEConfig: ICEConfig{ + InterfaceBlackList: nil, + }, + WgConfig: WgConfig{ + WgListenPort: 51820, + RemoteKey: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + AllowedIps: "172.16.254.0/16", + }, + } + relayManagerAlice := relayClient.NewManager(ctx, "127.0.0.1:1234", connConf.LocalKey) + connAlice, err := NewConn(ctx, connConfAlice, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, relayManagerAlice) + if err != nil { + log.Fatalf("failed to create conn: %v", err) + } + connAlice.Open() + + connConfbob := ConnConfig{ + Key: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + LocalKey: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + Timeout: time.Second, + LocalWgPort: 51820, + ICEConfig: ICEConfig{ + InterfaceBlackList: nil, + }, + WgConfig: WgConfig{ + WgListenPort: 51820, + RemoteKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", + AllowedIps: "172.16.254.0/16", + }, + } + relayManagerBob := relayClient.NewManager(ctx, "127.0.0.1:1234", connConf.LocalKey) + connBob, err := NewConn(ctx, connConfbob, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, relayManagerBob) + if err != nil { + log.Fatalf("failed to create conn: %v", err) + } + connBob.Open() +} diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 8441fe64a..ae96c858f 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -87,6 +87,7 @@ func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { h.mu.Lock() defer h.mu.Unlock() + h.log.Infof("start handshake with remote peer") h.handshakeArgs = args cachedOfferAnswer, ok := h.cachedHandshake() @@ -195,6 +196,7 @@ func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { case remoteOfferAnswer := <-h.remoteAnswerCh: return &remoteOfferAnswer, nil case <-timeout.C: + h.log.Debugf("handshake timeout") return nil, NewConnectionTimeoutError(h.config.Key, h.config.Timeout) case <-h.ctx.Done(): // closed externally diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 601f88934..611bcd72d 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -113,6 +113,7 @@ func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, config // It is trying to reconnection in a loop until the context is canceled. // In case of success connection it will call the onICEConnReady callback. func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { + time.Sleep(20 * time.Second) for { if !w.waitForReconnectTry() { return diff --git a/client/internal/wgproxy/factory_linux.go b/client/internal/wgproxy/factory_linux.go index 79f5cd548..ba1ef8c45 100644 --- a/client/internal/wgproxy/factory_linux.go +++ b/client/internal/wgproxy/factory_linux.go @@ -8,6 +8,17 @@ import ( func NewFactory(ctx context.Context, wgPort int) *Factory { f := &Factory{wgPort: wgPort} + // todo: put it back + /* + ebpfProxy := NewWGEBPFProxy(ctx, wgPort) + err := ebpfProxy.listen() + if err != nil { + log.Warnf("failed to initialize ebpf proxy, fallback to user space proxy: %s", err) + return f + } + f.ebpfProxy = ebpfProxy + + */ return f } diff --git a/relay/client/manager.go b/relay/client/manager.go index a57e3ce55..1810b8167 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -91,10 +91,10 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { } if !foreign { - log.Debugf("open connection to permanent server: %s", peerKey) + log.Debugf("open peer connection via permanent server: %s", peerKey) return m.relayClient.OpenConn(peerKey) } else { - log.Debugf("open connection to foreign server: %s", serverAddress) + log.Debugf("open peer connection via foreign server: %s", serverAddress) return m.openConnVia(serverAddress, peerKey) } } From 06ceac65de1d003d56c0d6d30944064d1efa4764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 21 Jun 2024 00:55:07 +0200 Subject: [PATCH 054/199] - Fix reconnect guard - Avoid double client creation --- relay/client/client.go | 9 +++++++++ relay/client/dialer/wsnhooyr/ws.go | 1 + relay/client/guard.go | 23 ++++++++++++++++------- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index d63629030..bb2ae5b63 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -19,6 +19,10 @@ const ( serverResponseTimeout = 8 * time.Second ) +var ( + ErrConnAlreadyExists = fmt.Errorf("connection already exists") +) + // Msg carry the payload from the server to the client. With this sturct, the net.Conn can free the buffer. type Msg struct { Payload []byte @@ -157,6 +161,11 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { } hashedID, hashedStringID := messages.HashID(dstPeerID) + _, ok := c.conns[hashedStringID] + if ok { + return nil, ErrConnAlreadyExists + } + log.Infof("open connection to peer: %s", hashedStringID) msgChannel := make(chan Msg, 2) conn := NewConn(c, hashedID, hashedStringID, msgChannel) diff --git a/relay/client/dialer/wsnhooyr/ws.go b/relay/client/dialer/wsnhooyr/ws.go index be78cf034..4fe0f0a26 100644 --- a/relay/client/dialer/wsnhooyr/ws.go +++ b/relay/client/dialer/wsnhooyr/ws.go @@ -26,6 +26,7 @@ func Dial(address string) (net.Conn, error) { wsConn, _, err := websocket.Dial(context.Background(), url, opts) if err != nil { + log.Errorf("failed to dial to Relay server '%s': %s", url, err) return nil, err } diff --git a/relay/client/guard.go b/relay/client/guard.go index 12fae4c54..5cea25653 100644 --- a/relay/client/guard.go +++ b/relay/client/guard.go @@ -3,6 +3,8 @@ package client import ( "context" "time" + + log "github.com/sirupsen/logrus" ) var ( @@ -24,13 +26,20 @@ func NewGuard(context context.Context, relayClient *Client) *Guard { } func (g *Guard) OnDisconnected() { - timeout := time.NewTimer(reconnectingTimeout) - defer timeout.Stop() + ticker := time.NewTicker(reconnectingTimeout) + defer ticker.Stop() - select { - case <-timeout.C: - _ = g.relayClient.Connect() - case <-g.ctx.Done(): - return + for { + select { + case <-ticker.C: + err := g.relayClient.Connect() + if err != nil { + log.Errorf("failed to reconnect to relay server: %s", err) + continue + } + return + case <-g.ctx.Done(): + return + } } } From bfe60c01ba2b1283123556bd7194238ba0d0ee4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 21 Jun 2024 00:55:30 +0200 Subject: [PATCH 055/199] Close proxy reading in case of eof --- client/internal/wgproxy/proxy_userspace.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/client/internal/wgproxy/proxy_userspace.go b/client/internal/wgproxy/proxy_userspace.go index 234ea2a42..e51bfbdcd 100644 --- a/client/internal/wgproxy/proxy_userspace.go +++ b/client/internal/wgproxy/proxy_userspace.go @@ -3,6 +3,7 @@ package wgproxy import ( "context" "fmt" + "io" "net" log "github.com/sirupsen/logrus" @@ -64,7 +65,6 @@ func (p *WGUserSpaceProxy) Free() error { // proxyToRemote proxies everything from Wireguard to the RemoteKey peer // blocks func (p *WGUserSpaceProxy) proxyToRemote() { - buf := make([]byte, 1500) for { select { @@ -78,6 +78,9 @@ func (p *WGUserSpaceProxy) proxyToRemote() { _, err = p.remoteConn.Write(buf[:n]) if err != nil { + if err == io.EOF { + p.cancel() + } continue } } @@ -96,6 +99,10 @@ func (p *WGUserSpaceProxy) proxyToLocal() { default: n, err := p.remoteConn.Read(buf) if err != nil { + if err == io.EOF { + p.cancel() + return + } continue } From 4a08f1a1e968756d095f10ef3c097487bb85b85f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 21 Jun 2024 12:35:28 +0200 Subject: [PATCH 056/199] Refactor handshaker loop --- client/internal/peer/conn.go | 27 ++- client/internal/peer/handshaker.go | 148 +++++++------- client/internal/peer/worker_ice.go | 288 ++++++++++++++++----------- client/internal/peer/worker_relay.go | 92 +++------ 4 files changed, 283 insertions(+), 272 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index d4a0dc5bf..a84c84890 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -125,7 +125,6 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu wgProxyFactory: wgProxyFactory, signaler: signaler, allowedIPsIP: allowedIPsIP.String(), - handshaker: NewHandshaker(ctx, connLog, config, signaler), statusRelay: StatusDisconnected, statusICE: StatusDisconnected, } @@ -133,7 +132,6 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu rFns := WorkerRelayCallbacks{ OnConnReady: conn.relayConnectionIsReady, OnStatusChanged: conn.onWorkerRelayStateChanged, - DoHandshake: conn.doHandshake, } wFns := WorkerICECallbacks{ @@ -142,8 +140,13 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu DoHandshake: conn.doHandshake, } + conn.handshaker = NewHandshaker(ctx, connLog, config, signaler, conn.onNewOffer) + go conn.handshaker.Listen() conn.workerRelay = NewWorkerRelay(ctx, connLog, relayManager, config, rFns) - conn.workerICE = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) + conn.workerICE, err = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) + if err != nil { + return nil, err + } return conn, nil } @@ -525,25 +528,21 @@ func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, re return } -func (conn *Conn) doHandshake() (*OfferAnswer, error) { +func (conn *Conn) doHandshake() error { if !conn.signaler.Ready() { - return nil, ErrSignalIsNotReady + return ErrSignalIsNotReady } var ( ha HandshakeArgs err error ) - ha.IceUFrag, ha.IcePwd, err = conn.workerICE.GetLocalUserCredentials() - if err != nil { - conn.log.Errorf("failed to get local user credentials: %v", err) - } - + ha.IceUFrag, ha.IcePwd = conn.workerICE.GetLocalUserCredentials() addr, err := conn.workerRelay.RelayAddress() if err == nil { ha.RelayAddr = addr.String() } - return conn.handshaker.Handshake(ha) + return conn.handshaker.SendOffer(ha) } func (conn *Conn) evalStatus() ConnStatus { @@ -558,6 +557,12 @@ func (conn *Conn) evalStatus() ConnStatus { return StatusDisconnected } +func (conn *Conn) onNewOffer(answer *OfferAnswer) { + // todo move to this callback into handshaker + go conn.workerRelay.OnNewOffer(answer) + go conn.workerICE.OnNewOffer(answer) +} + func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index ae96c858f..18b9408f4 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -54,12 +54,27 @@ type HandshakeArgs struct { RelayAddr string } +func (a HandshakeArgs) Equal(args HandshakeArgs) bool { + if a.IceUFrag != args.IceUFrag { + return false + } + + if a.IcePwd != args.IcePwd { + return false + } + if a.RelayAddr != args.RelayAddr { + return false + } + return true +} + type Handshaker struct { - mu sync.Mutex - ctx context.Context - log *log.Entry - config ConnConfig - signaler *Signaler + mu sync.Mutex + ctx context.Context + log *log.Entry + config ConnConfig + signaler *Signaler + onNewOfferListener func(*OfferAnswer) // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection remoteOffersCh chan OfferAnswer @@ -69,50 +84,54 @@ type Handshaker struct { remoteOfferAnswer *OfferAnswer remoteOfferAnswerCreated time.Time - handshakeArgs HandshakeArgs + lastSentOffer time.Time + lastOfferArgs HandshakeArgs } -func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler) *Handshaker { +func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, onNewOfferListener func(*OfferAnswer)) *Handshaker { return &Handshaker{ - ctx: ctx, - log: log, - config: config, - signaler: signaler, - remoteOffersCh: make(chan OfferAnswer), - remoteAnswerCh: make(chan OfferAnswer), + ctx: ctx, + log: log, + config: config, + signaler: signaler, + remoteOffersCh: make(chan OfferAnswer), + remoteAnswerCh: make(chan OfferAnswer), + onNewOfferListener: onNewOfferListener, } } -func (h *Handshaker) Handshake(args HandshakeArgs) (*OfferAnswer, error) { +func (h *Handshaker) Listen() { + for { + remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() + if err != nil { + if _, ok := err.(*ConnectionClosedError); ok { + return + } + log.Errorf("failed to received remote offer confirmation: %s", err) + continue + } + + h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) + go h.onNewOfferListener(remoteOfferAnswer) + } +} + +func (h *Handshaker) SendOffer(args HandshakeArgs) error { h.mu.Lock() defer h.mu.Unlock() - h.log.Infof("start handshake with remote peer") - h.handshakeArgs = args - - cachedOfferAnswer, ok := h.cachedHandshake() - if ok { - return cachedOfferAnswer, nil + if h.lastOfferArgs.Equal(args) && h.lastSentOffer.After(time.Now().Add(-time.Second)) { + return nil } err := h.sendOffer(args) if err != nil { - return nil, err + return err } - // Only continue once we got a connection confirmation from the remote peer. - // The connection timeout could have happened before a confirmation received from the remote. - // The connection could have also been closed externally (e.g. when we received an update from the management that peer shouldn't be connected) - remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() - if err != nil { - return nil, err - } - h.storeRemoteOfferAnswer(remoteOfferAnswer) - - h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", - remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) - - return remoteOfferAnswer, nil + h.lastOfferArgs = args + h.lastSentOffer = time.Now() + return nil } // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise @@ -149,6 +168,23 @@ func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { } } +func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { + select { + case remoteOfferAnswer := <-h.remoteOffersCh: + // received confirmation from the remote peer -> ready to proceed + err := h.sendAnswer() + if err != nil { + return nil, err + } + return &remoteOfferAnswer, nil + case remoteOfferAnswer := <-h.remoteAnswerCh: + return &remoteOfferAnswer, nil + case <-h.ctx.Done(): + // closed externally + return nil, NewConnectionClosedError(h.config.Key) + } +} + // sendOffer prepares local user credentials and signals them to the remote peer func (h *Handshaker) sendOffer(args HandshakeArgs) error { offer := OfferAnswer{ @@ -166,12 +202,12 @@ func (h *Handshaker) sendOffer(args HandshakeArgs) error { func (h *Handshaker) sendAnswer() error { h.log.Debugf("sending answer") answer := OfferAnswer{ - IceCredentials: IceCredentials{h.handshakeArgs.IceUFrag, h.handshakeArgs.IcePwd}, + IceCredentials: IceCredentials{h.lastOfferArgs.IceUFrag, h.lastOfferArgs.IcePwd}, WgListenPort: h.config.LocalWgPort, Version: version.NetbirdVersion(), RosenpassPubKey: h.config.RosenpassPubKey, RosenpassAddr: h.config.RosenpassAddr, - RelaySrvAddress: h.handshakeArgs.RelayAddr, + RelaySrvAddress: h.lastOfferArgs.RelayAddr, } err := h.signaler.SignalAnswer(answer, h.config.Key) if err != nil { @@ -180,43 +216,3 @@ func (h *Handshaker) sendAnswer() error { return nil } - -func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { - timeout := time.NewTimer(h.config.Timeout) - defer timeout.Stop() - - select { - case remoteOfferAnswer := <-h.remoteOffersCh: - // received confirmation from the remote peer -> ready to proceed - err := h.sendAnswer() - if err != nil { - return nil, err - } - return &remoteOfferAnswer, nil - case remoteOfferAnswer := <-h.remoteAnswerCh: - return &remoteOfferAnswer, nil - case <-timeout.C: - h.log.Debugf("handshake timeout") - return nil, NewConnectionTimeoutError(h.config.Key, h.config.Timeout) - case <-h.ctx.Done(): - // closed externally - return nil, NewConnectionClosedError(h.config.Key) - } -} - -func (h *Handshaker) storeRemoteOfferAnswer(answer *OfferAnswer) { - h.remoteOfferAnswer = answer - h.remoteOfferAnswerCreated = time.Now() -} - -func (h *Handshaker) cachedHandshake() (*OfferAnswer, bool) { - if h.remoteOfferAnswer == nil { - return nil, false - } - - if time.Since(h.remoteOfferAnswerCreated) > handshakeCacheTimeout { - return nil, false - } - - return h.remoteOfferAnswer, true -} diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 611bcd72d..859e5d393 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -2,7 +2,6 @@ package peer import ( "context" - "errors" "fmt" "math/rand" "net" @@ -69,7 +68,7 @@ type ICEConnInfo struct { type WorkerICECallbacks struct { OnConnReady func(ConnPriority, ICEConnInfo) OnStatusChanged func(ConnStatus) - DoHandshake func() (*OfferAnswer, error) + DoHandshake func() error } type WorkerICE struct { @@ -90,12 +89,19 @@ type WorkerICE struct { StunTurn []*stun.URI sentExtraSrflx bool - localUfrag string - localPwd string + + localUfrag string + localPwd string + creadantialHasUsed bool + hasRelayOnLocally bool + onDisconnected context.CancelFunc + onOfferReceived context.CancelFunc + tickerCancel context.CancelFunc + ticker *time.Ticker } -func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, callBacks WorkerICECallbacks) *WorkerICE { - cice := &WorkerICE{ +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, callBacks WorkerICECallbacks) (*WorkerICE, error) { + w := &WorkerICE{ ctx: ctx, log: log, config: config, @@ -105,113 +111,141 @@ func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, config statusRecorder: statusRecorder, conn: callBacks, } - return cice + + localUfrag, localPwd, err := generateICECredentials() + if err != nil { + return nil, err + } + w.localUfrag = localUfrag + w.localPwd = localPwd + return w, nil } -// SetupICEConnection sets up an ICE connection with the remote peer. -// If the relay mode is supported then try to connect in p2p way only. -// It is trying to reconnection in a loop until the context is canceled. -// In case of success connection it will call the onICEConnReady callback. func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { - time.Sleep(20 * time.Second) + w.muxAgent.Lock() + defer w.muxAgent.Unlock() + if w.agent != nil { + return + } + + w.hasRelayOnLocally = hasRelayOnLocally + go w.sendOffer() +} + +func (w *WorkerICE) sendOffer() { + w.ticker = time.NewTicker(w.config.Timeout) + defer w.ticker.Stop() + + tickerCtx, tickerCancel := context.WithCancel(w.ctx) + w.tickerCancel = tickerCancel + w.conn.OnStatusChanged(StatusConnecting) + + w.log.Debugf("ICE trigger a new handshake") + err := w.conn.DoHandshake() + if err != nil { + w.log.Errorf("%s", err) + } + for { - if !w.waitForReconnectTry() { + w.log.Debugf("ICE trigger new reconnect handshake") + select { + case <-w.ticker.C: + err := w.conn.DoHandshake() + if err != nil { + w.log.Errorf("%s", err) + } + case <-tickerCtx.Done(): + w.log.Debugf("left reconnect loop") + return + case <-w.ctx.Done(): return } - - w.conn.OnStatusChanged(StatusConnecting) - - w.log.Debugf("trying to establish ICE connection with peer %s", w.config.Key) - - remoteOfferAnswer, err := w.conn.DoHandshake() - if err != nil { - if errors.Is(err, ErrSignalIsNotReady) { - w.log.Infof("signal client isn't ready, skipping connection attempt") - } - continue - } - - var preferredCandidateTypes []ice.CandidateType - if hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { - w.selectedPriority = connPriorityICEP2P - preferredCandidateTypes = candidateTypesP2P() - } else { - w.selectedPriority = connPriorityICETurn - preferredCandidateTypes = candidateTypes() - } - - ctx, ctxCancel := context.WithCancel(w.ctx) - w.muxAgent.Lock() - agent, err := w.reCreateAgent(ctxCancel, preferredCandidateTypes) - if err != nil { - ctxCancel() - w.muxAgent.Unlock() - continue - } - w.agent = agent - // generate credentials for the next loop. Important the credentials are generated before handshake, because - // the handshake could provide a cached offer-answer - w.localUfrag, w.localPwd, err = generateICECredentials() - if err != nil { - ctxCancel() - w.muxAgent.Unlock() - continue - } - w.muxAgent.Unlock() - - err = w.agent.GatherCandidates() - if err != nil { - ctxCancel() - continue - } - - // will block until connection succeeded - // but it won't release if ICE Agent went into Disconnected or Failed state, - // so we have to cancel it with the provided context once agent detected a broken connection - remoteConn, err := w.turnAgentDial(remoteOfferAnswer) - if err != nil { - ctxCancel() - continue - } - - pair, err := w.agent.GetSelectedCandidatePair() - if err != nil { - ctxCancel() - continue - } - - if !isRelayCandidate(pair.Local) { - // dynamically set remote WireGuard port if other side specified a different one from the default one - remoteWgPort := iface.DefaultWgPort - if remoteOfferAnswer.WgListenPort != 0 { - remoteWgPort = remoteOfferAnswer.WgListenPort - } - - // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port - go w.punchRemoteWGPort(pair, remoteWgPort) - } - - ci := ICEConnInfo{ - RemoteConn: remoteConn, - RosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, - RosenpassAddr: remoteOfferAnswer.RosenpassAddr, - LocalIceCandidateType: pair.Local.Type().String(), - RemoteIceCandidateType: pair.Remote.Type().String(), - LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), - RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), - Direct: !isRelayCandidate(pair.Local), - Relayed: isRelayed(pair), - RelayedOnLocal: isRelayCandidate(pair.Local), - } - go w.conn.OnConnReady(w.selectedPriority, ci) - - <-ctx.Done() - ctxCancel() - _ = w.agent.Close() - w.conn.OnStatusChanged(StatusDisconnected) } } +func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { + log.Debugf("OnNewOffer for ICE") + w.muxAgent.Lock() + + if w.agent != nil { + log.Debugf("agent already exists, skipping the offer") + w.muxAgent.Unlock() + return + } + + // cancel reconnection loop + w.log.Debugf("canceling reconnection loop") + w.tickerCancel() + + var preferredCandidateTypes []ice.CandidateType + if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { + w.selectedPriority = connPriorityICEP2P + preferredCandidateTypes = candidateTypesP2P() + } else { + w.selectedPriority = connPriorityICETurn + preferredCandidateTypes = candidateTypes() + } + + w.log.Debugf("recreate agent") + agentCtx, agentCancel := context.WithCancel(w.ctx) + agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes) + if err != nil { + w.log.Errorf("failed to recreate ICE Agent: %s", err) + return + } + w.agent = agent + w.muxAgent.Unlock() + + w.log.Debugf("gather candidates") + err = w.agent.GatherCandidates() + if err != nil { + w.log.Debugf("failed to gather candidates: %s", err) + return + } + + // will block until connection succeeded + // but it won't release if ICE Agent went into Disconnected or Failed state, + // so we have to cancel it with the provided context once agent detected a broken connection + w.log.Debugf("turnAgentDial") + remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer) + if err != nil { + w.log.Debugf("failed to dial the remote peer: %s", err) + return + } + + w.log.Debugf("GetSelectedCandidatePair") + pair, err := w.agent.GetSelectedCandidatePair() + if err != nil { + return + } + + if !isRelayCandidate(pair.Local) { + // dynamically set remote WireGuard port if other side specified a different one from the default one + remoteWgPort := iface.DefaultWgPort + if remoteOfferAnswer.WgListenPort != 0 { + remoteWgPort = remoteOfferAnswer.WgListenPort + } + + // To support old version's with direct mode we attempt to punch an additional role with the remote WireGuard port + go w.punchRemoteWGPort(pair, remoteWgPort) + } + + ci := ICEConnInfo{ + RemoteConn: remoteConn, + RosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + RosenpassAddr: remoteOfferAnswer.RosenpassAddr, + LocalIceCandidateType: pair.Local.Type().String(), + RemoteIceCandidateType: pair.Remote.Type().String(), + LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), + RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), + Direct: !isRelayCandidate(pair.Local), + Relayed: isRelayed(pair), + RelayedOnLocal: isRelayCandidate(pair.Local), + } + w.log.Debugf("on conn ready") + go w.conn.OnConnReady(w.selectedPriority, ci) +} + // OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer. func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HAMap) { w.muxAgent.Lock() @@ -233,19 +267,14 @@ func (w *WorkerICE) OnRemoteCandidate(candidate ice.Candidate, haRoutes route.HA } } -func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string, err error) { +func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) { w.muxAgent.Lock() defer w.muxAgent.Unlock() - - if w.localUfrag != "" && w.localPwd != "" { - return w.localUfrag, w.localPwd, nil - } - - w.localUfrag, w.localPwd, err = generateICECredentials() - return w.localUfrag, w.localPwd, err + return w.localUfrag, w.localPwd } -func (w *WorkerICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { +func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { + log.Debugf("--RECREATE AGENT-----") transportNet, err := w.newStdNet() if err != nil { w.log.Errorf("failed to create pion's stdnet: %s", err) @@ -256,9 +285,9 @@ func (w *WorkerICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []i iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() agentConfig := &ice.AgentConfig{ - MulticastDNSMode: ice.MulticastDNSModeDisabled, - NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: w.configICE.StunTurn.Load().([]*stun.URI), + MulticastDNSMode: ice.MulticastDNSModeDisabled, + NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, + //Urls: w.configICE.StunTurn.Load().([]*stun.URI), CandidateTypes: relaySupport, InterfaceFilter: stdnet.InterfaceFilter(w.configICE.InterfaceBlackList), UDPMux: w.configICE.UDPMux, @@ -291,7 +320,23 @@ func (w *WorkerICE) reCreateAgent(ctxCancel context.CancelFunc, relaySupport []i err = agent.OnConnectionStateChange(func(state ice.ConnectionState) { w.log.Debugf("ICE ConnectionState has changed to %s", state.String()) if state == ice.ConnectionStateFailed || state == ice.ConnectionStateDisconnected { - ctxCancel() + w.conn.OnStatusChanged(StatusDisconnected) + + w.muxAgent.Lock() + agentCancel() + _ = agent.Close() + w.agent = nil + + // generate credentials for the next agent creation loop + localUfrag, localPwd, err := generateICECredentials() + if err != nil { + log.Errorf("failed to generate new ICE credentials: %s", err) + } + w.localUfrag = localUfrag + w.localPwd = localPwd + + w.muxAgent.Unlock() + go w.sendOffer() } }) if err != nil { @@ -387,12 +432,12 @@ func (w *WorkerICE) shouldSendExtraSrflxCandidate(candidate ice.Candidate) bool return false } -func (w *WorkerICE) turnAgentDial(remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { +func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) { isControlling := w.config.LocalKey > w.config.Key if isControlling { - return w.agent.Dial(w.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + return w.agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) } else { - return w.agent.Accept(w.ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) + return w.agent.Accept(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd) } } @@ -465,7 +510,7 @@ func candidateTypes() []ice.CandidateType { } func candidateTypesP2P() []ice.CandidateType { - return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} + return []ice.CandidateType{ice.CandidateTypeHost} } func isRelayCandidate(candidate ice.Candidate) bool { @@ -480,6 +525,7 @@ func isRelayed(pair *ice.CandidatePair) bool { } func generateICECredentials() (string, string, error) { + log.Debugf("-----GENERATE CREDENTIALS------") ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha) if err != nil { return "", "", err diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index a00e8ef90..53b32dc22 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -2,10 +2,7 @@ package peer import ( "context" - "errors" - "math/rand" "net" - "time" log "github.com/sirupsen/logrus" @@ -21,7 +18,6 @@ type RelayConnInfo struct { type WorkerRelayCallbacks struct { OnConnReady func(RelayConnInfo) OnStatusChanged func(ConnStatus) - DoHandshake func() (*OfferAnswer, error) } type WorkerRelay struct { @@ -44,52 +40,37 @@ func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClie // SetupRelayConnection todo: this function is not completed. Make no sense to put it in a for loop because we are not waiting for any event func (w *WorkerRelay) SetupRelayConnection() { - for { - if !w.waitForReconnectTry() { - return - } - w.log.Debugf("trying to establish Relay connection with peer %s", w.config.Key) +} - remoteOfferAnswer, err := w.conn.DoHandshake() - if err != nil { - if errors.Is(err, ErrSignalIsNotReady) { - w.log.Infof("signal client isn't ready, skipping connection attempt") - } - w.log.Errorf("%s", err) - continue - } - - if !w.isRelaySupported(remoteOfferAnswer) { - w.log.Infof("Relay is not supported by remote peer") - // todo should we retry? - // if the remote peer doesn't support relay make no sense to retry infinity - // but if the remote peer supports relay just the connection is lost we should retry - continue - } - - // the relayManager will return with error in case if the connection has lost with relay server - currentRelayAddress, err := w.relayManager.RelayAddress() - if err != nil { - w.log.Infof("local Relay connection is lost, skipping connection attempt") - continue - } - - srv := w.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) - relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) - if err != nil { - w.log.Infof("failed to open relay connection: %s", err) - continue - } - - go w.conn.OnConnReady(RelayConnInfo{ - relayedConn: relayedConn, - rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, - rosenpassAddr: remoteOfferAnswer.RosenpassAddr, - }) - - <-w.ctx.Done() +func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { + if !w.isRelaySupported(remoteOfferAnswer) { + w.log.Infof("Relay is not supported by remote peer") + // todo should we retry? + // if the remote peer doesn't support relay make no sense to retry infinity + // but if the remote peer supports relay just the connection is lost we should retry + return } + + // the relayManager will return with error in case if the connection has lost with relay server + currentRelayAddress, err := w.relayManager.RelayAddress() + if err != nil { + w.log.Infof("local Relay connection is lost, skipping connection attempt") + return + } + + srv := w.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) + if err != nil { + w.log.Infof("do not need to reopen relay connection: %s", err) + return + } + + go w.conn.OnConnReady(RelayConnInfo{ + relayedConn: relayedConn, + rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, + rosenpassAddr: remoteOfferAnswer.RosenpassAddr, + }) } func (w *WorkerRelay) RelayAddress() (net.Addr, error) { @@ -113,20 +94,3 @@ func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress st func (w *WorkerRelay) RelayIsSupportedLocally() bool { return w.relayManager.HasRelayAddress() } - -// waitForReconnectTry waits for a random duration before trying to reconnect -func (w *WorkerRelay) waitForReconnectTry() bool { - minWait := 500 - maxWait := 2000 - duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond - - timeout := time.NewTimer(duration) - defer timeout.Stop() - - select { - case <-w.ctx.Done(): - return false - case <-timeout.C: - return true - } -} From 4d67d72785a97a9f64f8696b8b96c98d8474670f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 21 Jun 2024 15:02:54 +0200 Subject: [PATCH 057/199] Use permanent credentials --- client/internal/peer/handshaker.go | 25 +++---------------------- client/internal/peer/worker_ice.go | 20 ++++---------------- 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 18b9408f4..04a93e454 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -19,8 +19,6 @@ var ( ErrSignalIsNotReady = errors.New("signal is not ready") ) -type DoHandshake func() (*OfferAnswer, error) - // IceCredentials ICE protocol credentials struct type IceCredentials struct { UFrag string @@ -54,20 +52,6 @@ type HandshakeArgs struct { RelayAddr string } -func (a HandshakeArgs) Equal(args HandshakeArgs) bool { - if a.IceUFrag != args.IceUFrag { - return false - } - - if a.IcePwd != args.IcePwd { - return false - } - if a.RelayAddr != args.RelayAddr { - return false - } - return true -} - type Handshaker struct { mu sync.Mutex ctx context.Context @@ -84,7 +68,6 @@ type Handshaker struct { remoteOfferAnswer *OfferAnswer remoteOfferAnswerCreated time.Time - lastSentOffer time.Time lastOfferArgs HandshakeArgs } @@ -105,6 +88,7 @@ func (h *Handshaker) Listen() { remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() if err != nil { if _, ok := err.(*ConnectionClosedError); ok { + log.Tracef("stop handshaker") return } log.Errorf("failed to received remote offer confirmation: %s", err) @@ -120,17 +104,12 @@ func (h *Handshaker) SendOffer(args HandshakeArgs) error { h.mu.Lock() defer h.mu.Unlock() - if h.lastOfferArgs.Equal(args) && h.lastSentOffer.After(time.Now().Add(-time.Second)) { - return nil - } - err := h.sendOffer(args) if err != nil { return err } h.lastOfferArgs = args - h.lastSentOffer = time.Now() return nil } @@ -187,6 +166,7 @@ func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { // sendOffer prepares local user credentials and signals them to the remote peer func (h *Handshaker) sendOffer(args HandshakeArgs) error { + log.Debugf("SEND OFFER: %s", args.IceUFrag) offer := OfferAnswer{ IceCredentials: IceCredentials{args.IceUFrag, args.IcePwd}, WgListenPort: h.config.LocalWgPort, @@ -201,6 +181,7 @@ func (h *Handshaker) sendOffer(args HandshakeArgs) error { func (h *Handshaker) sendAnswer() error { h.log.Debugf("sending answer") + log.Debugf("SEND ANSWER: %s", h.lastOfferArgs.IceUFrag) answer := OfferAnswer{ IceCredentials: IceCredentials{h.lastOfferArgs.IceUFrag, h.lastOfferArgs.IcePwd}, WgListenPort: h.config.LocalWgPort, diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 859e5d393..93ad070ae 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -94,8 +94,6 @@ type WorkerICE struct { localPwd string creadantialHasUsed bool hasRelayOnLocally bool - onDisconnected context.CancelFunc - onOfferReceived context.CancelFunc tickerCancel context.CancelFunc ticker *time.Ticker } @@ -274,7 +272,6 @@ func (w *WorkerICE) GetLocalUserCredentials() (frag string, pwd string) { } func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { - log.Debugf("--RECREATE AGENT-----") transportNet, err := w.newStdNet() if err != nil { w.log.Errorf("failed to create pion's stdnet: %s", err) @@ -285,9 +282,9 @@ func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport [ iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() agentConfig := &ice.AgentConfig{ - MulticastDNSMode: ice.MulticastDNSModeDisabled, - NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - //Urls: w.configICE.StunTurn.Load().([]*stun.URI), + MulticastDNSMode: ice.MulticastDNSModeDisabled, + NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, + Urls: w.configICE.StunTurn.Load().([]*stun.URI), CandidateTypes: relaySupport, InterfaceFilter: stdnet.InterfaceFilter(w.configICE.InterfaceBlackList), UDPMux: w.configICE.UDPMux, @@ -327,14 +324,6 @@ func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport [ _ = agent.Close() w.agent = nil - // generate credentials for the next agent creation loop - localUfrag, localPwd, err := generateICECredentials() - if err != nil { - log.Errorf("failed to generate new ICE credentials: %s", err) - } - w.localUfrag = localUfrag - w.localPwd = localPwd - w.muxAgent.Unlock() go w.sendOffer() } @@ -510,7 +499,7 @@ func candidateTypes() []ice.CandidateType { } func candidateTypesP2P() []ice.CandidateType { - return []ice.CandidateType{ice.CandidateTypeHost} + return []ice.CandidateType{ice.CandidateTypeHost, ice.CandidateTypeServerReflexive} } func isRelayCandidate(candidate ice.Candidate) bool { @@ -525,7 +514,6 @@ func isRelayed(pair *ice.CandidatePair) bool { } func generateICECredentials() (string, string, error) { - log.Debugf("-----GENERATE CREDENTIALS------") ufrag, err := randutil.GenerateCryptoRandomString(lenUFrag, runesAlpha) if err != nil { return "", "", err From 7581bbd925d0040d59eedbc45e28fc412549f6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 21 Jun 2024 15:35:15 +0200 Subject: [PATCH 058/199] Handle on offer listener in handshaker --- client/internal/peer/conn.go | 20 +++++++--------- client/internal/peer/handshaker.go | 35 ++++++++++++++++------------ client/internal/peer/worker_relay.go | 11 +++------ 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index a84c84890..783f6cba4 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -140,13 +140,18 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu DoHandshake: conn.doHandshake, } - conn.handshaker = NewHandshaker(ctx, connLog, config, signaler, conn.onNewOffer) - go conn.handshaker.Listen() - conn.workerRelay = NewWorkerRelay(ctx, connLog, relayManager, config, rFns) + conn.handshaker = NewHandshaker(ctx, connLog, config, signaler) + conn.workerRelay = NewWorkerRelay(ctx, connLog, config, relayManager, rFns) conn.workerICE, err = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) if err != nil { return nil, err } + + conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer) + conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer) + + go conn.handshaker.Listen() + return conn, nil } @@ -169,9 +174,6 @@ func (conn *Conn) Open() { } relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() - if relayIsSupportedLocally { - go conn.workerRelay.SetupRelayConnection() - } go conn.workerICE.SetupICEConnection(relayIsSupportedLocally) } @@ -557,12 +559,6 @@ func (conn *Conn) evalStatus() ConnStatus { return StatusDisconnected } -func (conn *Conn) onNewOffer(answer *OfferAnswer) { - // todo move to this callback into handshaker - go conn.workerRelay.OnNewOffer(answer) - go conn.workerICE.OnNewOffer(answer) -} - func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 04a93e454..881d985be 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -53,12 +53,12 @@ type HandshakeArgs struct { } type Handshaker struct { - mu sync.Mutex - ctx context.Context - log *log.Entry - config ConnConfig - signaler *Signaler - onNewOfferListener func(*OfferAnswer) + mu sync.Mutex + ctx context.Context + log *log.Entry + config ConnConfig + signaler *Signaler + onNewOfferListeners []func(*OfferAnswer) // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection remoteOffersCh chan OfferAnswer @@ -71,18 +71,21 @@ type Handshaker struct { lastOfferArgs HandshakeArgs } -func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, onNewOfferListener func(*OfferAnswer)) *Handshaker { +func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler) *Handshaker { return &Handshaker{ - ctx: ctx, - log: log, - config: config, - signaler: signaler, - remoteOffersCh: make(chan OfferAnswer), - remoteAnswerCh: make(chan OfferAnswer), - onNewOfferListener: onNewOfferListener, + ctx: ctx, + log: log, + config: config, + signaler: signaler, + remoteOffersCh: make(chan OfferAnswer), + remoteAnswerCh: make(chan OfferAnswer), } } +func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAnswer)) { + h.onNewOfferListeners = append(h.onNewOfferListeners, offer) +} + func (h *Handshaker) Listen() { for { remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() @@ -96,7 +99,9 @@ func (h *Handshaker) Listen() { } h.log.Debugf("received connection confirmation, running version %s and with remote WireGuard listen port %d", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort) - go h.onNewOfferListener(remoteOfferAnswer) + for _, listener := range h.onNewOfferListeners { + go listener(remoteOfferAnswer) + } } } diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 53b32dc22..158b8e068 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -23,26 +23,21 @@ type WorkerRelayCallbacks struct { type WorkerRelay struct { ctx context.Context log *log.Entry - relayManager *relayClient.Manager config ConnConfig + relayManager *relayClient.Manager conn WorkerRelayCallbacks } -func NewWorkerRelay(ctx context.Context, log *log.Entry, relayManager *relayClient.Manager, config ConnConfig, callbacks WorkerRelayCallbacks) *WorkerRelay { +func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager *relayClient.Manager, callbacks WorkerRelayCallbacks) *WorkerRelay { return &WorkerRelay{ ctx: ctx, log: log, - relayManager: relayManager, config: config, + relayManager: relayManager, conn: callbacks, } } -// SetupRelayConnection todo: this function is not completed. Make no sense to put it in a for loop because we are not waiting for any event -func (w *WorkerRelay) SetupRelayConnection() { - -} - func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { if !w.isRelaySupported(remoteOfferAnswer) { w.log.Infof("Relay is not supported by remote peer") From 69d8d5aa86773c24da2ea84719b3fb95c48c8516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 21 Jun 2024 19:13:41 +0200 Subject: [PATCH 059/199] Fix the active conn type logic --- client/internal/peer/conn.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 783f6cba4..3f7003187 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -89,8 +89,9 @@ type Conn struct { onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) onDisconnected func(remotePeer string, wgIP string) - statusRelay ConnStatus - statusICE ConnStatus + statusRelay ConnStatus + statusICE ConnStatus + currentConnType ConnPriority workerICE *WorkerICE workerRelay *WorkerRelay @@ -99,8 +100,6 @@ type Conn struct { beforeAddPeerHooks []BeforeAddPeerHookFunc afterRemovePeerHooks []AfterRemovePeerHookFunc - currentConnType ConnPriority - endpointRelay *net.UDPAddr } @@ -366,8 +365,12 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.statusRelay = stateConnected + // todo review this condition if conn.currentConnType > connPriorityRelay { - return + if conn.statusICE == StatusConnected { + log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnType) + return + } } if conn.currentConnType != 0 { @@ -430,6 +433,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.statusICE = stateConnected + // todo review this condition if conn.currentConnType > priority { return } From 54dc78aab87e3dab4cdc80153bb3dcf9fa89ef91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 24 Jun 2024 15:30:25 +0200 Subject: [PATCH 060/199] Remove debug log --- client/internal/peer/handshaker.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 881d985be..8d4377e36 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -171,7 +171,6 @@ func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { // sendOffer prepares local user credentials and signals them to the remote peer func (h *Handshaker) sendOffer(args HandshakeArgs) error { - log.Debugf("SEND OFFER: %s", args.IceUFrag) offer := OfferAnswer{ IceCredentials: IceCredentials{args.IceUFrag, args.IcePwd}, WgListenPort: h.config.LocalWgPort, @@ -186,7 +185,6 @@ func (h *Handshaker) sendOffer(args HandshakeArgs) error { func (h *Handshaker) sendAnswer() error { h.log.Debugf("sending answer") - log.Debugf("SEND ANSWER: %s", h.lastOfferArgs.IceUFrag) answer := OfferAnswer{ IceCredentials: IceCredentials{h.lastOfferArgs.IceUFrag, h.lastOfferArgs.IcePwd}, WgListenPort: h.config.LocalWgPort, From f72e852ccb91145af981af4eb262230e9f0cabcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 24 Jun 2024 18:54:03 +0200 Subject: [PATCH 061/199] Remove duplicated code --- relay/client/client.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index bb2ae5b63..612080abd 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -401,16 +401,6 @@ func (c *Client) closeConn(id string) error { return nil } -func (c *Client) onDisconnect() { - c.listenerMutex.Lock() - defer c.listenerMutex.Unlock() - - if c.onDisconnectListener == nil { - return - } - c.onDisconnectListener() -} - func (c *Client) notifyDisconnected() { c.listenerMutex.Lock() defer c.listenerMutex.Unlock() From 0a67f5be1a3251924e4ded6e182906b7f6ecf896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 25 Jun 2024 15:13:08 +0200 Subject: [PATCH 062/199] Fix logic --- client/internal/peer/conn.go | 370 ++++++++++++--------- client/internal/peer/handshaker.go | 9 +- client/internal/peer/worker_ice.go | 122 ++----- client/internal/peer/worker_relay.go | 31 +- client/internal/wgproxy/proxy_userspace.go | 5 + relay/client/client.go | 2 +- relay/client/guard.go | 1 - relay/client/manager.go | 77 ++++- relay/client/manager_test.go | 12 +- relay/cmd/main.go | 2 +- relay/messages/id.go | 2 +- relay/server/server.go | 3 +- 12 files changed, 345 insertions(+), 291 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 3f7003187..f93d4e8a6 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -2,6 +2,7 @@ package peer import ( "context" + "math/rand" "net" "runtime" "strings" @@ -26,8 +27,8 @@ const ( defaultWgKeepAlive = 25 * time.Second connPriorityRelay ConnPriority = 1 - connPriorityICETurn = 1 - connPriorityICEP2P = 2 + connPriorityICETurn ConnPriority = 1 + connPriorityICEP2P ConnPriority = 2 ) type WgConfig struct { @@ -69,7 +70,6 @@ type WorkerCallbacks struct { OnICEConnReadyCallback func(ConnPriority, ICEConnInfo) OnICEStatusChanged func(ConnStatus) - DoHandshake func(*OfferAnswer, error) } type Conn struct { @@ -83,6 +83,7 @@ type Conn struct { wgProxyICE wgproxy.Proxy wgProxyRelay wgproxy.Proxy signaler *Signaler + relayManager *relayClient.Manager allowedIPsIP string handshaker *Handshaker @@ -101,6 +102,9 @@ type Conn struct { afterRemovePeerHooks []AfterRemovePeerHookFunc endpointRelay *net.UDPAddr + + iCEDisconnected chan struct{} + relayDisconnected chan struct{} } // NewConn creates a new not opened Conn to the remote peer. @@ -116,32 +120,36 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu connLog := log.WithField("peer", config.Key) var conn = &Conn{ - log: connLog, - ctx: ctx, - ctxCancel: ctxCancel, - config: config, - statusRecorder: statusRecorder, - wgProxyFactory: wgProxyFactory, - signaler: signaler, - allowedIPsIP: allowedIPsIP.String(), - statusRelay: StatusDisconnected, - statusICE: StatusDisconnected, + log: connLog, + ctx: ctx, + ctxCancel: ctxCancel, + config: config, + statusRecorder: statusRecorder, + wgProxyFactory: wgProxyFactory, + signaler: signaler, + relayManager: relayManager, + allowedIPsIP: allowedIPsIP.String(), + statusRelay: StatusDisconnected, + statusICE: StatusDisconnected, + iCEDisconnected: make(chan struct{}), + relayDisconnected: make(chan struct{}), } rFns := WorkerRelayCallbacks{ - OnConnReady: conn.relayConnectionIsReady, - OnStatusChanged: conn.onWorkerRelayStateChanged, + OnConnReady: conn.relayConnectionIsReady, + OnDisconnected: conn.onWorkerRelayStateDisconnected, } wFns := WorkerICECallbacks{ OnConnReady: conn.iCEConnectionIsReady, - OnStatusChanged: conn.onWorkerICEStateChanged, - DoHandshake: conn.doHandshake, + OnStatusChanged: conn.onWorkerICEStateDisconnected, } conn.handshaker = NewHandshaker(ctx, connLog, config, signaler) conn.workerRelay = NewWorkerRelay(ctx, connLog, config, relayManager, rFns) - conn.workerICE, err = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, wFns) + + relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() + conn.workerICE, err = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns) if err != nil { return nil, err } @@ -172,14 +180,21 @@ func (conn *Conn) Open() { conn.log.Warnf("error while updating the state err: %v", err) } - relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() - go conn.workerICE.SetupICEConnection(relayIsSupportedLocally) + conn.waitRandomSleepTime() + + err = conn.doHandshake() + if err != nil { + conn.log.Errorf("failed to send offer: %v", err) + } + + go conn.reconnectLoop() } // Close closes this peer Conn issuing a close event to the Conn closeCh func (conn *Conn) Close() { conn.mu.Lock() defer conn.mu.Unlock() + conn.ctxCancel() if conn.wgProxyRelay != nil { @@ -198,7 +213,6 @@ func (conn *Conn) Close() { conn.wgProxyICE = nil } - // todo: is it problem if we try to remove a peer what is never existed? err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) if err != nil { conn.log.Errorf("failed to remove wg endpoint: %v", err) @@ -268,7 +282,7 @@ func (conn *Conn) SetOnDisconnected(handler func(remotePeer string, wgIP string) } func (conn *Conn) OnRemoteOffer(offer OfferAnswer) bool { - conn.log.Debugf("OnRemoteOffer, on status ICE: %s, status relay: %s", conn.statusICE, conn.statusRelay) + conn.log.Debugf("OnRemoteOffer, on status ICE: %s, status Relay: %s", conn.statusICE, conn.statusRelay) return conn.handshaker.OnRemoteOffer(offer) } @@ -288,136 +302,39 @@ func (conn *Conn) GetKey() string { return conn.config.Key } -func (conn *Conn) onWorkerICEStateChanged(newState ConnStatus) { - conn.mu.Lock() - defer conn.mu.Unlock() - log.Debugf("ICE connection state changed to %s", newState) - defer func() { - conn.statusICE = newState - }() - - if conn.statusRelay == StatusConnected { - return +func (conn *Conn) reconnectLoop() { + ticker := time.NewTicker(conn.config.Timeout) // todo use the interval from config + if !conn.workerRelay.IsController() { + ticker.Stop() + } else { + defer ticker.Stop() } - if conn.evalStatus() == newState { - return - } - - if conn.endpointRelay != nil { - err := conn.configureWGEndpoint(conn.endpointRelay) - if err != nil { - conn.log.Errorf("failed to switch back to relay conn: %v", err) - } - // todo update status to relay related things - log.Debugf("switched back to relay connection") - return - } - - if newState > conn.statusICE { - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: newState, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - _ = conn.statusRecorder.UpdatePeerState(peerState) - } -} - -func (conn *Conn) onWorkerRelayStateChanged(newState ConnStatus) { - conn.mu.Lock() - defer conn.mu.Unlock() - defer func() { - conn.statusRelay = newState - }() - - conn.log.Debugf("Relay connection state changed to %s", newState) - - if conn.statusICE == StatusConnected { - return - } - - if conn.evalStatus() == newState { - return - } - - if newState > conn.statusRelay { - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: newState, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - _ = conn.statusRecorder.UpdatePeerState(peerState) - } -} - -func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { - conn.mu.Lock() - defer conn.mu.Unlock() - - if conn.ctx.Err() != nil { - return - } - - conn.log.Debugf("relay connection is ready") - - conn.statusRelay = stateConnected - - // todo review this condition - if conn.currentConnType > connPriorityRelay { - if conn.statusICE == StatusConnected { - log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnType) + for { + select { + case <-ticker.C: + // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer + // todo check wg handshake + if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { + continue + } + case <-conn.relayDisconnected: + conn.log.Debugf("Relay connection is disconnected, start to send new offer") + ticker.Reset(10 * time.Second) + conn.waitRandomSleepTime() + case <-conn.iCEDisconnected: + conn.log.Debugf("ICE connection is disconnected, start to send new offer") + ticker.Reset(10 * time.Second) + conn.waitRandomSleepTime() + case <-conn.ctx.Done(): return } - } - if conn.currentConnType != 0 { - conn.log.Infof("update connection to Relay type") - } - - wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) - endpoint, err := wgProxy.AddTurnConn(rci.relayedConn) - if err != nil { - conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err) - return - } - - endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) - conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) - - conn.connID = nbnet.GenerateConnID() - for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { - conn.log.Errorf("Before add peer hook failed: %v", err) + err := conn.doHandshake() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) } } - - err = conn.configureWGEndpoint(endpointUdpAddr) - if err != nil { - if err := wgProxy.CloseConn(); err != nil { - conn.log.Warnf("Failed to close relay connection: %v", err) - } - conn.log.Errorf("Failed to update wg peer configuration: %v", err) - return - } - conn.endpointRelay = endpointUdpAddr - - if conn.wgProxyRelay != nil { - if err := conn.wgProxyRelay.CloseConn(); err != nil { - conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) - } - } - conn.wgProxyRelay = wgProxy - conn.currentConnType = connPriorityRelay - - peerState := State{ - Direct: false, - Relayed: true, - } - - conn.updateStatus(peerState, rci.rosenpassPubKey, rci.rosenpassAddr) } // configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected @@ -431,9 +348,8 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.log.Debugf("ICE connection is ready") - conn.statusICE = stateConnected + conn.statusICE = StatusConnected - // todo review this condition if conn.currentConnType > priority { return } @@ -503,6 +419,150 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.updateStatus(peerState, iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) } +// todo review to make sense to handle connection and disconnected status also? +func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { + conn.mu.Lock() + defer conn.mu.Unlock() + + conn.log.Tracef("ICE connection state changed to %s", newState) + defer func() { + conn.statusICE = newState + + select { + case conn.iCEDisconnected <- struct{}{}: + default: + } + }() + + // switch back to relay connection + if conn.endpointRelay != nil { + conn.log.Debugf("ICE disconnected, set Relay to active connection") + err := conn.configureWGEndpoint(conn.endpointRelay) + if err != nil { + conn.log.Errorf("failed to switch to relay conn: %v", err) + } + // todo update status to relay related things + return + } + + if conn.statusRelay == StatusConnected { + return + } + + if conn.evalStatus() == newState { + return + } + + if newState > conn.statusICE { + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: newState, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + _ = conn.statusRecorder.UpdatePeerState(peerState) + } +} + +func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.ctx.Err() != nil { + return + } + + conn.log.Debugf("Relay connection is ready to use") + conn.statusRelay = StatusConnected + + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + endpoint, err := wgProxy.AddTurnConn(rci.relayedConn) + if err != nil { + conn.log.Errorf("failed to add relayed net.Conn to local proxy: %v", err) + return + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + conn.endpointRelay = endpointUdpAddr + conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + + if conn.currentConnType > connPriorityRelay { + if conn.statusICE == StatusConnected { + log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnType) + return + } + } + + conn.connID = nbnet.GenerateConnID() + for _, hook := range conn.beforeAddPeerHooks { + if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { + conn.log.Errorf("Before add peer hook failed: %v", err) + } + } + + err = conn.configureWGEndpoint(endpointUdpAddr) + if err != nil { + if err := wgProxy.CloseConn(); err != nil { + conn.log.Warnf("Failed to close relay connection: %v", err) + } + conn.log.Errorf("Failed to update wg peer configuration: %v", err) + return + } + + if conn.wgProxyRelay != nil { + if err := conn.wgProxyRelay.CloseConn(); err != nil { + conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) + } + } + conn.wgProxyRelay = wgProxy + conn.currentConnType = connPriorityRelay + + peerState := State{ + Direct: false, + Relayed: true, + } + + conn.log.Infof("start to communicate with peer via relay") + conn.updateStatus(peerState, rci.rosenpassPubKey, rci.rosenpassAddr) +} + +func (conn *Conn) onWorkerRelayStateDisconnected() { + conn.mu.Lock() + defer conn.mu.Unlock() + defer func() { + conn.statusRelay = StatusDisconnected + + select { + case conn.relayDisconnected <- struct{}{}: + default: + } + }() + + if conn.wgProxyRelay != nil { + conn.endpointRelay = nil + _ = conn.wgProxyRelay.CloseConn() + conn.wgProxyRelay = nil + } + + if conn.statusICE == StatusConnected { + return + } + + if conn.evalStatus() == StatusDisconnected { + return + } + + if StatusDisconnected > conn.statusRelay { + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: StatusDisconnected, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + _ = conn.statusRecorder.UpdatePeerState(peerState) + } +} + func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error { return conn.config.WgConfig.WgInterface.UpdatePeer( conn.config.WgConfig.RemoteKey, @@ -531,7 +591,6 @@ func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, re if conn.onConnected != nil { conn.onConnected(conn.config.Key, remoteRosenpassPubKey, conn.allowedIPsIP, remoteRosenpassAddr) } - return } func (conn *Conn) doHandshake() error { @@ -548,9 +607,24 @@ func (conn *Conn) doHandshake() error { if err == nil { ha.RelayAddr = addr.String() } + conn.log.Tracef("send new offer: %#v", ha) return conn.handshaker.SendOffer(ha) } +func (conn *Conn) waitRandomSleepTime() { + minWait := 500 + maxWait := 2000 + duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond + + timeout := time.NewTimer(duration) + defer timeout.Stop() + + select { + case <-conn.ctx.Done(): + case <-timeout.C: + } +} + func (conn *Conn) evalStatus() ConnStatus { if conn.statusRelay == StatusConnected || conn.statusICE == StatusConnected { return StatusConnected diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 8d4377e36..0ec615291 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -4,17 +4,12 @@ import ( "context" "errors" "sync" - "time" log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/version" ) -const ( - handshakeCacheTimeout = 3 * time.Second -) - var ( ErrSignalIsNotReady = errors.New("signal is not ready") ) @@ -65,9 +60,6 @@ type Handshaker struct { // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection remoteAnswerCh chan OfferAnswer - remoteOfferAnswer *OfferAnswer - remoteOfferAnswerCreated time.Time - lastOfferArgs HandshakeArgs } @@ -88,6 +80,7 @@ func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAn func (h *Handshaker) Listen() { for { + log.Debugf("wait for remote offer confirmation") remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() if err != nil { if _, ok := err.(*ConnectionClosedError); ok { diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 93ad070ae..75892ca52 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -3,7 +3,6 @@ package peer import ( "context" "fmt" - "math/rand" "net" "net/netip" "runtime" @@ -68,18 +67,18 @@ type ICEConnInfo struct { type WorkerICECallbacks struct { OnConnReady func(ConnPriority, ICEConnInfo) OnStatusChanged func(ConnStatus) - DoHandshake func() error } type WorkerICE struct { - ctx context.Context - log *log.Entry - config ConnConfig - configICE ICEConfig - signaler *Signaler - iFaceDiscover stdnet.ExternalIFaceDiscover - statusRecorder *Status - conn WorkerICECallbacks + ctx context.Context + log *log.Entry + config ConnConfig + configICE ICEConfig + signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + statusRecorder *Status + hasRelayOnLocally bool + conn WorkerICECallbacks selectedPriority ConnPriority @@ -90,24 +89,21 @@ type WorkerICE struct { sentExtraSrflx bool - localUfrag string - localPwd string - creadantialHasUsed bool - hasRelayOnLocally bool - tickerCancel context.CancelFunc - ticker *time.Ticker + localUfrag string + localPwd string } -func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, callBacks WorkerICECallbacks) (*WorkerICE, error) { +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) { w := &WorkerICE{ - ctx: ctx, - log: log, - config: config, - configICE: configICE, - signaler: signaler, - iFaceDiscover: ifaceDiscover, - statusRecorder: statusRecorder, - conn: callBacks, + ctx: ctx, + log: log, + config: config, + configICE: configICE, + signaler: signaler, + iFaceDiscover: ifaceDiscover, + statusRecorder: statusRecorder, + hasRelayOnLocally: hasRelayOnLocally, + conn: callBacks, } localUfrag, localPwd, err := generateICECredentials() @@ -119,62 +115,16 @@ func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, config return w, nil } -func (w *WorkerICE) SetupICEConnection(hasRelayOnLocally bool) { - w.muxAgent.Lock() - defer w.muxAgent.Unlock() - if w.agent != nil { - return - } - - w.hasRelayOnLocally = hasRelayOnLocally - go w.sendOffer() -} - -func (w *WorkerICE) sendOffer() { - w.ticker = time.NewTicker(w.config.Timeout) - defer w.ticker.Stop() - - tickerCtx, tickerCancel := context.WithCancel(w.ctx) - w.tickerCancel = tickerCancel - w.conn.OnStatusChanged(StatusConnecting) - - w.log.Debugf("ICE trigger a new handshake") - err := w.conn.DoHandshake() - if err != nil { - w.log.Errorf("%s", err) - } - - for { - w.log.Debugf("ICE trigger new reconnect handshake") - select { - case <-w.ticker.C: - err := w.conn.DoHandshake() - if err != nil { - w.log.Errorf("%s", err) - } - case <-tickerCtx.Done(): - w.log.Debugf("left reconnect loop") - return - case <-w.ctx.Done(): - return - } - } -} - func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { - log.Debugf("OnNewOffer for ICE") + w.log.Debugf("OnNewOffer for ICE") w.muxAgent.Lock() if w.agent != nil { - log.Debugf("agent already exists, skipping the offer") + w.log.Debugf("agent already exists, skipping the offer") w.muxAgent.Unlock() return } - // cancel reconnection loop - w.log.Debugf("canceling reconnection loop") - w.tickerCancel() - var preferredCandidateTypes []ice.CandidateType if w.hasRelayOnLocally && remoteOfferAnswer.RelaySrvAddress != "" { w.selectedPriority = connPriorityICEP2P @@ -184,7 +134,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { preferredCandidateTypes = candidateTypes() } - w.log.Debugf("recreate agent") + w.log.Debugf("recreate ICE agent") agentCtx, agentCancel := context.WithCancel(w.ctx) agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes) if err != nil { @@ -204,14 +154,14 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { // will block until connection succeeded // but it won't release if ICE Agent went into Disconnected or Failed state, // so we have to cancel it with the provided context once agent detected a broken connection - w.log.Debugf("turnAgentDial") + w.log.Debugf("turn agent dial") remoteConn, err := w.turnAgentDial(agentCtx, remoteOfferAnswer) if err != nil { w.log.Debugf("failed to dial the remote peer: %s", err) return } + w.log.Debugf("agent dial succeeded") - w.log.Debugf("GetSelectedCandidatePair") pair, err := w.agent.GetSelectedCandidatePair() if err != nil { return @@ -240,7 +190,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { Relayed: isRelayed(pair), RelayedOnLocal: isRelayCandidate(pair.Local), } - w.log.Debugf("on conn ready") + w.log.Debugf("on ICE conn read to use ready") go w.conn.OnConnReady(w.selectedPriority, ci) } @@ -325,7 +275,6 @@ func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport [ w.agent = nil w.muxAgent.Unlock() - go w.sendOffer() } }) if err != nil { @@ -430,23 +379,6 @@ func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferA } } -// waitForReconnectTry waits for a random duration before trying to reconnect -func (w *WorkerICE) waitForReconnectTry() bool { - minWait := 500 - maxWait := 2000 - duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond - - timeout := time.NewTimer(duration) - defer timeout.Stop() - - select { - case <-w.ctx.Done(): - return false - case <-timeout.C: - return true - } -} - func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { relatedAdd := candidate.RelatedAddress() return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 158b8e068..5ad4a1fb5 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -2,6 +2,7 @@ package peer import ( "context" + "errors" "net" log "github.com/sirupsen/logrus" @@ -16,8 +17,8 @@ type RelayConnInfo struct { } type WorkerRelayCallbacks struct { - OnConnReady func(RelayConnInfo) - OnStatusChanged func(ConnStatus) + OnConnReady func(RelayConnInfo) + OnDisconnected func() } type WorkerRelay struct { @@ -41,9 +42,6 @@ func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, rela func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { if !w.isRelaySupported(remoteOfferAnswer) { w.log.Infof("Relay is not supported by remote peer") - // todo should we retry? - // if the remote peer doesn't support relay make no sense to retry infinity - // but if the remote peer supports relay just the connection is lost we should retry return } @@ -55,12 +53,19 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { } srv := w.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) - relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) + + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key, w.conn.OnDisconnected) if err != nil { + // todo handle all type errors + if errors.Is(err, relayClient.ErrConnAlreadyExists) { + w.log.Infof("do not need to reopen relay connection") + return + } w.log.Infof("do not need to reopen relay connection: %s", err) return } + w.log.Debugf("Relay connection established with %s", srv) go w.conn.OnConnReady(RelayConnInfo{ relayedConn: relayedConn, rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, @@ -72,6 +77,14 @@ func (w *WorkerRelay) RelayAddress() (net.Addr, error) { return w.relayManager.RelayAddress() } +func (w *WorkerRelay) IsController() bool { + return w.config.LocalKey > w.config.Key +} + +func (w *WorkerRelay) RelayIsSupportedLocally() bool { + return w.relayManager.HasRelayAddress() +} + func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { if !w.relayManager.HasRelayAddress() { return false @@ -80,12 +93,8 @@ func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { } func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string { - if w.config.LocalKey > w.config.Key { + if w.IsController() { return myRelayAddress } return remoteRelayAddress } - -func (w *WorkerRelay) RelayIsSupportedLocally() bool { - return w.relayManager.HasRelayAddress() -} diff --git a/client/internal/wgproxy/proxy_userspace.go b/client/internal/wgproxy/proxy_userspace.go index e51bfbdcd..c2c8a9b51 100644 --- a/client/internal/wgproxy/proxy_userspace.go +++ b/client/internal/wgproxy/proxy_userspace.go @@ -73,6 +73,7 @@ func (p *WGUserSpaceProxy) proxyToRemote() { default: n, err := p.localConn.Read(buf) if err != nil { + log.Debugf("failed to read from wg interface conn: %s", err) continue } @@ -80,6 +81,8 @@ func (p *WGUserSpaceProxy) proxyToRemote() { if err != nil { if err == io.EOF { p.cancel() + } else { + log.Debugf("failed to write to remote conn: %s", err) } continue } @@ -103,11 +106,13 @@ func (p *WGUserSpaceProxy) proxyToLocal() { p.cancel() return } + log.Errorf("failed to read from remote conn: %s", err) continue } _, err = p.localConn.Write(buf[:n]) if err != nil { + log.Debugf("failed to write to wg interface conn: %s", err) continue } } diff --git a/relay/client/client.go b/relay/client/client.go index 612080abd..1ba815018 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -115,7 +115,7 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { // Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. func (c *Client) Connect() error { - log.Infof("connecting to relay server: %s", c.serverAddress) + c.log.Infof("connecting to relay server: %s", c.serverAddress) c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() diff --git a/relay/client/guard.go b/relay/client/guard.go index 5cea25653..050a9f0e7 100644 --- a/relay/client/guard.go +++ b/relay/client/guard.go @@ -21,7 +21,6 @@ func NewGuard(context context.Context, relayClient *Client) *Guard { ctx: context, relayClient: relayClient, } - return g } diff --git a/relay/client/manager.go b/relay/client/manager.go index 1810b8167..ae44be4b9 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -44,19 +44,22 @@ type Manager struct { relayClients map[string]*RelayTrack relayClientsMutex sync.RWMutex + + onDisconnectedListeners map[string]map[*func()]struct{} + listenerLock sync.Mutex } func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { return &Manager{ - ctx: ctx, - srvAddress: serverAddress, - peerID: peerID, - relayClients: make(map[string]*RelayTrack), + ctx: ctx, + srvAddress: serverAddress, + peerID: peerID, + relayClients: make(map[string]*RelayTrack), + onDisconnectedListeners: make(map[string]map[*func()]struct{}), } } // Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop. -// todo: consider to return an error if the initial connection to the relay server is not established. func (m *Manager) Serve() error { if m.relayClient != nil { return fmt.Errorf("manager already serving") @@ -70,8 +73,9 @@ func (m *Manager) Serve() error { } m.reconnectGuard = NewGuard(m.ctx, m.relayClient) - m.relayClient.SetOnDisconnectListener(m.reconnectGuard.OnDisconnected) - + m.relayClient.SetOnDisconnectListener(func() { + m.onServerDisconnected(m.srvAddress) + }) m.startCleanupLoop() return nil @@ -80,7 +84,7 @@ func (m *Manager) Serve() error { // OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be // established via the relay server. If the peer is on a different relay server, the manager will establish a new // connection to the relay server. -func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { +func (m *Manager) OpenConn(serverAddress, peerKey string, onClosedListener func()) (net.Conn, error) { if m.relayClient == nil { return nil, errRelayClientNotConnected } @@ -90,13 +94,26 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { return nil, err } + var ( + netConn net.Conn + ) if !foreign { log.Debugf("open peer connection via permanent server: %s", peerKey) - return m.relayClient.OpenConn(peerKey) + netConn, err = m.relayClient.OpenConn(peerKey) } else { log.Debugf("open peer connection via foreign server: %s", serverAddress) - return m.openConnVia(serverAddress, peerKey) + netConn, err = m.openConnVia(serverAddress, peerKey) } + + if err != nil { + return nil, err + } + + if onClosedListener != nil { + m.addListener(serverAddress, onClosedListener) + } + + return netConn, err } // RelayAddress returns the address of the permanent relay server. It could change if the network connection is lost. @@ -152,7 +169,7 @@ func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { } // if connection closed then delete the relay client from the list relayClient.SetOnDisconnectListener(func() { - m.deleteRelayConn(serverAddress) + m.onServerDisconnected(serverAddress) }) rt.relayClient = relayClient rt.Unlock() @@ -164,11 +181,12 @@ func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { return conn, nil } -func (m *Manager) deleteRelayConn(address string) { - log.Infof("deleting relay client for %s", address) - m.relayClientsMutex.Lock() - delete(m.relayClients, address) - m.relayClientsMutex.Unlock() +func (m *Manager) onServerDisconnected(serverAddress string) { + if serverAddress == m.srvAddress { + m.reconnectGuard.OnDisconnected() + } + + m.notifyOnDisconnectListeners(serverAddress) } func (m *Manager) isForeignServer(address string) (bool, error) { @@ -212,8 +230,33 @@ func (m *Manager) cleanUpUnusedRelays() { go func() { _ = rt.relayClient.Close() }() - log.Debugf("clean up relay client: %s", addr) + log.Debugf("clean up unused relay server connection: %s", addr) delete(m.relayClients, addr) rt.Unlock() } } + +func (m *Manager) addListener(serverAddress string, onClosedListener func()) { + m.listenerLock.Lock() + l, ok := m.onDisconnectedListeners[serverAddress] + if !ok { + l = make(map[*func()]struct{}) + } + l[&onClosedListener] = struct{}{} + m.onDisconnectedListeners[serverAddress] = l + m.listenerLock.Unlock() +} + +func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { + m.listenerLock.Lock() + l, ok := m.onDisconnectedListeners[serverAddress] + if !ok { + return + } + for f := range l { + go (*f)() + } + delete(m.onDisconnectedListeners, serverAddress) + m.listenerLock.Unlock() + +} diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index f4de9b4de..69539cc5e 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -61,11 +61,11 @@ func TestForeignConn(t *testing.T) { if err != nil { t.Fatalf("failed to get relay address: %s", err) } - connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr.String(), idBob) + connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr.String(), idBob, nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } - connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr.String(), idAlice) + connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr.String(), idAlice, nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -139,7 +139,7 @@ func TestForeginConnClose(t *testing.T) { mgr := NewManager(mCtx, addr1, idAlice) mgr.Serve() - conn, err := mgr.OpenConn(addr2, "anotherpeer") + conn, err := mgr.OpenConn(addr2, "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -198,7 +198,7 @@ func TestForeginAutoClose(t *testing.T) { mgr.Serve() t.Log("open connection to another peer") - conn, err := mgr.OpenConn(addr2, "anotherpeer") + conn, err := mgr.OpenConn(addr2, "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -246,7 +246,7 @@ func TestAutoReconnect(t *testing.T) { if err != nil { t.Errorf("failed to get relay address: %s", err) } - conn, err := clientAlice.OpenConn(ra.String(), "bob") + conn, err := clientAlice.OpenConn(ra.String(), "bob", nil) if err != nil { t.Errorf("failed to bind channel: %s", err) } @@ -264,7 +264,7 @@ func TestAutoReconnect(t *testing.T) { time.Sleep(reconnectingTimeout + 1*time.Second) log.Infof("reopent the connection") - _, err = clientAlice.OpenConn(ra.String(), "bob") + _, err = clientAlice.OpenConn(ra.String(), "bob", nil) if err != nil { t.Errorf("failed to open channel: %s", err) } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index dc5cfe69d..eb7d87a1f 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -32,7 +32,7 @@ func init() { func waitForExitSignal() { osSigs := make(chan os.Signal, 1) signal.Notify(osSigs, syscall.SIGINT, syscall.SIGTERM) - _ = <-osSigs + <-osSigs } func execute(cmd *cobra.Command, args []string) { diff --git a/relay/messages/id.go b/relay/messages/id.go index d37dc37f8..c2248fb2b 100644 --- a/relay/messages/id.go +++ b/relay/messages/id.go @@ -16,5 +16,5 @@ func HashID(peerID string) ([]byte, string) { } func HashIDToString(idHash []byte) string { - return base64.StdEncoding.EncodeToString(idHash[:]) + return base64.StdEncoding.EncodeToString(idHash) } diff --git a/relay/server/server.go b/relay/server/server.go index bb9666a53..e27fd0f4f 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -80,7 +80,7 @@ func (r *Server) Close() error { func (r *Server) accept(conn net.Conn) { peer, err := handShake(conn) if err != nil { - log.Errorf("failed to handshake wiht %s: %s", conn.RemoteAddr(), err) + log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err) cErr := conn.Close() if cErr != nil { log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr) @@ -134,7 +134,6 @@ func (r *Server) accept(conn net.Conn) { if err != nil { peer.Log.Errorf("failed to write transport message to: %s", dp.String()) } - return }() case messages.MsgClose: peer.Log.Infof("peer disconnected gracefully") From 085d072b17dc7d19f8f63b86c19a4dd220fac6e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 25 Jun 2024 17:36:04 +0200 Subject: [PATCH 063/199] - Add sha prefix for peer id in protocol - Add magic cookie in hello msg - Add tests --- relay/client/client.go | 6 ++++- relay/client/manager.go | 1 + relay/messages/id.go | 15 ++++++++---- relay/messages/id_test.go | 26 ++++++++++++++++++++ relay/messages/message.go | 38 ++++++++++++++++-------------- relay/messages/message_test.go | 43 ++++++++++++++++++++++++++++++++++ 6 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 relay/messages/id_test.go create mode 100644 relay/messages/message_test.go diff --git a/relay/client/client.go b/relay/client/client.go index 1ba815018..6b8dac9df 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -371,7 +371,11 @@ func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { return 0, io.EOF } */ - msg := messages.MarshalTransportMsg(dstID, payload) + msg, err := messages.MarshalTransportMsg(dstID, payload) + if err != nil { + log.Errorf("failed to marshal transport message: %s", err) + return 0, err + } n, err := c.relayConn.Write(msg) if err != nil { log.Errorf("failed to write transport message: %s", err) diff --git a/relay/client/manager.go b/relay/client/manager.go index ae44be4b9..42cbcd280 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -194,6 +194,7 @@ func (m *Manager) isForeignServer(address string) (bool, error) { if err != nil { return false, fmt.Errorf("relay client not connected") } + log.Debugf("check if foreign server: %s != %s", rAddr.String(), address) return rAddr.String() != address, nil } diff --git a/relay/messages/id.go b/relay/messages/id.go index c2248fb2b..dbb49cc58 100644 --- a/relay/messages/id.go +++ b/relay/messages/id.go @@ -3,18 +3,25 @@ package messages import ( "crypto/sha256" "encoding/base64" + "fmt" ) const ( - IDSize = sha256.Size + prefixLength = 4 + IDSize = sha256.Size + 4 // 4 is equal with len(prefix) +) + +var ( + prefix = []byte("sha-") // 4 bytes ) func HashID(peerID string) ([]byte, string) { idHash := sha256.Sum256([]byte(peerID)) - idHashString := base64.StdEncoding.EncodeToString(idHash[:]) - return idHash[:], idHashString + idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:]) + prefixedHash := append(prefix, idHash[:]...) + return prefixedHash, idHashString } func HashIDToString(idHash []byte) string { - return base64.StdEncoding.EncodeToString(idHash) + return fmt.Sprintf("%s%s", idHash[:prefixLength], base64.StdEncoding.EncodeToString(idHash[prefixLength:])) } diff --git a/relay/messages/id_test.go b/relay/messages/id_test.go new file mode 100644 index 000000000..25c64955b --- /dev/null +++ b/relay/messages/id_test.go @@ -0,0 +1,26 @@ +package messages + +import ( + "encoding/binary" + "testing" + + log "github.com/sirupsen/logrus" +) + +func TestHashID(t *testing.T) { + hashedID, hashedStringId := HashID("abc") + enc := HashIDToString(hashedID) + if enc != hashedStringId { + t.Errorf("expected %s, got %s", hashedStringId, enc) + } + + var magicHeader uint32 = 0x2112A442 // size 4 byte + + msg := make([]byte, 4) + binary.BigEndian.PutUint32(msg, magicHeader) + + magicHeader2 := []byte{0x21, 0x12, 0xA4, 0x42} + + log.Infof("msg: %v, %v", msg, magicHeader2) + +} diff --git a/relay/messages/message.go b/relay/messages/message.go index b865687ce..92b8bced7 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -1,6 +1,7 @@ package messages import ( + "bytes" "fmt" log "github.com/sirupsen/logrus" @@ -11,10 +12,15 @@ const ( MsgTypeHelloResponse MsgType = 1 MsgTypeTransport MsgType = 2 MsgClose MsgType = 3 + + headerSizeTransport = 1 + IDSize // 1 byte for msg type, IDSize for peerID + headerSizeHello = 1 + 4 + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID ) var ( ErrInvalidMessageLength = fmt.Errorf("invalid message length") + + magicHeader = []byte{0x21, 0x12, 0xA4, 0x42} ) type MsgType byte @@ -35,7 +41,6 @@ func (m MsgType) String() string { } func DetermineClientMsgType(msg []byte) (MsgType, error) { - // todo: validate magic byte msgType := MsgType(msg[0]) switch msgType { case MsgTypeHello: @@ -50,7 +55,6 @@ func DetermineClientMsgType(msg []byte) (MsgType, error) { } func DetermineServerMsgType(msg []byte) (MsgType, error) { - // todo: validate magic byte msgType := MsgType(msg[0]) switch msgType { case MsgTypeHelloResponse: @@ -67,19 +71,21 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { // MarshalHelloMsg initial hello message func MarshalHelloMsg(peerID []byte) ([]byte, error) { if len(peerID) != IDSize { - return nil, fmt.Errorf("invalid peerID length") + return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) } - msg := make([]byte, 1, 1+len(peerID)) + msg := make([]byte, 5, headerSizeHello) msg[0] = byte(MsgTypeHello) + copy(msg[1:5], magicHeader) msg = append(msg, peerID...) return msg, nil } func UnmarshalHelloMsg(msg []byte) ([]byte, error) { - if len(msg) < 2 { + if len(msg) < headerSizeHello { return nil, fmt.Errorf("invalid 'hello' messge") } - return msg[1:], nil + bytes.Equal(msg[1:5], magicHeader) + return msg[5:], nil } func MarshalHelloResponse() []byte { @@ -98,34 +104,32 @@ func MarshalCloseMsg() []byte { // Transport message -func MarshalTransportMsg(peerID []byte, payload []byte) []byte { +func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) { if len(peerID) != IDSize { - return nil + return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) } - msg := make([]byte, 1+IDSize, 1+IDSize+len(payload)) + msg := make([]byte, headerSizeTransport, headerSizeTransport+len(payload)) msg[0] = byte(MsgTypeTransport) copy(msg[1:], peerID) msg = append(msg, payload...) - return msg + return msg, nil } func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) { - headerSize := 1 + IDSize - if len(buf) < headerSize { + if len(buf) < headerSizeTransport { return nil, nil, ErrInvalidMessageLength } - return buf[1:headerSize], buf[headerSize:], nil + return buf[1:headerSizeTransport], buf[headerSizeTransport:], nil } func UnmarshalTransportID(buf []byte) ([]byte, error) { - headerSize := 1 + IDSize - if len(buf) < headerSize { - log.Debugf("invalid message length: %d, expected: %d, %x", len(buf), headerSize, buf) + if len(buf) < headerSizeTransport { + log.Debugf("invalid message length: %d, expected: %d, %x", len(buf), headerSizeTransport, buf) return nil, ErrInvalidMessageLength } - return buf[1:headerSize], nil + return buf[1:headerSizeTransport], nil } func UpdateTransportMsg(msg []byte, peerID []byte) error { diff --git a/relay/messages/message_test.go b/relay/messages/message_test.go new file mode 100644 index 000000000..d40963d8b --- /dev/null +++ b/relay/messages/message_test.go @@ -0,0 +1,43 @@ +package messages + +import ( + "testing" +) + +func TestMarshalHelloMsg(t *testing.T) { + peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") + bHello, err := MarshalHelloMsg(peerID) + if err != nil { + t.Fatalf("error: %v", err) + } + + receivedPeerID, err := UnmarshalHelloMsg(bHello) + if err != nil { + t.Fatalf("error: %v", err) + } + if string(receivedPeerID) != string(peerID) { + t.Errorf("expected %s, got %s", peerID, receivedPeerID) + } +} + +func TestMarshalTransportMsg(t *testing.T) { + peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") + payload := []byte("payload") + msg, err := MarshalTransportMsg(peerID, payload) + if err != nil { + t.Fatalf("error: %v", err) + } + + id, respPayload, err := UnmarshalTransportMsg(msg) + if err != nil { + t.Fatalf("error: %v", err) + } + + if string(id) != string(peerID) { + t.Errorf("expected %s, got %s", peerID, id) + } + + if string(respPayload) != string(payload) { + t.Errorf("expected %s, got %s", payload, respPayload) + } +} From 745e4f76b1c59ee98f2a8ca4452078c93d695528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 26 Jun 2024 15:25:01 +0200 Subject: [PATCH 064/199] Remove gorilla lib --- relay/client/dialer/ws/client_conn.go | 59 ----------------- relay/client/dialer/ws/ws.go | 22 ------- relay/server/listener/ws/conn.go | 74 --------------------- relay/server/listener/ws/listener.go | 92 --------------------------- 4 files changed, 247 deletions(-) delete mode 100644 relay/client/dialer/ws/client_conn.go delete mode 100644 relay/client/dialer/ws/ws.go delete mode 100644 relay/server/listener/ws/conn.go delete mode 100644 relay/server/listener/ws/listener.go diff --git a/relay/client/dialer/ws/client_conn.go b/relay/client/dialer/ws/client_conn.go deleted file mode 100644 index ee3283ecd..000000000 --- a/relay/client/dialer/ws/client_conn.go +++ /dev/null @@ -1,59 +0,0 @@ -package ws - -import ( - "fmt" - "net" - "sync" - "time" - - "github.com/gorilla/websocket" -) - -type Conn struct { - *websocket.Conn - mu sync.Mutex -} - -func NewConn(wsConn *websocket.Conn) net.Conn { - return &Conn{ - Conn: wsConn, - } -} - -func (c *Conn) Read(b []byte) (n int, err error) { - t, r, err := c.NextReader() - if err != nil { - return 0, err - } - - if t != websocket.BinaryMessage { - return 0, fmt.Errorf("unexpected message type") - } - - return r.Read(b) -} - -func (c *Conn) Write(b []byte) (int, error) { - c.mu.Lock() - err := c.WriteMessage(websocket.BinaryMessage, b) - c.mu.Unlock() - return len(b), err -} - -func (c *Conn) SetDeadline(t time.Time) error { - errR := c.SetReadDeadline(t) - errW := c.SetWriteDeadline(t) - - if errR != nil { - return errR - } - - if errW != nil { - return errW - } - return nil -} - -func (c *Conn) Close() error { - return c.Conn.Close() -} diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go deleted file mode 100644 index b526bd315..000000000 --- a/relay/client/dialer/ws/ws.go +++ /dev/null @@ -1,22 +0,0 @@ -package ws - -import ( - "fmt" - "net" - "time" - - "github.com/gorilla/websocket" -) - -func Dial(address string) (net.Conn, error) { - addr := fmt.Sprintf("ws://" + address) - wsDialer := websocket.Dialer{ - HandshakeTimeout: 3 * time.Second, - } - wsConn, _, err := wsDialer.Dial(addr, nil) - if err != nil { - return nil, err - } - conn := NewConn(wsConn) - return conn, nil -} diff --git a/relay/server/listener/ws/conn.go b/relay/server/listener/ws/conn.go deleted file mode 100644 index 8734293ac..000000000 --- a/relay/server/listener/ws/conn.go +++ /dev/null @@ -1,74 +0,0 @@ -package ws - -import ( - "errors" - "fmt" - "io" - "sync" - "time" - - "github.com/gorilla/websocket" - log "github.com/sirupsen/logrus" -) - -type Conn struct { - *websocket.Conn - - mu sync.Mutex -} - -func NewConn(wsConn *websocket.Conn) *Conn { - return &Conn{ - Conn: wsConn, - } -} - -func (c *Conn) Read(b []byte) (n int, err error) { - t, r, err := c.NextReader() - if err != nil { - return 0, ioErrHandling(err) - } - - if t != websocket.BinaryMessage { - log.Errorf("unexpected message type: %d", t) - return 0, fmt.Errorf("unexpected message type") - } - - n, err = r.Read(b) - if err != nil { - return 0, ioErrHandling(err) - } - return n, err -} - -func (c *Conn) Write(b []byte) (int, error) { - c.mu.Lock() - err := c.WriteMessage(websocket.BinaryMessage, b) - c.mu.Unlock() - return len(b), err -} - -func (c *Conn) SetDeadline(t time.Time) error { - errR := c.SetReadDeadline(t) - errW := c.SetWriteDeadline(t) - - if errR != nil { - return errR - } - - if errW != nil { - return errW - } - return nil -} - -func ioErrHandling(err error) error { - var wErr *websocket.CloseError - if !errors.As(err, &wErr) { - return err - } - if wErr.Code == websocket.CloseNormalClosure { - return io.EOF - } - return err -} diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go deleted file mode 100644 index 632de153f..000000000 --- a/relay/server/listener/ws/listener.go +++ /dev/null @@ -1,92 +0,0 @@ -package ws - -import ( - "context" - "errors" - "fmt" - "net" - "net/http" - "sync" - "time" - - "github.com/gorilla/websocket" - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/relay/server/listener" -) - -var ( - upgrader = websocket.Upgrader{} // use default options -) - -type Listener struct { - address string - - wg sync.WaitGroup - server *http.Server - acceptFn func(conn net.Conn) -} - -func NewListener(address string) listener.Listener { - return &Listener{ - address: address, - } -} - -func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { - if l.server != nil { - return errors.New("server is already running") - } - - l.acceptFn = acceptFn - mux := http.NewServeMux() - mux.HandleFunc("/", l.onAccept) - - l.server = &http.Server{ - Addr: l.address, - Handler: mux, - } - - log.Infof("WS server is listening on address: %s", l.address) - err := l.server.ListenAndServe() - if errors.Is(err, http.ErrServerClosed) { - return nil - } - return err -} - -func (l *Listener) Close() error { - if l.server == nil { - return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - log.Debugf("closing WS server") - if err := l.server.Shutdown(ctx); err != nil { - return fmt.Errorf("server shutdown failed: %v", err) - } - - l.wg.Wait() - return nil -} - -func (l *Listener) WaitForExitAcceptedConns() { - l.wg.Wait() -} - -func (l *Listener) onAccept(writer http.ResponseWriter, request *http.Request) { - l.wg.Add(1) - defer l.wg.Done() - - wsConn, err := upgrader.Upgrade(writer, request, nil) - if err != nil { - log.Errorf("failed to upgrade connection: %s", err) - return - } - conn := NewConn(wsConn) - log.Infof("new connection from: %s", conn.RemoteAddr()) - l.acceptFn(conn) - return -} From 0a59f120127fee867d0c5d4c03730d2278c28ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 26 Jun 2024 15:25:32 +0200 Subject: [PATCH 065/199] Env var to force relay usage --- client/internal/peer/conn.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index f93d4e8a6..e8dcb069b 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -4,6 +4,7 @@ import ( "context" "math/rand" "net" + "os" "runtime" "strings" "sync" @@ -155,7 +156,9 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu } conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer) - conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer) + if os.Getenv("NB_FORCE_RELAY") != "true" { + conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer) + } go conn.handshaker.Listen() From f0eb00458279a5cbb0530790f88f13070924603c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 26 Jun 2024 15:26:19 +0200 Subject: [PATCH 066/199] Single thread on server sending --- relay/client/client.go | 1 + relay/server/server.go | 47 +++++++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 6b8dac9df..d972207ea 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -371,6 +371,7 @@ func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { return 0, io.EOF } */ + // todo: use buffer pool instead of create new transport msg. msg, err := messages.MarshalTransportMsg(dstID, payload) if err != nil { log.Errorf("failed to marshal transport message: %s", err) diff --git a/relay/server/server.go b/relay/server/server.go index e27fd0f4f..c7085c3d1 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -15,6 +15,11 @@ import ( ws "github.com/netbirdio/netbird/relay/server/listener/wsnhooyr" ) +const ( + bufferSize = 8820 + maxHandshakeSize = 90 +) + type Server struct { store *Store storeMu sync.RWMutex @@ -95,8 +100,8 @@ func (r *Server) accept(conn net.Conn) { peer.Log.Infof("relay connection closed") }() + buf := make([]byte, bufferSize) for { - buf := make([]byte, 1500) // todo: optimize buffer size n, err := conn.Read(buf) if err != nil { if err != io.EOF { @@ -105,36 +110,36 @@ func (r *Server) accept(conn net.Conn) { return } - msgType, err := messages.DetermineClientMsgType(buf[:n]) + msg := buf[:n] + + msgType, err := messages.DetermineClientMsgType(msg) if err != nil { peer.Log.Errorf("failed to determine message type: %s", err) return } switch msgType { case messages.MsgTypeTransport: - msg := buf[:n] peerID, err := messages.UnmarshalTransportID(msg) if err != nil { peer.Log.Errorf("failed to unmarshal transport message: %s", err) continue } - go func() { - stringPeerID := messages.HashIDToString(peerID) - dp, ok := r.store.Peer(stringPeerID) - if !ok { - peer.Log.Errorf("peer not found: %s", stringPeerID) - return - } - err := messages.UpdateTransportMsg(msg, peer.ID()) - if err != nil { - peer.Log.Errorf("failed to update transport message: %s", err) - return - } - _, err = dp.conn.Write(msg) - if err != nil { - peer.Log.Errorf("failed to write transport message to: %s", dp.String()) - } - }() + stringPeerID := messages.HashIDToString(peerID) + dp, ok := r.store.Peer(stringPeerID) + if !ok { + peer.Log.Errorf("peer not found: %s", stringPeerID) + continue + } + err = messages.UpdateTransportMsg(msg, peer.ID()) + if err != nil { + peer.Log.Errorf("failed to update transport message: %s", err) + continue + } + peer.Log.Infof("write transport msg!!!!") + _, err = dp.conn.Write(msg) + if err != nil { + peer.Log.Errorf("failed to write transport message to: %s", dp.String()) + } case messages.MsgClose: peer.Log.Infof("peer disconnected gracefully") _ = conn.Close() @@ -163,7 +168,7 @@ func (r *Server) sendCloseMsgs() { } func handShake(conn net.Conn) (*Peer, error) { - buf := make([]byte, 1500) + buf := make([]byte, maxHandshakeSize) n, err := conn.Read(buf) if err != nil { log.Errorf("failed to read message: %s", err) From c0efce65566ee71238adef97fa6ae81df62b237f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 26 Jun 2024 16:22:26 +0200 Subject: [PATCH 067/199] Fix msg delivery timeouts --- relay/client/client.go | 41 +++++++++++++-------- relay/client/dialer/wsnhooyr/client_conn.go | 13 ------- relay/client/manager.go | 1 - relay/messages/message.go | 2 + relay/server/listener/wsnhooyr/conn.go | 12 +++++- relay/server/listener/wsnhooyr/listener.go | 1 - relay/server/server.go | 6 +-- 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index d972207ea..3f689e6dd 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -250,13 +250,6 @@ func (c *Client) connect() error { } func (c *Client) handShake() error { - defer func() { - err := c.relayConn.SetReadDeadline(time.Time{}) - if err != nil { - log.Errorf("failed to reset read deadline: %s", err) - } - }() - msg, err := messages.MarshalHelloMsg(c.hashedID) if err != nil { log.Errorf("failed to marshal hello message: %s", err) @@ -267,15 +260,8 @@ func (c *Client) handShake() error { log.Errorf("failed to send hello message: %s", err) return err } - - err = c.relayConn.SetReadDeadline(time.Now().Add(serverResponseTimeout)) - if err != nil { - log.Errorf("failed to set read deadline: %s", err) - return err - } - - buf := make([]byte, 1500) // todo: optimise buffer size - n, err := c.relayConn.Read(buf) + buf := make([]byte, messages.MaxHandshakeSize) + n, err := c.readWithTimeout(buf) if err != nil { log.Errorf("failed to read hello response: %s", err) return err @@ -391,6 +377,29 @@ func (c *Client) closeAllConns() { c.conns = make(map[string]*connContainer) } +func (c *Client) readWithTimeout(buf []byte) (int, error) { + ctx, cancel := context.WithTimeout(c.parentCtx, serverResponseTimeout) + defer cancel() + + readDone := make(chan struct{}) + var ( + n int + err error + ) + + go func() { + n, err = c.relayConn.Read(buf) + close(readDone) + }() + + select { + case <-ctx.Done(): + return 0, fmt.Errorf("read operation timed out") + case <-readDone: + return n, err + } +} + // todo check by reference too, the id is not enought because the id come from the outer conn func (c *Client) closeConn(id string) error { c.mu.Lock() diff --git a/relay/client/dialer/wsnhooyr/client_conn.go b/relay/client/dialer/wsnhooyr/client_conn.go index 0aa995286..219425460 100644 --- a/relay/client/dialer/wsnhooyr/client_conn.go +++ b/relay/client/dialer/wsnhooyr/client_conn.go @@ -51,27 +51,14 @@ func (c *Conn) LocalAddr() net.Addr { } func (c *Conn) SetReadDeadline(t time.Time) error { - // todo: implement me return nil } func (c *Conn) SetWriteDeadline(t time.Time) error { - // todo: implement me return nil } func (c *Conn) SetDeadline(t time.Time) error { - // todo: implement me - errR := c.SetReadDeadline(t) - errW := c.SetWriteDeadline(t) - - if errR != nil { - return errR - } - - if errW != nil { - return errW - } return nil } diff --git a/relay/client/manager.go b/relay/client/manager.go index 42cbcd280..ae44be4b9 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -194,7 +194,6 @@ func (m *Manager) isForeignServer(address string) (bool, error) { if err != nil { return false, fmt.Errorf("relay client not connected") } - log.Debugf("check if foreign server: %s != %s", rAddr.String(), address) return rAddr.String() != address, nil } diff --git a/relay/messages/message.go b/relay/messages/message.go index 92b8bced7..d2a6d46d7 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -15,6 +15,8 @@ const ( headerSizeTransport = 1 + IDSize // 1 byte for msg type, IDSize for peerID headerSizeHello = 1 + 4 + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID + + MaxHandshakeSize = 90 ) var ( diff --git a/relay/server/listener/wsnhooyr/conn.go b/relay/server/listener/wsnhooyr/conn.go index b52e5d082..7a96c6529 100644 --- a/relay/server/listener/wsnhooyr/conn.go +++ b/relay/server/listener/wsnhooyr/conn.go @@ -13,6 +13,10 @@ import ( "nhooyr.io/websocket" ) +const ( + writeTimeout = 10 * time.Second +) + type Conn struct { *websocket.Conn lAddr *net.TCPAddr @@ -50,8 +54,14 @@ func (c *Conn) Read(b []byte) (n int, err error) { return n, err } +// Write writes a binary message with the given payload. +// It does not block until fill the internal buffer. +// If the buffer filled up, wait until the buffer is drained or timeout. func (c *Conn) Write(b []byte) (int, error) { - err := c.Conn.Write(c.ctx, websocket.MessageBinary, b) + ctx, ctxCancel := context.WithTimeout(c.ctx, writeTimeout) + defer ctxCancel() + + err := c.Conn.Write(ctx, websocket.MessageBinary, b) return len(b), err } diff --git a/relay/server/listener/wsnhooyr/listener.go b/relay/server/listener/wsnhooyr/listener.go index e47a60b47..b6bcd12c0 100644 --- a/relay/server/listener/wsnhooyr/listener.go +++ b/relay/server/listener/wsnhooyr/listener.go @@ -29,7 +29,6 @@ func NewListener(address string) listener.Listener { } } -// Listen todo: prevent multiple call func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { l.acceptFn = acceptFn mux := http.NewServeMux() diff --git a/relay/server/server.go b/relay/server/server.go index c7085c3d1..f52f7eab8 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -16,8 +16,7 @@ import ( ) const ( - bufferSize = 8820 - maxHandshakeSize = 90 + bufferSize = 8820 ) type Server struct { @@ -135,7 +134,6 @@ func (r *Server) accept(conn net.Conn) { peer.Log.Errorf("failed to update transport message: %s", err) continue } - peer.Log.Infof("write transport msg!!!!") _, err = dp.conn.Write(msg) if err != nil { peer.Log.Errorf("failed to write transport message to: %s", dp.String()) @@ -168,7 +166,7 @@ func (r *Server) sendCloseMsgs() { } func handShake(conn net.Conn) (*Peer, error) { - buf := make([]byte, maxHandshakeSize) + buf := make([]byte, messages.MaxHandshakeSize) n, err := conn.Read(buf) if err != nil { log.Errorf("failed to read message: %s", err) From 3fcdb51376c51cfe4b50574f2418e086456663c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 26 Jun 2024 16:23:50 +0200 Subject: [PATCH 068/199] Error handling --- relay/client/dialer/wsnhooyr/client_conn.go | 6 +++--- relay/server/listener/wsnhooyr/conn.go | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/relay/client/dialer/wsnhooyr/client_conn.go b/relay/client/dialer/wsnhooyr/client_conn.go index 219425460..261edd04c 100644 --- a/relay/client/dialer/wsnhooyr/client_conn.go +++ b/relay/client/dialer/wsnhooyr/client_conn.go @@ -51,15 +51,15 @@ func (c *Conn) LocalAddr() net.Addr { } func (c *Conn) SetReadDeadline(t time.Time) error { - return nil + return fmt.Errorf("not implemented") } func (c *Conn) SetWriteDeadline(t time.Time) error { - return nil + return fmt.Errorf("not implemented") } func (c *Conn) SetDeadline(t time.Time) error { - return nil + return fmt.Errorf("not implemented") } func (c *Conn) Close() error { diff --git a/relay/server/listener/wsnhooyr/conn.go b/relay/server/listener/wsnhooyr/conn.go index 7a96c6529..c43546b96 100644 --- a/relay/server/listener/wsnhooyr/conn.go +++ b/relay/server/listener/wsnhooyr/conn.go @@ -74,18 +74,15 @@ func (c *Conn) RemoteAddr() net.Addr { } func (c *Conn) SetReadDeadline(t time.Time) error { - // todo: implement me - return nil + return fmt.Errorf("not implemented") } func (c *Conn) SetWriteDeadline(t time.Time) error { - // todo: implement me - return nil + return fmt.Errorf("not implemented") } func (c *Conn) SetDeadline(t time.Time) error { - // todo: implement me - return nil + return fmt.Errorf("not implemented") } func (c *Conn) Close() error { From 4d0e16f2d0cf8af17bfb8c71a138c10c7c5c95c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 27 Jun 2024 02:36:44 +0200 Subject: [PATCH 069/199] - Remove WaitForExitAcceptedConns logic from server - Implement thread safe gracefully close logic - organise the server code --- relay/server/listener/listener.go | 1 - relay/server/listener/udp/listener.go | 5 - .../server/listener/{wsnhooyr => ws}/conn.go | 0 .../listener/{wsnhooyr => ws}/listener.go | 10 -- relay/server/peer.go | 126 ++++++++++++-- relay/server/relay.go | 99 +++++++++++ relay/server/server.go | 159 +++--------------- 7 files changed, 234 insertions(+), 166 deletions(-) rename relay/server/listener/{wsnhooyr => ws}/conn.go (100%) rename relay/server/listener/{wsnhooyr => ws}/listener.go (92%) create mode 100644 relay/server/relay.go diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go index 3336e0ad8..66e6d357e 100644 --- a/relay/server/listener/listener.go +++ b/relay/server/listener/listener.go @@ -5,5 +5,4 @@ import "net" type Listener interface { Listen(func(conn net.Conn)) error Close() error - WaitForExitAcceptedConns() } diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go index 3c2dfc070..31b6098e5 100644 --- a/relay/server/listener/udp/listener.go +++ b/relay/server/listener/udp/listener.go @@ -21,11 +21,6 @@ type Listener struct { lock sync.Mutex } -func (l *Listener) WaitForExitAcceptedConns() { - l.wg.Wait() - return -} - func NewListener(address string) listener.Listener { return &Listener{ address: address, diff --git a/relay/server/listener/wsnhooyr/conn.go b/relay/server/listener/ws/conn.go similarity index 100% rename from relay/server/listener/wsnhooyr/conn.go rename to relay/server/listener/ws/conn.go diff --git a/relay/server/listener/wsnhooyr/listener.go b/relay/server/listener/ws/listener.go similarity index 92% rename from relay/server/listener/wsnhooyr/listener.go rename to relay/server/listener/ws/listener.go index b6bcd12c0..26ba2276e 100644 --- a/relay/server/listener/wsnhooyr/listener.go +++ b/relay/server/listener/ws/listener.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "net/http" - "sync" "time" log "github.com/sirupsen/logrus" @@ -18,7 +17,6 @@ import ( type Listener struct { address string - wg sync.WaitGroup server *http.Server acceptFn func(conn net.Conn) } @@ -63,14 +61,7 @@ func (l *Listener) Close() error { return nil } -func (l *Listener) WaitForExitAcceptedConns() { - l.wg.Wait() -} - func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { - l.wg.Add(1) - defer l.wg.Done() - wsConn, err := websocket.Accept(w, r, nil) if err != nil { log.Errorf("failed to accept ws connection: %s", err) @@ -91,5 +82,4 @@ func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { conn := NewConn(wsConn, lAddr, rAddr) l.acceptFn(conn) - return } diff --git a/relay/server/peer.go b/relay/server/peer.go index 7af113079..14a86a8ab 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -1,33 +1,137 @@ package server import ( + "context" + "fmt" + "io" "net" + "sync" + "time" log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/relay/messages" ) +const ( + bufferSize = 8820 +) + type Peer struct { - Log *log.Entry - idS string - idB []byte - conn net.Conn + log *log.Entry + idS string + idB []byte + conn net.Conn + connMu sync.RWMutex + store *Store } -func NewPeer(id []byte, conn net.Conn) *Peer { +func NewPeer(id []byte, conn net.Conn, store *Store) *Peer { stringID := messages.HashIDToString(id) return &Peer{ - Log: log.WithField("peer_id", stringID), - idB: id, - idS: stringID, - conn: conn, + log: log.WithField("peer_id", stringID), + idS: stringID, + idB: id, + conn: conn, + store: store, } } -func (p *Peer) ID() []byte { - return p.idB + +func (p *Peer) Work() { + buf := make([]byte, bufferSize) + for { + n, err := p.conn.Read(buf) + if err != nil { + if err != io.EOF { + p.log.Errorf("failed to read message: %s", err) + } + return + } + + msg := buf[:n] + + msgType, err := messages.DetermineClientMsgType(msg) + if err != nil { + p.log.Errorf("failed to determine message type: %s", err) + return + } + switch msgType { + case messages.MsgHealthCheck: + case messages.MsgTypeTransport: + peerID, err := messages.UnmarshalTransportID(msg) + if err != nil { + p.log.Errorf("failed to unmarshal transport message: %s", err) + continue + } + stringPeerID := messages.HashIDToString(peerID) + dp, ok := p.store.Peer(stringPeerID) + if !ok { + p.log.Errorf("peer not found: %s", stringPeerID) + continue + } + err = messages.UpdateTransportMsg(msg, p.idB) + if err != nil { + p.log.Errorf("failed to update transport message: %s", err) + continue + } + _, err = dp.Write(msg) + if err != nil { + p.log.Errorf("failed to write transport message to: %s", dp.String()) + } + case messages.MsgClose: + p.log.Infof("peer exited gracefully") + _ = p.conn.Close() + return + } + } +} + +// Write writes data to the connection +// it has been called by the remote peer +func (p *Peer) Write(b []byte) (int, error) { + p.connMu.RLock() + defer p.connMu.RUnlock() + return p.conn.Write(b) +} + +func (p *Peer) CloseGracefully(ctx context.Context) { + p.connMu.Lock() + _, err := p.writeWithTimeout(ctx, messages.MarshalCloseMsg()) + if err != nil { + log.Errorf("failed to send close message to peer: %s", p.String()) + } + + err = p.conn.Close() + if err != nil { + log.Errorf("failed to close connection to peer: %s", err) + } + + defer p.connMu.Unlock() } func (p *Peer) String() string { return p.idS } + +func (p *Peer) writeWithTimeout(ctx context.Context, buf []byte) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + writeDone := make(chan struct{}) + var ( + n int + err error + ) + + go func() { + _, err = p.conn.Write(buf) + close(writeDone) + }() + + select { + case <-ctx.Done(): + return 0, fmt.Errorf("write operation timed out") + case <-writeDone: + return n, err + } +} diff --git a/relay/server/relay.go b/relay/server/relay.go new file mode 100644 index 000000000..39719f4a9 --- /dev/null +++ b/relay/server/relay.go @@ -0,0 +1,99 @@ +package server + +import ( + "context" + "fmt" + "net" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/messages" +) + +type Relay struct { + store *Store + + closed bool + closeMu sync.RWMutex +} + +func NewRelay() *Relay { + return &Relay{ + store: NewStore(), + } +} + +func (r *Relay) Accept(conn net.Conn) { + r.closeMu.RLock() + defer r.closeMu.RUnlock() + if r.closed { + return + } + + peerID, err := handShake(conn) + if err != nil { + log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err) + cErr := conn.Close() + if cErr != nil { + log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr) + } + return + } + + peer := NewPeer(peerID, conn, r.store) + peer.log.Infof("peer connected from: %s", conn.RemoteAddr()) + r.store.AddPeer(peer) + + go func() { + peer.Work() + r.store.DeletePeer(peer) + peer.log.Debugf("relay connection closed") + }() +} + +func (r *Relay) Close(ctx context.Context) { + log.Infof("closeing connection with all peers") + r.closeMu.Lock() + wg := sync.WaitGroup{} + peers := r.store.Peers() + for _, peer := range peers { + wg.Add(1) + go func(p *Peer) { + p.CloseGracefully(ctx) + wg.Done() + }(peer) + } + wg.Wait() + r.closeMu.Unlock() +} + +func handShake(conn net.Conn) ([]byte, error) { + buf := make([]byte, messages.MaxHandshakeSize) + n, err := conn.Read(buf) + if err != nil { + log.Errorf("failed to read message: %s", err) + return nil, err + } + msgType, err := messages.DetermineClientMsgType(buf[:n]) + if err != nil { + return nil, err + } + if msgType != messages.MsgTypeHello { + tErr := fmt.Errorf("invalid message type") + log.Errorf("failed to handshake: %s", tErr) + return nil, tErr + } + peerID, err := messages.UnmarshalHelloMsg(buf[:n]) + if err != nil { + log.Errorf("failed to handshake: %s", err) + return nil, err + } + + msg := messages.MarshalHelloResponse() + _, err = conn.Write(msg) + if err != nil { + return nil, err + } + return peerID, nil +} diff --git a/relay/server/server.go b/relay/server/server.go index f52f7eab8..cf48e19e3 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -1,36 +1,27 @@ package server import ( + "context" "errors" - "fmt" - "io" - "net" "sync" + "time" log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/relay/messages" "github.com/netbirdio/netbird/relay/server/listener" "github.com/netbirdio/netbird/relay/server/listener/udp" - ws "github.com/netbirdio/netbird/relay/server/listener/wsnhooyr" -) - -const ( - bufferSize = 8820 + "github.com/netbirdio/netbird/relay/server/listener/ws" ) type Server struct { - store *Store - storeMu sync.RWMutex - - UDPListener listener.Listener - WSListener listener.Listener + relay *Relay + uDPListener listener.Listener + wSListener listener.Listener } func NewServer() *Server { return &Server{ - store: NewStore(), - storeMu: sync.RWMutex{}, + relay: NewRelay(), } } @@ -38,21 +29,21 @@ func (r *Server) Listen(address string) error { wg := sync.WaitGroup{} wg.Add(2) - r.WSListener = ws.NewListener(address) + r.wSListener = ws.NewListener(address) var wslErr error go func() { defer wg.Done() - wslErr = r.WSListener.Listen(r.accept) + wslErr = r.wSListener.Listen(r.relay.Accept) if wslErr != nil { log.Errorf("failed to bind ws server: %s", wslErr) } }() - r.UDPListener = udp.NewListener(address) + r.uDPListener = udp.NewListener(address) var udpLErr error go func() { defer wg.Done() - udpLErr = r.UDPListener.Listen(r.accept) + udpLErr = r.uDPListener.Listen(r.relay.Accept) if udpLErr != nil { log.Errorf("failed to bind ws server: %s", udpLErr) } @@ -64,131 +55,21 @@ func (r *Server) Listen(address string) error { func (r *Server) Close() error { var wErr error - if r.WSListener != nil { - wErr = r.WSListener.Close() + // stop service new connections + if r.wSListener != nil { + wErr = r.wSListener.Close() } var uErr error - if r.UDPListener != nil { - uErr = r.UDPListener.Close() + if r.uDPListener != nil { + uErr = r.uDPListener.Close() } - r.sendCloseMsgs() - - r.WSListener.WaitForExitAcceptedConns() + // close accepted connections gracefully + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + r.relay.Close(ctx) err := errors.Join(wErr, uErr) return err } - -func (r *Server) accept(conn net.Conn) { - peer, err := handShake(conn) - if err != nil { - log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err) - cErr := conn.Close() - if cErr != nil { - log.Errorf("failed to close connection, %s: %s", conn.RemoteAddr(), cErr) - } - return - } - peer.Log.Infof("peer connected from: %s", conn.RemoteAddr()) - - r.store.AddPeer(peer) - defer func() { - r.store.DeletePeer(peer) - peer.Log.Infof("relay connection closed") - }() - - buf := make([]byte, bufferSize) - for { - n, err := conn.Read(buf) - if err != nil { - if err != io.EOF { - peer.Log.Errorf("failed to read message: %s", err) - } - return - } - - msg := buf[:n] - - msgType, err := messages.DetermineClientMsgType(msg) - if err != nil { - peer.Log.Errorf("failed to determine message type: %s", err) - return - } - switch msgType { - case messages.MsgTypeTransport: - peerID, err := messages.UnmarshalTransportID(msg) - if err != nil { - peer.Log.Errorf("failed to unmarshal transport message: %s", err) - continue - } - stringPeerID := messages.HashIDToString(peerID) - dp, ok := r.store.Peer(stringPeerID) - if !ok { - peer.Log.Errorf("peer not found: %s", stringPeerID) - continue - } - err = messages.UpdateTransportMsg(msg, peer.ID()) - if err != nil { - peer.Log.Errorf("failed to update transport message: %s", err) - continue - } - _, err = dp.conn.Write(msg) - if err != nil { - peer.Log.Errorf("failed to write transport message to: %s", dp.String()) - } - case messages.MsgClose: - peer.Log.Infof("peer disconnected gracefully") - _ = conn.Close() - return - } - } -} - -func (r *Server) sendCloseMsgs() { - msg := messages.MarshalCloseMsg() - - r.storeMu.Lock() - log.Debugf("sending close messages to %d peers", len(r.store.peers)) - for _, p := range r.store.peers { - _, err := p.conn.Write(msg) - if err != nil { - log.Errorf("failed to send close message to peer: %s", p.String()) - } - - err = p.conn.Close() - if err != nil { - log.Errorf("failed to close connection to peer: %s", err) - } - } - r.storeMu.Unlock() -} - -func handShake(conn net.Conn) (*Peer, error) { - buf := make([]byte, messages.MaxHandshakeSize) - n, err := conn.Read(buf) - if err != nil { - log.Errorf("failed to read message: %s", err) - return nil, err - } - msgType, err := messages.DetermineClientMsgType(buf[:n]) - if err != nil { - return nil, err - } - if msgType != messages.MsgTypeHello { - tErr := fmt.Errorf("invalid message type") - log.Errorf("failed to handshake: %s", tErr) - return nil, tErr - } - peerId, err := messages.UnmarshalHelloMsg(buf[:n]) - if err != nil { - log.Errorf("failed to handshake: %s", err) - return nil, err - } - p := NewPeer(peerId, conn) - - msg := messages.MarshalHelloResponse() - _, err = conn.Write(msg) - return p, err -} From dd0d15c9d4aaadcd27fd6dd582c02f8274edc2d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 27 Jun 2024 18:40:12 +0200 Subject: [PATCH 070/199] Add healthcheck code --- relay/client/client.go | 9 +++- relay/healthcheck/receiver.go | 75 ++++++++++++++++++++++++++++++ relay/healthcheck/receiver_test.go | 42 +++++++++++++++++ relay/healthcheck/sender.go | 68 +++++++++++++++++++++++++++ relay/healthcheck/sender_test.go | 66 ++++++++++++++++++++++++++ relay/messages/message.go | 25 +++++++--- relay/server/peer.go | 31 +++++++++++- 7 files changed, 307 insertions(+), 9 deletions(-) create mode 100644 relay/healthcheck/receiver.go create mode 100644 relay/healthcheck/receiver_test.go create mode 100644 relay/healthcheck/sender.go create mode 100644 relay/healthcheck/sender_test.go diff --git a/relay/client/client.go b/relay/client/client.go index 3f689e6dd..2913550c7 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -141,6 +141,7 @@ func (c *Client) Connect() error { log.Errorf("failed to close relay connection: %s", cErr) } }) + c.wgReadLoop.Add(1) go c.readLoop(c.relayConn) @@ -306,6 +307,12 @@ func (c *Client) readLoop(relayConn net.Conn) { } switch msgType { + case messages.MsgTypeHealthCheck: + msg := messages.MarshalHealthcheck() + _, err := c.relayConn.Write(msg) + if err != nil { + c.log.Errorf("failed to send heartbeat response: %s", err) + } case messages.MsgTypeTransport: peerID, payload, err := messages.UnmarshalTransportMsg(buf[:n]) if err != nil { @@ -330,7 +337,7 @@ func (c *Client) readLoop(relayConn net.Conn) { bufPool: c.bufPool, bufPtr: bufPtr, Payload: payload}) - case messages.MsgClose: + case messages.MsgTypeClose: closedByServer = true log.Debugf("relay connection close by server") goto Exit diff --git a/relay/healthcheck/receiver.go b/relay/healthcheck/receiver.go new file mode 100644 index 000000000..157fb6684 --- /dev/null +++ b/relay/healthcheck/receiver.go @@ -0,0 +1,75 @@ +package healthcheck + +import ( + "context" + "time" +) + +var ( + heartbeatTimeout = healthCheckInterval + 3*time.Second +) + +// Receiver is a healthcheck receiver +// It will listen for heartbeat and check if the heartbeat is not received in a certain time +// If the heartbeat is not received in a certain time, it will send a timeout signal and stop to work +// It will also stop if the context is canceled +// The heartbeat timeout is a bit longer than the sender's healthcheck interval +type Receiver struct { + OnTimeout chan struct{} + + ctx context.Context + ctxCancel context.CancelFunc + heartbeat chan struct{} + live bool +} + +func NewReceiver() *Receiver { + ctx, ctxCancel := context.WithCancel(context.Background()) + + r := &Receiver{ + OnTimeout: make(chan struct{}, 1), + ctx: ctx, + ctxCancel: ctxCancel, + heartbeat: make(chan struct{}, 1), + } + + go r.waitForHealthcheck() + return r +} + +func (r *Receiver) Heartbeat() { + select { + case r.heartbeat <- struct{}{}: + default: + } +} + +func (r *Receiver) Stop() { + r.ctxCancel() +} + +func (r *Receiver) waitForHealthcheck() { + ticker := time.NewTicker(heartbeatTimeout) + defer ticker.Stop() + defer r.ctxCancel() + defer close(r.OnTimeout) + + for { + select { + case <-r.heartbeat: + r.live = true + case <-ticker.C: + if r.live { + r.live = false + continue + } + select { + case r.OnTimeout <- struct{}{}: + default: + } + return + case <-r.ctx.Done(): + return + } + } +} diff --git a/relay/healthcheck/receiver_test.go b/relay/healthcheck/receiver_test.go new file mode 100644 index 000000000..4b4123416 --- /dev/null +++ b/relay/healthcheck/receiver_test.go @@ -0,0 +1,42 @@ +package healthcheck + +import ( + "testing" + "time" +) + +func TestNewReceiver(t *testing.T) { + heartbeatTimeout = 5 * time.Second + r := NewReceiver() + + select { + case <-r.OnTimeout: + t.Error("unexpected timeout") + case <-time.After(1 * time.Second): + + } +} + +func TestNewReceiverNotReceive(t *testing.T) { + heartbeatTimeout = 1 * time.Second + r := NewReceiver() + + select { + case <-r.OnTimeout: + case <-time.After(2 * time.Second): + t.Error("timeout not received") + } +} + +func TestNewReceiverAck(t *testing.T) { + heartbeatTimeout = 2 * time.Second + r := NewReceiver() + + r.Heartbeat() + + select { + case <-r.OnTimeout: + t.Error("unexpected timeout") + case <-time.After(3 * time.Second): + } +} diff --git a/relay/healthcheck/sender.go b/relay/healthcheck/sender.go new file mode 100644 index 000000000..401170ec9 --- /dev/null +++ b/relay/healthcheck/sender.go @@ -0,0 +1,68 @@ +package healthcheck + +import ( + "context" + "time" +) + +var ( + healthCheckInterval = 25 * time.Second + healthCheckTimeout = 5 * time.Second +) + +// Sender is a healthcheck sender +// It will send healthcheck signal to the receiver +// If the receiver does not receive the signal in a certain time, it will send a timeout signal and stop to work +// It will also stop if the context is canceled +type Sender struct { + HealthCheck chan struct{} + Timeout chan struct{} + + ctx context.Context + ack chan struct{} +} + +// NewSender creates a new healthcheck sender +func NewSender(ctx context.Context) *Sender { + hc := &Sender{ + HealthCheck: make(chan struct{}, 1), + Timeout: make(chan struct{}, 1), + ctx: ctx, + ack: make(chan struct{}, 1), + } + + go hc.healthCheck() + return hc +} + +func (hc *Sender) OnHCResponse() { + select { + case hc.ack <- struct{}{}: + default: + } +} + +func (hc *Sender) healthCheck() { + ticker := time.NewTicker(healthCheckInterval) + defer ticker.Stop() + + timeoutTimer := time.NewTimer(healthCheckInterval + healthCheckTimeout) + defer timeoutTimer.Stop() + + defer close(hc.HealthCheck) + defer close(hc.Timeout) + + for { + select { + case <-ticker.C: + hc.HealthCheck <- struct{}{} + case <-timeoutTimer.C: + hc.Timeout <- struct{}{} + return + case <-hc.ack: + timeoutTimer.Stop() + case <-hc.ctx.Done(): + return + } + } +} diff --git a/relay/healthcheck/sender_test.go b/relay/healthcheck/sender_test.go new file mode 100644 index 000000000..a571437ef --- /dev/null +++ b/relay/healthcheck/sender_test.go @@ -0,0 +1,66 @@ +package healthcheck + +import ( + "context" + "testing" + "time" +) + +func TestNewHealthPeriod(t *testing.T) { + // override the health check interval to speed up the test + healthCheckInterval = 1 * time.Second + healthCheckTimeout = 100 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender(ctx) + + iterations := 0 + for i := 0; i < 3; i++ { + select { + case <-hc.HealthCheck: + iterations++ + hc.OnHCResponse() + case <-hc.Timeout: + t.Fatalf("health check is timed out") + case <-time.After(healthCheckInterval + 100*time.Millisecond): + t.Fatalf("health check not received") + } + } +} + +func TestNewHealthFailed(t *testing.T) { + // override the health check interval to speed up the test + healthCheckInterval = 1 * time.Second + healthCheckTimeout = 500 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + hc := NewSender(ctx) + + select { + case <-hc.Timeout: + case <-time.After(healthCheckInterval + healthCheckTimeout + 100*time.Millisecond): + t.Fatalf("health check is not timed out") + } +} + +func TestNewHealthcheckStop(t *testing.T) { + + ctx, cancel := context.WithCancel(context.Background()) + hc := NewSender(ctx) + + time.Sleep(1 * time.Second) + cancel() + + select { + case <-hc.HealthCheck: + t.Fatalf("is not closed") + case <-hc.Timeout: + t.Fatalf("is not closed") + case <-ctx.Done(): + // expected + case <-time.After(1 * time.Second): + t.Fatalf("is not exited") + } +} diff --git a/relay/messages/message.go b/relay/messages/message.go index d2a6d46d7..aa62fe867 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -11,7 +11,8 @@ const ( MsgTypeHello MsgType = 0 MsgTypeHelloResponse MsgType = 1 MsgTypeTransport MsgType = 2 - MsgClose MsgType = 3 + MsgTypeClose MsgType = 3 + MsgTypeHealthCheck MsgType = 4 headerSizeTransport = 1 + IDSize // 1 byte for msg type, IDSize for peerID headerSizeHello = 1 + 4 + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID @@ -23,6 +24,8 @@ var ( ErrInvalidMessageLength = fmt.Errorf("invalid message length") magicHeader = []byte{0x21, 0x12, 0xA4, 0x42} + + healthCheckMsg = []byte{byte(MsgTypeHealthCheck)} ) type MsgType byte @@ -35,7 +38,7 @@ func (m MsgType) String() string { return "hello response" case MsgTypeTransport: return "transport" - case MsgClose: + case MsgTypeClose: return "close" default: return "unknown" @@ -49,7 +52,9 @@ func DetermineClientMsgType(msg []byte) (MsgType, error) { return msgType, nil case MsgTypeTransport: return msgType, nil - case MsgClose: + case MsgTypeClose: + return msgType, nil + case MsgTypeHealthCheck: return msgType, nil default: return 0, fmt.Errorf("invalid msg type, len: %d", len(msg)) @@ -63,7 +68,9 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { return msgType, nil case MsgTypeTransport: return msgType, nil - case MsgClose: + case MsgTypeClose: + return msgType, nil + case MsgTypeHealthCheck: return msgType, nil default: return 0, fmt.Errorf("invalid msg type (len: %d)", len(msg)) @@ -100,8 +107,8 @@ func MarshalHelloResponse() []byte { func MarshalCloseMsg() []byte { msg := make([]byte, 1) - msg[0] = byte(MsgClose) - return msg + msg[0] = byte(MsgTypeClose) + return healthCheckMsg } // Transport message @@ -141,3 +148,9 @@ func UpdateTransportMsg(msg []byte, peerID []byte) error { copy(msg[1:], peerID) return nil } + +// health check message + +func MarshalHealthcheck() []byte { + return healthCheckMsg +} diff --git a/relay/server/peer.go b/relay/server/peer.go index 14a86a8ab..2869340a9 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -10,6 +10,7 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/relay/healthcheck" "github.com/netbirdio/netbird/relay/messages" ) @@ -38,6 +39,11 @@ func NewPeer(id []byte, conn net.Conn, store *Store) *Peer { } func (p *Peer) Work() { + ctx, cancel := context.WithCancel(context.Background()) + hc := healthcheck.NewSender(ctx) + go p.healthcheck(ctx, hc) + defer cancel() + buf := make([]byte, bufferSize) for { n, err := p.conn.Read(buf) @@ -56,7 +62,8 @@ func (p *Peer) Work() { return } switch msgType { - case messages.MsgHealthCheck: + case messages.MsgTypeHealthCheck: + hc.OnHCResponse() case messages.MsgTypeTransport: peerID, err := messages.UnmarshalTransportID(msg) if err != nil { @@ -78,7 +85,7 @@ func (p *Peer) Work() { if err != nil { p.log.Errorf("failed to write transport message to: %s", dp.String()) } - case messages.MsgClose: + case messages.MsgTypeClose: p.log.Infof("peer exited gracefully") _ = p.conn.Close() return @@ -135,3 +142,23 @@ func (p *Peer) writeWithTimeout(ctx context.Context, buf []byte) (int, error) { return n, err } } + +func (p *Peer) healthcheck(ctx context.Context, hc *healthcheck.Sender) { + for { + select { + case <-hc.HealthCheck: + p.log.Debugf("sending healthcheck message") + _, err := p.Write(messages.MarshalHealthcheck()) + if err != nil { + p.log.Errorf("failed to send healthcheck message: %s", err) + return + } + case <-hc.Timeout: + p.log.Errorf("peer healthcheck timeout") + _ = p.conn.Close() + return + case <-ctx.Done(): + return + } + } +} From 183f746158a6a83d0464647def338fa445348802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 27 Jun 2024 18:42:40 +0200 Subject: [PATCH 071/199] Order the source code --- relay/client/client.go | 116 ++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 2913550c7..40421e2b9 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -203,31 +203,6 @@ func (c *Client) Close() error { return c.close(false) } -func (c *Client) close(byServer bool) error { - c.readLoopMutex.Lock() - defer c.readLoopMutex.Unlock() - - c.mu.Lock() - var err error - if !c.serviceIsRunning { - c.mu.Unlock() - return nil - } - - c.serviceIsRunning = false - c.closeAllConns() - if !byServer { - c.writeCloseMsg() - err = c.relayConn.Close() - } - c.mu.Unlock() - - c.wgReadLoop.Wait() - c.log.Infof("relay connection closed with: %s", c.serverAddress) - c.ctxCancel() - return err -} - func (c *Client) connect() error { conn, err := ws.Dial(c.serverAddress) if err != nil { @@ -384,6 +359,64 @@ func (c *Client) closeAllConns() { c.conns = make(map[string]*connContainer) } +// todo check by reference too, the id is not enought because the id come from the outer conn +func (c *Client) closeConn(id string) error { + c.mu.Lock() + defer c.mu.Unlock() + + container, ok := c.conns[id] + if !ok { + return fmt.Errorf("connection already closed") + } + container.close() + delete(c.conns, id) + + return nil +} + +func (c *Client) close(byServer bool) error { + c.readLoopMutex.Lock() + defer c.readLoopMutex.Unlock() + + c.mu.Lock() + var err error + if !c.serviceIsRunning { + c.mu.Unlock() + return nil + } + + c.serviceIsRunning = false + c.closeAllConns() + if !byServer { + c.writeCloseMsg() + err = c.relayConn.Close() + } + c.mu.Unlock() + + c.wgReadLoop.Wait() + c.log.Infof("relay connection closed with: %s", c.serverAddress) + c.ctxCancel() + return err +} + +func (c *Client) notifyDisconnected() { + c.listenerMutex.Lock() + defer c.listenerMutex.Unlock() + + if c.onDisconnectListener == nil { + return + } + go c.onDisconnectListener() +} + +func (c *Client) writeCloseMsg() { + msg := messages.MarshalCloseMsg() + _, err := c.relayConn.Write(msg) + if err != nil { + c.log.Errorf("failed to send close message: %s", err) + } +} + func (c *Client) readWithTimeout(buf []byte) (int, error) { ctx, cancel := context.WithTimeout(c.parentCtx, serverResponseTimeout) defer cancel() @@ -406,36 +439,3 @@ func (c *Client) readWithTimeout(buf []byte) (int, error) { return n, err } } - -// todo check by reference too, the id is not enought because the id come from the outer conn -func (c *Client) closeConn(id string) error { - c.mu.Lock() - defer c.mu.Unlock() - - container, ok := c.conns[id] - if !ok { - return fmt.Errorf("connection already closed") - } - container.close() - delete(c.conns, id) - - return nil -} - -func (c *Client) notifyDisconnected() { - c.listenerMutex.Lock() - defer c.listenerMutex.Unlock() - - if c.onDisconnectListener == nil { - return - } - go c.onDisconnectListener() -} - -func (c *Client) writeCloseMsg() { - msg := messages.MarshalCloseMsg() - _, err := c.relayConn.Write(msg) - if err != nil { - c.log.Errorf("failed to send close message: %s", err) - } -} From c94c949173b87034b5d4375f60bac409417d35a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 28 Jun 2024 11:12:53 +0200 Subject: [PATCH 072/199] Add comment --- relay/healthcheck/receiver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/relay/healthcheck/receiver.go b/relay/healthcheck/receiver.go index 157fb6684..147fe2d5f 100644 --- a/relay/healthcheck/receiver.go +++ b/relay/healthcheck/receiver.go @@ -23,6 +23,7 @@ type Receiver struct { live bool } +// NewReceiver creates a new healthcheck receiver and start the timer in the background func NewReceiver() *Receiver { ctx, ctxCancel := context.WithCancel(context.Background()) @@ -37,6 +38,7 @@ func NewReceiver() *Receiver { return r } +// Heartbeat acknowledge the heartbeat has been received func (r *Receiver) Heartbeat() { select { case r.heartbeat <- struct{}{}: @@ -44,6 +46,7 @@ func (r *Receiver) Heartbeat() { } } +// Stop check the timeout and do not send new notifications func (r *Receiver) Stop() { r.ctxCancel() } From 98aa830831f41af36278580006b2bb3f655a88b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 28 Jun 2024 11:17:06 +0200 Subject: [PATCH 073/199] Rename client ws package --- relay/client/client.go | 2 +- relay/client/dialer/{wsnhooyr => ws}/client_conn.go | 2 +- relay/client/dialer/{wsnhooyr => ws}/ws.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename relay/client/dialer/{wsnhooyr => ws}/client_conn.go (98%) rename relay/client/dialer/{wsnhooyr => ws}/ws.go (97%) diff --git a/relay/client/client.go b/relay/client/client.go index 40421e2b9..b5d4392b3 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -10,7 +10,7 @@ import ( log "github.com/sirupsen/logrus" - ws "github.com/netbirdio/netbird/relay/client/dialer/wsnhooyr" + "github.com/netbirdio/netbird/relay/client/dialer/ws" "github.com/netbirdio/netbird/relay/messages" ) diff --git a/relay/client/dialer/wsnhooyr/client_conn.go b/relay/client/dialer/ws/client_conn.go similarity index 98% rename from relay/client/dialer/wsnhooyr/client_conn.go rename to relay/client/dialer/ws/client_conn.go index 261edd04c..b0b8f5aba 100644 --- a/relay/client/dialer/wsnhooyr/client_conn.go +++ b/relay/client/dialer/ws/client_conn.go @@ -1,4 +1,4 @@ -package wsnhooyr +package ws import ( "context" diff --git a/relay/client/dialer/wsnhooyr/ws.go b/relay/client/dialer/ws/ws.go similarity index 97% rename from relay/client/dialer/wsnhooyr/ws.go rename to relay/client/dialer/ws/ws.go index 4fe0f0a26..c1d1f74f1 100644 --- a/relay/client/dialer/wsnhooyr/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -1,4 +1,4 @@ -package wsnhooyr +package ws import ( "context" From 9ae03046e7eba9bdf73ad5ea95ff55b0b524c905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 28 Jun 2024 11:17:21 +0200 Subject: [PATCH 074/199] rename file --- relay/client/dialer/ws/{client_conn.go => conn.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename relay/client/dialer/ws/{client_conn.go => conn.go} (100%) diff --git a/relay/client/dialer/ws/client_conn.go b/relay/client/dialer/ws/conn.go similarity index 100% rename from relay/client/dialer/ws/client_conn.go rename to relay/client/dialer/ws/conn.go From faeae523292e0492065ab1569e6c6877d24eac16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 28 Jun 2024 11:44:50 +0200 Subject: [PATCH 075/199] Support exit node in ws client --- relay/client/dialer/ws/ws.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index c1d1f74f1..90175ebf9 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "net" + "net/http" log "github.com/sirupsen/logrus" "nhooyr.io/websocket" + + nbnet "github.com/netbirdio/netbird/util/net" ) func Dial(address string) (net.Conn, error) { @@ -21,7 +24,8 @@ func Dial(address string) (net.Conn, error) { url := fmt.Sprintf("ws://%s:%d", addr.IP.String(), addr.Port) opts := &websocket.DialOptions{ - Host: hostName, + Host: hostName, + HTTPClient: httpClientNbDialer(), } wsConn, _, err := websocket.Dial(context.Background(), url, opts) @@ -34,3 +38,17 @@ func Dial(address string) (net.Conn, error) { return conn, nil } + +func httpClientNbDialer() *http.Client { + customDialer := nbnet.NewDialer() + + customTransport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return customDialer.DialContext(ctx, network, addr) + }, + } + + return &http.Client{ + Transport: customTransport, + } +} From aa55fba5eecfc7cb1daabdfc779e86709940e3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Sat, 29 Jun 2024 14:13:05 +0200 Subject: [PATCH 076/199] Add client side heartbeat handling --- relay/client/client.go | 90 +++++++++++++++++++++++++---------- relay/healthcheck/receiver.go | 21 ++++---- 2 files changed, 77 insertions(+), 34 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index b5d4392b3..5d5116477 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -11,6 +11,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/relay/client/dialer/ws" + "github.com/netbirdio/netbird/relay/healthcheck" "github.com/netbirdio/netbird/relay/messages" ) @@ -23,6 +24,27 @@ var ( ErrConnAlreadyExists = fmt.Errorf("connection already exists") ) +type internalStopFlag struct { + sync.Mutex + stop bool +} + +func newInternalStopFlag() *internalStopFlag { + return &internalStopFlag{} +} + +func (isf *internalStopFlag) set() { + isf.Lock() + defer isf.Unlock() + isf.stop = true +} + +func (isf *internalStopFlag) isSet() bool { + isf.Lock() + defer isf.Unlock() + return isf.stop +} + // Msg carry the payload from the server to the client. With this sturct, the net.Conn can free the buffer. type Msg struct { Payload []byte @@ -75,7 +97,6 @@ func (cc *connContainer) close() { type Client struct { log *log.Entry parentCtx context.Context - ctxCancel context.CancelFunc serverAddress string hashedID []byte @@ -84,7 +105,7 @@ type Client struct { relayConn net.Conn conns map[string]*connContainer serviceIsRunning bool - mu sync.Mutex + mu sync.Mutex // protect serviceIsRunning and conns readLoopMutex sync.Mutex wgReadLoop sync.WaitGroup @@ -100,7 +121,6 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { return &Client{ log: log.WithField("client_id", hashedStringId), parentCtx: ctx, - ctxCancel: func() {}, serverAddress: serverAddress, hashedID: hashedID, bufPool: &sync.Pool{ @@ -133,15 +153,6 @@ func (c *Client) Connect() error { c.serviceIsRunning = true - var ctx context.Context - ctx, c.ctxCancel = context.WithCancel(c.parentCtx) - context.AfterFunc(ctx, func() { - cErr := c.close(false) - if cErr != nil { - log.Errorf("failed to close relay connection: %s", cErr) - } - }) - c.wgReadLoop.Add(1) go c.readLoop(c.relayConn) @@ -200,7 +211,7 @@ func (c *Client) HasConns() bool { // Close closes the connection to the relay server and all connections to other peers. func (c *Client) Close() error { - return c.close(false) + return c.close(true) } func (c *Client) connect() error { @@ -257,10 +268,13 @@ func (c *Client) handShake() error { } func (c *Client) readLoop(relayConn net.Conn) { + internallyStoppedFlag := newInternalStopFlag() + hc := healthcheck.NewReceiver() + go c.listenForStopEvents(hc, relayConn, internallyStoppedFlag) + var ( - errExit error - n int - closedByServer bool + errExit error + n int ) for { bufPtr := c.bufPool.Get().(*[]byte) @@ -268,7 +282,7 @@ func (c *Client) readLoop(relayConn net.Conn) { n, errExit = relayConn.Read(buf) if errExit != nil { c.mu.Lock() - if c.serviceIsRunning { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { c.log.Debugf("failed to read message from relay server: %s", errExit) } c.mu.Unlock() @@ -283,15 +297,19 @@ func (c *Client) readLoop(relayConn net.Conn) { switch msgType { case messages.MsgTypeHealthCheck: + log.Debugf("on new heartbeat") msg := messages.MarshalHealthcheck() - _, err := c.relayConn.Write(msg) - if err != nil { - c.log.Errorf("failed to send heartbeat response: %s", err) + _, wErr := c.relayConn.Write(msg) + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to send heartbeat: %s", wErr) } + hc.Heartbeat() case messages.MsgTypeTransport: peerID, payload, err := messages.UnmarshalTransportMsg(buf[:n]) if err != nil { - c.log.Errorf("failed to parse transport message: %v", err) + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to parse transport message: %v", err) + } continue } stringID := messages.HashIDToString(peerID) @@ -313,16 +331,16 @@ func (c *Client) readLoop(relayConn net.Conn) { bufPtr: bufPtr, Payload: payload}) case messages.MsgTypeClose: - closedByServer = true log.Debugf("relay connection close by server") goto Exit } } Exit: + hc.Stop() c.notifyDisconnected() c.wgReadLoop.Done() - _ = c.close(closedByServer) + _ = c.close(false) } // todo check by reference too, the id is not enought because the id come from the outer conn @@ -352,6 +370,27 @@ func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { return n, err } +func (c *Client) listenForStopEvents(hc *healthcheck.Receiver, conn net.Conn, internalStopFlag *internalStopFlag) { + for { + select { + case _, ok := <-hc.OnTimeout: + if !ok { + return + } + c.log.Errorf("health check timeout") + internalStopFlag.set() + _ = conn.Close() // ignore the err because the readLoop will handle it + return + case <-c.parentCtx.Done(): + err := c.close(true) + if err != nil { + log.Errorf("failed to teardown connection: %s", err) + } + return + } + } +} + func (c *Client) closeAllConns() { for _, container := range c.conns { container.close() @@ -374,7 +413,7 @@ func (c *Client) closeConn(id string) error { return nil } -func (c *Client) close(byServer bool) error { +func (c *Client) close(gracefullyExit bool) error { c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() @@ -387,7 +426,7 @@ func (c *Client) close(byServer bool) error { c.serviceIsRunning = false c.closeAllConns() - if !byServer { + if gracefullyExit { c.writeCloseMsg() err = c.relayConn.Close() } @@ -395,7 +434,6 @@ func (c *Client) close(byServer bool) error { c.wgReadLoop.Wait() c.log.Infof("relay connection closed with: %s", c.serverAddress) - c.ctxCancel() return err } diff --git a/relay/healthcheck/receiver.go b/relay/healthcheck/receiver.go index 147fe2d5f..e1ef17e0e 100644 --- a/relay/healthcheck/receiver.go +++ b/relay/healthcheck/receiver.go @@ -20,7 +20,7 @@ type Receiver struct { ctx context.Context ctxCancel context.CancelFunc heartbeat chan struct{} - live bool + alive bool } // NewReceiver creates a new healthcheck receiver and start the timer in the background @@ -60,19 +60,24 @@ func (r *Receiver) waitForHealthcheck() { for { select { case <-r.heartbeat: - r.live = true + r.alive = true case <-ticker.C: - if r.live { - r.live = false + if r.alive { + r.alive = false continue } - select { - case r.OnTimeout <- struct{}{}: - default: - } + + r.notifyTimeout() return case <-r.ctx.Done(): return } } } + +func (r *Receiver) notifyTimeout() { + select { + case r.OnTimeout <- struct{}{}: + default: + } +} From ed82ef7fe4b389f02e24b806fb9e4c39c7fd575b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Sun, 30 Jun 2024 10:43:12 +0200 Subject: [PATCH 077/199] Fix error logging --- relay/client/client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 5d5116477..2401f972e 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -300,8 +300,10 @@ func (c *Client) readLoop(relayConn net.Conn) { log.Debugf("on new heartbeat") msg := messages.MarshalHealthcheck() _, wErr := c.relayConn.Write(msg) - if c.serviceIsRunning && !internallyStoppedFlag.isSet() { - c.log.Errorf("failed to send heartbeat: %s", wErr) + if wErr != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to send heartbeat: %s", wErr) + } } hc.Heartbeat() case messages.MsgTypeTransport: From d3785dc1fa36f237474df18abe6631bd2aff1987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 1 Jul 2024 11:50:18 +0200 Subject: [PATCH 078/199] Fix ssl configuration --- relay/client/client_test.go | 64 ++++++++++++++-------------- relay/client/dialer/ws/conn.go | 15 ++++--- relay/client/dialer/ws/ws.go | 37 +++++++++++----- relay/client/manager_test.go | 56 +++++++++++++++--------- relay/cmd/main.go | 36 ++++++++++++++-- relay/server/listener/ws/listener.go | 29 +++++++------ relay/server/server.go | 16 +++++-- 7 files changed, 164 insertions(+), 89 deletions(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index f5d122276..278d46d08 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -23,10 +23,10 @@ func TestMain(m *testing.M) { func TestClient(t *testing.T) { ctx := context.Background() - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -39,21 +39,21 @@ func TestClient(t *testing.T) { } }() - clientAlice := NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, srvCfg.Address, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientAlice.Close() - clientPlaceHolder := NewClient(ctx, addr, "clientPlaceHolder") + clientPlaceHolder := NewClient(ctx, srvCfg.Address, "clientPlaceHolder") err = clientPlaceHolder.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientPlaceHolder.Close() - clientBob := NewClient(ctx, addr, "bob") + clientBob := NewClient(ctx, srvCfg.Address, "bob") err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -91,16 +91,16 @@ func TestClient(t *testing.T) { func TestRegistration(t *testing.T) { ctx := context.Background() - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } }() - clientAlice := NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, srvCfg.Address, "alice") err := clientAlice.Connect() if err != nil { _ = srv.Close() @@ -156,10 +156,10 @@ func TestEcho(t *testing.T) { ctx := context.Background() idAlice := "alice" idBob := "bob" - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -172,7 +172,7 @@ func TestEcho(t *testing.T) { } }() - clientAlice := NewClient(ctx, addr, idAlice) + clientAlice := NewClient(ctx, srvCfg.Address, idAlice) err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -184,7 +184,7 @@ func TestEcho(t *testing.T) { } }() - clientBob := NewClient(ctx, addr, idBob) + clientBob := NewClient(ctx, srvCfg.Address, idBob) err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -236,10 +236,10 @@ func TestEcho(t *testing.T) { func TestBindToUnavailabePeer(t *testing.T) { ctx := context.Background() - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -253,7 +253,7 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() - clientAlice := NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, srvCfg.Address, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -273,10 +273,10 @@ func TestBindToUnavailabePeer(t *testing.T) { func TestBindReconnect(t *testing.T) { ctx := context.Background() - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Errorf("failed to bind server: %s", err) } @@ -290,7 +290,7 @@ func TestBindReconnect(t *testing.T) { } }() - clientAlice := NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, srvCfg.Address, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -301,7 +301,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to bind channel: %s", err) } - clientBob := NewClient(ctx, addr, "bob") + clientBob := NewClient(ctx, srvCfg.Address, "bob") err = clientBob.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -318,7 +318,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to close client: %s", err) } - clientAlice = NewClient(ctx, addr, "alice") + clientAlice = NewClient(ctx, srvCfg.Address, "alice") err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -355,10 +355,10 @@ func TestBindReconnect(t *testing.T) { func TestCloseConn(t *testing.T) { ctx := context.Background() - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Errorf("failed to bind server: %s", err) } @@ -372,7 +372,7 @@ func TestCloseConn(t *testing.T) { } }() - clientAlice := NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, srvCfg.Address, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -403,10 +403,10 @@ func TestCloseConn(t *testing.T) { func TestCloseRelayConn(t *testing.T) { ctx := context.Background() - addr := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Errorf("failed to bind server: %s", err) } @@ -419,7 +419,7 @@ func TestCloseRelayConn(t *testing.T) { } }() - clientAlice := NewClient(ctx, addr, "alice") + clientAlice := NewClient(ctx, srvCfg.Address, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -446,10 +446,10 @@ func TestCloseRelayConn(t *testing.T) { func TestCloseByServer(t *testing.T) { ctx := context.Background() - addr1 := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv1 := server.NewServer() go func() { - err := srv1.Listen(addr1) + err := srv1.Listen(srvCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -457,7 +457,7 @@ func TestCloseByServer(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") - relayClient := NewClient(ctx, addr1, idAlice) + relayClient := NewClient(ctx, srvCfg.Address, idAlice) err := relayClient.Connect() if err != nil { log.Fatalf("failed to connect to server: %s", err) @@ -489,10 +489,10 @@ func TestCloseByServer(t *testing.T) { func TestCloseByClient(t *testing.T) { ctx := context.Background() - addr1 := "localhost:1234" + srvCfg := server.Config{Address: "localhost:1234"} srv := server.NewServer() go func() { - err := srv.Listen(addr1) + err := srv.Listen(srvCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -500,7 +500,7 @@ func TestCloseByClient(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") - relayClient := NewClient(ctx, addr1, idAlice) + relayClient := NewClient(ctx, srvCfg.Address, idAlice) err := relayClient.Connect() if err != nil { log.Fatalf("failed to connect to server: %s", err) diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go index b0b8f5aba..44d86c1bb 100644 --- a/relay/client/dialer/ws/conn.go +++ b/relay/client/dialer/ws/conn.go @@ -12,14 +12,16 @@ import ( type Conn struct { ctx context.Context *websocket.Conn - srvAddr *net.TCPAddr + srvAddr net.Addr + localAddr net.Addr } -func NewConn(wsConn *websocket.Conn, srvAddr *net.TCPAddr) net.Conn { +func NewConn(wsConn *websocket.Conn, srvAddr, localAddr net.Addr) net.Conn { return &Conn{ - ctx: context.Background(), - Conn: wsConn, - srvAddr: srvAddr, + ctx: context.Background(), + Conn: wsConn, + srvAddr: srvAddr, + localAddr: localAddr, } } @@ -46,8 +48,7 @@ func (c *Conn) RemoteAddr() net.Addr { } func (c *Conn) LocalAddr() net.Addr { - // todo: implement me - return nil + return c.localAddr } func (c *Conn) SetReadDeadline(t time.Time) error { diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index 90175ebf9..070a02362 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/http" + "strings" log "github.com/sirupsen/logrus" "nhooyr.io/websocket" @@ -13,32 +14,48 @@ import ( ) func Dial(address string) (net.Conn, error) { - - hostName, _, err := net.SplitHostPort(address) - - addr, err := net.ResolveTCPAddr("tcp", address) + wsURL, err := prepareURL(address) if err != nil { - log.Errorf("failed to resolve address of Relay server: %s", address) return nil, err } - url := fmt.Sprintf("ws://%s:%d", addr.IP.String(), addr.Port) opts := &websocket.DialOptions{ - Host: hostName, HTTPClient: httpClientNbDialer(), } - wsConn, _, err := websocket.Dial(context.Background(), url, opts) + wsConn, _, err := websocket.Dial(context.Background(), wsURL, opts) if err != nil { - log.Errorf("failed to dial to Relay server '%s': %s", url, err) + log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err) return nil, err } - conn := NewConn(wsConn, addr) + /* + response.Body.(net.Conn).LocalAddr() + unc, ok := response.Body.(net.Conn) + if !ok { + log.Errorf("failed to get local address: %s", err) + return nil, fmt.Errorf("failed to get local address") + } + */ + // todo figure out the proper address + dummy := &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 8080, + } + + conn := NewConn(wsConn, dummy, dummy) return conn, nil } +func prepareURL(address string) (string, error) { + if !strings.HasPrefix(address, "rel") { + return "", fmt.Errorf("unsupported scheme: %s", address) + } + + return strings.Replace(address, "rel", "ws", 1), nil +} + func httpClientNbDialer() *http.Client { customDialer := nbnet.NewDialer() diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 69539cc5e..71e4a416f 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -13,10 +13,12 @@ import ( func TestForeignConn(t *testing.T) { ctx := context.Background() - addr1 := "localhost:1234" + srvCfg1 := server.Config{ + Address: "localhost:1234", + } srv1 := server.NewServer() go func() { - err := srv1.Listen(addr1) + err := srv1.Listen(srvCfg1) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -29,10 +31,12 @@ func TestForeignConn(t *testing.T) { } }() - addr2 := "localhost:2234" + srvCfg2 := server.Config{ + Address: "localhost:2234", + } srv2 := server.NewServer() go func() { - err := srv2.Listen(addr2) + err := srv2.Listen(srvCfg2) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -49,12 +53,12 @@ func TestForeignConn(t *testing.T) { log.Debugf("connect by alice") mCtx, cancel := context.WithCancel(ctx) defer cancel() - clientAlice := NewManager(mCtx, addr1, idAlice) + clientAlice := NewManager(mCtx, srvCfg1.Address, idAlice) clientAlice.Serve() idBob := "bob" log.Debugf("connect by bob") - clientBob := NewManager(mCtx, addr2, idBob) + clientBob := NewManager(mCtx, srvCfg2.Address, idBob) clientBob.Serve() bobsSrvAddr, err := clientBob.RelayAddress() @@ -100,10 +104,12 @@ func TestForeignConn(t *testing.T) { func TestForeginConnClose(t *testing.T) { ctx := context.Background() - addr1 := "localhost:1234" + srvCfg1 := server.Config{ + Address: "localhost:1234", + } srv1 := server.NewServer() go func() { - err := srv1.Listen(addr1) + err := srv1.Listen(srvCfg1) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -116,10 +122,12 @@ func TestForeginConnClose(t *testing.T) { } }() - addr2 := "localhost:2234" + srvCfg2 := server.Config{ + Address: "localhost:2234", + } srv2 := server.NewServer() go func() { - err := srv2.Listen(addr2) + err := srv2.Listen(srvCfg2) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -136,10 +144,10 @@ func TestForeginConnClose(t *testing.T) { log.Debugf("connect by alice") mCtx, cancel := context.WithCancel(ctx) defer cancel() - mgr := NewManager(mCtx, addr1, idAlice) + mgr := NewManager(mCtx, srvCfg1.Address, idAlice) mgr.Serve() - conn, err := mgr.OpenConn(addr2, "anotherpeer", nil) + conn, err := mgr.OpenConn(srvCfg2.Address, "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -153,11 +161,13 @@ func TestForeginConnClose(t *testing.T) { func TestForeginAutoClose(t *testing.T) { ctx := context.Background() relayCleanupInterval = 1 * time.Second - addr1 := "localhost:1234" + srvCfg1 := server.Config{ + Address: "localhost:1234", + } srv1 := server.NewServer() go func() { t.Log("binding server 1.") - err := srv1.Listen(addr1) + err := srv1.Listen(srvCfg1) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -172,11 +182,13 @@ func TestForeginAutoClose(t *testing.T) { t.Logf("server 1. closed") }() - addr2 := "localhost:2234" + srvCfg2 := server.Config{ + Address: "localhost:2234", + } srv2 := server.NewServer() go func() { t.Log("binding server 2.") - err := srv2.Listen(addr2) + err := srv2.Listen(srvCfg2) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -194,11 +206,11 @@ func TestForeginAutoClose(t *testing.T) { t.Log("connect to server 1.") mCtx, cancel := context.WithCancel(ctx) defer cancel() - mgr := NewManager(mCtx, addr1, idAlice) + mgr := NewManager(mCtx, srvCfg1.Address, idAlice) mgr.Serve() t.Log("open connection to another peer") - conn, err := mgr.OpenConn(addr2, "anotherpeer", nil) + conn, err := mgr.OpenConn(srvCfg2.Address, "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -222,10 +234,12 @@ func TestAutoReconnect(t *testing.T) { ctx := context.Background() reconnectingTimeout = 2 * time.Second - addr := "localhost:1234" + srvCfg := server.Config{ + Address: "localhost:1234", + } srv := server.NewServer() go func() { - err := srv.Listen(addr) + err := srv.Listen(srvCfg) if err != nil { t.Errorf("failed to bind server: %s", err) } @@ -240,7 +254,7 @@ func TestAutoReconnect(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() - clientAlice := NewManager(mCtx, addr, "alice") + clientAlice := NewManager(mCtx, srvCfg.Address, "alice") clientAlice.Serve() ra, err := clientAlice.RelayAddress() if err != nil { diff --git a/relay/cmd/main.go b/relay/cmd/main.go index eb7d87a1f..9d1802076 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -1,6 +1,8 @@ package main import ( + "crypto/tls" + "fmt" "os" "os/signal" "syscall" @@ -8,12 +10,15 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/netbirdio/netbird/encryption" "github.com/netbirdio/netbird/relay/server" "github.com/netbirdio/netbird/util" ) var ( - listenAddress string + listenAddress string + letsencryptDataDir string + letsencryptDomain string rootCmd = &cobra.Command{ Use: "relay", @@ -26,7 +31,8 @@ var ( func init() { _ = util.InitLog("trace", "console") rootCmd.PersistentFlags().StringVarP(&listenAddress, "listen-address", "l", ":1235", "listen address") - + rootCmd.PersistentFlags().StringVarP(&letsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") + rootCmd.PersistentFlags().StringVarP(&letsencryptDomain, "letsencrypt-domain", "a", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") } func waitForExitSignal() { @@ -36,8 +42,20 @@ func waitForExitSignal() { } func execute(cmd *cobra.Command, args []string) { + srvCfg := server.Config{ + Address: listenAddress, + } + if hasLetsEncrypt() { + tlscfg, err := setupTLS() + if err != nil { + log.Errorf("%s", err) + os.Exit(1) + } + srvCfg.TLSConfig = tlscfg + } + srv := server.NewServer() - err := srv.Listen(listenAddress) + err := srv.Listen(srvCfg) if err != nil { log.Errorf("failed to bind server: %s", err) os.Exit(1) @@ -52,6 +70,18 @@ func execute(cmd *cobra.Command, args []string) { } } +func hasLetsEncrypt() bool { + return letsencryptDataDir != "" && letsencryptDomain != "" +} + +func setupTLS() (*tls.Config, error) { + certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomain) + if err != nil { + return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err) + } + return certManager.TLSConfig(), nil +} + func main() { err := rootCmd.Execute() if err != nil { diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index 26ba2276e..6b2f669f4 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -2,6 +2,7 @@ package ws import ( "context" + "crypto/tls" "errors" "fmt" "net" @@ -10,35 +11,37 @@ import ( log "github.com/sirupsen/logrus" "nhooyr.io/websocket" - - "github.com/netbirdio/netbird/relay/server/listener" ) type Listener struct { - address string + // Address is the address to listen on. + Address string + // TLSConfig is the TLS configuration for the server. + TLSConfig *tls.Config server *http.Server acceptFn func(conn net.Conn) } -func NewListener(address string) listener.Listener { - return &Listener{ - address: address, - } -} - func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { l.acceptFn = acceptFn mux := http.NewServeMux() mux.HandleFunc("/", l.onAccept) l.server = &http.Server{ - Addr: l.address, - Handler: mux, + Addr: l.Address, + Handler: mux, + TLSConfig: l.TLSConfig, } - log.Infof("WS server is listening on address: %s", l.address) - err := l.server.ListenAndServe() + log.Infof("WS server is listening on address: %s", l.Address) + var err error + if l.TLSConfig != nil { + err = l.server.ListenAndServeTLS("", "") + + } else { + err = l.server.ListenAndServe() + } if errors.Is(err, http.ErrServerClosed) { return nil } diff --git a/relay/server/server.go b/relay/server/server.go index cf48e19e3..687486138 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -2,6 +2,7 @@ package server import ( "context" + "crypto/tls" "errors" "sync" "time" @@ -13,6 +14,11 @@ import ( "github.com/netbirdio/netbird/relay/server/listener/ws" ) +type Config struct { + Address string + TLSConfig *tls.Config +} + type Server struct { relay *Relay uDPListener listener.Listener @@ -25,11 +31,15 @@ func NewServer() *Server { } } -func (r *Server) Listen(address string) error { +func (r *Server) Listen(cfg Config) error { wg := sync.WaitGroup{} wg.Add(2) - r.wSListener = ws.NewListener(address) + r.wSListener = &ws.Listener{ + Address: cfg.Address, + TLSConfig: cfg.TLSConfig, + } + var wslErr error go func() { defer wg.Done() @@ -39,7 +49,7 @@ func (r *Server) Listen(address string) error { } }() - r.uDPListener = udp.NewListener(address) + r.uDPListener = udp.NewListener(cfg.Address) var udpLErr error go func() { defer wg.Done() From 15a7b7629b447b915b0b6982b8589d5ca65a92f9 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 2 Jul 2024 11:57:17 +0200 Subject: [PATCH 079/199] Add exposed address --- client/internal/peer/conn.go | 4 +-- client/internal/peer/worker_relay.go | 8 +++--- encryption/letsencrypt.go | 4 +-- go.mod | 4 +-- go.sum | 4 +-- relay/client/client.go | 35 +++++++++++++----------- relay/client/client_test.go | 36 ++++++++++++------------- relay/client/dialer/ws/conn.go | 14 ++++------ relay/client/dialer/ws/ws.go | 17 +----------- relay/client/manager.go | 30 ++++++++++----------- relay/client/manager_test.go | 40 ++++++++++++++-------------- relay/cmd/main.go | 22 ++++++++------- relay/messages/message.go | 34 ++++++++++++++++++++--- relay/server/relay.go | 22 ++++++++++----- relay/server/server.go | 8 +++--- 15 files changed, 154 insertions(+), 128 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index e8dcb069b..ac5c46f83 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -606,9 +606,9 @@ func (conn *Conn) doHandshake() error { err error ) ha.IceUFrag, ha.IcePwd = conn.workerICE.GetLocalUserCredentials() - addr, err := conn.workerRelay.RelayAddress() + addr, err := conn.workerRelay.RelayInstanceAddress() if err == nil { - ha.RelayAddr = addr.String() + ha.RelayAddr = addr } conn.log.Tracef("send new offer: %#v", ha) return conn.handshaker.SendOffer(ha) diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 5ad4a1fb5..beea912aa 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -46,13 +46,13 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { } // the relayManager will return with error in case if the connection has lost with relay server - currentRelayAddress, err := w.relayManager.RelayAddress() + currentRelayAddress, err := w.relayManager.RelayInstanceAddress() if err != nil { w.log.Infof("local Relay connection is lost, skipping connection attempt") return } - srv := w.preferredRelayServer(currentRelayAddress.String(), remoteOfferAnswer.RelaySrvAddress) + srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress) relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key, w.conn.OnDisconnected) if err != nil { @@ -73,8 +73,8 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { }) } -func (w *WorkerRelay) RelayAddress() (net.Addr, error) { - return w.relayManager.RelayAddress() +func (w *WorkerRelay) RelayInstanceAddress() (string, error) { + return w.relayManager.RelayInstanceAddress() } func (w *WorkerRelay) IsController() bool { diff --git a/encryption/letsencrypt.go b/encryption/letsencrypt.go index cfe54ec5a..27a5e3110 100644 --- a/encryption/letsencrypt.go +++ b/encryption/letsencrypt.go @@ -9,7 +9,7 @@ import ( ) // CreateCertManager wraps common logic of generating Let's encrypt certificate. -func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Manager, error) { +func CreateCertManager(datadir string, letsencryptDomain ...string) (*autocert.Manager, error) { certDir := filepath.Join(datadir, "letsencrypt") if _, err := os.Stat(certDir); os.IsNotExist(err) { @@ -24,7 +24,7 @@ func CreateCertManager(datadir string, letsencryptDomain string) (*autocert.Mana certManager := &autocert.Manager{ Prompt: autocert.AcceptTOS, Cache: autocert.DirCache(certDir), - HostPolicy: autocert.HostWhitelist(letsencryptDomain), + HostPolicy: autocert.HostWhitelist(letsencryptDomain...), } return certManager, nil diff --git a/go.mod b/go.mod index 67655fc7b..9955fe703 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( github.com/google/martian/v3 v3.0.0 github.com/google/nftables v0.0.0-20220808154552-2eca00135732 github.com/gopacket/gopacket v1.1.1 - github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 @@ -64,6 +63,7 @@ require ( github.com/oschwald/maxminddb-golang v1.12.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pion/logging v0.2.2 + github.com/pion/randutil v0.1.0 github.com/pion/stun/v2 v2.0.0 github.com/pion/transport/v3 v3.0.1 github.com/pion/turn/v3 v3.0.1 @@ -77,6 +77,7 @@ require ( github.com/things-go/go-socks5 v0.0.4 github.com/yusufpapurcu/wmi v1.2.4 github.com/zcalusic/sysinfo v1.0.2 + go.mongodb.org/mongo-driver v1.16.0 go.opentelemetry.io/otel v1.26.0 go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.26.0 @@ -173,7 +174,6 @@ require ( github.com/pegasus-kv/thrift v0.13.0 // indirect github.com/pion/dtls/v2 v2.2.10 // indirect github.com/pion/mdns v0.0.12 // indirect - github.com/pion/randutil v0.1.0 // indirect github.com/pion/transport/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 590915767..db50be34a 100644 --- a/go.sum +++ b/go.sum @@ -237,8 +237,6 @@ github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0 h1:fWY+zXdWhvWnd github.com/gopherjs/gopherjs v0.0.0-20220410123724-9e86199038b0/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357 h1:Fkzd8ktnpOR9h47SXHe2AYPwelXLH2GjGsjlAloiWfo= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.2-0.20240212192251-757544f21357/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= @@ -513,6 +511,8 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= +go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= +go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= diff --git a/relay/client/client.go b/relay/client/client.go index 2401f972e..9fc5b84b8 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -97,7 +97,7 @@ func (cc *connContainer) close() { type Client struct { log *log.Entry parentCtx context.Context - serverAddress string + connectionURL string hashedID []byte bufPool *sync.Pool @@ -108,20 +108,19 @@ type Client struct { mu sync.Mutex // protect serviceIsRunning and conns readLoopMutex sync.Mutex wgReadLoop sync.WaitGroup - - remoteAddr net.Addr + instanceURL string onDisconnectListener func() listenerMutex sync.Mutex } // NewClient creates a new client for the relay server. The client is not connected to the server until the Connect -func NewClient(ctx context.Context, serverAddress, peerID string) *Client { +func NewClient(ctx context.Context, serverURL, peerID string) *Client { hashedID, hashedStringId := messages.HashID(peerID) return &Client{ log: log.WithField("client_id", hashedStringId), parentCtx: ctx, - serverAddress: serverAddress, + connectionURL: serverURL, hashedID: hashedID, bufPool: &sync.Pool{ New: func() any { @@ -135,7 +134,7 @@ func NewClient(ctx context.Context, serverAddress, peerID string) *Client { // Connect establishes a connection to the relay server. It blocks until the connection is established or an error occurs. func (c *Client) Connect() error { - c.log.Infof("connecting to relay server: %s", c.serverAddress) + c.log.Infof("connecting to relay server: %s", c.connectionURL) c.readLoopMutex.Lock() defer c.readLoopMutex.Unlock() @@ -156,7 +155,7 @@ func (c *Client) Connect() error { c.wgReadLoop.Add(1) go c.readLoop(c.relayConn) - log.Infof("relay connection established with: %s", c.serverAddress) + log.Infof("relay connection established with: %s", c.connectionURL) return nil } @@ -186,14 +185,14 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { return conn, nil } -// RelayRemoteAddress returns the IP address of the relay server. It could change after the close and reopen the connection. -func (c *Client) RelayRemoteAddress() (net.Addr, error) { +// ServerInstanceURL returns the address of the relay server. It could change after the close and reopen the connection. +func (c *Client) ServerInstanceURL() (string, error) { c.mu.Lock() defer c.mu.Unlock() - if c.remoteAddr == nil { - return nil, fmt.Errorf("relay connection is not established") + if c.instanceURL == "" { + return "", fmt.Errorf("relay connection is not established") } - return c.remoteAddr, nil + return c.instanceURL, nil } // SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. @@ -215,7 +214,7 @@ func (c *Client) Close() error { } func (c *Client) connect() error { - conn, err := ws.Dial(c.serverAddress) + conn, err := ws.Dial(c.connectionURL) if err != nil { return err } @@ -231,8 +230,6 @@ func (c *Client) connect() error { return err } - c.remoteAddr = conn.RemoteAddr() - return nil } @@ -264,6 +261,12 @@ func (c *Client) handShake() error { log.Errorf("unexpected message type: %s", msgType) return fmt.Errorf("unexpected message type") } + + domain, err := messages.UnmarshalHelloResponse(buf[:n]) + if err != nil { + return err + } + c.instanceURL = domain return nil } @@ -435,7 +438,7 @@ func (c *Client) close(gracefullyExit bool) error { c.mu.Unlock() c.wgReadLoop.Wait() - c.log.Infof("relay connection closed with: %s", c.serverAddress) + c.log.Infof("relay connection closed with: %s", c.connectionURL) return err } diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 278d46d08..7b1ee5c62 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -23,8 +23,8 @@ func TestMain(m *testing.M) { func TestClient(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -91,8 +91,8 @@ func TestClient(t *testing.T) { func TestRegistration(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -156,8 +156,8 @@ func TestEcho(t *testing.T) { ctx := context.Background() idAlice := "alice" idBob := "bob" - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -236,8 +236,8 @@ func TestEcho(t *testing.T) { func TestBindToUnavailabePeer(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -273,8 +273,8 @@ func TestBindToUnavailabePeer(t *testing.T) { func TestBindReconnect(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -355,8 +355,8 @@ func TestBindReconnect(t *testing.T) { func TestCloseConn(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -403,8 +403,8 @@ func TestCloseConn(t *testing.T) { func TestCloseRelayConn(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -446,8 +446,8 @@ func TestCloseRelayConn(t *testing.T) { func TestCloseByServer(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv1 := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv1 := server.NewServer(srvCfg.Address, false) go func() { err := srv1.Listen(srvCfg) if err != nil { @@ -489,8 +489,8 @@ func TestCloseByServer(t *testing.T) { func TestCloseByClient(t *testing.T) { ctx := context.Background() - srvCfg := server.Config{Address: "localhost:1234"} - srv := server.NewServer() + srvCfg := server.ListenerConfig{Address: "localhost:1234"} + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go index 44d86c1bb..1632fcb45 100644 --- a/relay/client/dialer/ws/conn.go +++ b/relay/client/dialer/ws/conn.go @@ -12,16 +12,12 @@ import ( type Conn struct { ctx context.Context *websocket.Conn - srvAddr net.Addr - localAddr net.Addr } -func NewConn(wsConn *websocket.Conn, srvAddr, localAddr net.Addr) net.Conn { +func NewConn(wsConn *websocket.Conn) net.Conn { return &Conn{ - ctx: context.Background(), - Conn: wsConn, - srvAddr: srvAddr, - localAddr: localAddr, + ctx: context.Background(), + Conn: wsConn, } } @@ -44,11 +40,11 @@ func (c *Conn) Write(b []byte) (n int, err error) { } func (c *Conn) RemoteAddr() net.Addr { - return c.srvAddr + panic("not implemented") } func (c *Conn) LocalAddr() net.Addr { - return c.localAddr + panic("not implemented") } func (c *Conn) SetReadDeadline(t time.Time) error { diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index 070a02362..3a2bf48c8 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -29,22 +29,7 @@ func Dial(address string) (net.Conn, error) { return nil, err } - /* - response.Body.(net.Conn).LocalAddr() - unc, ok := response.Body.(net.Conn) - if !ok { - log.Errorf("failed to get local address: %s", err) - return nil, fmt.Errorf("failed to get local address") - } - - */ - // todo figure out the proper address - dummy := &net.TCPAddr{ - IP: net.ParseIP("127.0.0.1"), - Port: 8080, - } - - conn := NewConn(wsConn, dummy, dummy) + conn := NewConn(wsConn) return conn, nil } diff --git a/relay/client/manager.go b/relay/client/manager.go index ae44be4b9..109d3b139 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -35,9 +35,9 @@ func NewRelayTrack() *RelayTrack { // relay servers will be closed if there is no active connection. Periodically the manager will check if there is any // unused relay connection and close it. type Manager struct { - ctx context.Context - srvAddress string - peerID string + ctx context.Context + serverURL string + peerID string relayClient *Client reconnectGuard *Guard @@ -49,10 +49,10 @@ type Manager struct { listenerLock sync.Mutex } -func NewManager(ctx context.Context, serverAddress string, peerID string) *Manager { +func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { return &Manager{ ctx: ctx, - srvAddress: serverAddress, + serverURL: serverURL, peerID: peerID, relayClients: make(map[string]*RelayTrack), onDisconnectedListeners: make(map[string]map[*func()]struct{}), @@ -65,7 +65,7 @@ func (m *Manager) Serve() error { return fmt.Errorf("manager already serving") } - m.relayClient = NewClient(m.ctx, m.srvAddress, m.peerID) + m.relayClient = NewClient(m.ctx, m.serverURL, m.peerID) err := m.relayClient.Connect() if err != nil { log.Errorf("failed to connect to relay server: %s", err) @@ -74,7 +74,7 @@ func (m *Manager) Serve() error { m.reconnectGuard = NewGuard(m.ctx, m.relayClient) m.relayClient.SetOnDisconnectListener(func() { - m.onServerDisconnected(m.srvAddress) + m.onServerDisconnected(m.serverURL) }) m.startCleanupLoop() @@ -116,17 +116,17 @@ func (m *Manager) OpenConn(serverAddress, peerKey string, onClosedListener func( return netConn, err } -// RelayAddress returns the address of the permanent relay server. It could change if the network connection is lost. +// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is lost. // This address will be sent to the target peer to choose the common relay server for the communication. -func (m *Manager) RelayAddress() (net.Addr, error) { +func (m *Manager) RelayInstanceAddress() (string, error) { if m.relayClient == nil { - return nil, errRelayClientNotConnected + return "", errRelayClientNotConnected } - return m.relayClient.RelayRemoteAddress() + return m.relayClient.ServerInstanceURL() } func (m *Manager) HasRelayAddress() bool { - return m.srvAddress != "" + return m.serverURL != "" } func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { @@ -182,7 +182,7 @@ func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { } func (m *Manager) onServerDisconnected(serverAddress string) { - if serverAddress == m.srvAddress { + if serverAddress == m.serverURL { m.reconnectGuard.OnDisconnected() } @@ -190,11 +190,11 @@ func (m *Manager) onServerDisconnected(serverAddress string) { } func (m *Manager) isForeignServer(address string) (bool, error) { - rAddr, err := m.relayClient.RelayRemoteAddress() + rAddr, err := m.relayClient.ServerInstanceURL() if err != nil { return false, fmt.Errorf("relay client not connected") } - return rAddr.String() != address, nil + return rAddr != address, nil } func (m *Manager) startCleanupLoop() { diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 71e4a416f..3192c1a09 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -13,10 +13,10 @@ import ( func TestForeignConn(t *testing.T) { ctx := context.Background() - srvCfg1 := server.Config{ + srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer() + srv1 := server.NewServer(srvCfg1.Address, false) go func() { err := srv1.Listen(srvCfg1) if err != nil { @@ -31,10 +31,10 @@ func TestForeignConn(t *testing.T) { } }() - srvCfg2 := server.Config{ + srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer() + srv2 := server.NewServer(srvCfg2.Address, false) go func() { err := srv2.Listen(srvCfg2) if err != nil { @@ -61,15 +61,15 @@ func TestForeignConn(t *testing.T) { clientBob := NewManager(mCtx, srvCfg2.Address, idBob) clientBob.Serve() - bobsSrvAddr, err := clientBob.RelayAddress() + bobsSrvAddr, err := clientBob.RelayInstanceAddress() if err != nil { t.Fatalf("failed to get relay address: %s", err) } - connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr.String(), idBob, nil) + connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob, nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } - connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr.String(), idAlice, nil) + connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice, nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -104,10 +104,10 @@ func TestForeignConn(t *testing.T) { func TestForeginConnClose(t *testing.T) { ctx := context.Background() - srvCfg1 := server.Config{ + srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer() + srv1 := server.NewServer(srvCfg1.Address, false) go func() { err := srv1.Listen(srvCfg1) if err != nil { @@ -122,10 +122,10 @@ func TestForeginConnClose(t *testing.T) { } }() - srvCfg2 := server.Config{ + srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer() + srv2 := server.NewServer(srvCfg2.Address, false) go func() { err := srv2.Listen(srvCfg2) if err != nil { @@ -161,10 +161,10 @@ func TestForeginConnClose(t *testing.T) { func TestForeginAutoClose(t *testing.T) { ctx := context.Background() relayCleanupInterval = 1 * time.Second - srvCfg1 := server.Config{ + srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer() + srv1 := server.NewServer(srvCfg1.Address, false) go func() { t.Log("binding server 1.") err := srv1.Listen(srvCfg1) @@ -182,10 +182,10 @@ func TestForeginAutoClose(t *testing.T) { t.Logf("server 1. closed") }() - srvCfg2 := server.Config{ + srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer() + srv2 := server.NewServer(srvCfg2.Address, false) go func() { t.Log("binding server 2.") err := srv2.Listen(srvCfg2) @@ -234,10 +234,10 @@ func TestAutoReconnect(t *testing.T) { ctx := context.Background() reconnectingTimeout = 2 * time.Second - srvCfg := server.Config{ + srvCfg := server.ListenerConfig{ Address: "localhost:1234", } - srv := server.NewServer() + srv := server.NewServer(srvCfg.Address, false) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -256,11 +256,11 @@ func TestAutoReconnect(t *testing.T) { defer cancel() clientAlice := NewManager(mCtx, srvCfg.Address, "alice") clientAlice.Serve() - ra, err := clientAlice.RelayAddress() + ra, err := clientAlice.RelayInstanceAddress() if err != nil { t.Errorf("failed to get relay address: %s", err) } - conn, err := clientAlice.OpenConn(ra.String(), "bob", nil) + conn, err := clientAlice.OpenConn(ra, "bob", nil) if err != nil { t.Errorf("failed to bind channel: %s", err) } @@ -278,7 +278,7 @@ func TestAutoReconnect(t *testing.T) { time.Sleep(reconnectingTimeout + 1*time.Second) log.Infof("reopent the connection") - _, err = clientAlice.OpenConn(ra.String(), "bob", nil) + _, err = clientAlice.OpenConn(ra, "bob", nil) if err != nil { t.Errorf("failed to open channel: %s", err) } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 9d1802076..fb437e78d 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -16,9 +16,11 @@ import ( ) var ( - listenAddress string + listenAddress string + // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection + exposedAddress string letsencryptDataDir string - letsencryptDomain string + letsencryptDomains []string rootCmd = &cobra.Command{ Use: "relay", @@ -31,8 +33,9 @@ var ( func init() { _ = util.InitLog("trace", "console") rootCmd.PersistentFlags().StringVarP(&listenAddress, "listen-address", "l", ":1235", "listen address") + rootCmd.PersistentFlags().StringVarP(&exposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") rootCmd.PersistentFlags().StringVarP(&letsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") - rootCmd.PersistentFlags().StringVarP(&letsencryptDomain, "letsencrypt-domain", "a", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringArrayVarP(&letsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") } func waitForExitSignal() { @@ -42,7 +45,7 @@ func waitForExitSignal() { } func execute(cmd *cobra.Command, args []string) { - srvCfg := server.Config{ + srvListenerCfg := server.ListenerConfig{ Address: listenAddress, } if hasLetsEncrypt() { @@ -51,11 +54,12 @@ func execute(cmd *cobra.Command, args []string) { log.Errorf("%s", err) os.Exit(1) } - srvCfg.TLSConfig = tlscfg + srvListenerCfg.TLSConfig = tlscfg } - srv := server.NewServer() - err := srv.Listen(srvCfg) + tlsSupport := srvListenerCfg.TLSConfig != nil + srv := server.NewServer(exposedAddress, tlsSupport) + err := srv.Listen(srvListenerCfg) if err != nil { log.Errorf("failed to bind server: %s", err) os.Exit(1) @@ -71,11 +75,11 @@ func execute(cmd *cobra.Command, args []string) { } func hasLetsEncrypt() bool { - return letsencryptDataDir != "" && letsencryptDomain != "" + return letsencryptDataDir != "" && letsencryptDomains != nil && len(letsencryptDomains) > 0 } func setupTLS() (*tls.Config, error) { - certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomain) + certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...) if err != nil { return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err) } diff --git a/relay/messages/message.go b/relay/messages/message.go index aa62fe867..5177b691d 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -4,6 +4,8 @@ import ( "bytes" "fmt" + "go.mongodb.org/mongo-driver/bson" + log "github.com/sirupsen/logrus" ) @@ -45,6 +47,10 @@ func (m MsgType) String() string { } } +type HelloResponse struct { + DomainAddress string +} + func DetermineClientMsgType(msg []byte) (MsgType, error) { msgType := MsgType(msg[0]) switch msgType { @@ -97,10 +103,32 @@ func UnmarshalHelloMsg(msg []byte) ([]byte, error) { return msg[5:], nil } -func MarshalHelloResponse() []byte { - msg := make([]byte, 1) +func MarshalHelloResponse(DomainAddress string) ([]byte, error) { + payload := HelloResponse{ + DomainAddress: DomainAddress, + } + helloResponse, err := bson.Marshal(payload) + if err != nil { + log.Errorf("failed to marshal hello response: %s", err) + return nil, err + } + msg := make([]byte, 1, 1+len(helloResponse)) msg[0] = byte(MsgTypeHelloResponse) - return msg + msg = append(msg, helloResponse...) + return msg, nil +} + +func UnmarshalHelloResponse(msg []byte) (string, error) { + if len(msg) < 2 { + return "", fmt.Errorf("invalid 'hello response' message") + } + payload := HelloResponse{} + err := bson.Unmarshal(msg[1:], &payload) + if err != nil { + log.Errorf("failed to unmarshal hello response: %s", err) + return "", err + } + return payload.DomainAddress, nil } // Close message diff --git a/relay/server/relay.go b/relay/server/relay.go index 39719f4a9..9f0528b25 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -12,16 +12,24 @@ import ( ) type Relay struct { - store *Store + store *Store + instaceURL string // domain:port closed bool closeMu sync.RWMutex } -func NewRelay() *Relay { - return &Relay{ +func NewRelay(exposedAddress string, tlsSupport bool) *Relay { + r := &Relay{ store: NewStore(), } + + if tlsSupport { + r.instaceURL = fmt.Sprintf("rels://%s", exposedAddress) + } else { + r.instaceURL = fmt.Sprintf("rel://%s", exposedAddress) + } + return r } func (r *Relay) Accept(conn net.Conn) { @@ -31,7 +39,7 @@ func (r *Relay) Accept(conn net.Conn) { return } - peerID, err := handShake(conn) + peerID, err := r.handShake(conn) if err != nil { log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err) cErr := conn.Close() @@ -68,7 +76,7 @@ func (r *Relay) Close(ctx context.Context) { r.closeMu.Unlock() } -func handShake(conn net.Conn) ([]byte, error) { +func (r *Relay) handShake(conn net.Conn) ([]byte, error) { buf := make([]byte, messages.MaxHandshakeSize) n, err := conn.Read(buf) if err != nil { @@ -79,18 +87,20 @@ func handShake(conn net.Conn) ([]byte, error) { if err != nil { return nil, err } + if msgType != messages.MsgTypeHello { tErr := fmt.Errorf("invalid message type") log.Errorf("failed to handshake: %s", tErr) return nil, tErr } + peerID, err := messages.UnmarshalHelloMsg(buf[:n]) if err != nil { log.Errorf("failed to handshake: %s", err) return nil, err } - msg := messages.MarshalHelloResponse() + msg, _ := messages.MarshalHelloResponse(r.instaceURL) _, err = conn.Write(msg) if err != nil { return nil, err diff --git a/relay/server/server.go b/relay/server/server.go index 687486138..449819e61 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -14,7 +14,7 @@ import ( "github.com/netbirdio/netbird/relay/server/listener/ws" ) -type Config struct { +type ListenerConfig struct { Address string TLSConfig *tls.Config } @@ -25,13 +25,13 @@ type Server struct { wSListener listener.Listener } -func NewServer() *Server { +func NewServer(exposedAddress string, tlsSupport bool) *Server { return &Server{ - relay: NewRelay(), + relay: NewRelay(exposedAddress, tlsSupport), } } -func (r *Server) Listen(cfg Config) error { +func (r *Server) Listen(cfg ListenerConfig) error { wg := sync.WaitGroup{} wg.Add(2) From 1a5ee744a8624013b0209d683bca6bbeac7c3cf4 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 3 Jul 2024 15:03:57 +0200 Subject: [PATCH 080/199] - add file based cert - print out the exposed address - handle empty exposed address --- encryption/cert.go | 19 ++++++++++++++++++ relay/cmd/main.go | 32 +++++++++++++++++++++++++++---- relay/server/relay.go | 4 ++++ relay/server/server.go | 4 ++++ signal/proto/signalexchange.proto | 2 +- 5 files changed, 56 insertions(+), 5 deletions(-) create mode 100644 encryption/cert.go diff --git a/encryption/cert.go b/encryption/cert.go new file mode 100644 index 000000000..3f6d5c679 --- /dev/null +++ b/encryption/cert.go @@ -0,0 +1,19 @@ +package encryption + +import "crypto/tls" + +func LoadTLSConfig(certFile, keyFile string) (*tls.Config, error) { + serverCert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + + config := &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientAuth: tls.NoClientCert, + NextProtos: []string{ + "h2", "http/1.1", // enable HTTP/2 + }, + } + return config, nil +} diff --git a/relay/cmd/main.go b/relay/cmd/main.go index fb437e78d..a7cd5efa8 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -18,9 +18,12 @@ import ( var ( listenAddress string // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection + // it is a domain:port or ip:port exposedAddress string letsencryptDataDir string letsencryptDomains []string + tlsCertFile string + tlsKeyFile string rootCmd = &cobra.Command{ Use: "relay", @@ -32,10 +35,13 @@ var ( func init() { _ = util.InitLog("trace", "console") - rootCmd.PersistentFlags().StringVarP(&listenAddress, "listen-address", "l", ":1235", "listen address") + rootCmd.PersistentFlags().StringVarP(&listenAddress, "listen-address", "l", ":443", "listen address") rootCmd.PersistentFlags().StringVarP(&exposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") rootCmd.PersistentFlags().StringVarP(&letsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") rootCmd.PersistentFlags().StringArrayVarP(&letsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringVarP(&tlsCertFile, "tls-cert-file", "c", "", "") + rootCmd.PersistentFlags().StringVarP(&tlsKeyFile, "tls-key-file", "k", "", "") + } func waitForExitSignal() { @@ -45,20 +51,33 @@ func waitForExitSignal() { } func execute(cmd *cobra.Command, args []string) { + if exposedAddress == "" { + log.Errorf("exposed address is required") + os.Exit(1) + } + srvListenerCfg := server.ListenerConfig{ Address: listenAddress, } if hasLetsEncrypt() { - tlscfg, err := setupTLS() + tlsCfg, err := setupTLSCertManager() if err != nil { log.Errorf("%s", err) os.Exit(1) } - srvListenerCfg.TLSConfig = tlscfg + srvListenerCfg.TLSConfig = tlsCfg + } else if hasCertConfig() { + tlsCfg, err := encryption.LoadTLSConfig(tlsCertFile, tlsKeyFile) + if err != nil { + log.Errorf("%s", err) + os.Exit(1) + } + srvListenerCfg.TLSConfig = tlsCfg } tlsSupport := srvListenerCfg.TLSConfig != nil srv := server.NewServer(exposedAddress, tlsSupport) + log.Infof("server will be available on: %s", srv.InstanceURL()) err := srv.Listen(srvListenerCfg) if err != nil { log.Errorf("failed to bind server: %s", err) @@ -74,11 +93,16 @@ func execute(cmd *cobra.Command, args []string) { } } +func hasCertConfig() bool { + return tlsCertFile != "" && tlsKeyFile != "" + +} + func hasLetsEncrypt() bool { return letsencryptDataDir != "" && letsencryptDomains != nil && len(letsencryptDomains) > 0 } -func setupTLS() (*tls.Config, error) { +func setupTLSCertManager() (*tls.Config, error) { certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...) if err != nil { return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err) diff --git a/relay/server/relay.go b/relay/server/relay.go index 9f0528b25..aff15a5fb 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -107,3 +107,7 @@ func (r *Relay) handShake(conn net.Conn) ([]byte, error) { } return peerID, nil } + +func (r *Relay) InstanceURL() string { + return r.instaceURL +} diff --git a/relay/server/server.go b/relay/server/server.go index 449819e61..5341cad2e 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -83,3 +83,7 @@ func (r *Server) Close() error { err := errors.Join(wErr, uErr) return err } + +func (r *Server) InstanceURL() string { + return r.relay.instaceURL +} diff --git a/signal/proto/signalexchange.proto b/signal/proto/signalexchange.proto index 5aa24d893..4431edd7c 100644 --- a/signal/proto/signalexchange.proto +++ b/signal/proto/signalexchange.proto @@ -61,7 +61,7 @@ message Body { // RosenpassConfig is a Rosenpass config of the remote peer our peer tries to connect to RosenpassConfig rosenpassConfig = 7; - // relayServerAddress is an IP:port of the relay server + // relayServerAddress is url of the relay server string relayServerAddress = 8; } From 8845e8fbc7a68adfda063bb4d8a0b0d3e97e5b5b Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 4 Jul 2024 13:42:27 +0200 Subject: [PATCH 081/199] replace bson to gob --- go.mod | 1 - go.sum | 2 -- relay/messages/message.go | 23 +++++++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 9955fe703..327476c4f 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,6 @@ require ( github.com/things-go/go-socks5 v0.0.4 github.com/yusufpapurcu/wmi v1.2.4 github.com/zcalusic/sysinfo v1.0.2 - go.mongodb.org/mongo-driver v1.16.0 go.opentelemetry.io/otel v1.26.0 go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.26.0 diff --git a/go.sum b/go.sum index db50be34a..84eec1545 100644 --- a/go.sum +++ b/go.sum @@ -511,8 +511,6 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= -go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= -go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 h1:Xs2Ncz0gNihqu9iosIZ5SkBbWo5T8JhhLJFMQL1qmLI= diff --git a/relay/messages/message.go b/relay/messages/message.go index 5177b691d..991bc36d0 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -2,10 +2,9 @@ package messages import ( "bytes" + "encoding/gob" "fmt" - "go.mongodb.org/mongo-driver/bson" - log "github.com/sirupsen/logrus" ) @@ -107,14 +106,19 @@ func MarshalHelloResponse(DomainAddress string) ([]byte, error) { payload := HelloResponse{ DomainAddress: DomainAddress, } - helloResponse, err := bson.Marshal(payload) + + buf := new(bytes.Buffer) + enc := gob.NewEncoder(buf) + + err := enc.Encode(payload) if err != nil { - log.Errorf("failed to marshal hello response: %s", err) + log.Errorf("failed to gob encode hello response: %s", err) return nil, err } - msg := make([]byte, 1, 1+len(helloResponse)) + + msg := make([]byte, 1, 1+buf.Len()) msg[0] = byte(MsgTypeHelloResponse) - msg = append(msg, helloResponse...) + msg = append(msg, buf.Bytes()...) return msg, nil } @@ -123,9 +127,12 @@ func UnmarshalHelloResponse(msg []byte) (string, error) { return "", fmt.Errorf("invalid 'hello response' message") } payload := HelloResponse{} - err := bson.Unmarshal(msg[1:], &payload) + buf := bytes.NewBuffer(msg[1:]) + dec := gob.NewDecoder(buf) + + err := dec.Decode(&payload) if err != nil { - log.Errorf("failed to unmarshal hello response: %s", err) + log.Errorf("failed to gob decode hello response: %s", err) return "", err } return payload.DomainAddress, nil From 836072098b04597b8ead0452bff4c83d666ae938 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 5 Jul 2024 16:12:30 +0200 Subject: [PATCH 082/199] Integrate the relay authentication --- client/cmd/testutil.go | 2 +- client/internal/connect.go | 30 +- client/internal/engine.go | 17 +- client/internal/engine_test.go | 2 +- client/server/server_test.go | 2 +- management/client/client_test.go | 2 +- management/cmd/management.go | 4 +- management/proto/management.pb.go | 3836 +++++++++++------ management/proto/management.proto | 9 +- management/server/dns_test.go | 16 +- management/server/grpcserver.go | 87 +- management/server/management_proto_test.go | 4 +- management/server/management_test.go | 2 +- management/server/peer.go | 2 +- management/server/token_mgr.go | 126 + ...ncredentials_test.go => token_mgr_test.go} | 14 +- management/server/turncredentials.go | 125 - relay/auth/allow_all.go | 9 + relay/auth/hmac/store.go | 24 + relay/auth/hmac/token.go | 104 + relay/auth/hmac/token_test.go | 103 + relay/auth/hmac/validator.go | 27 + relay/auth/validator.go | 5 + relay/client/client.go | 16 +- relay/client/manager.go | 18 +- relay/cmd/main.go | 10 +- relay/messages/message.go | 23 +- relay/messages/message_test.go | 4 +- relay/server/relay.go | 18 +- relay/server/server.go | 8 +- 30 files changed, 3055 insertions(+), 1594 deletions(-) create mode 100644 management/server/token_mgr.go rename management/server/{turncredentials_test.go => token_mgr_test.go} (94%) delete mode 100644 management/server/turncredentials.go create mode 100644 relay/auth/allow_all.go create mode 100644 relay/auth/hmac/store.go create mode 100644 relay/auth/hmac/token.go create mode 100644 relay/auth/hmac/token_test.go create mode 100644 relay/auth/hmac/validator.go create mode 100644 relay/auth/validator.go diff --git a/client/cmd/testutil.go b/client/cmd/testutil.go index 35fd7c537..f032884df 100644 --- a/client/cmd/testutil.go +++ b/client/cmd/testutil.go @@ -86,7 +86,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { t.Fatal(err) diff --git a/client/internal/connect.go b/client/internal/connect.go index 49544a921..4d79ba72b 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -26,6 +26,7 @@ import ( "github.com/netbirdio/netbird/iface" mgm "github.com/netbirdio/netbird/management/client" mgmProto "github.com/netbirdio/netbird/management/proto" + "github.com/netbirdio/netbird/relay/auth/hmac" relayClient "github.com/netbirdio/netbird/relay/client" signal "github.com/netbirdio/netbird/signal/client" "github.com/netbirdio/netbird/util" @@ -245,9 +246,10 @@ func (c *ConnectClient) run( c.statusRecorder.MarkSignalConnected() - relayAddress := relayAddress(loginResp) - relayManager := relayClient.NewManager(engineCtx, relayAddress, myPrivateKey.PublicKey().String()) - if relayAddress != "" { + relayURL, token := parseRelayInfo(loginResp) + relayManager := relayClient.NewManager(engineCtx, relayURL, myPrivateKey.PublicKey().String()) + if relayURL != "" { + relayManager.UpdateToken(token) if err = relayManager.Serve(); err != nil { log.Error(err) return wrapErr(err) @@ -307,15 +309,27 @@ func (c *ConnectClient) run( return nil } -func relayAddress(resp *mgmProto.LoginResponse) string { +func parseRelayInfo(resp *mgmProto.LoginResponse) (string, hmac.Token) { + // todo remove this if ra := peer.ForcedRelayAddress(); ra != "" { - return ra + return ra, hmac.Token{} } - if resp.GetWiretrusteeConfig().GetRelayAddress() != "" { - return resp.GetWiretrusteeConfig().GetRelayAddress() + msg := resp.GetWiretrusteeConfig().GetRelay() + if msg == nil { + return "", hmac.Token{} } - return "" + + var url string + if msg.GetUrls() != nil && len(msg.GetUrls()) > 0 { + url = msg.GetUrls()[0] + } + + token := hmac.Token{ + Payload: msg.GetTokenPayload(), + Signature: msg.GetTokenSignature(), + } + return url, token } func (c *ConnectClient) Engine() *Engine { diff --git a/client/internal/engine.go b/client/internal/engine.go index 2438533a1..cb624bf43 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -24,6 +24,7 @@ import ( "github.com/netbirdio/netbird/client/firewall/manager" "github.com/netbirdio/netbird/client/internal/acl" "github.com/netbirdio/netbird/client/internal/dns" + "github.com/netbirdio/netbird/client/internal/networkmonitor" "github.com/netbirdio/netbird/client/internal/peer" "github.com/netbirdio/netbird/client/internal/relay" @@ -36,6 +37,7 @@ import ( "github.com/netbirdio/netbird/iface/bind" mgm "github.com/netbirdio/netbird/management/client" mgmProto "github.com/netbirdio/netbird/management/proto" + auth "github.com/netbirdio/netbird/relay/auth/hmac" relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/route" signal "github.com/netbirdio/netbird/signal/client" @@ -467,12 +469,13 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { defer e.syncMsgMux.Unlock() if update.GetWiretrusteeConfig() != nil { - err := e.updateTURNs(update.GetWiretrusteeConfig().GetTurns()) + wCfg := update.GetWiretrusteeConfig() + err := e.updateTURNs(wCfg.GetTurns()) if err != nil { return err } - err = e.updateSTUNs(update.GetWiretrusteeConfig().GetStuns()) + err = e.updateSTUNs(wCfg.GetStuns()) if err != nil { return err } @@ -482,8 +485,16 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { stunTurn = append(stunTurn, e.TURNs...) e.StunTurn.Store(stunTurn) - // todo update relay address in the relay manager + relayMsg := wCfg.GetRelay() + if relayMsg != nil { + c := auth.Token{ + Payload: relayMsg.GetTokenPayload(), + Signature: relayMsg.GetTokenSignature(), + } + e.relayManager.UpdateToken(c) + } + // todo update relay address in the relay manager // todo update signal } diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 172216e73..af0662541 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -1071,7 +1071,7 @@ func startManagement(dataDir string) (*grpc.Server, string, error) { if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { return nil, "", err diff --git a/client/server/server_test.go b/client/server/server_test.go index a9f23ce7c..2337e972d 100644 --- a/client/server/server_test.go +++ b/client/server/server_test.go @@ -122,7 +122,7 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { return nil, "", err diff --git a/management/client/client_test.go b/management/client/client_test.go index 32ad8fce4..050b90356 100644 --- a/management/client/client_test.go +++ b/management/client/client_test.go @@ -75,7 +75,7 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) { if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { t.Fatal(err) diff --git a/management/cmd/management.go b/management/cmd/management.go index 366935802..2c24fae18 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -183,7 +183,7 @@ var ( return fmt.Errorf("failed to build default manager: %v", err) } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnRelayTokenManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.RelayAddress) trustedPeers := config.ReverseProxy.TrustedPeers defaultTrustedPeers := []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0")} @@ -260,7 +260,7 @@ var ( ephemeralManager.LoadInitialPeers() gRPCAPIHandler := grpc.NewServer(gRPCOpts...) - srv, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, appMetrics, ephemeralManager) + srv, err := server.NewServer(config, accountManager, peersUpdateManager, turnRelayTokenManager, appMetrics, ephemeralManager) if err != nil { return fmt.Errorf("failed creating gRPC API handler: %v", err) } diff --git a/management/proto/management.pb.go b/management/proto/management.pb.go index f5dab4d81..bd097f9b3 100644 --- a/management/proto/management.pb.go +++ b/management/proto/management.pb.go @@ -1,25 +1,25 @@ // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.21.12 // source: management.proto package proto import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" - math "math" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type HostConfig_Protocol int32 @@ -31,28 +31,49 @@ const ( HostConfig_DTLS HostConfig_Protocol = 4 ) -var HostConfig_Protocol_name = map[int32]string{ - 0: "UDP", - 1: "TCP", - 2: "HTTP", - 3: "HTTPS", - 4: "DTLS", -} +// Enum value maps for HostConfig_Protocol. +var ( + HostConfig_Protocol_name = map[int32]string{ + 0: "UDP", + 1: "TCP", + 2: "HTTP", + 3: "HTTPS", + 4: "DTLS", + } + HostConfig_Protocol_value = map[string]int32{ + "UDP": 0, + "TCP": 1, + "HTTP": 2, + "HTTPS": 3, + "DTLS": 4, + } +) -var HostConfig_Protocol_value = map[string]int32{ - "UDP": 0, - "TCP": 1, - "HTTP": 2, - "HTTPS": 3, - "DTLS": 4, +func (x HostConfig_Protocol) Enum() *HostConfig_Protocol { + p := new(HostConfig_Protocol) + *p = x + return p } func (x HostConfig_Protocol) String() string { - return proto.EnumName(HostConfig_Protocol_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } +func (HostConfig_Protocol) Descriptor() protoreflect.EnumDescriptor { + return file_management_proto_enumTypes[0].Descriptor() +} + +func (HostConfig_Protocol) Type() protoreflect.EnumType { + return &file_management_proto_enumTypes[0] +} + +func (x HostConfig_Protocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use HostConfig_Protocol.Descriptor instead. func (HostConfig_Protocol) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{11, 0} + return file_management_proto_rawDescGZIP(), []int{11, 0} } type DeviceAuthorizationFlowProvider int32 @@ -61,20 +82,41 @@ const ( DeviceAuthorizationFlow_HOSTED DeviceAuthorizationFlowProvider = 0 ) -var DeviceAuthorizationFlowProvider_name = map[int32]string{ - 0: "HOSTED", -} +// Enum value maps for DeviceAuthorizationFlowProvider. +var ( + DeviceAuthorizationFlowProvider_name = map[int32]string{ + 0: "HOSTED", + } + DeviceAuthorizationFlowProvider_value = map[string]int32{ + "HOSTED": 0, + } +) -var DeviceAuthorizationFlowProvider_value = map[string]int32{ - "HOSTED": 0, +func (x DeviceAuthorizationFlowProvider) Enum() *DeviceAuthorizationFlowProvider { + p := new(DeviceAuthorizationFlowProvider) + *p = x + return p } func (x DeviceAuthorizationFlowProvider) String() string { - return proto.EnumName(DeviceAuthorizationFlowProvider_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } +func (DeviceAuthorizationFlowProvider) Descriptor() protoreflect.EnumDescriptor { + return file_management_proto_enumTypes[1].Descriptor() +} + +func (DeviceAuthorizationFlowProvider) Type() protoreflect.EnumType { + return &file_management_proto_enumTypes[1] +} + +func (x DeviceAuthorizationFlowProvider) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DeviceAuthorizationFlowProvider.Descriptor instead. func (DeviceAuthorizationFlowProvider) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{18, 0} + return file_management_proto_rawDescGZIP(), []int{19, 0} } type FirewallRuleDirection int32 @@ -84,22 +126,43 @@ const ( FirewallRule_OUT FirewallRuleDirection = 1 ) -var FirewallRuleDirection_name = map[int32]string{ - 0: "IN", - 1: "OUT", -} +// Enum value maps for FirewallRuleDirection. +var ( + FirewallRuleDirection_name = map[int32]string{ + 0: "IN", + 1: "OUT", + } + FirewallRuleDirection_value = map[string]int32{ + "IN": 0, + "OUT": 1, + } +) -var FirewallRuleDirection_value = map[string]int32{ - "IN": 0, - "OUT": 1, +func (x FirewallRuleDirection) Enum() *FirewallRuleDirection { + p := new(FirewallRuleDirection) + *p = x + return p } func (x FirewallRuleDirection) String() string { - return proto.EnumName(FirewallRuleDirection_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } +func (FirewallRuleDirection) Descriptor() protoreflect.EnumDescriptor { + return file_management_proto_enumTypes[2].Descriptor() +} + +func (FirewallRuleDirection) Type() protoreflect.EnumType { + return &file_management_proto_enumTypes[2] +} + +func (x FirewallRuleDirection) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FirewallRuleDirection.Descriptor instead. func (FirewallRuleDirection) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{28, 0} + return file_management_proto_rawDescGZIP(), []int{29, 0} } type FirewallRuleAction int32 @@ -109,22 +172,43 @@ const ( FirewallRule_DROP FirewallRuleAction = 1 ) -var FirewallRuleAction_name = map[int32]string{ - 0: "ACCEPT", - 1: "DROP", -} +// Enum value maps for FirewallRuleAction. +var ( + FirewallRuleAction_name = map[int32]string{ + 0: "ACCEPT", + 1: "DROP", + } + FirewallRuleAction_value = map[string]int32{ + "ACCEPT": 0, + "DROP": 1, + } +) -var FirewallRuleAction_value = map[string]int32{ - "ACCEPT": 0, - "DROP": 1, +func (x FirewallRuleAction) Enum() *FirewallRuleAction { + p := new(FirewallRuleAction) + *p = x + return p } func (x FirewallRuleAction) String() string { - return proto.EnumName(FirewallRuleAction_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } +func (FirewallRuleAction) Descriptor() protoreflect.EnumDescriptor { + return file_management_proto_enumTypes[3].Descriptor() +} + +func (FirewallRuleAction) Type() protoreflect.EnumType { + return &file_management_proto_enumTypes[3] +} + +func (x FirewallRuleAction) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FirewallRuleAction.Descriptor instead. func (FirewallRuleAction) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{28, 1} + return file_management_proto_rawDescGZIP(), []int{29, 1} } type FirewallRuleProtocol int32 @@ -137,121 +221,161 @@ const ( FirewallRule_ICMP FirewallRuleProtocol = 4 ) -var FirewallRuleProtocol_name = map[int32]string{ - 0: "UNKNOWN", - 1: "ALL", - 2: "TCP", - 3: "UDP", - 4: "ICMP", -} +// Enum value maps for FirewallRuleProtocol. +var ( + FirewallRuleProtocol_name = map[int32]string{ + 0: "UNKNOWN", + 1: "ALL", + 2: "TCP", + 3: "UDP", + 4: "ICMP", + } + FirewallRuleProtocol_value = map[string]int32{ + "UNKNOWN": 0, + "ALL": 1, + "TCP": 2, + "UDP": 3, + "ICMP": 4, + } +) -var FirewallRuleProtocol_value = map[string]int32{ - "UNKNOWN": 0, - "ALL": 1, - "TCP": 2, - "UDP": 3, - "ICMP": 4, +func (x FirewallRuleProtocol) Enum() *FirewallRuleProtocol { + p := new(FirewallRuleProtocol) + *p = x + return p } func (x FirewallRuleProtocol) String() string { - return proto.EnumName(FirewallRuleProtocol_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } +func (FirewallRuleProtocol) Descriptor() protoreflect.EnumDescriptor { + return file_management_proto_enumTypes[4].Descriptor() +} + +func (FirewallRuleProtocol) Type() protoreflect.EnumType { + return &file_management_proto_enumTypes[4] +} + +func (x FirewallRuleProtocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FirewallRuleProtocol.Descriptor instead. func (FirewallRuleProtocol) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{28, 2} + return file_management_proto_rawDescGZIP(), []int{29, 2} } type EncryptedMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Wireguard public key WgPubKey string `protobuf:"bytes,1,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` // encrypted message Body Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` // Version of the Wiretrustee Management Service protocol - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` } -func (m *EncryptedMessage) Reset() { *m = EncryptedMessage{} } -func (m *EncryptedMessage) String() string { return proto.CompactTextString(m) } -func (*EncryptedMessage) ProtoMessage() {} +func (x *EncryptedMessage) Reset() { + *x = EncryptedMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EncryptedMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EncryptedMessage) ProtoMessage() {} + +func (x *EncryptedMessage) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EncryptedMessage.ProtoReflect.Descriptor instead. func (*EncryptedMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{0} + return file_management_proto_rawDescGZIP(), []int{0} } -func (m *EncryptedMessage) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EncryptedMessage.Unmarshal(m, b) -} -func (m *EncryptedMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EncryptedMessage.Marshal(b, m, deterministic) -} -func (m *EncryptedMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_EncryptedMessage.Merge(m, src) -} -func (m *EncryptedMessage) XXX_Size() int { - return xxx_messageInfo_EncryptedMessage.Size(m) -} -func (m *EncryptedMessage) XXX_DiscardUnknown() { - xxx_messageInfo_EncryptedMessage.DiscardUnknown(m) -} - -var xxx_messageInfo_EncryptedMessage proto.InternalMessageInfo - -func (m *EncryptedMessage) GetWgPubKey() string { - if m != nil { - return m.WgPubKey +func (x *EncryptedMessage) GetWgPubKey() string { + if x != nil { + return x.WgPubKey } return "" } -func (m *EncryptedMessage) GetBody() []byte { - if m != nil { - return m.Body +func (x *EncryptedMessage) GetBody() []byte { + if x != nil { + return x.Body } return nil } -func (m *EncryptedMessage) GetVersion() int32 { - if m != nil { - return m.Version +func (x *EncryptedMessage) GetVersion() int32 { + if x != nil { + return x.Version } return 0 } type SyncRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *SyncRequest) Reset() { *m = SyncRequest{} } -func (m *SyncRequest) String() string { return proto.CompactTextString(m) } -func (*SyncRequest) ProtoMessage() {} +func (x *SyncRequest) Reset() { + *x = SyncRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncRequest) ProtoMessage() {} + +func (x *SyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SyncRequest.ProtoReflect.Descriptor instead. func (*SyncRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{1} + return file_management_proto_rawDescGZIP(), []int{1} } -func (m *SyncRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SyncRequest.Unmarshal(m, b) -} -func (m *SyncRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SyncRequest.Marshal(b, m, deterministic) -} -func (m *SyncRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SyncRequest.Merge(m, src) -} -func (m *SyncRequest) XXX_Size() int { - return xxx_messageInfo_SyncRequest.Size(m) -} -func (m *SyncRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SyncRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SyncRequest proto.InternalMessageInfo - // SyncResponse represents a state that should be applied to the local peer (e.g. Wiretrustee servers config as well as local peer and remote peers configs) type SyncResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Global config WiretrusteeConfig *WiretrusteeConfig `protobuf:"bytes,1,opt,name=wiretrusteeConfig,proto3" json:"wiretrusteeConfig,omitempty"` // Deprecated. Use NetworkMap.PeerConfig @@ -260,74 +384,82 @@ type SyncResponse struct { RemotePeers []*RemotePeerConfig `protobuf:"bytes,3,rep,name=remotePeers,proto3" json:"remotePeers,omitempty"` // Indicates whether remotePeers array is empty or not to bypass protobuf null and empty array equality. // Deprecated. Use NetworkMap.remotePeersIsEmpty - RemotePeersIsEmpty bool `protobuf:"varint,4,opt,name=remotePeersIsEmpty,proto3" json:"remotePeersIsEmpty,omitempty"` - NetworkMap *NetworkMap `protobuf:"bytes,5,opt,name=NetworkMap,proto3" json:"NetworkMap,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RemotePeersIsEmpty bool `protobuf:"varint,4,opt,name=remotePeersIsEmpty,proto3" json:"remotePeersIsEmpty,omitempty"` + NetworkMap *NetworkMap `protobuf:"bytes,5,opt,name=NetworkMap,proto3" json:"NetworkMap,omitempty"` } -func (m *SyncResponse) Reset() { *m = SyncResponse{} } -func (m *SyncResponse) String() string { return proto.CompactTextString(m) } -func (*SyncResponse) ProtoMessage() {} +func (x *SyncResponse) Reset() { + *x = SyncResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SyncResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SyncResponse) ProtoMessage() {} + +func (x *SyncResponse) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SyncResponse.ProtoReflect.Descriptor instead. func (*SyncResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{2} + return file_management_proto_rawDescGZIP(), []int{2} } -func (m *SyncResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SyncResponse.Unmarshal(m, b) -} -func (m *SyncResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SyncResponse.Marshal(b, m, deterministic) -} -func (m *SyncResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SyncResponse.Merge(m, src) -} -func (m *SyncResponse) XXX_Size() int { - return xxx_messageInfo_SyncResponse.Size(m) -} -func (m *SyncResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SyncResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SyncResponse proto.InternalMessageInfo - -func (m *SyncResponse) GetWiretrusteeConfig() *WiretrusteeConfig { - if m != nil { - return m.WiretrusteeConfig +func (x *SyncResponse) GetWiretrusteeConfig() *WiretrusteeConfig { + if x != nil { + return x.WiretrusteeConfig } return nil } -func (m *SyncResponse) GetPeerConfig() *PeerConfig { - if m != nil { - return m.PeerConfig +func (x *SyncResponse) GetPeerConfig() *PeerConfig { + if x != nil { + return x.PeerConfig } return nil } -func (m *SyncResponse) GetRemotePeers() []*RemotePeerConfig { - if m != nil { - return m.RemotePeers +func (x *SyncResponse) GetRemotePeers() []*RemotePeerConfig { + if x != nil { + return x.RemotePeers } return nil } -func (m *SyncResponse) GetRemotePeersIsEmpty() bool { - if m != nil { - return m.RemotePeersIsEmpty +func (x *SyncResponse) GetRemotePeersIsEmpty() bool { + if x != nil { + return x.RemotePeersIsEmpty } return false } -func (m *SyncResponse) GetNetworkMap() *NetworkMap { - if m != nil { - return m.NetworkMap +func (x *SyncResponse) GetNetworkMap() *NetworkMap { + if x != nil { + return x.NetworkMap } return nil } type LoginRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Pre-authorized setup key (can be empty) SetupKey string `protobuf:"bytes,1,opt,name=setupKey,proto3" json:"setupKey,omitempty"` // Meta data of the peer (e.g. name, os_name, os_version, @@ -335,61 +467,65 @@ type LoginRequest struct { // SSO token (can be empty) JwtToken string `protobuf:"bytes,3,opt,name=jwtToken,proto3" json:"jwtToken,omitempty"` // Can be absent for now. - PeerKeys *PeerKeys `protobuf:"bytes,4,opt,name=peerKeys,proto3" json:"peerKeys,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + PeerKeys *PeerKeys `protobuf:"bytes,4,opt,name=peerKeys,proto3" json:"peerKeys,omitempty"` } -func (m *LoginRequest) Reset() { *m = LoginRequest{} } -func (m *LoginRequest) String() string { return proto.CompactTextString(m) } -func (*LoginRequest) ProtoMessage() {} +func (x *LoginRequest) Reset() { + *x = LoginRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoginRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoginRequest) ProtoMessage() {} + +func (x *LoginRequest) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoginRequest.ProtoReflect.Descriptor instead. func (*LoginRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{3} + return file_management_proto_rawDescGZIP(), []int{3} } -func (m *LoginRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoginRequest.Unmarshal(m, b) -} -func (m *LoginRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoginRequest.Marshal(b, m, deterministic) -} -func (m *LoginRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoginRequest.Merge(m, src) -} -func (m *LoginRequest) XXX_Size() int { - return xxx_messageInfo_LoginRequest.Size(m) -} -func (m *LoginRequest) XXX_DiscardUnknown() { - xxx_messageInfo_LoginRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_LoginRequest proto.InternalMessageInfo - -func (m *LoginRequest) GetSetupKey() string { - if m != nil { - return m.SetupKey +func (x *LoginRequest) GetSetupKey() string { + if x != nil { + return x.SetupKey } return "" } -func (m *LoginRequest) GetMeta() *PeerSystemMeta { - if m != nil { - return m.Meta +func (x *LoginRequest) GetMeta() *PeerSystemMeta { + if x != nil { + return x.Meta } return nil } -func (m *LoginRequest) GetJwtToken() string { - if m != nil { - return m.JwtToken +func (x *LoginRequest) GetJwtToken() string { + if x != nil { + return x.JwtToken } return "" } -func (m *LoginRequest) GetPeerKeys() *PeerKeys { - if m != nil { - return m.PeerKeys +func (x *LoginRequest) GetPeerKeys() *PeerKeys { + if x != nil { + return x.PeerKeys } return nil } @@ -397,563 +533,697 @@ func (m *LoginRequest) GetPeerKeys() *PeerKeys { // PeerKeys is additional peer info like SSH pub key and WireGuard public key. // This message is sent on Login or register requests, or when a key rotation has to happen. type PeerKeys struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // sshPubKey represents a public SSH key of the peer. Can be absent. SshPubKey []byte `protobuf:"bytes,1,opt,name=sshPubKey,proto3" json:"sshPubKey,omitempty"` // wgPubKey represents a public WireGuard key of the peer. Can be absent. - WgPubKey []byte `protobuf:"bytes,2,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + WgPubKey []byte `protobuf:"bytes,2,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` } -func (m *PeerKeys) Reset() { *m = PeerKeys{} } -func (m *PeerKeys) String() string { return proto.CompactTextString(m) } -func (*PeerKeys) ProtoMessage() {} +func (x *PeerKeys) Reset() { + *x = PeerKeys{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PeerKeys) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerKeys) ProtoMessage() {} + +func (x *PeerKeys) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerKeys.ProtoReflect.Descriptor instead. func (*PeerKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{4} + return file_management_proto_rawDescGZIP(), []int{4} } -func (m *PeerKeys) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PeerKeys.Unmarshal(m, b) -} -func (m *PeerKeys) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PeerKeys.Marshal(b, m, deterministic) -} -func (m *PeerKeys) XXX_Merge(src proto.Message) { - xxx_messageInfo_PeerKeys.Merge(m, src) -} -func (m *PeerKeys) XXX_Size() int { - return xxx_messageInfo_PeerKeys.Size(m) -} -func (m *PeerKeys) XXX_DiscardUnknown() { - xxx_messageInfo_PeerKeys.DiscardUnknown(m) -} - -var xxx_messageInfo_PeerKeys proto.InternalMessageInfo - -func (m *PeerKeys) GetSshPubKey() []byte { - if m != nil { - return m.SshPubKey +func (x *PeerKeys) GetSshPubKey() []byte { + if x != nil { + return x.SshPubKey } return nil } -func (m *PeerKeys) GetWgPubKey() []byte { - if m != nil { - return m.WgPubKey +func (x *PeerKeys) GetWgPubKey() []byte { + if x != nil { + return x.WgPubKey } return nil } // Environment is part of the PeerSystemMeta and describes the environment the agent is running in. type Environment struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // cloud is the cloud provider the agent is running in if applicable. Cloud string `protobuf:"bytes,1,opt,name=cloud,proto3" json:"cloud,omitempty"` // platform is the platform the agent is running on if applicable. - Platform string `protobuf:"bytes,2,opt,name=platform,proto3" json:"platform,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Platform string `protobuf:"bytes,2,opt,name=platform,proto3" json:"platform,omitempty"` } -func (m *Environment) Reset() { *m = Environment{} } -func (m *Environment) String() string { return proto.CompactTextString(m) } -func (*Environment) ProtoMessage() {} +func (x *Environment) Reset() { + *x = Environment{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Environment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Environment) ProtoMessage() {} + +func (x *Environment) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Environment.ProtoReflect.Descriptor instead. func (*Environment) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{5} + return file_management_proto_rawDescGZIP(), []int{5} } -func (m *Environment) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Environment.Unmarshal(m, b) -} -func (m *Environment) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Environment.Marshal(b, m, deterministic) -} -func (m *Environment) XXX_Merge(src proto.Message) { - xxx_messageInfo_Environment.Merge(m, src) -} -func (m *Environment) XXX_Size() int { - return xxx_messageInfo_Environment.Size(m) -} -func (m *Environment) XXX_DiscardUnknown() { - xxx_messageInfo_Environment.DiscardUnknown(m) -} - -var xxx_messageInfo_Environment proto.InternalMessageInfo - -func (m *Environment) GetCloud() string { - if m != nil { - return m.Cloud +func (x *Environment) GetCloud() string { + if x != nil { + return x.Cloud } return "" } -func (m *Environment) GetPlatform() string { - if m != nil { - return m.Platform +func (x *Environment) GetPlatform() string { + if x != nil { + return x.Platform } return "" } // PeerSystemMeta is machine meta data like OS and version. type PeerSystemMeta struct { - Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` - GoOS string `protobuf:"bytes,2,opt,name=goOS,proto3" json:"goOS,omitempty"` - Kernel string `protobuf:"bytes,3,opt,name=kernel,proto3" json:"kernel,omitempty"` - Core string `protobuf:"bytes,4,opt,name=core,proto3" json:"core,omitempty"` - Platform string `protobuf:"bytes,5,opt,name=platform,proto3" json:"platform,omitempty"` - OS string `protobuf:"bytes,6,opt,name=OS,proto3" json:"OS,omitempty"` - WiretrusteeVersion string `protobuf:"bytes,7,opt,name=wiretrusteeVersion,proto3" json:"wiretrusteeVersion,omitempty"` - UiVersion string `protobuf:"bytes,8,opt,name=uiVersion,proto3" json:"uiVersion,omitempty"` - KernelVersion string `protobuf:"bytes,9,opt,name=kernelVersion,proto3" json:"kernelVersion,omitempty"` - OSVersion string `protobuf:"bytes,10,opt,name=OSVersion,proto3" json:"OSVersion,omitempty"` - NetworkAddresses []*NetworkAddress `protobuf:"bytes,11,rep,name=networkAddresses,proto3" json:"networkAddresses,omitempty"` - SysSerialNumber string `protobuf:"bytes,12,opt,name=sysSerialNumber,proto3" json:"sysSerialNumber,omitempty"` - SysProductName string `protobuf:"bytes,13,opt,name=sysProductName,proto3" json:"sysProductName,omitempty"` - SysManufacturer string `protobuf:"bytes,14,opt,name=sysManufacturer,proto3" json:"sysManufacturer,omitempty"` - Environment *Environment `protobuf:"bytes,15,opt,name=environment,proto3" json:"environment,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` + GoOS string `protobuf:"bytes,2,opt,name=goOS,proto3" json:"goOS,omitempty"` + Kernel string `protobuf:"bytes,3,opt,name=kernel,proto3" json:"kernel,omitempty"` + Core string `protobuf:"bytes,4,opt,name=core,proto3" json:"core,omitempty"` + Platform string `protobuf:"bytes,5,opt,name=platform,proto3" json:"platform,omitempty"` + OS string `protobuf:"bytes,6,opt,name=OS,proto3" json:"OS,omitempty"` + WiretrusteeVersion string `protobuf:"bytes,7,opt,name=wiretrusteeVersion,proto3" json:"wiretrusteeVersion,omitempty"` + UiVersion string `protobuf:"bytes,8,opt,name=uiVersion,proto3" json:"uiVersion,omitempty"` + KernelVersion string `protobuf:"bytes,9,opt,name=kernelVersion,proto3" json:"kernelVersion,omitempty"` + OSVersion string `protobuf:"bytes,10,opt,name=OSVersion,proto3" json:"OSVersion,omitempty"` + NetworkAddresses []*NetworkAddress `protobuf:"bytes,11,rep,name=networkAddresses,proto3" json:"networkAddresses,omitempty"` + SysSerialNumber string `protobuf:"bytes,12,opt,name=sysSerialNumber,proto3" json:"sysSerialNumber,omitempty"` + SysProductName string `protobuf:"bytes,13,opt,name=sysProductName,proto3" json:"sysProductName,omitempty"` + SysManufacturer string `protobuf:"bytes,14,opt,name=sysManufacturer,proto3" json:"sysManufacturer,omitempty"` + Environment *Environment `protobuf:"bytes,15,opt,name=environment,proto3" json:"environment,omitempty"` } -func (m *PeerSystemMeta) Reset() { *m = PeerSystemMeta{} } -func (m *PeerSystemMeta) String() string { return proto.CompactTextString(m) } -func (*PeerSystemMeta) ProtoMessage() {} +func (x *PeerSystemMeta) Reset() { + *x = PeerSystemMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PeerSystemMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerSystemMeta) ProtoMessage() {} + +func (x *PeerSystemMeta) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerSystemMeta.ProtoReflect.Descriptor instead. func (*PeerSystemMeta) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{6} + return file_management_proto_rawDescGZIP(), []int{6} } -func (m *PeerSystemMeta) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PeerSystemMeta.Unmarshal(m, b) -} -func (m *PeerSystemMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PeerSystemMeta.Marshal(b, m, deterministic) -} -func (m *PeerSystemMeta) XXX_Merge(src proto.Message) { - xxx_messageInfo_PeerSystemMeta.Merge(m, src) -} -func (m *PeerSystemMeta) XXX_Size() int { - return xxx_messageInfo_PeerSystemMeta.Size(m) -} -func (m *PeerSystemMeta) XXX_DiscardUnknown() { - xxx_messageInfo_PeerSystemMeta.DiscardUnknown(m) -} - -var xxx_messageInfo_PeerSystemMeta proto.InternalMessageInfo - -func (m *PeerSystemMeta) GetHostname() string { - if m != nil { - return m.Hostname +func (x *PeerSystemMeta) GetHostname() string { + if x != nil { + return x.Hostname } return "" } -func (m *PeerSystemMeta) GetGoOS() string { - if m != nil { - return m.GoOS +func (x *PeerSystemMeta) GetGoOS() string { + if x != nil { + return x.GoOS } return "" } -func (m *PeerSystemMeta) GetKernel() string { - if m != nil { - return m.Kernel +func (x *PeerSystemMeta) GetKernel() string { + if x != nil { + return x.Kernel } return "" } -func (m *PeerSystemMeta) GetCore() string { - if m != nil { - return m.Core +func (x *PeerSystemMeta) GetCore() string { + if x != nil { + return x.Core } return "" } -func (m *PeerSystemMeta) GetPlatform() string { - if m != nil { - return m.Platform +func (x *PeerSystemMeta) GetPlatform() string { + if x != nil { + return x.Platform } return "" } -func (m *PeerSystemMeta) GetOS() string { - if m != nil { - return m.OS +func (x *PeerSystemMeta) GetOS() string { + if x != nil { + return x.OS } return "" } -func (m *PeerSystemMeta) GetWiretrusteeVersion() string { - if m != nil { - return m.WiretrusteeVersion +func (x *PeerSystemMeta) GetWiretrusteeVersion() string { + if x != nil { + return x.WiretrusteeVersion } return "" } -func (m *PeerSystemMeta) GetUiVersion() string { - if m != nil { - return m.UiVersion +func (x *PeerSystemMeta) GetUiVersion() string { + if x != nil { + return x.UiVersion } return "" } -func (m *PeerSystemMeta) GetKernelVersion() string { - if m != nil { - return m.KernelVersion +func (x *PeerSystemMeta) GetKernelVersion() string { + if x != nil { + return x.KernelVersion } return "" } -func (m *PeerSystemMeta) GetOSVersion() string { - if m != nil { - return m.OSVersion +func (x *PeerSystemMeta) GetOSVersion() string { + if x != nil { + return x.OSVersion } return "" } -func (m *PeerSystemMeta) GetNetworkAddresses() []*NetworkAddress { - if m != nil { - return m.NetworkAddresses +func (x *PeerSystemMeta) GetNetworkAddresses() []*NetworkAddress { + if x != nil { + return x.NetworkAddresses } return nil } -func (m *PeerSystemMeta) GetSysSerialNumber() string { - if m != nil { - return m.SysSerialNumber +func (x *PeerSystemMeta) GetSysSerialNumber() string { + if x != nil { + return x.SysSerialNumber } return "" } -func (m *PeerSystemMeta) GetSysProductName() string { - if m != nil { - return m.SysProductName +func (x *PeerSystemMeta) GetSysProductName() string { + if x != nil { + return x.SysProductName } return "" } -func (m *PeerSystemMeta) GetSysManufacturer() string { - if m != nil { - return m.SysManufacturer +func (x *PeerSystemMeta) GetSysManufacturer() string { + if x != nil { + return x.SysManufacturer } return "" } -func (m *PeerSystemMeta) GetEnvironment() *Environment { - if m != nil { - return m.Environment +func (x *PeerSystemMeta) GetEnvironment() *Environment { + if x != nil { + return x.Environment } return nil } type LoginResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Global config WiretrusteeConfig *WiretrusteeConfig `protobuf:"bytes,1,opt,name=wiretrusteeConfig,proto3" json:"wiretrusteeConfig,omitempty"` // Peer local config - PeerConfig *PeerConfig `protobuf:"bytes,2,opt,name=peerConfig,proto3" json:"peerConfig,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + PeerConfig *PeerConfig `protobuf:"bytes,2,opt,name=peerConfig,proto3" json:"peerConfig,omitempty"` } -func (m *LoginResponse) Reset() { *m = LoginResponse{} } -func (m *LoginResponse) String() string { return proto.CompactTextString(m) } -func (*LoginResponse) ProtoMessage() {} +func (x *LoginResponse) Reset() { + *x = LoginResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoginResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoginResponse) ProtoMessage() {} + +func (x *LoginResponse) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoginResponse.ProtoReflect.Descriptor instead. func (*LoginResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{7} + return file_management_proto_rawDescGZIP(), []int{7} } -func (m *LoginResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoginResponse.Unmarshal(m, b) -} -func (m *LoginResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoginResponse.Marshal(b, m, deterministic) -} -func (m *LoginResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoginResponse.Merge(m, src) -} -func (m *LoginResponse) XXX_Size() int { - return xxx_messageInfo_LoginResponse.Size(m) -} -func (m *LoginResponse) XXX_DiscardUnknown() { - xxx_messageInfo_LoginResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_LoginResponse proto.InternalMessageInfo - -func (m *LoginResponse) GetWiretrusteeConfig() *WiretrusteeConfig { - if m != nil { - return m.WiretrusteeConfig +func (x *LoginResponse) GetWiretrusteeConfig() *WiretrusteeConfig { + if x != nil { + return x.WiretrusteeConfig } return nil } -func (m *LoginResponse) GetPeerConfig() *PeerConfig { - if m != nil { - return m.PeerConfig +func (x *LoginResponse) GetPeerConfig() *PeerConfig { + if x != nil { + return x.PeerConfig } return nil } type ServerKeyResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Server's Wireguard public key Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // Key expiration timestamp after which the key should be fetched again by the client ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=expiresAt,proto3" json:"expiresAt,omitempty"` // Version of the Wiretrustee Management Service protocol - Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Version int32 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"` } -func (m *ServerKeyResponse) Reset() { *m = ServerKeyResponse{} } -func (m *ServerKeyResponse) String() string { return proto.CompactTextString(m) } -func (*ServerKeyResponse) ProtoMessage() {} +func (x *ServerKeyResponse) Reset() { + *x = ServerKeyResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerKeyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerKeyResponse) ProtoMessage() {} + +func (x *ServerKeyResponse) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerKeyResponse.ProtoReflect.Descriptor instead. func (*ServerKeyResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{8} + return file_management_proto_rawDescGZIP(), []int{8} } -func (m *ServerKeyResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerKeyResponse.Unmarshal(m, b) -} -func (m *ServerKeyResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerKeyResponse.Marshal(b, m, deterministic) -} -func (m *ServerKeyResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerKeyResponse.Merge(m, src) -} -func (m *ServerKeyResponse) XXX_Size() int { - return xxx_messageInfo_ServerKeyResponse.Size(m) -} -func (m *ServerKeyResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ServerKeyResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_ServerKeyResponse proto.InternalMessageInfo - -func (m *ServerKeyResponse) GetKey() string { - if m != nil { - return m.Key +func (x *ServerKeyResponse) GetKey() string { + if x != nil { + return x.Key } return "" } -func (m *ServerKeyResponse) GetExpiresAt() *timestamppb.Timestamp { - if m != nil { - return m.ExpiresAt +func (x *ServerKeyResponse) GetExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAt } return nil } -func (m *ServerKeyResponse) GetVersion() int32 { - if m != nil { - return m.Version +func (x *ServerKeyResponse) GetVersion() int32 { + if x != nil { + return x.Version } return 0 } type Empty struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *Empty) Reset() { *m = Empty{} } -func (m *Empty) String() string { return proto.CompactTextString(m) } -func (*Empty) ProtoMessage() {} +func (x *Empty) Reset() { + *x = Empty{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Empty) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Empty) ProtoMessage() {} + +func (x *Empty) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Empty.ProtoReflect.Descriptor instead. func (*Empty) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{9} + return file_management_proto_rawDescGZIP(), []int{9} } -func (m *Empty) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Empty.Unmarshal(m, b) -} -func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Empty.Marshal(b, m, deterministic) -} -func (m *Empty) XXX_Merge(src proto.Message) { - xxx_messageInfo_Empty.Merge(m, src) -} -func (m *Empty) XXX_Size() int { - return xxx_messageInfo_Empty.Size(m) -} -func (m *Empty) XXX_DiscardUnknown() { - xxx_messageInfo_Empty.DiscardUnknown(m) -} - -var xxx_messageInfo_Empty proto.InternalMessageInfo - // WiretrusteeConfig is a common configuration of any Wiretrustee peer. It contains STUN, TURN, Signal and Management servers configurations type WiretrusteeConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // a list of STUN servers Stuns []*HostConfig `protobuf:"bytes,1,rep,name=stuns,proto3" json:"stuns,omitempty"` // a list of TURN servers Turns []*ProtectedHostConfig `protobuf:"bytes,2,rep,name=turns,proto3" json:"turns,omitempty"` // a Signal server config - Signal *HostConfig `protobuf:"bytes,3,opt,name=signal,proto3" json:"signal,omitempty"` - RelayAddress string `protobuf:"bytes,4,opt,name=RelayAddress,proto3" json:"RelayAddress,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Signal *HostConfig `protobuf:"bytes,3,opt,name=signal,proto3" json:"signal,omitempty"` + Relay *RelayConfig `protobuf:"bytes,4,opt,name=relay,proto3" json:"relay,omitempty"` } -func (m *WiretrusteeConfig) Reset() { *m = WiretrusteeConfig{} } -func (m *WiretrusteeConfig) String() string { return proto.CompactTextString(m) } -func (*WiretrusteeConfig) ProtoMessage() {} +func (x *WiretrusteeConfig) Reset() { + *x = WiretrusteeConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *WiretrusteeConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WiretrusteeConfig) ProtoMessage() {} + +func (x *WiretrusteeConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WiretrusteeConfig.ProtoReflect.Descriptor instead. func (*WiretrusteeConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{10} + return file_management_proto_rawDescGZIP(), []int{10} } -func (m *WiretrusteeConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_WiretrusteeConfig.Unmarshal(m, b) -} -func (m *WiretrusteeConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_WiretrusteeConfig.Marshal(b, m, deterministic) -} -func (m *WiretrusteeConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_WiretrusteeConfig.Merge(m, src) -} -func (m *WiretrusteeConfig) XXX_Size() int { - return xxx_messageInfo_WiretrusteeConfig.Size(m) -} -func (m *WiretrusteeConfig) XXX_DiscardUnknown() { - xxx_messageInfo_WiretrusteeConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_WiretrusteeConfig proto.InternalMessageInfo - -func (m *WiretrusteeConfig) GetStuns() []*HostConfig { - if m != nil { - return m.Stuns +func (x *WiretrusteeConfig) GetStuns() []*HostConfig { + if x != nil { + return x.Stuns } return nil } -func (m *WiretrusteeConfig) GetTurns() []*ProtectedHostConfig { - if m != nil { - return m.Turns +func (x *WiretrusteeConfig) GetTurns() []*ProtectedHostConfig { + if x != nil { + return x.Turns } return nil } -func (m *WiretrusteeConfig) GetSignal() *HostConfig { - if m != nil { - return m.Signal +func (x *WiretrusteeConfig) GetSignal() *HostConfig { + if x != nil { + return x.Signal } return nil } -func (m *WiretrusteeConfig) GetRelayAddress() string { - if m != nil { - return m.RelayAddress +func (x *WiretrusteeConfig) GetRelay() *RelayConfig { + if x != nil { + return x.Relay } - return "" + return nil } // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) type HostConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // URI of the resource e.g. turns://stun.wiretrustee.com:4430 or signal.wiretrustee.com:10000 - Uri string `protobuf:"bytes,1,opt,name=uri,proto3" json:"uri,omitempty"` - Protocol HostConfig_Protocol `protobuf:"varint,2,opt,name=protocol,proto3,enum=management.HostConfig_Protocol" json:"protocol,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Uri string `protobuf:"bytes,1,opt,name=uri,proto3" json:"uri,omitempty"` + Protocol HostConfig_Protocol `protobuf:"varint,2,opt,name=protocol,proto3,enum=management.HostConfig_Protocol" json:"protocol,omitempty"` } -func (m *HostConfig) Reset() { *m = HostConfig{} } -func (m *HostConfig) String() string { return proto.CompactTextString(m) } -func (*HostConfig) ProtoMessage() {} +func (x *HostConfig) Reset() { + *x = HostConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HostConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HostConfig) ProtoMessage() {} + +func (x *HostConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HostConfig.ProtoReflect.Descriptor instead. func (*HostConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{11} + return file_management_proto_rawDescGZIP(), []int{11} } -func (m *HostConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HostConfig.Unmarshal(m, b) -} -func (m *HostConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HostConfig.Marshal(b, m, deterministic) -} -func (m *HostConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_HostConfig.Merge(m, src) -} -func (m *HostConfig) XXX_Size() int { - return xxx_messageInfo_HostConfig.Size(m) -} -func (m *HostConfig) XXX_DiscardUnknown() { - xxx_messageInfo_HostConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_HostConfig proto.InternalMessageInfo - -func (m *HostConfig) GetUri() string { - if m != nil { - return m.Uri +func (x *HostConfig) GetUri() string { + if x != nil { + return x.Uri } return "" } -func (m *HostConfig) GetProtocol() HostConfig_Protocol { - if m != nil { - return m.Protocol +func (x *HostConfig) GetProtocol() HostConfig_Protocol { + if x != nil { + return x.Protocol } return HostConfig_UDP } +type RelayConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Urls []string `protobuf:"bytes,1,rep,name=urls,proto3" json:"urls,omitempty"` + TokenPayload string `protobuf:"bytes,2,opt,name=tokenPayload,proto3" json:"tokenPayload,omitempty"` + TokenSignature string `protobuf:"bytes,3,opt,name=tokenSignature,proto3" json:"tokenSignature,omitempty"` +} + +func (x *RelayConfig) Reset() { + *x = RelayConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RelayConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RelayConfig) ProtoMessage() {} + +func (x *RelayConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RelayConfig.ProtoReflect.Descriptor instead. +func (*RelayConfig) Descriptor() ([]byte, []int) { + return file_management_proto_rawDescGZIP(), []int{12} +} + +func (x *RelayConfig) GetUrls() []string { + if x != nil { + return x.Urls + } + return nil +} + +func (x *RelayConfig) GetTokenPayload() string { + if x != nil { + return x.TokenPayload + } + return "" +} + +func (x *RelayConfig) GetTokenSignature() string { + if x != nil { + return x.TokenSignature + } + return "" +} + // ProtectedHostConfig is similar to HostConfig but has additional user and password // Mostly used for TURN servers type ProtectedHostConfig struct { - HostConfig *HostConfig `protobuf:"bytes,1,opt,name=hostConfig,proto3" json:"hostConfig,omitempty"` - User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` - Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + HostConfig *HostConfig `protobuf:"bytes,1,opt,name=hostConfig,proto3" json:"hostConfig,omitempty"` + User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` + Password string `protobuf:"bytes,3,opt,name=password,proto3" json:"password,omitempty"` } -func (m *ProtectedHostConfig) Reset() { *m = ProtectedHostConfig{} } -func (m *ProtectedHostConfig) String() string { return proto.CompactTextString(m) } -func (*ProtectedHostConfig) ProtoMessage() {} +func (x *ProtectedHostConfig) Reset() { + *x = ProtectedHostConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProtectedHostConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProtectedHostConfig) ProtoMessage() {} + +func (x *ProtectedHostConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProtectedHostConfig.ProtoReflect.Descriptor instead. func (*ProtectedHostConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{12} + return file_management_proto_rawDescGZIP(), []int{13} } -func (m *ProtectedHostConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ProtectedHostConfig.Unmarshal(m, b) -} -func (m *ProtectedHostConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ProtectedHostConfig.Marshal(b, m, deterministic) -} -func (m *ProtectedHostConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProtectedHostConfig.Merge(m, src) -} -func (m *ProtectedHostConfig) XXX_Size() int { - return xxx_messageInfo_ProtectedHostConfig.Size(m) -} -func (m *ProtectedHostConfig) XXX_DiscardUnknown() { - xxx_messageInfo_ProtectedHostConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_ProtectedHostConfig proto.InternalMessageInfo - -func (m *ProtectedHostConfig) GetHostConfig() *HostConfig { - if m != nil { - return m.HostConfig +func (x *ProtectedHostConfig) GetHostConfig() *HostConfig { + if x != nil { + return x.HostConfig } return nil } -func (m *ProtectedHostConfig) GetUser() string { - if m != nil { - return m.User +func (x *ProtectedHostConfig) GetUser() string { + if x != nil { + return x.User } return "" } -func (m *ProtectedHostConfig) GetPassword() string { - if m != nil { - return m.Password +func (x *ProtectedHostConfig) GetPassword() string { + if x != nil { + return x.Password } return "" } @@ -961,6 +1231,10 @@ func (m *ProtectedHostConfig) GetPassword() string { // PeerConfig represents a configuration of a "our" peer. // The properties are used to configure local Wireguard type PeerConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Peer's virtual IP address within the Wiretrustee VPN (a Wireguard address config) Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"` // Wiretrustee DNS server (a Wireguard DNS config) @@ -968,67 +1242,75 @@ type PeerConfig struct { // SSHConfig of the peer. SshConfig *SSHConfig `protobuf:"bytes,3,opt,name=sshConfig,proto3" json:"sshConfig,omitempty"` // Peer fully qualified domain name - Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` } -func (m *PeerConfig) Reset() { *m = PeerConfig{} } -func (m *PeerConfig) String() string { return proto.CompactTextString(m) } -func (*PeerConfig) ProtoMessage() {} +func (x *PeerConfig) Reset() { + *x = PeerConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PeerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PeerConfig) ProtoMessage() {} + +func (x *PeerConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PeerConfig.ProtoReflect.Descriptor instead. func (*PeerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{13} + return file_management_proto_rawDescGZIP(), []int{14} } -func (m *PeerConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PeerConfig.Unmarshal(m, b) -} -func (m *PeerConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PeerConfig.Marshal(b, m, deterministic) -} -func (m *PeerConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_PeerConfig.Merge(m, src) -} -func (m *PeerConfig) XXX_Size() int { - return xxx_messageInfo_PeerConfig.Size(m) -} -func (m *PeerConfig) XXX_DiscardUnknown() { - xxx_messageInfo_PeerConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_PeerConfig proto.InternalMessageInfo - -func (m *PeerConfig) GetAddress() string { - if m != nil { - return m.Address +func (x *PeerConfig) GetAddress() string { + if x != nil { + return x.Address } return "" } -func (m *PeerConfig) GetDns() string { - if m != nil { - return m.Dns +func (x *PeerConfig) GetDns() string { + if x != nil { + return x.Dns } return "" } -func (m *PeerConfig) GetSshConfig() *SSHConfig { - if m != nil { - return m.SshConfig +func (x *PeerConfig) GetSshConfig() *SSHConfig { + if x != nil { + return x.SshConfig } return nil } -func (m *PeerConfig) GetFqdn() string { - if m != nil { - return m.Fqdn +func (x *PeerConfig) GetFqdn() string { + if x != nil { + return x.Fqdn } return "" } // NetworkMap represents a network state of the peer with the corresponding configuration parameters to establish peer-to-peer connections type NetworkMap struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Serial is an ID of the network state to be used by clients to order updates. // The larger the Serial the newer the configuration. // E.g. the client app should keep track of this id locally and discard all the configurations with a lower value @@ -1048,96 +1330,100 @@ type NetworkMap struct { // FirewallRule represents a list of firewall rules to be applied to peer FirewallRules []*FirewallRule `protobuf:"bytes,8,rep,name=FirewallRules,proto3" json:"FirewallRules,omitempty"` // firewallRulesIsEmpty indicates whether FirewallRule array is empty or not to bypass protobuf null and empty array equality. - FirewallRulesIsEmpty bool `protobuf:"varint,9,opt,name=firewallRulesIsEmpty,proto3" json:"firewallRulesIsEmpty,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + FirewallRulesIsEmpty bool `protobuf:"varint,9,opt,name=firewallRulesIsEmpty,proto3" json:"firewallRulesIsEmpty,omitempty"` } -func (m *NetworkMap) Reset() { *m = NetworkMap{} } -func (m *NetworkMap) String() string { return proto.CompactTextString(m) } -func (*NetworkMap) ProtoMessage() {} +func (x *NetworkMap) Reset() { + *x = NetworkMap{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NetworkMap) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkMap) ProtoMessage() {} + +func (x *NetworkMap) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkMap.ProtoReflect.Descriptor instead. func (*NetworkMap) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{14} + return file_management_proto_rawDescGZIP(), []int{15} } -func (m *NetworkMap) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NetworkMap.Unmarshal(m, b) -} -func (m *NetworkMap) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NetworkMap.Marshal(b, m, deterministic) -} -func (m *NetworkMap) XXX_Merge(src proto.Message) { - xxx_messageInfo_NetworkMap.Merge(m, src) -} -func (m *NetworkMap) XXX_Size() int { - return xxx_messageInfo_NetworkMap.Size(m) -} -func (m *NetworkMap) XXX_DiscardUnknown() { - xxx_messageInfo_NetworkMap.DiscardUnknown(m) -} - -var xxx_messageInfo_NetworkMap proto.InternalMessageInfo - -func (m *NetworkMap) GetSerial() uint64 { - if m != nil { - return m.Serial +func (x *NetworkMap) GetSerial() uint64 { + if x != nil { + return x.Serial } return 0 } -func (m *NetworkMap) GetPeerConfig() *PeerConfig { - if m != nil { - return m.PeerConfig +func (x *NetworkMap) GetPeerConfig() *PeerConfig { + if x != nil { + return x.PeerConfig } return nil } -func (m *NetworkMap) GetRemotePeers() []*RemotePeerConfig { - if m != nil { - return m.RemotePeers +func (x *NetworkMap) GetRemotePeers() []*RemotePeerConfig { + if x != nil { + return x.RemotePeers } return nil } -func (m *NetworkMap) GetRemotePeersIsEmpty() bool { - if m != nil { - return m.RemotePeersIsEmpty +func (x *NetworkMap) GetRemotePeersIsEmpty() bool { + if x != nil { + return x.RemotePeersIsEmpty } return false } -func (m *NetworkMap) GetRoutes() []*Route { - if m != nil { - return m.Routes +func (x *NetworkMap) GetRoutes() []*Route { + if x != nil { + return x.Routes } return nil } -func (m *NetworkMap) GetDNSConfig() *DNSConfig { - if m != nil { - return m.DNSConfig +func (x *NetworkMap) GetDNSConfig() *DNSConfig { + if x != nil { + return x.DNSConfig } return nil } -func (m *NetworkMap) GetOfflinePeers() []*RemotePeerConfig { - if m != nil { - return m.OfflinePeers +func (x *NetworkMap) GetOfflinePeers() []*RemotePeerConfig { + if x != nil { + return x.OfflinePeers } return nil } -func (m *NetworkMap) GetFirewallRules() []*FirewallRule { - if m != nil { - return m.FirewallRules +func (x *NetworkMap) GetFirewallRules() []*FirewallRule { + if x != nil { + return x.FirewallRules } return nil } -func (m *NetworkMap) GetFirewallRulesIsEmpty() bool { - if m != nil { - return m.FirewallRulesIsEmpty +func (x *NetworkMap) GetFirewallRulesIsEmpty() bool { + if x != nil { + return x.FirewallRulesIsEmpty } return false } @@ -1145,6 +1431,10 @@ func (m *NetworkMap) GetFirewallRulesIsEmpty() bool { // RemotePeerConfig represents a configuration of a remote peer. // The properties are used to configure WireGuard Peers sections type RemotePeerConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // A WireGuard public key of a remote peer WgPubKey string `protobuf:"bytes,1,opt,name=wgPubKey,proto3" json:"wgPubKey,omitempty"` // WireGuard allowed IPs of a remote peer e.g. [10.30.30.1/32] @@ -1152,275 +1442,321 @@ type RemotePeerConfig struct { // SSHConfig is a SSH config of the remote peer. SSHConfig.sshPubKey should be ignored because peer knows it's SSH key. SshConfig *SSHConfig `protobuf:"bytes,3,opt,name=sshConfig,proto3" json:"sshConfig,omitempty"` // Peer fully qualified domain name - Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Fqdn string `protobuf:"bytes,4,opt,name=fqdn,proto3" json:"fqdn,omitempty"` } -func (m *RemotePeerConfig) Reset() { *m = RemotePeerConfig{} } -func (m *RemotePeerConfig) String() string { return proto.CompactTextString(m) } -func (*RemotePeerConfig) ProtoMessage() {} +func (x *RemotePeerConfig) Reset() { + *x = RemotePeerConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RemotePeerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RemotePeerConfig) ProtoMessage() {} + +func (x *RemotePeerConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RemotePeerConfig.ProtoReflect.Descriptor instead. func (*RemotePeerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{15} + return file_management_proto_rawDescGZIP(), []int{16} } -func (m *RemotePeerConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RemotePeerConfig.Unmarshal(m, b) -} -func (m *RemotePeerConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RemotePeerConfig.Marshal(b, m, deterministic) -} -func (m *RemotePeerConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_RemotePeerConfig.Merge(m, src) -} -func (m *RemotePeerConfig) XXX_Size() int { - return xxx_messageInfo_RemotePeerConfig.Size(m) -} -func (m *RemotePeerConfig) XXX_DiscardUnknown() { - xxx_messageInfo_RemotePeerConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_RemotePeerConfig proto.InternalMessageInfo - -func (m *RemotePeerConfig) GetWgPubKey() string { - if m != nil { - return m.WgPubKey +func (x *RemotePeerConfig) GetWgPubKey() string { + if x != nil { + return x.WgPubKey } return "" } -func (m *RemotePeerConfig) GetAllowedIps() []string { - if m != nil { - return m.AllowedIps +func (x *RemotePeerConfig) GetAllowedIps() []string { + if x != nil { + return x.AllowedIps } return nil } -func (m *RemotePeerConfig) GetSshConfig() *SSHConfig { - if m != nil { - return m.SshConfig +func (x *RemotePeerConfig) GetSshConfig() *SSHConfig { + if x != nil { + return x.SshConfig } return nil } -func (m *RemotePeerConfig) GetFqdn() string { - if m != nil { - return m.Fqdn +func (x *RemotePeerConfig) GetFqdn() string { + if x != nil { + return x.Fqdn } return "" } // SSHConfig represents SSH configurations of a peer. type SSHConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // sshEnabled indicates whether a SSH server is enabled on this peer SshEnabled bool `protobuf:"varint,1,opt,name=sshEnabled,proto3" json:"sshEnabled,omitempty"` // sshPubKey is a SSH public key of a peer to be added to authorized_hosts. // This property should be ignore if SSHConfig comes from PeerConfig. - SshPubKey []byte `protobuf:"bytes,2,opt,name=sshPubKey,proto3" json:"sshPubKey,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + SshPubKey []byte `protobuf:"bytes,2,opt,name=sshPubKey,proto3" json:"sshPubKey,omitempty"` } -func (m *SSHConfig) Reset() { *m = SSHConfig{} } -func (m *SSHConfig) String() string { return proto.CompactTextString(m) } -func (*SSHConfig) ProtoMessage() {} +func (x *SSHConfig) Reset() { + *x = SSHConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SSHConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SSHConfig) ProtoMessage() {} + +func (x *SSHConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SSHConfig.ProtoReflect.Descriptor instead. func (*SSHConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{16} + return file_management_proto_rawDescGZIP(), []int{17} } -func (m *SSHConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SSHConfig.Unmarshal(m, b) -} -func (m *SSHConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SSHConfig.Marshal(b, m, deterministic) -} -func (m *SSHConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_SSHConfig.Merge(m, src) -} -func (m *SSHConfig) XXX_Size() int { - return xxx_messageInfo_SSHConfig.Size(m) -} -func (m *SSHConfig) XXX_DiscardUnknown() { - xxx_messageInfo_SSHConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_SSHConfig proto.InternalMessageInfo - -func (m *SSHConfig) GetSshEnabled() bool { - if m != nil { - return m.SshEnabled +func (x *SSHConfig) GetSshEnabled() bool { + if x != nil { + return x.SshEnabled } return false } -func (m *SSHConfig) GetSshPubKey() []byte { - if m != nil { - return m.SshPubKey +func (x *SSHConfig) GetSshPubKey() []byte { + if x != nil { + return x.SshPubKey } return nil } // DeviceAuthorizationFlowRequest empty struct for future expansion type DeviceAuthorizationFlowRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *DeviceAuthorizationFlowRequest) Reset() { *m = DeviceAuthorizationFlowRequest{} } -func (m *DeviceAuthorizationFlowRequest) String() string { return proto.CompactTextString(m) } -func (*DeviceAuthorizationFlowRequest) ProtoMessage() {} +func (x *DeviceAuthorizationFlowRequest) Reset() { + *x = DeviceAuthorizationFlowRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeviceAuthorizationFlowRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeviceAuthorizationFlowRequest) ProtoMessage() {} + +func (x *DeviceAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeviceAuthorizationFlowRequest.ProtoReflect.Descriptor instead. func (*DeviceAuthorizationFlowRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{17} + return file_management_proto_rawDescGZIP(), []int{18} } -func (m *DeviceAuthorizationFlowRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeviceAuthorizationFlowRequest.Unmarshal(m, b) -} -func (m *DeviceAuthorizationFlowRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeviceAuthorizationFlowRequest.Marshal(b, m, deterministic) -} -func (m *DeviceAuthorizationFlowRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeviceAuthorizationFlowRequest.Merge(m, src) -} -func (m *DeviceAuthorizationFlowRequest) XXX_Size() int { - return xxx_messageInfo_DeviceAuthorizationFlowRequest.Size(m) -} -func (m *DeviceAuthorizationFlowRequest) XXX_DiscardUnknown() { - xxx_messageInfo_DeviceAuthorizationFlowRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_DeviceAuthorizationFlowRequest proto.InternalMessageInfo - // DeviceAuthorizationFlow represents Device Authorization Flow information // that can be used by the client to login initiate a Oauth 2.0 device authorization grant flow // see https://datatracker.ietf.org/doc/html/rfc8628 type DeviceAuthorizationFlow struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // An IDP provider , (eg. Auth0) - Provider DeviceAuthorizationFlowProvider `protobuf:"varint,1,opt,name=Provider,proto3,enum=management.DeviceAuthorizationFlowProvider" json:"Provider,omitempty"` - ProviderConfig *ProviderConfig `protobuf:"bytes,2,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Provider DeviceAuthorizationFlowProvider `protobuf:"varint,1,opt,name=Provider,proto3,enum=management.DeviceAuthorizationFlowProvider" json:"Provider,omitempty"` + ProviderConfig *ProviderConfig `protobuf:"bytes,2,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` } -func (m *DeviceAuthorizationFlow) Reset() { *m = DeviceAuthorizationFlow{} } -func (m *DeviceAuthorizationFlow) String() string { return proto.CompactTextString(m) } -func (*DeviceAuthorizationFlow) ProtoMessage() {} +func (x *DeviceAuthorizationFlow) Reset() { + *x = DeviceAuthorizationFlow{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeviceAuthorizationFlow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeviceAuthorizationFlow) ProtoMessage() {} + +func (x *DeviceAuthorizationFlow) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeviceAuthorizationFlow.ProtoReflect.Descriptor instead. func (*DeviceAuthorizationFlow) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{18} + return file_management_proto_rawDescGZIP(), []int{19} } -func (m *DeviceAuthorizationFlow) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeviceAuthorizationFlow.Unmarshal(m, b) -} -func (m *DeviceAuthorizationFlow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeviceAuthorizationFlow.Marshal(b, m, deterministic) -} -func (m *DeviceAuthorizationFlow) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeviceAuthorizationFlow.Merge(m, src) -} -func (m *DeviceAuthorizationFlow) XXX_Size() int { - return xxx_messageInfo_DeviceAuthorizationFlow.Size(m) -} -func (m *DeviceAuthorizationFlow) XXX_DiscardUnknown() { - xxx_messageInfo_DeviceAuthorizationFlow.DiscardUnknown(m) -} - -var xxx_messageInfo_DeviceAuthorizationFlow proto.InternalMessageInfo - -func (m *DeviceAuthorizationFlow) GetProvider() DeviceAuthorizationFlowProvider { - if m != nil { - return m.Provider +func (x *DeviceAuthorizationFlow) GetProvider() DeviceAuthorizationFlowProvider { + if x != nil { + return x.Provider } return DeviceAuthorizationFlow_HOSTED } -func (m *DeviceAuthorizationFlow) GetProviderConfig() *ProviderConfig { - if m != nil { - return m.ProviderConfig +func (x *DeviceAuthorizationFlow) GetProviderConfig() *ProviderConfig { + if x != nil { + return x.ProviderConfig } return nil } // PKCEAuthorizationFlowRequest empty struct for future expansion type PKCEAuthorizationFlowRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *PKCEAuthorizationFlowRequest) Reset() { *m = PKCEAuthorizationFlowRequest{} } -func (m *PKCEAuthorizationFlowRequest) String() string { return proto.CompactTextString(m) } -func (*PKCEAuthorizationFlowRequest) ProtoMessage() {} +func (x *PKCEAuthorizationFlowRequest) Reset() { + *x = PKCEAuthorizationFlowRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PKCEAuthorizationFlowRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PKCEAuthorizationFlowRequest) ProtoMessage() {} + +func (x *PKCEAuthorizationFlowRequest) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PKCEAuthorizationFlowRequest.ProtoReflect.Descriptor instead. func (*PKCEAuthorizationFlowRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{19} + return file_management_proto_rawDescGZIP(), []int{20} } -func (m *PKCEAuthorizationFlowRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PKCEAuthorizationFlowRequest.Unmarshal(m, b) -} -func (m *PKCEAuthorizationFlowRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PKCEAuthorizationFlowRequest.Marshal(b, m, deterministic) -} -func (m *PKCEAuthorizationFlowRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_PKCEAuthorizationFlowRequest.Merge(m, src) -} -func (m *PKCEAuthorizationFlowRequest) XXX_Size() int { - return xxx_messageInfo_PKCEAuthorizationFlowRequest.Size(m) -} -func (m *PKCEAuthorizationFlowRequest) XXX_DiscardUnknown() { - xxx_messageInfo_PKCEAuthorizationFlowRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_PKCEAuthorizationFlowRequest proto.InternalMessageInfo - // PKCEAuthorizationFlow represents Authorization Code Flow information // that can be used by the client to login initiate a Oauth 2.0 authorization code grant flow // with Proof Key for Code Exchange (PKCE). See https://datatracker.ietf.org/doc/html/rfc7636 type PKCEAuthorizationFlow struct { - ProviderConfig *ProviderConfig `protobuf:"bytes,1,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ProviderConfig *ProviderConfig `protobuf:"bytes,1,opt,name=ProviderConfig,proto3" json:"ProviderConfig,omitempty"` } -func (m *PKCEAuthorizationFlow) Reset() { *m = PKCEAuthorizationFlow{} } -func (m *PKCEAuthorizationFlow) String() string { return proto.CompactTextString(m) } -func (*PKCEAuthorizationFlow) ProtoMessage() {} +func (x *PKCEAuthorizationFlow) Reset() { + *x = PKCEAuthorizationFlow{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PKCEAuthorizationFlow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PKCEAuthorizationFlow) ProtoMessage() {} + +func (x *PKCEAuthorizationFlow) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PKCEAuthorizationFlow.ProtoReflect.Descriptor instead. func (*PKCEAuthorizationFlow) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{20} + return file_management_proto_rawDescGZIP(), []int{21} } -func (m *PKCEAuthorizationFlow) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PKCEAuthorizationFlow.Unmarshal(m, b) -} -func (m *PKCEAuthorizationFlow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PKCEAuthorizationFlow.Marshal(b, m, deterministic) -} -func (m *PKCEAuthorizationFlow) XXX_Merge(src proto.Message) { - xxx_messageInfo_PKCEAuthorizationFlow.Merge(m, src) -} -func (m *PKCEAuthorizationFlow) XXX_Size() int { - return xxx_messageInfo_PKCEAuthorizationFlow.Size(m) -} -func (m *PKCEAuthorizationFlow) XXX_DiscardUnknown() { - xxx_messageInfo_PKCEAuthorizationFlow.DiscardUnknown(m) -} - -var xxx_messageInfo_PKCEAuthorizationFlow proto.InternalMessageInfo - -func (m *PKCEAuthorizationFlow) GetProviderConfig() *ProviderConfig { - if m != nil { - return m.ProviderConfig +func (x *PKCEAuthorizationFlow) GetProviderConfig() *ProviderConfig { + if x != nil { + return x.ProviderConfig } return nil } // ProviderConfig has all attributes needed to initiate a device/pkce authorization flow type ProviderConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // An IDP application client id ClientID string `protobuf:"bytes,1,opt,name=ClientID,proto3" json:"ClientID,omitempty"` // An IDP application client secret @@ -1441,773 +1777,1535 @@ type ProviderConfig struct { // AuthorizationEndpoint is the endpoint of an IDP manager where clients can obtain authorization code. AuthorizationEndpoint string `protobuf:"bytes,9,opt,name=AuthorizationEndpoint,proto3" json:"AuthorizationEndpoint,omitempty"` // RedirectURLs handles authorization code from IDP manager - RedirectURLs []string `protobuf:"bytes,10,rep,name=RedirectURLs,proto3" json:"RedirectURLs,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RedirectURLs []string `protobuf:"bytes,10,rep,name=RedirectURLs,proto3" json:"RedirectURLs,omitempty"` } -func (m *ProviderConfig) Reset() { *m = ProviderConfig{} } -func (m *ProviderConfig) String() string { return proto.CompactTextString(m) } -func (*ProviderConfig) ProtoMessage() {} +func (x *ProviderConfig) Reset() { + *x = ProviderConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProviderConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProviderConfig) ProtoMessage() {} + +func (x *ProviderConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProviderConfig.ProtoReflect.Descriptor instead. func (*ProviderConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{21} + return file_management_proto_rawDescGZIP(), []int{22} } -func (m *ProviderConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ProviderConfig.Unmarshal(m, b) -} -func (m *ProviderConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ProviderConfig.Marshal(b, m, deterministic) -} -func (m *ProviderConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProviderConfig.Merge(m, src) -} -func (m *ProviderConfig) XXX_Size() int { - return xxx_messageInfo_ProviderConfig.Size(m) -} -func (m *ProviderConfig) XXX_DiscardUnknown() { - xxx_messageInfo_ProviderConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_ProviderConfig proto.InternalMessageInfo - -func (m *ProviderConfig) GetClientID() string { - if m != nil { - return m.ClientID +func (x *ProviderConfig) GetClientID() string { + if x != nil { + return x.ClientID } return "" } -func (m *ProviderConfig) GetClientSecret() string { - if m != nil { - return m.ClientSecret +func (x *ProviderConfig) GetClientSecret() string { + if x != nil { + return x.ClientSecret } return "" } -func (m *ProviderConfig) GetDomain() string { - if m != nil { - return m.Domain +func (x *ProviderConfig) GetDomain() string { + if x != nil { + return x.Domain } return "" } -func (m *ProviderConfig) GetAudience() string { - if m != nil { - return m.Audience +func (x *ProviderConfig) GetAudience() string { + if x != nil { + return x.Audience } return "" } -func (m *ProviderConfig) GetDeviceAuthEndpoint() string { - if m != nil { - return m.DeviceAuthEndpoint +func (x *ProviderConfig) GetDeviceAuthEndpoint() string { + if x != nil { + return x.DeviceAuthEndpoint } return "" } -func (m *ProviderConfig) GetTokenEndpoint() string { - if m != nil { - return m.TokenEndpoint +func (x *ProviderConfig) GetTokenEndpoint() string { + if x != nil { + return x.TokenEndpoint } return "" } -func (m *ProviderConfig) GetScope() string { - if m != nil { - return m.Scope +func (x *ProviderConfig) GetScope() string { + if x != nil { + return x.Scope } return "" } -func (m *ProviderConfig) GetUseIDToken() bool { - if m != nil { - return m.UseIDToken +func (x *ProviderConfig) GetUseIDToken() bool { + if x != nil { + return x.UseIDToken } return false } -func (m *ProviderConfig) GetAuthorizationEndpoint() string { - if m != nil { - return m.AuthorizationEndpoint +func (x *ProviderConfig) GetAuthorizationEndpoint() string { + if x != nil { + return x.AuthorizationEndpoint } return "" } -func (m *ProviderConfig) GetRedirectURLs() []string { - if m != nil { - return m.RedirectURLs +func (x *ProviderConfig) GetRedirectURLs() []string { + if x != nil { + return x.RedirectURLs } return nil } // Route represents a route.Route object type Route struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty"` - NetworkType int64 `protobuf:"varint,3,opt,name=NetworkType,proto3" json:"NetworkType,omitempty"` - Peer string `protobuf:"bytes,4,opt,name=Peer,proto3" json:"Peer,omitempty"` - Metric int64 `protobuf:"varint,5,opt,name=Metric,proto3" json:"Metric,omitempty"` - Masquerade bool `protobuf:"varint,6,opt,name=Masquerade,proto3" json:"Masquerade,omitempty"` - NetID string `protobuf:"bytes,7,opt,name=NetID,proto3" json:"NetID,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` + Network string `protobuf:"bytes,2,opt,name=Network,proto3" json:"Network,omitempty"` + NetworkType int64 `protobuf:"varint,3,opt,name=NetworkType,proto3" json:"NetworkType,omitempty"` + Peer string `protobuf:"bytes,4,opt,name=Peer,proto3" json:"Peer,omitempty"` + Metric int64 `protobuf:"varint,5,opt,name=Metric,proto3" json:"Metric,omitempty"` + Masquerade bool `protobuf:"varint,6,opt,name=Masquerade,proto3" json:"Masquerade,omitempty"` + NetID string `protobuf:"bytes,7,opt,name=NetID,proto3" json:"NetID,omitempty"` } -func (m *Route) Reset() { *m = Route{} } -func (m *Route) String() string { return proto.CompactTextString(m) } -func (*Route) ProtoMessage() {} +func (x *Route) Reset() { + *x = Route{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Route) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Route) ProtoMessage() {} + +func (x *Route) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Route.ProtoReflect.Descriptor instead. func (*Route) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{22} + return file_management_proto_rawDescGZIP(), []int{23} } -func (m *Route) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Route.Unmarshal(m, b) -} -func (m *Route) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Route.Marshal(b, m, deterministic) -} -func (m *Route) XXX_Merge(src proto.Message) { - xxx_messageInfo_Route.Merge(m, src) -} -func (m *Route) XXX_Size() int { - return xxx_messageInfo_Route.Size(m) -} -func (m *Route) XXX_DiscardUnknown() { - xxx_messageInfo_Route.DiscardUnknown(m) -} - -var xxx_messageInfo_Route proto.InternalMessageInfo - -func (m *Route) GetID() string { - if m != nil { - return m.ID +func (x *Route) GetID() string { + if x != nil { + return x.ID } return "" } -func (m *Route) GetNetwork() string { - if m != nil { - return m.Network +func (x *Route) GetNetwork() string { + if x != nil { + return x.Network } return "" } -func (m *Route) GetNetworkType() int64 { - if m != nil { - return m.NetworkType +func (x *Route) GetNetworkType() int64 { + if x != nil { + return x.NetworkType } return 0 } -func (m *Route) GetPeer() string { - if m != nil { - return m.Peer +func (x *Route) GetPeer() string { + if x != nil { + return x.Peer } return "" } -func (m *Route) GetMetric() int64 { - if m != nil { - return m.Metric +func (x *Route) GetMetric() int64 { + if x != nil { + return x.Metric } return 0 } -func (m *Route) GetMasquerade() bool { - if m != nil { - return m.Masquerade +func (x *Route) GetMasquerade() bool { + if x != nil { + return x.Masquerade } return false } -func (m *Route) GetNetID() string { - if m != nil { - return m.NetID +func (x *Route) GetNetID() string { + if x != nil { + return x.NetID } return "" } // DNSConfig represents a dns.Update type DNSConfig struct { - ServiceEnable bool `protobuf:"varint,1,opt,name=ServiceEnable,proto3" json:"ServiceEnable,omitempty"` - NameServerGroups []*NameServerGroup `protobuf:"bytes,2,rep,name=NameServerGroups,proto3" json:"NameServerGroups,omitempty"` - CustomZones []*CustomZone `protobuf:"bytes,3,rep,name=CustomZones,proto3" json:"CustomZones,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ServiceEnable bool `protobuf:"varint,1,opt,name=ServiceEnable,proto3" json:"ServiceEnable,omitempty"` + NameServerGroups []*NameServerGroup `protobuf:"bytes,2,rep,name=NameServerGroups,proto3" json:"NameServerGroups,omitempty"` + CustomZones []*CustomZone `protobuf:"bytes,3,rep,name=CustomZones,proto3" json:"CustomZones,omitempty"` } -func (m *DNSConfig) Reset() { *m = DNSConfig{} } -func (m *DNSConfig) String() string { return proto.CompactTextString(m) } -func (*DNSConfig) ProtoMessage() {} +func (x *DNSConfig) Reset() { + *x = DNSConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DNSConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DNSConfig) ProtoMessage() {} + +func (x *DNSConfig) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DNSConfig.ProtoReflect.Descriptor instead. func (*DNSConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{23} + return file_management_proto_rawDescGZIP(), []int{24} } -func (m *DNSConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DNSConfig.Unmarshal(m, b) -} -func (m *DNSConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DNSConfig.Marshal(b, m, deterministic) -} -func (m *DNSConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_DNSConfig.Merge(m, src) -} -func (m *DNSConfig) XXX_Size() int { - return xxx_messageInfo_DNSConfig.Size(m) -} -func (m *DNSConfig) XXX_DiscardUnknown() { - xxx_messageInfo_DNSConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_DNSConfig proto.InternalMessageInfo - -func (m *DNSConfig) GetServiceEnable() bool { - if m != nil { - return m.ServiceEnable +func (x *DNSConfig) GetServiceEnable() bool { + if x != nil { + return x.ServiceEnable } return false } -func (m *DNSConfig) GetNameServerGroups() []*NameServerGroup { - if m != nil { - return m.NameServerGroups +func (x *DNSConfig) GetNameServerGroups() []*NameServerGroup { + if x != nil { + return x.NameServerGroups } return nil } -func (m *DNSConfig) GetCustomZones() []*CustomZone { - if m != nil { - return m.CustomZones +func (x *DNSConfig) GetCustomZones() []*CustomZone { + if x != nil { + return x.CustomZones } return nil } // CustomZone represents a dns.CustomZone type CustomZone struct { - Domain string `protobuf:"bytes,1,opt,name=Domain,proto3" json:"Domain,omitempty"` - Records []*SimpleRecord `protobuf:"bytes,2,rep,name=Records,proto3" json:"Records,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Domain string `protobuf:"bytes,1,opt,name=Domain,proto3" json:"Domain,omitempty"` + Records []*SimpleRecord `protobuf:"bytes,2,rep,name=Records,proto3" json:"Records,omitempty"` } -func (m *CustomZone) Reset() { *m = CustomZone{} } -func (m *CustomZone) String() string { return proto.CompactTextString(m) } -func (*CustomZone) ProtoMessage() {} +func (x *CustomZone) Reset() { + *x = CustomZone{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CustomZone) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CustomZone) ProtoMessage() {} + +func (x *CustomZone) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CustomZone.ProtoReflect.Descriptor instead. func (*CustomZone) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{24} + return file_management_proto_rawDescGZIP(), []int{25} } -func (m *CustomZone) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CustomZone.Unmarshal(m, b) -} -func (m *CustomZone) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CustomZone.Marshal(b, m, deterministic) -} -func (m *CustomZone) XXX_Merge(src proto.Message) { - xxx_messageInfo_CustomZone.Merge(m, src) -} -func (m *CustomZone) XXX_Size() int { - return xxx_messageInfo_CustomZone.Size(m) -} -func (m *CustomZone) XXX_DiscardUnknown() { - xxx_messageInfo_CustomZone.DiscardUnknown(m) -} - -var xxx_messageInfo_CustomZone proto.InternalMessageInfo - -func (m *CustomZone) GetDomain() string { - if m != nil { - return m.Domain +func (x *CustomZone) GetDomain() string { + if x != nil { + return x.Domain } return "" } -func (m *CustomZone) GetRecords() []*SimpleRecord { - if m != nil { - return m.Records +func (x *CustomZone) GetRecords() []*SimpleRecord { + if x != nil { + return x.Records } return nil } // SimpleRecord represents a dns.SimpleRecord type SimpleRecord struct { - Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` - Type int64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` - Class string `protobuf:"bytes,3,opt,name=Class,proto3" json:"Class,omitempty"` - TTL int64 `protobuf:"varint,4,opt,name=TTL,proto3" json:"TTL,omitempty"` - RData string `protobuf:"bytes,5,opt,name=RData,proto3" json:"RData,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Type int64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + Class string `protobuf:"bytes,3,opt,name=Class,proto3" json:"Class,omitempty"` + TTL int64 `protobuf:"varint,4,opt,name=TTL,proto3" json:"TTL,omitempty"` + RData string `protobuf:"bytes,5,opt,name=RData,proto3" json:"RData,omitempty"` } -func (m *SimpleRecord) Reset() { *m = SimpleRecord{} } -func (m *SimpleRecord) String() string { return proto.CompactTextString(m) } -func (*SimpleRecord) ProtoMessage() {} +func (x *SimpleRecord) Reset() { + *x = SimpleRecord{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleRecord) ProtoMessage() {} + +func (x *SimpleRecord) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleRecord.ProtoReflect.Descriptor instead. func (*SimpleRecord) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{25} + return file_management_proto_rawDescGZIP(), []int{26} } -func (m *SimpleRecord) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleRecord.Unmarshal(m, b) -} -func (m *SimpleRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleRecord.Marshal(b, m, deterministic) -} -func (m *SimpleRecord) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleRecord.Merge(m, src) -} -func (m *SimpleRecord) XXX_Size() int { - return xxx_messageInfo_SimpleRecord.Size(m) -} -func (m *SimpleRecord) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleRecord.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleRecord proto.InternalMessageInfo - -func (m *SimpleRecord) GetName() string { - if m != nil { - return m.Name +func (x *SimpleRecord) GetName() string { + if x != nil { + return x.Name } return "" } -func (m *SimpleRecord) GetType() int64 { - if m != nil { - return m.Type +func (x *SimpleRecord) GetType() int64 { + if x != nil { + return x.Type } return 0 } -func (m *SimpleRecord) GetClass() string { - if m != nil { - return m.Class +func (x *SimpleRecord) GetClass() string { + if x != nil { + return x.Class } return "" } -func (m *SimpleRecord) GetTTL() int64 { - if m != nil { - return m.TTL +func (x *SimpleRecord) GetTTL() int64 { + if x != nil { + return x.TTL } return 0 } -func (m *SimpleRecord) GetRData() string { - if m != nil { - return m.RData +func (x *SimpleRecord) GetRData() string { + if x != nil { + return x.RData } return "" } // NameServerGroup represents a dns.NameServerGroup type NameServerGroup struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + NameServers []*NameServer `protobuf:"bytes,1,rep,name=NameServers,proto3" json:"NameServers,omitempty"` Primary bool `protobuf:"varint,2,opt,name=Primary,proto3" json:"Primary,omitempty"` Domains []string `protobuf:"bytes,3,rep,name=Domains,proto3" json:"Domains,omitempty"` SearchDomainsEnabled bool `protobuf:"varint,4,opt,name=SearchDomainsEnabled,proto3" json:"SearchDomainsEnabled,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *NameServerGroup) Reset() { *m = NameServerGroup{} } -func (m *NameServerGroup) String() string { return proto.CompactTextString(m) } -func (*NameServerGroup) ProtoMessage() {} +func (x *NameServerGroup) Reset() { + *x = NameServerGroup{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NameServerGroup) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NameServerGroup) ProtoMessage() {} + +func (x *NameServerGroup) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NameServerGroup.ProtoReflect.Descriptor instead. func (*NameServerGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{26} + return file_management_proto_rawDescGZIP(), []int{27} } -func (m *NameServerGroup) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NameServerGroup.Unmarshal(m, b) -} -func (m *NameServerGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NameServerGroup.Marshal(b, m, deterministic) -} -func (m *NameServerGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_NameServerGroup.Merge(m, src) -} -func (m *NameServerGroup) XXX_Size() int { - return xxx_messageInfo_NameServerGroup.Size(m) -} -func (m *NameServerGroup) XXX_DiscardUnknown() { - xxx_messageInfo_NameServerGroup.DiscardUnknown(m) -} - -var xxx_messageInfo_NameServerGroup proto.InternalMessageInfo - -func (m *NameServerGroup) GetNameServers() []*NameServer { - if m != nil { - return m.NameServers +func (x *NameServerGroup) GetNameServers() []*NameServer { + if x != nil { + return x.NameServers } return nil } -func (m *NameServerGroup) GetPrimary() bool { - if m != nil { - return m.Primary +func (x *NameServerGroup) GetPrimary() bool { + if x != nil { + return x.Primary } return false } -func (m *NameServerGroup) GetDomains() []string { - if m != nil { - return m.Domains +func (x *NameServerGroup) GetDomains() []string { + if x != nil { + return x.Domains } return nil } -func (m *NameServerGroup) GetSearchDomainsEnabled() bool { - if m != nil { - return m.SearchDomainsEnabled +func (x *NameServerGroup) GetSearchDomainsEnabled() bool { + if x != nil { + return x.SearchDomainsEnabled } return false } // NameServer represents a dns.NameServer type NameServer struct { - IP string `protobuf:"bytes,1,opt,name=IP,proto3" json:"IP,omitempty"` - NSType int64 `protobuf:"varint,2,opt,name=NSType,proto3" json:"NSType,omitempty"` - Port int64 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IP string `protobuf:"bytes,1,opt,name=IP,proto3" json:"IP,omitempty"` + NSType int64 `protobuf:"varint,2,opt,name=NSType,proto3" json:"NSType,omitempty"` + Port int64 `protobuf:"varint,3,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *NameServer) Reset() { *m = NameServer{} } -func (m *NameServer) String() string { return proto.CompactTextString(m) } -func (*NameServer) ProtoMessage() {} +func (x *NameServer) Reset() { + *x = NameServer{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NameServer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NameServer) ProtoMessage() {} + +func (x *NameServer) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NameServer.ProtoReflect.Descriptor instead. func (*NameServer) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{27} + return file_management_proto_rawDescGZIP(), []int{28} } -func (m *NameServer) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NameServer.Unmarshal(m, b) -} -func (m *NameServer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NameServer.Marshal(b, m, deterministic) -} -func (m *NameServer) XXX_Merge(src proto.Message) { - xxx_messageInfo_NameServer.Merge(m, src) -} -func (m *NameServer) XXX_Size() int { - return xxx_messageInfo_NameServer.Size(m) -} -func (m *NameServer) XXX_DiscardUnknown() { - xxx_messageInfo_NameServer.DiscardUnknown(m) -} - -var xxx_messageInfo_NameServer proto.InternalMessageInfo - -func (m *NameServer) GetIP() string { - if m != nil { - return m.IP +func (x *NameServer) GetIP() string { + if x != nil { + return x.IP } return "" } -func (m *NameServer) GetNSType() int64 { - if m != nil { - return m.NSType +func (x *NameServer) GetNSType() int64 { + if x != nil { + return x.NSType } return 0 } -func (m *NameServer) GetPort() int64 { - if m != nil { - return m.Port +func (x *NameServer) GetPort() int64 { + if x != nil { + return x.Port } return 0 } // FirewallRule represents a firewall rule type FirewallRule struct { - PeerIP string `protobuf:"bytes,1,opt,name=PeerIP,proto3" json:"PeerIP,omitempty"` - Direction FirewallRuleDirection `protobuf:"varint,2,opt,name=Direction,proto3,enum=management.FirewallRuleDirection" json:"Direction,omitempty"` - Action FirewallRuleAction `protobuf:"varint,3,opt,name=Action,proto3,enum=management.FirewallRuleAction" json:"Action,omitempty"` - Protocol FirewallRuleProtocol `protobuf:"varint,4,opt,name=Protocol,proto3,enum=management.FirewallRuleProtocol" json:"Protocol,omitempty"` - Port string `protobuf:"bytes,5,opt,name=Port,proto3" json:"Port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + PeerIP string `protobuf:"bytes,1,opt,name=PeerIP,proto3" json:"PeerIP,omitempty"` + Direction FirewallRuleDirection `protobuf:"varint,2,opt,name=Direction,proto3,enum=management.FirewallRuleDirection" json:"Direction,omitempty"` + Action FirewallRuleAction `protobuf:"varint,3,opt,name=Action,proto3,enum=management.FirewallRuleAction" json:"Action,omitempty"` + Protocol FirewallRuleProtocol `protobuf:"varint,4,opt,name=Protocol,proto3,enum=management.FirewallRuleProtocol" json:"Protocol,omitempty"` + Port string `protobuf:"bytes,5,opt,name=Port,proto3" json:"Port,omitempty"` } -func (m *FirewallRule) Reset() { *m = FirewallRule{} } -func (m *FirewallRule) String() string { return proto.CompactTextString(m) } -func (*FirewallRule) ProtoMessage() {} +func (x *FirewallRule) Reset() { + *x = FirewallRule{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FirewallRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FirewallRule) ProtoMessage() {} + +func (x *FirewallRule) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FirewallRule.ProtoReflect.Descriptor instead. func (*FirewallRule) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{28} + return file_management_proto_rawDescGZIP(), []int{29} } -func (m *FirewallRule) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_FirewallRule.Unmarshal(m, b) -} -func (m *FirewallRule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_FirewallRule.Marshal(b, m, deterministic) -} -func (m *FirewallRule) XXX_Merge(src proto.Message) { - xxx_messageInfo_FirewallRule.Merge(m, src) -} -func (m *FirewallRule) XXX_Size() int { - return xxx_messageInfo_FirewallRule.Size(m) -} -func (m *FirewallRule) XXX_DiscardUnknown() { - xxx_messageInfo_FirewallRule.DiscardUnknown(m) -} - -var xxx_messageInfo_FirewallRule proto.InternalMessageInfo - -func (m *FirewallRule) GetPeerIP() string { - if m != nil { - return m.PeerIP +func (x *FirewallRule) GetPeerIP() string { + if x != nil { + return x.PeerIP } return "" } -func (m *FirewallRule) GetDirection() FirewallRuleDirection { - if m != nil { - return m.Direction +func (x *FirewallRule) GetDirection() FirewallRuleDirection { + if x != nil { + return x.Direction } return FirewallRule_IN } -func (m *FirewallRule) GetAction() FirewallRuleAction { - if m != nil { - return m.Action +func (x *FirewallRule) GetAction() FirewallRuleAction { + if x != nil { + return x.Action } return FirewallRule_ACCEPT } -func (m *FirewallRule) GetProtocol() FirewallRuleProtocol { - if m != nil { - return m.Protocol +func (x *FirewallRule) GetProtocol() FirewallRuleProtocol { + if x != nil { + return x.Protocol } return FirewallRule_UNKNOWN } -func (m *FirewallRule) GetPort() string { - if m != nil { - return m.Port +func (x *FirewallRule) GetPort() string { + if x != nil { + return x.Port } return "" } type NetworkAddress struct { - NetIP string `protobuf:"bytes,1,opt,name=netIP,proto3" json:"netIP,omitempty"` - Mac string `protobuf:"bytes,2,opt,name=mac,proto3" json:"mac,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NetIP string `protobuf:"bytes,1,opt,name=netIP,proto3" json:"netIP,omitempty"` + Mac string `protobuf:"bytes,2,opt,name=mac,proto3" json:"mac,omitempty"` } -func (m *NetworkAddress) Reset() { *m = NetworkAddress{} } -func (m *NetworkAddress) String() string { return proto.CompactTextString(m) } -func (*NetworkAddress) ProtoMessage() {} +func (x *NetworkAddress) Reset() { + *x = NetworkAddress{} + if protoimpl.UnsafeEnabled { + mi := &file_management_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NetworkAddress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkAddress) ProtoMessage() {} + +func (x *NetworkAddress) ProtoReflect() protoreflect.Message { + mi := &file_management_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkAddress.ProtoReflect.Descriptor instead. func (*NetworkAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_edc174f991dc0a25, []int{29} + return file_management_proto_rawDescGZIP(), []int{30} } -func (m *NetworkAddress) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NetworkAddress.Unmarshal(m, b) -} -func (m *NetworkAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NetworkAddress.Marshal(b, m, deterministic) -} -func (m *NetworkAddress) XXX_Merge(src proto.Message) { - xxx_messageInfo_NetworkAddress.Merge(m, src) -} -func (m *NetworkAddress) XXX_Size() int { - return xxx_messageInfo_NetworkAddress.Size(m) -} -func (m *NetworkAddress) XXX_DiscardUnknown() { - xxx_messageInfo_NetworkAddress.DiscardUnknown(m) -} - -var xxx_messageInfo_NetworkAddress proto.InternalMessageInfo - -func (m *NetworkAddress) GetNetIP() string { - if m != nil { - return m.NetIP +func (x *NetworkAddress) GetNetIP() string { + if x != nil { + return x.NetIP } return "" } -func (m *NetworkAddress) GetMac() string { - if m != nil { - return m.Mac +func (x *NetworkAddress) GetMac() string { + if x != nil { + return x.Mac } return "" } -func init() { - proto.RegisterEnum("management.HostConfig_Protocol", HostConfig_Protocol_name, HostConfig_Protocol_value) - proto.RegisterEnum("management.DeviceAuthorizationFlowProvider", DeviceAuthorizationFlowProvider_name, DeviceAuthorizationFlowProvider_value) - proto.RegisterEnum("management.FirewallRuleDirection", FirewallRuleDirection_name, FirewallRuleDirection_value) - proto.RegisterEnum("management.FirewallRuleAction", FirewallRuleAction_name, FirewallRuleAction_value) - proto.RegisterEnum("management.FirewallRuleProtocol", FirewallRuleProtocol_name, FirewallRuleProtocol_value) - proto.RegisterType((*EncryptedMessage)(nil), "management.EncryptedMessage") - proto.RegisterType((*SyncRequest)(nil), "management.SyncRequest") - proto.RegisterType((*SyncResponse)(nil), "management.SyncResponse") - proto.RegisterType((*LoginRequest)(nil), "management.LoginRequest") - proto.RegisterType((*PeerKeys)(nil), "management.PeerKeys") - proto.RegisterType((*Environment)(nil), "management.Environment") - proto.RegisterType((*PeerSystemMeta)(nil), "management.PeerSystemMeta") - proto.RegisterType((*LoginResponse)(nil), "management.LoginResponse") - proto.RegisterType((*ServerKeyResponse)(nil), "management.ServerKeyResponse") - proto.RegisterType((*Empty)(nil), "management.Empty") - proto.RegisterType((*WiretrusteeConfig)(nil), "management.WiretrusteeConfig") - proto.RegisterType((*HostConfig)(nil), "management.HostConfig") - proto.RegisterType((*ProtectedHostConfig)(nil), "management.ProtectedHostConfig") - proto.RegisterType((*PeerConfig)(nil), "management.PeerConfig") - proto.RegisterType((*NetworkMap)(nil), "management.NetworkMap") - proto.RegisterType((*RemotePeerConfig)(nil), "management.RemotePeerConfig") - proto.RegisterType((*SSHConfig)(nil), "management.SSHConfig") - proto.RegisterType((*DeviceAuthorizationFlowRequest)(nil), "management.DeviceAuthorizationFlowRequest") - proto.RegisterType((*DeviceAuthorizationFlow)(nil), "management.DeviceAuthorizationFlow") - proto.RegisterType((*PKCEAuthorizationFlowRequest)(nil), "management.PKCEAuthorizationFlowRequest") - proto.RegisterType((*PKCEAuthorizationFlow)(nil), "management.PKCEAuthorizationFlow") - proto.RegisterType((*ProviderConfig)(nil), "management.ProviderConfig") - proto.RegisterType((*Route)(nil), "management.Route") - proto.RegisterType((*DNSConfig)(nil), "management.DNSConfig") - proto.RegisterType((*CustomZone)(nil), "management.CustomZone") - proto.RegisterType((*SimpleRecord)(nil), "management.SimpleRecord") - proto.RegisterType((*NameServerGroup)(nil), "management.NameServerGroup") - proto.RegisterType((*NameServer)(nil), "management.NameServer") - proto.RegisterType((*FirewallRule)(nil), "management.FirewallRule") - proto.RegisterType((*NetworkAddress)(nil), "management.NetworkAddress") +var File_management_proto protoreflect.FileDescriptor + +var file_management_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x0a, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x1a, 0x1f, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x5c, 0x0a, 0x10, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, + 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62, + 0x6f, 0x64, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x0d, 0x0a, + 0x0b, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbb, 0x02, 0x0a, + 0x0c, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, + 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, + 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, + 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, + 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x36, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x52, 0x0a, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x22, 0xa8, 0x01, 0x0a, 0x0c, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, + 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, + 0x65, 0x74, 0x75, 0x70, 0x4b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, + 0x61, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x1a, 0x0a, 0x08, 0x6a, 0x77, 0x74, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6a, 0x77, 0x74, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x30, 0x0a, 0x08, 0x70, 0x65, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x73, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x08, 0x70, 0x65, 0x65, + 0x72, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x44, 0x0a, 0x08, 0x50, 0x65, 0x65, 0x72, 0x4b, 0x65, 0x79, + 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, + 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x22, 0x3f, 0x0a, 0x0b, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x63, 0x6c, 0x6f, 0x75, 0x64, + 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x22, 0xa9, 0x04, 0x0a, + 0x0e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x4d, 0x65, 0x74, 0x61, 0x12, + 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x67, + 0x6f, 0x4f, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x6f, 0x4f, 0x53, 0x12, + 0x16, 0x0a, 0x06, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x72, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x53, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x4f, 0x53, 0x12, 0x2e, 0x0a, 0x12, 0x77, 0x69, 0x72, 0x65, 0x74, + 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x12, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x69, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x75, 0x69, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6b, 0x65, + 0x72, 0x6e, 0x65, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x4f, + 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x4f, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x10, 0x6e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x0b, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, + 0x10, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x65, + 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x4e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x79, 0x73, 0x53, + 0x65, 0x72, 0x69, 0x61, 0x6c, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x73, + 0x79, 0x73, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0d, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x79, 0x73, 0x50, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x73, 0x79, 0x73, 0x4d, 0x61, 0x6e, 0x75, 0x66, 0x61, + 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x79, + 0x73, 0x4d, 0x61, 0x6e, 0x75, 0x66, 0x61, 0x63, 0x74, 0x75, 0x72, 0x65, 0x72, 0x12, 0x39, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x0f, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x0d, 0x4c, 0x6f, 0x67, + 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x11, 0x77, 0x69, + 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, 0x65, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, 0x77, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, 0x74, 0x65, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, + 0x79, 0x0a, 0x11, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, + 0x73, 0x41, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x22, 0xd7, 0x01, 0x0a, 0x11, 0x57, 0x69, 0x72, 0x65, 0x74, 0x72, 0x75, 0x73, + 0x74, 0x65, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2c, 0x0a, 0x05, 0x73, 0x74, 0x75, + 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x05, 0x73, 0x74, 0x75, 0x6e, 0x73, 0x12, 0x35, 0x0a, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x65, 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x74, 0x75, 0x72, 0x6e, 0x73, 0x12, 0x2e, + 0x0a, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, 0x73, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x12, 0x2d, + 0x0a, 0x05, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x05, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x22, 0x98, 0x01, + 0x0a, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x72, 0x69, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x69, 0x12, 0x3b, + 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x48, 0x6f, + 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0x3b, 0x0a, 0x08, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x00, + 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x48, 0x54, 0x54, + 0x50, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x48, 0x54, 0x54, 0x50, 0x53, 0x10, 0x03, 0x12, 0x08, + 0x0a, 0x04, 0x44, 0x54, 0x4c, 0x53, 0x10, 0x04, 0x22, 0x6d, 0x0a, 0x0b, 0x52, 0x65, 0x6c, 0x61, + 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x75, 0x72, 0x6c, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, + 0x26, 0x0a, 0x0e, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x22, 0x7d, 0x0a, 0x13, 0x50, 0x72, 0x6f, 0x74, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x36, + 0x0a, 0x0a, 0x68, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x48, 0x6f, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x68, 0x6f, 0x73, 0x74, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, + 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, + 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x10, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x6e, + 0x73, 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x68, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0xe2, 0x03, 0x0a, 0x0a, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4d, 0x61, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x65, 0x72, + 0x69, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x53, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x12, 0x36, 0x0a, 0x0a, 0x70, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, + 0x6e, 0x74, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0a, 0x70, + 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3e, 0x0a, 0x0b, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0b, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x72, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, + 0x72, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x29, 0x0a, 0x06, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x09, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, + 0x44, 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x40, 0x0a, 0x0c, 0x6f, 0x66, 0x66, + 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x6f, + 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x50, 0x65, 0x65, 0x72, 0x73, 0x12, 0x3e, 0x0a, 0x0d, 0x46, + 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x52, 0x0d, 0x46, 0x69, + 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x66, + 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x66, 0x69, 0x72, 0x65, 0x77, + 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x73, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, + 0x97, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x50, 0x65, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x67, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, + 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x49, 0x70, 0x73, + 0x12, 0x33, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x53, 0x53, 0x48, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x09, 0x73, 0x73, 0x68, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x22, 0x49, 0x0a, 0x09, 0x53, 0x53, 0x48, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x73, 0x68, 0x45, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x73, 0x73, 0x68, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, 0x62, + 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x73, 0x68, 0x50, 0x75, + 0x62, 0x4b, 0x65, 0x79, 0x22, 0x20, 0x0a, 0x1e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xbf, 0x01, 0x0a, 0x17, 0x44, 0x65, 0x76, 0x69, 0x63, + 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, + 0x6f, 0x77, 0x12, 0x48, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x42, 0x0a, 0x0e, + 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x22, 0x16, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0a, 0x0a, 0x06, + 0x48, 0x4f, 0x53, 0x54, 0x45, 0x44, 0x10, 0x00, 0x22, 0x1e, 0x0a, 0x1c, 0x50, 0x4b, 0x43, 0x45, + 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, + 0x77, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x5b, 0x0a, 0x15, 0x50, 0x4b, 0x43, 0x45, + 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, + 0x77, 0x12, 0x42, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xea, 0x02, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x49, 0x44, 0x12, 0x22, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, + 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x12, 0x1a, 0x0a, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x41, 0x75, 0x64, 0x69, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2e, 0x0a, 0x12, + 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x41, 0x75, 0x74, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0d, + 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, 0x65, 0x49, + 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x55, 0x73, + 0x65, 0x49, 0x44, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x34, 0x0a, 0x15, 0x41, 0x75, 0x74, 0x68, + 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, + 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x22, + 0x0a, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, 0x4c, 0x73, 0x18, 0x0a, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x52, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x55, 0x52, + 0x4c, 0x73, 0x22, 0xb5, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x4e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x65, 0x65, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x50, 0x65, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x12, 0x1e, 0x0a, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, 0x72, 0x61, + 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4d, 0x61, 0x73, 0x71, 0x75, 0x65, + 0x72, 0x61, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x49, 0x44, 0x22, 0xb4, 0x01, 0x0a, 0x09, 0x44, + 0x4e, 0x53, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x47, + 0x0a, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x38, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, + 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x5a, 0x6f, 0x6e, 0x65, 0x52, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, + 0x73, 0x22, 0x58, 0x0a, 0x0a, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5a, 0x6f, 0x6e, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x32, 0x0a, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, + 0x72, 0x64, 0x52, 0x07, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x22, 0x74, 0x0a, 0x0c, 0x53, + 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x4e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x54, 0x54, 0x4c, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x54, 0x54, 0x4c, 0x12, 0x14, 0x0a, 0x05, 0x52, + 0x44, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x52, 0x44, 0x61, 0x74, + 0x61, 0x22, 0xb3, 0x01, 0x0a, 0x0f, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x38, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x52, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, + 0x18, 0x0a, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x50, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x44, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x44, 0x6f, 0x6d, 0x61, + 0x69, 0x6e, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x14, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, + 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x48, 0x0a, 0x0a, 0x4e, 0x61, 0x6d, 0x65, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x4e, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x50, 0x6f, 0x72, + 0x74, 0x22, 0xf0, 0x02, 0x0a, 0x0c, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, + 0x6c, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x50, 0x65, 0x65, 0x72, 0x49, 0x50, 0x12, 0x40, 0x0a, 0x09, 0x44, 0x69, + 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, + 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x06, + 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x6d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, + 0x6c, 0x6c, 0x52, 0x75, 0x6c, 0x65, 0x2e, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x41, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3d, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x46, 0x69, 0x72, 0x65, 0x77, 0x61, 0x6c, 0x6c, 0x52, 0x75, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x06, 0x0a, 0x02, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x07, 0x0a, + 0x03, 0x4f, 0x55, 0x54, 0x10, 0x01, 0x22, 0x1e, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x0a, 0x0a, 0x06, 0x41, 0x43, 0x43, 0x45, 0x50, 0x54, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, + 0x44, 0x52, 0x4f, 0x50, 0x10, 0x01, 0x22, 0x3c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, + 0x07, 0x0a, 0x03, 0x41, 0x4c, 0x4c, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, + 0x02, 0x12, 0x07, 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x43, + 0x4d, 0x50, 0x10, 0x04, 0x22, 0x38, 0x0a, 0x0e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6e, 0x65, 0x74, 0x49, 0x50, 0x12, 0x10, 0x0a, 0x03, + 0x6d, 0x61, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, 0x32, 0xd1, + 0x03, 0x0a, 0x11, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1c, 0x2e, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, + 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x46, 0x0a, 0x04, 0x53, + 0x79, 0x6e, 0x63, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, + 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, + 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x4b, 0x65, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x69, 0x73, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x79, 0x12, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x11, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, + 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x1a, + 0x47, 0x65, 0x74, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, + 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, + 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x50, + 0x4b, 0x43, 0x45, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x46, 0x6c, 0x6f, 0x77, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, + 0x74, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x2e, + 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } -func init() { - proto.RegisterFile("management.proto", fileDescriptor_edc174f991dc0a25) +var ( + file_management_proto_rawDescOnce sync.Once + file_management_proto_rawDescData = file_management_proto_rawDesc +) + +func file_management_proto_rawDescGZIP() []byte { + file_management_proto_rawDescOnce.Do(func() { + file_management_proto_rawDescData = protoimpl.X.CompressGZIP(file_management_proto_rawDescData) + }) + return file_management_proto_rawDescData } -var fileDescriptor_edc174f991dc0a25 = []byte{ - // 1923 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x58, 0x4d, 0x6f, 0x23, 0x49, - 0x19, 0x4e, 0xdb, 0xb1, 0x63, 0xbf, 0x76, 0x32, 0x4e, 0x31, 0x33, 0xdb, 0x0a, 0x61, 0x36, 0xb4, - 0x10, 0x0a, 0xd2, 0x2a, 0xbb, 0xca, 0xf2, 0x31, 0x68, 0x61, 0xd9, 0x4c, 0x9c, 0x49, 0xa2, 0x24, - 0x8e, 0x55, 0xed, 0xb0, 0xa3, 0x81, 0x4b, 0xa7, 0xbb, 0xe2, 0x34, 0xd3, 0xee, 0xf2, 0x54, 0x55, - 0x27, 0x18, 0x89, 0x03, 0xff, 0x01, 0x09, 0x7e, 0x02, 0x9c, 0x81, 0x13, 0x07, 0xfe, 0x00, 0x17, - 0xfe, 0x02, 0x47, 0x4e, 0xfc, 0x04, 0x54, 0x5f, 0xfd, 0x61, 0x3b, 0x23, 0xd0, 0x70, 0x80, 0x53, - 0xea, 0xfd, 0xec, 0xb7, 0x9e, 0x7a, 0xeb, 0xad, 0xc7, 0x81, 0xde, 0x24, 0x48, 0x83, 0x31, 0x99, - 0x90, 0x54, 0xec, 0x4d, 0x19, 0x15, 0x14, 0x41, 0xa1, 0xd9, 0xfa, 0x70, 0x4c, 0xe9, 0x38, 0x21, - 0x1f, 0x2b, 0xcb, 0x75, 0x76, 0xf3, 0xb1, 0x88, 0x27, 0x84, 0x8b, 0x60, 0x32, 0xd5, 0xce, 0xde, - 0x4f, 0xa1, 0x77, 0x94, 0x86, 0x6c, 0x36, 0x15, 0x24, 0xba, 0x20, 0x9c, 0x07, 0x63, 0x82, 0xb6, - 0xa0, 0x75, 0x3f, 0x1e, 0x66, 0xd7, 0x67, 0x64, 0xe6, 0x3a, 0x3b, 0xce, 0x6e, 0x1b, 0xe7, 0x32, - 0x42, 0xb0, 0x7a, 0x4d, 0xa3, 0x99, 0x5b, 0xdb, 0x71, 0x76, 0xbb, 0x58, 0xad, 0x91, 0x0b, 0x6b, - 0x77, 0x84, 0xf1, 0x98, 0xa6, 0x6e, 0x7d, 0xc7, 0xd9, 0x6d, 0x60, 0x2b, 0x7a, 0xeb, 0xd0, 0xf1, - 0x67, 0x69, 0x88, 0xc9, 0xdb, 0x8c, 0x70, 0xe1, 0xfd, 0xb9, 0x06, 0x5d, 0x2d, 0xf3, 0x29, 0x4d, - 0x39, 0x41, 0x67, 0xb0, 0x79, 0x1f, 0x33, 0x22, 0x58, 0xc6, 0x05, 0x21, 0x87, 0x34, 0xbd, 0x89, - 0xc7, 0xea, 0x93, 0x9d, 0xfd, 0xaf, 0xed, 0x95, 0x36, 0xf6, 0xe5, 0xbc, 0x13, 0x5e, 0x8c, 0x43, - 0xdf, 0x05, 0x98, 0x12, 0xc2, 0x4c, 0x96, 0x9a, 0xca, 0xf2, 0xb4, 0x9c, 0x65, 0x98, 0x5b, 0x71, - 0xc9, 0x13, 0x7d, 0x0e, 0x1d, 0x46, 0x26, 0x54, 0x10, 0x69, 0xe7, 0x6e, 0x7d, 0xa7, 0xbe, 0xdb, - 0xd9, 0xdf, 0x2e, 0x07, 0xe2, 0xdc, 0x6c, 0xc2, 0xcb, 0x01, 0x68, 0x0f, 0x50, 0x49, 0x3c, 0xe5, - 0x47, 0x93, 0xa9, 0x98, 0xb9, 0xab, 0x3b, 0xce, 0x6e, 0x0b, 0x2f, 0xb1, 0xc8, 0x3a, 0x07, 0x44, - 0xdc, 0x53, 0xf6, 0xe6, 0x22, 0x98, 0xba, 0x8d, 0xc5, 0x3a, 0x0b, 0x2b, 0x2e, 0x79, 0x7a, 0xbf, - 0x73, 0xa0, 0x7b, 0x4e, 0xc7, 0x71, 0x6a, 0xe0, 0x94, 0xe7, 0xc4, 0x89, 0xc8, 0xa6, 0xa5, 0x73, - 0xb2, 0x32, 0xda, 0x83, 0xd5, 0x09, 0x11, 0x81, 0x81, 0x61, 0x6b, 0x1e, 0x06, 0x7f, 0xc6, 0x05, - 0x99, 0x5c, 0x10, 0x11, 0x60, 0xe5, 0x27, 0x73, 0xfd, 0xec, 0x5e, 0x8c, 0xe8, 0x1b, 0xa2, 0x0f, - 0xb1, 0x8d, 0x73, 0x19, 0x7d, 0x02, 0x2d, 0x09, 0xd7, 0x19, 0x99, 0x71, 0xb5, 0xad, 0xce, 0xfe, - 0xe3, 0xf9, 0x7c, 0xd2, 0x86, 0x73, 0x2f, 0xaf, 0x0f, 0x2d, 0xab, 0x45, 0xdb, 0xd0, 0xe6, 0xfc, - 0xb6, 0xd4, 0x4e, 0x5d, 0x5c, 0x28, 0x2a, 0xbd, 0xa6, 0x7b, 0x2a, 0x97, 0xbd, 0x1f, 0x41, 0xe7, - 0x28, 0xbd, 0x8b, 0x19, 0x4d, 0xe5, 0x77, 0xd0, 0x63, 0x68, 0x84, 0x09, 0xcd, 0x22, 0xb3, 0x57, - 0x2d, 0xc8, 0x04, 0xd3, 0x24, 0x10, 0x37, 0x94, 0x4d, 0x54, 0x82, 0x36, 0xce, 0x65, 0xef, 0xf7, - 0xab, 0xb0, 0x51, 0xdd, 0xad, 0x74, 0xbf, 0xa5, 0x5c, 0xa4, 0xc1, 0x84, 0x58, 0xcc, 0xac, 0x2c, - 0x7b, 0x7b, 0x4c, 0x2f, 0x7d, 0x93, 0x46, 0xad, 0xd1, 0x53, 0x68, 0xbe, 0x21, 0x2c, 0x25, 0x89, - 0x41, 0xc5, 0x48, 0xd2, 0x37, 0xa4, 0x8c, 0x28, 0x3c, 0xda, 0x58, 0xad, 0x2b, 0xa5, 0x34, 0xaa, - 0xa5, 0xa0, 0x0d, 0xa8, 0x5d, 0xfa, 0x6e, 0x53, 0x69, 0x6b, 0x97, 0xbe, 0x6c, 0x9a, 0x52, 0x07, - 0xff, 0xd8, 0x5c, 0x9f, 0x35, 0x65, 0x5f, 0x62, 0x91, 0x28, 0x66, 0xb1, 0x75, 0x6b, 0x29, 0xb7, - 0x42, 0x81, 0xbe, 0x01, 0xeb, 0xba, 0x2e, 0xeb, 0xd1, 0x56, 0x1e, 0x55, 0xa5, 0xcc, 0x71, 0xe9, - 0x5b, 0x0f, 0xd0, 0x39, 0x72, 0x05, 0x7a, 0x09, 0xbd, 0x54, 0x37, 0xdb, 0x41, 0x14, 0x31, 0xc2, - 0x39, 0xe1, 0x6e, 0x47, 0xdd, 0x85, 0xad, 0x25, 0xcd, 0x69, 0x7c, 0xf0, 0x42, 0x0c, 0xda, 0x85, - 0x47, 0x7c, 0xc6, 0x7d, 0xc2, 0xe2, 0x20, 0x19, 0x64, 0x93, 0x6b, 0xc2, 0xdc, 0xae, 0xfa, 0xd6, - 0xbc, 0x1a, 0x7d, 0x13, 0x36, 0xf8, 0x8c, 0x0f, 0x19, 0x8d, 0xb2, 0x50, 0x0c, 0xe4, 0x89, 0xac, - 0x2b, 0xc7, 0x39, 0xad, 0xc9, 0x78, 0x11, 0xa4, 0xd9, 0x4d, 0x10, 0x8a, 0x8c, 0x11, 0xe6, 0x6e, - 0xe4, 0x19, 0xcb, 0x6a, 0xf4, 0x7d, 0xe8, 0x90, 0xa2, 0x63, 0xdc, 0x47, 0xaa, 0x59, 0x3f, 0x28, - 0x97, 0x5f, 0x6a, 0x28, 0x5c, 0xf6, 0xf5, 0x7e, 0xed, 0xc0, 0xba, 0xb9, 0x5d, 0xff, 0x43, 0xc3, - 0xc9, 0x9b, 0xc1, 0xa6, 0x4f, 0xd8, 0x9d, 0xba, 0x4b, 0x79, 0x65, 0x3d, 0xa8, 0xbf, 0xc9, 0xef, - 0xbc, 0x5c, 0xa2, 0xe7, 0xd0, 0x26, 0x3f, 0x9f, 0xc6, 0x8c, 0xf0, 0x03, 0x91, 0xdf, 0x79, 0x3d, - 0xfb, 0xf7, 0xec, 0xec, 0xdf, 0x1b, 0xd9, 0xd9, 0x8f, 0x0b, 0xe7, 0x77, 0x0c, 0xef, 0x35, 0x68, - 0xa8, 0x81, 0xe5, 0xfd, 0xd5, 0x81, 0xcd, 0x85, 0x4d, 0xa2, 0x8f, 0xa0, 0xc1, 0x45, 0x96, 0x72, - 0xd7, 0x51, 0x4d, 0x52, 0xd9, 0xcc, 0x09, 0xe5, 0xc2, 0x6c, 0x46, 0x3b, 0xa1, 0xef, 0x40, 0x43, - 0x64, 0x2c, 0xe5, 0x6e, 0x4d, 0x79, 0x7f, 0x58, 0xd9, 0x3a, 0xa3, 0x82, 0x84, 0x82, 0x44, 0xe5, - 0x30, 0xe5, 0x8d, 0xf6, 0xa0, 0xc9, 0xe3, 0x71, 0x1a, 0xe8, 0xeb, 0xf7, 0xf0, 0x57, 0x8c, 0x17, - 0xf2, 0xa0, 0x8b, 0x49, 0x12, 0xcc, 0x4c, 0x3b, 0x9a, 0xeb, 0x59, 0xd1, 0x79, 0xbf, 0x75, 0x00, - 0x8a, 0x50, 0x09, 0x66, 0xc6, 0x62, 0x0b, 0x66, 0xc6, 0x62, 0xf4, 0x19, 0xb4, 0x14, 0x66, 0x21, - 0x4d, 0x14, 0x96, 0x1b, 0xd5, 0x72, 0x8b, 0x58, 0x55, 0xb9, 0x74, 0xc3, 0x79, 0x80, 0xf7, 0x19, - 0xb4, 0xac, 0x16, 0xad, 0x41, 0xfd, 0xaa, 0x3f, 0xec, 0xad, 0xc8, 0xc5, 0xe8, 0x70, 0xd8, 0x73, - 0x50, 0x0b, 0x56, 0x4f, 0x46, 0xa3, 0x61, 0xaf, 0x86, 0xda, 0xd0, 0x90, 0x2b, 0xbf, 0x57, 0x97, - 0xca, 0xfe, 0xe8, 0xdc, 0xef, 0xad, 0x7a, 0xbf, 0x84, 0xaf, 0x2c, 0x01, 0x43, 0x36, 0xcf, 0x6d, - 0x2e, 0x99, 0x16, 0x7c, 0x08, 0x89, 0x92, 0xa7, 0x1c, 0x52, 0x19, 0x27, 0xcc, 0x0e, 0x34, 0xb9, - 0x56, 0x43, 0x2a, 0xe0, 0xfc, 0x9e, 0xb2, 0xc8, 0x0e, 0x7a, 0x2b, 0x7b, 0xbf, 0x72, 0x00, 0x8a, - 0x3e, 0x94, 0xad, 0x11, 0x18, 0x1c, 0x35, 0x3a, 0x56, 0x94, 0x98, 0x45, 0xea, 0x2c, 0x15, 0x66, - 0x51, 0xca, 0xd1, 0xa7, 0x6a, 0xca, 0x9b, 0x0a, 0xf5, 0x59, 0x3d, 0x29, 0x57, 0xe8, 0xfb, 0x27, - 0xa6, 0xc0, 0xc2, 0x4f, 0xd6, 0x77, 0xf3, 0x36, 0x4a, 0xed, 0x10, 0x95, 0x6b, 0xef, 0xef, 0xf5, - 0xf2, 0xf3, 0x28, 0xe7, 0xaf, 0x9e, 0x19, 0xaa, 0x84, 0x55, 0x6c, 0xa4, 0xff, 0x9b, 0xc7, 0xfe, - 0x5b, 0xd0, 0xc4, 0x34, 0x13, 0x84, 0xbb, 0x0d, 0xf5, 0xa9, 0xcd, 0xca, 0xa7, 0xa4, 0x05, 0x1b, - 0x07, 0x09, 0x61, 0x7f, 0xe0, 0x9b, 0x1d, 0x35, 0x17, 0x21, 0xcc, 0x8d, 0xb8, 0xf0, 0x43, 0x5f, - 0x40, 0x97, 0xde, 0xdc, 0x24, 0x71, 0x6a, 0x36, 0xb4, 0xf6, 0x6f, 0x6c, 0xa8, 0x12, 0x81, 0x3e, - 0x87, 0xf5, 0x97, 0x31, 0x23, 0xf7, 0x41, 0x92, 0xe0, 0x2c, 0x21, 0xdc, 0x6d, 0xa9, 0x14, 0x6e, - 0x39, 0x45, 0xd9, 0x01, 0x57, 0xdd, 0xd1, 0x3e, 0x3c, 0xbe, 0x29, 0x2b, 0x2c, 0x26, 0x6d, 0x85, - 0xc9, 0x52, 0x9b, 0xf7, 0x1b, 0x07, 0x7a, 0xf3, 0x65, 0xbd, 0x93, 0x76, 0x3e, 0x03, 0x08, 0x92, - 0x84, 0xde, 0x93, 0xe8, 0x74, 0xaa, 0x67, 0x48, 0x1b, 0x97, 0x34, 0xff, 0xbd, 0xf6, 0x3b, 0x85, - 0x76, 0xee, 0x2b, 0xbf, 0xca, 0xf9, 0xed, 0x51, 0x1a, 0x5c, 0x27, 0x44, 0xd3, 0x8e, 0x16, 0x2e, - 0x69, 0xaa, 0xd4, 0xa6, 0x36, 0x47, 0x6d, 0xbc, 0x1d, 0x78, 0xd6, 0x27, 0x77, 0x71, 0x48, 0x0e, - 0x32, 0x71, 0x4b, 0x59, 0xfc, 0x8b, 0x40, 0xc4, 0x34, 0x7d, 0x99, 0xd0, 0x7b, 0xcb, 0x87, 0xff, - 0xe2, 0xc0, 0x07, 0x0f, 0xb8, 0xa0, 0x13, 0x35, 0x47, 0xee, 0xe2, 0x88, 0x30, 0xf5, 0xe5, 0x8d, - 0xfd, 0x8f, 0x2a, 0xcd, 0xb0, 0x3c, 0x4c, 0x0e, 0x7b, 0x15, 0x83, 0xf3, 0x68, 0xf4, 0x02, 0x36, - 0xec, 0xba, 0x72, 0x5d, 0xb6, 0xe6, 0x66, 0x70, 0xc9, 0x03, 0xcf, 0x45, 0x78, 0x4f, 0xd5, 0x48, - 0xd4, 0xf9, 0x00, 0x9a, 0x27, 0x97, 0xfe, 0xe8, 0xa8, 0xdf, 0x5b, 0xf1, 0x9e, 0xc1, 0xf6, 0xf0, - 0xec, 0xf0, 0xe8, 0xc1, 0x1d, 0xfe, 0x04, 0x9e, 0x2c, 0xb5, 0x2f, 0x29, 0xca, 0xf9, 0x8f, 0x8b, - 0xfa, 0x47, 0x6d, 0x3e, 0x89, 0xec, 0xa1, 0xc3, 0x24, 0x26, 0xa9, 0x38, 0xed, 0xdb, 0x1e, 0xb2, - 0xb2, 0x7c, 0x1b, 0xf4, 0xda, 0x27, 0x21, 0x23, 0xc2, 0x4c, 0xaf, 0x8a, 0x4e, 0x8e, 0x9b, 0x3e, - 0x9d, 0x04, 0xb1, 0x25, 0xc1, 0x46, 0x92, 0x79, 0x0f, 0xb2, 0x28, 0x26, 0x69, 0x68, 0x29, 0x5f, - 0x2e, 0xcb, 0x91, 0x50, 0x9c, 0xc6, 0x51, 0x1a, 0x4d, 0x69, 0x9c, 0x0a, 0x43, 0x00, 0x97, 0x58, - 0x24, 0x59, 0x53, 0xbc, 0x3a, 0x77, 0xd5, 0xac, 0xb0, 0xaa, 0x94, 0x6c, 0xd7, 0x0f, 0xe9, 0x94, - 0x18, 0x4e, 0xa8, 0x05, 0xd9, 0x91, 0x57, 0x9c, 0x9c, 0xf6, 0x35, 0x51, 0x6f, 0xe9, 0x8e, 0x2c, - 0x34, 0xe8, 0xdb, 0xf0, 0xa4, 0x82, 0x75, 0xfe, 0x0d, 0x4d, 0x08, 0x97, 0x1b, 0xf5, 0xab, 0x19, - 0xc5, 0x8c, 0x84, 0xe2, 0x0a, 0x9f, 0x73, 0x17, 0xd4, 0xfd, 0xaa, 0xe8, 0xbc, 0x3f, 0x39, 0xd0, - 0x50, 0x83, 0x4a, 0x52, 0xd9, 0x1c, 0xdd, 0xda, 0x69, 0x5f, 0x3e, 0x13, 0x66, 0x60, 0x1b, 0x48, - 0xad, 0x88, 0x76, 0xa0, 0x63, 0x96, 0xa3, 0xd9, 0x94, 0x28, 0x48, 0xeb, 0xb8, 0xac, 0x92, 0x57, - 0x50, 0x4e, 0x00, 0x7b, 0x05, 0xe5, 0x5a, 0x9e, 0xc1, 0x05, 0x11, 0x2c, 0x0e, 0x15, 0x86, 0x75, - 0x6c, 0x24, 0xb9, 0xf7, 0x8b, 0x80, 0xbf, 0xcd, 0x08, 0x0b, 0x22, 0xa2, 0x40, 0x6b, 0xe1, 0x92, - 0x46, 0x22, 0x36, 0x20, 0xf2, 0xe0, 0x0d, 0x62, 0x4a, 0xf0, 0xfe, 0xe8, 0x94, 0xc6, 0xaa, 0xc4, - 0x5e, 0xd2, 0xa9, 0x38, 0x24, 0xfa, 0x0e, 0x9b, 0x4b, 0x5d, 0x55, 0xa2, 0x63, 0xe8, 0x49, 0xe2, - 0xa9, 0x89, 0xd7, 0x31, 0xa3, 0xd9, 0xd4, 0xf2, 0x96, 0xaf, 0x56, 0xa8, 0x70, 0xd5, 0x07, 0x2f, - 0x04, 0xa1, 0xe7, 0xd0, 0x39, 0xcc, 0xb8, 0xa0, 0x93, 0xd7, 0x34, 0x25, 0xf6, 0xb5, 0xa9, 0x3c, - 0x53, 0x85, 0x19, 0x97, 0x5d, 0xbd, 0x57, 0x00, 0x85, 0x58, 0x6a, 0x4b, 0xa7, 0xd2, 0x96, 0xfb, - 0xb0, 0x86, 0x49, 0x48, 0x59, 0x64, 0xeb, 0xab, 0x4c, 0x6d, 0x3f, 0x9e, 0x4c, 0x13, 0xa2, 0x1d, - 0xb0, 0x75, 0xf4, 0x04, 0x74, 0xcb, 0x06, 0x79, 0x04, 0x83, 0xe2, 0xd7, 0x90, 0x5a, 0x4b, 0x9d, - 0x3a, 0xb1, 0x9a, 0x3a, 0x00, 0xb5, 0x96, 0xf0, 0x1e, 0x26, 0x01, 0xe7, 0xe6, 0x66, 0x68, 0x41, - 0x32, 0x81, 0xd1, 0xe8, 0x5c, 0x9d, 0x5f, 0x1d, 0xcb, 0xa5, 0xf4, 0xc3, 0xfd, 0x40, 0x04, 0xe6, - 0x06, 0x68, 0xc1, 0xfb, 0x83, 0x03, 0x8f, 0xe6, 0xe0, 0x91, 0xe8, 0x14, 0xaa, 0xa5, 0x3c, 0xb2, - 0x30, 0xe3, 0xb2, 0xab, 0x6c, 0xb9, 0x21, 0x8b, 0x27, 0x01, 0xd3, 0x63, 0xb7, 0x85, 0xad, 0x28, - 0x2d, 0x1a, 0x1b, 0x8d, 0x76, 0x1b, 0x5b, 0x51, 0xbe, 0x53, 0x3e, 0x09, 0x58, 0x78, 0x6b, 0x14, - 0x76, 0xac, 0xeb, 0xb7, 0x7b, 0xa9, 0xcd, 0x3b, 0x01, 0x28, 0x3e, 0xab, 0x1a, 0x7f, 0x98, 0x37, - 0xfe, 0x50, 0x9e, 0xca, 0xc0, 0x2f, 0xe1, 0x64, 0x24, 0xd5, 0xd4, 0x94, 0x09, 0xd3, 0xef, 0x6a, - 0xed, 0xfd, 0xb3, 0x06, 0xdd, 0xf2, 0xbb, 0x29, 0x83, 0x65, 0xb7, 0xe7, 0x09, 0x8d, 0x84, 0xbe, - 0x80, 0x76, 0x5f, 0xdd, 0x3a, 0xc9, 0xc8, 0x35, 0xfb, 0xf4, 0x1e, 0x7a, 0x8a, 0xf7, 0x22, 0xeb, - 0x89, 0x8b, 0x20, 0xf4, 0x3d, 0x68, 0x1e, 0xe8, 0xf0, 0xfa, 0x22, 0x79, 0xad, 0x84, 0x07, 0x3a, - 0xd6, 0xb8, 0xa3, 0x1f, 0x16, 0xd4, 0x55, 0xa1, 0xb2, 0xb1, 0xff, 0xf5, 0x07, 0x43, 0xa7, 0x39, - 0xf3, 0xcd, 0xd9, 0xae, 0xdd, 0x76, 0xc3, 0xdc, 0x65, 0xb9, 0xed, 0x6d, 0x68, 0xe7, 0x35, 0xa2, - 0x26, 0xd4, 0x4e, 0x07, 0x9a, 0x0d, 0x5f, 0x5e, 0x8d, 0x7a, 0x8e, 0xf7, 0x0c, 0x9a, 0xba, 0x04, - 0xf9, 0xa6, 0x1c, 0x1c, 0x1e, 0x1e, 0x0d, 0x47, 0xbd, 0x15, 0x45, 0x87, 0xf1, 0xe5, 0xb0, 0xe7, - 0x78, 0x3f, 0x28, 0x88, 0x38, 0xea, 0xc0, 0xda, 0xd5, 0xe0, 0x6c, 0x70, 0xf9, 0xa5, 0xc9, 0x70, - 0x70, 0x7e, 0xde, 0x73, 0x2c, 0xb1, 0xae, 0x59, 0xaa, 0xad, 0xc8, 0xf4, 0xe9, 0xe1, 0xc5, 0xb0, - 0xb7, 0xea, 0x3d, 0x87, 0x8d, 0xea, 0x8f, 0x55, 0xd9, 0x9a, 0x29, 0x11, 0x39, 0xe4, 0x5a, 0x90, - 0x2d, 0x3c, 0x09, 0x42, 0x4b, 0x66, 0x27, 0x41, 0xb8, 0xff, 0xb7, 0x3a, 0x6c, 0x5e, 0xe4, 0x1b, - 0x37, 0xb3, 0x01, 0x1d, 0x41, 0x43, 0xfd, 0x40, 0x44, 0xdb, 0xd5, 0x1f, 0x94, 0xd5, 0xff, 0x9e, - 0x6d, 0xbd, 0xd3, 0xea, 0xad, 0xa0, 0x97, 0xb0, 0xea, 0xcf, 0xd2, 0xf0, 0xfd, 0xb2, 0x7c, 0xe2, - 0xa0, 0x17, 0xd0, 0x3d, 0x26, 0x22, 0xff, 0x71, 0x88, 0x2a, 0xcc, 0x52, 0xd1, 0xac, 0xad, 0xca, - 0xcf, 0xd4, 0x85, 0x9f, 0x91, 0xde, 0x8a, 0xa4, 0x4d, 0x31, 0x3f, 0x21, 0x41, 0x22, 0x6e, 0x97, - 0x26, 0x58, 0x54, 0x79, 0x2b, 0xe8, 0x35, 0x6c, 0x1d, 0x13, 0xf1, 0x10, 0x6f, 0x79, 0x3f, 0x70, - 0x5e, 0x81, 0x7b, 0x4c, 0xc4, 0x72, 0xca, 0xf0, 0x5e, 0x99, 0x5f, 0xb4, 0x5e, 0x37, 0xcd, 0x7f, - 0x41, 0x9b, 0xea, 0xcf, 0xa7, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0xcb, 0xde, 0x32, 0x7a, 0x3a, - 0x15, 0x00, 0x00, +var file_management_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_management_proto_msgTypes = make([]protoimpl.MessageInfo, 31) +var file_management_proto_goTypes = []interface{}{ + (HostConfig_Protocol)(0), // 0: management.HostConfig.Protocol + (DeviceAuthorizationFlowProvider)(0), // 1: management.DeviceAuthorizationFlow.provider + (FirewallRuleDirection)(0), // 2: management.FirewallRule.direction + (FirewallRuleAction)(0), // 3: management.FirewallRule.action + (FirewallRuleProtocol)(0), // 4: management.FirewallRule.protocol + (*EncryptedMessage)(nil), // 5: management.EncryptedMessage + (*SyncRequest)(nil), // 6: management.SyncRequest + (*SyncResponse)(nil), // 7: management.SyncResponse + (*LoginRequest)(nil), // 8: management.LoginRequest + (*PeerKeys)(nil), // 9: management.PeerKeys + (*Environment)(nil), // 10: management.Environment + (*PeerSystemMeta)(nil), // 11: management.PeerSystemMeta + (*LoginResponse)(nil), // 12: management.LoginResponse + (*ServerKeyResponse)(nil), // 13: management.ServerKeyResponse + (*Empty)(nil), // 14: management.Empty + (*WiretrusteeConfig)(nil), // 15: management.WiretrusteeConfig + (*HostConfig)(nil), // 16: management.HostConfig + (*RelayConfig)(nil), // 17: management.RelayConfig + (*ProtectedHostConfig)(nil), // 18: management.ProtectedHostConfig + (*PeerConfig)(nil), // 19: management.PeerConfig + (*NetworkMap)(nil), // 20: management.NetworkMap + (*RemotePeerConfig)(nil), // 21: management.RemotePeerConfig + (*SSHConfig)(nil), // 22: management.SSHConfig + (*DeviceAuthorizationFlowRequest)(nil), // 23: management.DeviceAuthorizationFlowRequest + (*DeviceAuthorizationFlow)(nil), // 24: management.DeviceAuthorizationFlow + (*PKCEAuthorizationFlowRequest)(nil), // 25: management.PKCEAuthorizationFlowRequest + (*PKCEAuthorizationFlow)(nil), // 26: management.PKCEAuthorizationFlow + (*ProviderConfig)(nil), // 27: management.ProviderConfig + (*Route)(nil), // 28: management.Route + (*DNSConfig)(nil), // 29: management.DNSConfig + (*CustomZone)(nil), // 30: management.CustomZone + (*SimpleRecord)(nil), // 31: management.SimpleRecord + (*NameServerGroup)(nil), // 32: management.NameServerGroup + (*NameServer)(nil), // 33: management.NameServer + (*FirewallRule)(nil), // 34: management.FirewallRule + (*NetworkAddress)(nil), // 35: management.NetworkAddress + (*timestamppb.Timestamp)(nil), // 36: google.protobuf.Timestamp +} +var file_management_proto_depIdxs = []int32{ + 15, // 0: management.SyncResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig + 19, // 1: management.SyncResponse.peerConfig:type_name -> management.PeerConfig + 21, // 2: management.SyncResponse.remotePeers:type_name -> management.RemotePeerConfig + 20, // 3: management.SyncResponse.NetworkMap:type_name -> management.NetworkMap + 11, // 4: management.LoginRequest.meta:type_name -> management.PeerSystemMeta + 9, // 5: management.LoginRequest.peerKeys:type_name -> management.PeerKeys + 35, // 6: management.PeerSystemMeta.networkAddresses:type_name -> management.NetworkAddress + 10, // 7: management.PeerSystemMeta.environment:type_name -> management.Environment + 15, // 8: management.LoginResponse.wiretrusteeConfig:type_name -> management.WiretrusteeConfig + 19, // 9: management.LoginResponse.peerConfig:type_name -> management.PeerConfig + 36, // 10: management.ServerKeyResponse.expiresAt:type_name -> google.protobuf.Timestamp + 16, // 11: management.WiretrusteeConfig.stuns:type_name -> management.HostConfig + 18, // 12: management.WiretrusteeConfig.turns:type_name -> management.ProtectedHostConfig + 16, // 13: management.WiretrusteeConfig.signal:type_name -> management.HostConfig + 17, // 14: management.WiretrusteeConfig.relay:type_name -> management.RelayConfig + 0, // 15: management.HostConfig.protocol:type_name -> management.HostConfig.Protocol + 16, // 16: management.ProtectedHostConfig.hostConfig:type_name -> management.HostConfig + 22, // 17: management.PeerConfig.sshConfig:type_name -> management.SSHConfig + 19, // 18: management.NetworkMap.peerConfig:type_name -> management.PeerConfig + 21, // 19: management.NetworkMap.remotePeers:type_name -> management.RemotePeerConfig + 28, // 20: management.NetworkMap.Routes:type_name -> management.Route + 29, // 21: management.NetworkMap.DNSConfig:type_name -> management.DNSConfig + 21, // 22: management.NetworkMap.offlinePeers:type_name -> management.RemotePeerConfig + 34, // 23: management.NetworkMap.FirewallRules:type_name -> management.FirewallRule + 22, // 24: management.RemotePeerConfig.sshConfig:type_name -> management.SSHConfig + 1, // 25: management.DeviceAuthorizationFlow.Provider:type_name -> management.DeviceAuthorizationFlow.provider + 27, // 26: management.DeviceAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig + 27, // 27: management.PKCEAuthorizationFlow.ProviderConfig:type_name -> management.ProviderConfig + 32, // 28: management.DNSConfig.NameServerGroups:type_name -> management.NameServerGroup + 30, // 29: management.DNSConfig.CustomZones:type_name -> management.CustomZone + 31, // 30: management.CustomZone.Records:type_name -> management.SimpleRecord + 33, // 31: management.NameServerGroup.NameServers:type_name -> management.NameServer + 2, // 32: management.FirewallRule.Direction:type_name -> management.FirewallRule.direction + 3, // 33: management.FirewallRule.Action:type_name -> management.FirewallRule.action + 4, // 34: management.FirewallRule.Protocol:type_name -> management.FirewallRule.protocol + 5, // 35: management.ManagementService.Login:input_type -> management.EncryptedMessage + 5, // 36: management.ManagementService.Sync:input_type -> management.EncryptedMessage + 14, // 37: management.ManagementService.GetServerKey:input_type -> management.Empty + 14, // 38: management.ManagementService.isHealthy:input_type -> management.Empty + 5, // 39: management.ManagementService.GetDeviceAuthorizationFlow:input_type -> management.EncryptedMessage + 5, // 40: management.ManagementService.GetPKCEAuthorizationFlow:input_type -> management.EncryptedMessage + 5, // 41: management.ManagementService.Login:output_type -> management.EncryptedMessage + 5, // 42: management.ManagementService.Sync:output_type -> management.EncryptedMessage + 13, // 43: management.ManagementService.GetServerKey:output_type -> management.ServerKeyResponse + 14, // 44: management.ManagementService.isHealthy:output_type -> management.Empty + 5, // 45: management.ManagementService.GetDeviceAuthorizationFlow:output_type -> management.EncryptedMessage + 5, // 46: management.ManagementService.GetPKCEAuthorizationFlow:output_type -> management.EncryptedMessage + 41, // [41:47] is the sub-list for method output_type + 35, // [35:41] is the sub-list for method input_type + 35, // [35:35] is the sub-list for extension type_name + 35, // [35:35] is the sub-list for extension extendee + 0, // [0:35] is the sub-list for field type_name +} + +func init() { file_management_proto_init() } +func file_management_proto_init() { + if File_management_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_management_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EncryptedMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SyncRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SyncResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoginRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PeerKeys); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Environment); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PeerSystemMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoginResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerKeyResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Empty); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*WiretrusteeConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HostConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RelayConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProtectedHostConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PeerConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NetworkMap); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RemotePeerConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SSHConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeviceAuthorizationFlowRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeviceAuthorizationFlow); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PKCEAuthorizationFlowRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PKCEAuthorizationFlow); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProviderConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Route); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DNSConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CustomZone); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleRecord); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NameServerGroup); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NameServer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FirewallRule); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_management_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NetworkAddress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_management_proto_rawDesc, + NumEnums: 5, + NumMessages: 31, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_management_proto_goTypes, + DependencyIndexes: file_management_proto_depIdxs, + EnumInfos: file_management_proto_enumTypes, + MessageInfos: file_management_proto_msgTypes, + }.Build() + File_management_proto = out.File + file_management_proto_rawDesc = nil + file_management_proto_goTypes = nil + file_management_proto_depIdxs = nil } diff --git a/management/proto/management.proto b/management/proto/management.proto index 61304c8c4..c6695dd6a 100644 --- a/management/proto/management.proto +++ b/management/proto/management.proto @@ -147,7 +147,7 @@ message WiretrusteeConfig { // a Signal server config HostConfig signal = 3; - string RelayAddress = 4; + RelayConfig relay = 4; } // HostConfig describes connection properties of some server (e.g. STUN, Signal, Management) @@ -164,6 +164,13 @@ message HostConfig { DTLS = 4; } } + +message RelayConfig { + repeated string urls = 1; + string tokenPayload = 2; + string tokenSignature = 3; +} + // ProtectedHostConfig is similar to HostConfig but has additional user and password // Mostly used for TURN servers message ProtectedHostConfig { diff --git a/management/server/dns_test.go b/management/server/dns_test.go index b5074e50c..bfa50b1cf 100644 --- a/management/server/dns_test.go +++ b/management/server/dns_test.go @@ -166,9 +166,9 @@ func TestGetNetworkMap_DNSConfigSync(t *testing.T) { newAccountDNSConfig, err := am.GetNetworkMap(peer1.ID) require.NoError(t, err) - require.Len(t, newAccountDNSConfig.DNSConfig.CustomZones, 1, "default DNS config should have one custom zone for peers") - require.True(t, newAccountDNSConfig.DNSConfig.ServiceEnable, "default DNS config should have local DNS service enabled") - require.Len(t, newAccountDNSConfig.DNSConfig.NameServerGroups, 0, "updated DNS config should have no nameserver groups since peer 1 is NS for the only existing NS group") + require.Len(t, newAccountDNSConfig.DNSConfig.CustomZones, 1, "default DNS turnCfg should have one custom zone for peers") + require.True(t, newAccountDNSConfig.DNSConfig.ServiceEnable, "default DNS turnCfg should have local DNS service enabled") + require.Len(t, newAccountDNSConfig.DNSConfig.NameServerGroups, 0, "updated DNS turnCfg should have no nameserver groups since peer 1 is NS for the only existing NS group") dnsSettings := account.DNSSettings.Copy() dnsSettings.DisabledManagementGroups = append(dnsSettings.DisabledManagementGroups, dnsGroup1ID) @@ -178,13 +178,13 @@ func TestGetNetworkMap_DNSConfigSync(t *testing.T) { updatedAccountDNSConfig, err := am.GetNetworkMap(peer1.ID) require.NoError(t, err) - require.Len(t, updatedAccountDNSConfig.DNSConfig.CustomZones, 0, "updated DNS config should have no custom zone when peer belongs to a disabled group") - require.False(t, updatedAccountDNSConfig.DNSConfig.ServiceEnable, "updated DNS config should have local DNS service disabled when peer belongs to a disabled group") + require.Len(t, updatedAccountDNSConfig.DNSConfig.CustomZones, 0, "updated DNS turnCfg should have no custom zone when peer belongs to a disabled group") + require.False(t, updatedAccountDNSConfig.DNSConfig.ServiceEnable, "updated DNS turnCfg should have local DNS service disabled when peer belongs to a disabled group") peer2AccountDNSConfig, err := am.GetNetworkMap(peer2.ID) require.NoError(t, err) - require.Len(t, peer2AccountDNSConfig.DNSConfig.CustomZones, 1, "DNS config should have one custom zone for peers not in the disabled group") - require.True(t, peer2AccountDNSConfig.DNSConfig.ServiceEnable, "DNS config should have DNS service enabled for peers not in the disabled group") - require.Len(t, peer2AccountDNSConfig.DNSConfig.NameServerGroups, 1, "updated DNS config should have 1 nameserver groups since peer 2 is part of the group All") + require.Len(t, peer2AccountDNSConfig.DNSConfig.CustomZones, 1, "DNS turnCfg should have one custom zone for peers not in the disabled group") + require.True(t, peer2AccountDNSConfig.DNSConfig.ServiceEnable, "DNS turnCfg should have DNS service enabled for peers not in the disabled group") + require.Len(t, peer2AccountDNSConfig.DNSConfig.NameServerGroups, 1, "updated DNS turnCfg should have 1 nameserver groups since peer 2 is part of the group All") } func createDNSManager(t *testing.T) (*DefaultAccountManager, error) { diff --git a/management/server/grpcserver.go b/management/server/grpcserver.go index 8c8b8354f..a7d3b675d 100644 --- a/management/server/grpcserver.go +++ b/management/server/grpcserver.go @@ -29,17 +29,17 @@ type GRPCServer struct { accountManager AccountManager wgKey wgtypes.Key proto.UnimplementedManagementServiceServer - peersUpdateManager *PeersUpdateManager - config *Config - turnCredentialsManager TURNCredentialsManager - jwtValidator *jwtclaims.JWTValidator - jwtClaimsExtractor *jwtclaims.ClaimsExtractor - appMetrics telemetry.AppMetrics - ephemeralManager *EphemeralManager + peersUpdateManager *PeersUpdateManager + config *Config + turnRelayTokenManager TURNRelayTokenManager + jwtValidator *jwtclaims.JWTValidator + jwtClaimsExtractor *jwtclaims.ClaimsExtractor + appMetrics telemetry.AppMetrics + ephemeralManager *EphemeralManager } // NewServer creates a new Management server -func NewServer(config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnCredentialsManager TURNCredentialsManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) { +func NewServer(config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnRelayTokenManager TURNRelayTokenManager, appMetrics telemetry.AppMetrics, ephemeralManager *EphemeralManager) (*GRPCServer, error) { key, err := wgtypes.GeneratePrivateKey() if err != nil { return nil, err @@ -58,7 +58,7 @@ func NewServer(config *Config, accountManager AccountManager, peersUpdateManager return nil, status.Errorf(codes.Internal, "unable to create new jwt middleware, err: %v", err) } } else { - log.Debug("unable to use http config to create new jwt middleware") + log.Debug("unable to use http turnCfg to create new jwt middleware") } if appMetrics != nil { @@ -84,14 +84,14 @@ func NewServer(config *Config, accountManager AccountManager, peersUpdateManager return &GRPCServer{ wgKey: key, // peerKey -> event channel - peersUpdateManager: peersUpdateManager, - accountManager: accountManager, - config: config, - turnCredentialsManager: turnCredentialsManager, - jwtValidator: jwtValidator, - jwtClaimsExtractor: jwtClaimsExtractor, - appMetrics: appMetrics, - ephemeralManager: ephemeralManager, + peersUpdateManager: peersUpdateManager, + accountManager: accountManager, + config: config, + turnRelayTokenManager: turnRelayTokenManager, + jwtValidator: jwtValidator, + jwtClaimsExtractor: jwtClaimsExtractor, + appMetrics: appMetrics, + ephemeralManager: ephemeralManager, }, nil } @@ -150,7 +150,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi s.ephemeralManager.OnPeerConnected(peer) if s.config.TURNConfig.TimeBasedCredentials { - s.turnCredentialsManager.SetupRefresh(peer.ID) + s.turnRelayTokenManager.SetupRefresh(peer.ID) } if s.appMetrics != nil { @@ -201,7 +201,7 @@ func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementServi func (s *GRPCServer) cancelPeerRoutines(peer *nbpeer.Peer) { s.peersUpdateManager.CloseChannel(peer.ID) - s.turnCredentialsManager.CancelRefresh(peer.ID) + s.turnRelayTokenManager.CancelRefresh(peer.ID) _ = s.accountManager.CancelPeerRoutines(peer) s.ephemeralManager.OnPeerDisconnected(peer) } @@ -377,9 +377,14 @@ func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*p s.ephemeralManager.OnPeerDisconnected(peer) } + trt, err := s.turnRelayTokenManager.Generate() + if err != nil { + log.Errorf("failed generating TURN and Relay token: %v", err) + } + // if peer has reached this point then it has logged in loginResp := &proto.LoginResponse{ - WiretrusteeConfig: toWiretrusteeConfig(s.config, nil), + WiretrusteeConfig: toWiretrusteeConfig(s.config, nil, trt), PeerConfig: toPeerConfig(peer, netMap.Network, s.accountManager.GetDNSDomain()), } encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, loginResp) @@ -407,11 +412,11 @@ func ToResponseProto(configProto Protocol) proto.HostConfig_Protocol { case TCP: return proto.HostConfig_TCP default: - panic(fmt.Errorf("unexpected config protocol type %v", configProto)) + panic(fmt.Errorf("unexpected turnCfg protocol type %v", configProto)) } } -func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *proto.WiretrusteeConfig { +func toWiretrusteeConfig(config *Config, turnCredentials *TURNRelayToken, relayToken *TURNRelayToken) *proto.WiretrusteeConfig { if config == nil { return nil } @@ -427,8 +432,8 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot var username string var password string if turnCredentials != nil { - username = turnCredentials.Username - password = turnCredentials.Password + username = turnCredentials.Payload + password = turnCredentials.Signature } else { username = turn.Username password = turn.Password @@ -443,6 +448,18 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot }) } + var relayCfg *proto.RelayConfig + if config.RelayAddress != "" { + relayCfg = &proto.RelayConfig{ + Urls: []string{config.RelayAddress}, + } + + if relayToken != nil { + relayCfg.TokenPayload = relayToken.Payload + relayCfg.TokenSignature = relayToken.Signature + } + } + return &proto.WiretrusteeConfig{ Stuns: stuns, Turns: turns, @@ -450,7 +467,7 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNCredentials) *prot Uri: config.Signal.URI, Protocol: ToResponseProto(config.Signal.Proto), }, - RelayAddress: config.RelayAddress, + Relay: relayCfg, } } @@ -478,8 +495,8 @@ func toRemotePeerConfig(peers []*nbpeer.Peer, dnsName string) []*proto.RemotePee return remotePeers } -func toSyncResponse(config *Config, peer *nbpeer.Peer, turnCredentials *TURNCredentials, networkMap *NetworkMap, dnsName string) *proto.SyncResponse { - wtConfig := toWiretrusteeConfig(config, turnCredentials) +func toSyncResponse(config *Config, peer *nbpeer.Peer, turnCredentials *TURNRelayToken, relayCredentials *TURNRelayToken, networkMap *NetworkMap, dnsName string) *proto.SyncResponse { + wtConfig := toWiretrusteeConfig(config, turnCredentials, relayCredentials) pConfig := toPeerConfig(peer, networkMap.Network, dnsName) @@ -520,14 +537,16 @@ func (s *GRPCServer) IsHealthy(ctx context.Context, req *proto.Empty) (*proto.Em // sendInitialSync sends initial proto.SyncResponse to the peer requesting synchronization func (s *GRPCServer) sendInitialSync(peerKey wgtypes.Key, peer *nbpeer.Peer, networkMap *NetworkMap, srv proto.ManagementService_SyncServer) error { // make secret time based TURN credentials optional - var turnCredentials *TURNCredentials - if s.config.TURNConfig.TimeBasedCredentials { - creds := s.turnCredentialsManager.GenerateCredentials() - turnCredentials = &creds - } else { - turnCredentials = nil + var turnCredentials *TURNRelayToken + trt, err := s.turnRelayTokenManager.Generate() + if err != nil { + log.Errorf("failed generating TURN and Relay token: %v", err) } - plainResp := toSyncResponse(s.config, peer, turnCredentials, networkMap, s.accountManager.GetDNSDomain()) + if s.config.TURNConfig.TimeBasedCredentials { + turnCredentials = trt + } + + plainResp := toSyncResponse(s.config, peer, turnCredentials, trt, networkMap, s.accountManager.GetDNSDomain()) encryptedResp, err := encryption.EncryptMessage(peerKey, s.wgKey, plainResp) if err != nil { diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index c2672b1e9..1b7bced3c 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -169,7 +169,7 @@ func Test_SyncProtocol(t *testing.T) { } if wiretrusteeConfig.GetSignal() == nil { - t.Fatal("expecting SyncResponse to have WiretrusteeConfig with non-nil Signal config") + t.Fatal("expecting SyncResponse to have WiretrusteeConfig with non-nil Signal turnCfg") } expectedSignalConfig := &mgmtProto.HostConfig{ @@ -418,7 +418,7 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, string, error) if err != nil { return nil, "", err } - turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") ephemeralMgr := NewEphemeralManager(store, accountManager) mgmtServer, err := NewServer(config, accountManager, peersUpdateManager, turnManager, nil, ephemeralMgr) diff --git a/management/server/management_test.go b/management/server/management_test.go index 564afaf55..0ad8426cf 100644 --- a/management/server/management_test.go +++ b/management/server/management_test.go @@ -544,7 +544,7 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) { if err != nil { log.Fatalf("failed creating a manager: %v", err) } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig) + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil, nil) Expect(err).NotTo(HaveOccurred()) mgmtProto.RegisterManagementServiceServer(s, mgmtServer) diff --git a/management/server/peer.go b/management/server/peer.go index 13ac3801d..e6488aa3a 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -900,7 +900,7 @@ func (am *DefaultAccountManager) updateAccountPeers(account *Account) { continue } remotePeerNetworkMap := account.GetPeerNetworkMap(peer.ID, am.dnsDomain, approvedPeersMap) - update := toSyncResponse(nil, peer, nil, remotePeerNetworkMap, am.GetDNSDomain()) + update := toSyncResponse(nil, peer, nil, nil, remotePeerNetworkMap, am.GetDNSDomain()) am.peersUpdateManager.SendUpdate(peer.ID, &UpdateMessage{Update: update}) } } diff --git a/management/server/token_mgr.go b/management/server/token_mgr.go new file mode 100644 index 000000000..3f30d0494 --- /dev/null +++ b/management/server/token_mgr.go @@ -0,0 +1,126 @@ +package server + +import ( + "fmt" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/management/proto" + auth "github.com/netbirdio/netbird/relay/auth/hmac" +) + +// TURNRelayTokenManager used to manage TURN credentials +type TURNRelayTokenManager interface { + Generate() (*TURNRelayToken, error) + SetupRefresh(peerKey string) + CancelRefresh(peerKey string) +} + +// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server +type TimeBasedAuthSecretsManager struct { + mux sync.Mutex + turnCfg *TURNConfig + relayAddr string + hmacToken *auth.TimedHMAC + updateManager *PeersUpdateManager + cancelMap map[string]chan struct{} +} + +type TURNRelayToken auth.Token + +func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayAddress string) *TimeBasedAuthSecretsManager { + return &TimeBasedAuthSecretsManager{ + mux: sync.Mutex{}, + updateManager: updateManager, + turnCfg: turnCfg, + relayAddr: relayAddress, + hmacToken: auth.NewTimedHMAC(turnCfg.Secret, turnCfg.CredentialsTTL.Duration), + cancelMap: make(map[string]chan struct{}), + } +} + +// Generate generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret +func (m *TimeBasedAuthSecretsManager) Generate() (*TURNRelayToken, error) { + token, err := m.hmacToken.GenerateToken() + if err != nil { + return nil, fmt.Errorf("failed to generate token: %s", err) + } + + return (*TURNRelayToken)(token), nil +} + +func (m *TimeBasedAuthSecretsManager) cancel(peerID string) { + if channel, ok := m.cancelMap[peerID]; ok { + close(channel) + delete(m.cancelMap, peerID) + } +} + +// CancelRefresh cancels scheduled peer credentials refresh +func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) { + m.mux.Lock() + defer m.mux.Unlock() + m.cancel(peerID) +} + +// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer. +// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone. +func (m *TimeBasedAuthSecretsManager) SetupRefresh(peerID string) { + m.mux.Lock() + defer m.mux.Unlock() + m.cancel(peerID) + cancel := make(chan struct{}, 1) + m.cancelMap[peerID] = cancel + log.Debugf("starting turn refresh for %s", peerID) + + go func() { + // we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL) + ticker := time.NewTicker(m.turnCfg.CredentialsTTL.Duration / 4 * 3) + defer ticker.Stop() + + for { + select { + case <-cancel: + log.Debugf("stopping turn refresh for %s", peerID) + return + case <-ticker.C: + m.pushNewTokens(peerID) + } + } + }() +} + +func (m *TimeBasedAuthSecretsManager) pushNewTokens(peerID string) { + token, err := m.hmacToken.GenerateToken() + if err != nil { + log.Errorf("failed to generate token for peer '%s': %s", peerID, err) + return + } + + var turns []*proto.ProtectedHostConfig + for _, host := range m.turnCfg.Turns { + turns = append(turns, &proto.ProtectedHostConfig{ + HostConfig: &proto.HostConfig{ + Uri: host.URI, + Protocol: ToResponseProto(host.Proto), + }, + User: token.Payload, + Password: token.Signature, + }) + } + + update := &proto.SyncResponse{ + WiretrusteeConfig: &proto.WiretrusteeConfig{ + Turns: turns, + Relay: &proto.RelayConfig{ + Urls: []string{m.relayAddr}, + TokenPayload: token.Payload, + TokenSignature: token.Signature, + }, + }, + } + log.Debugf("sending new TURN credentials to peer %s", peerID) + m.updateManager.SendUpdate(peerID, &UpdateMessage{Update: update}) +} diff --git a/management/server/turncredentials_test.go b/management/server/token_mgr_test.go similarity index 94% rename from management/server/turncredentials_test.go rename to management/server/token_mgr_test.go index 5066fdbe9..70314f4fc 100644 --- a/management/server/turncredentials_test.go +++ b/management/server/token_mgr_test.go @@ -26,18 +26,18 @@ func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) { CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }) + }, "") - credentials := tested.GenerateCredentials() + credentials, _ := tested.Generate() - if credentials.Username == "" { + if credentials.Payload == "" { t.Errorf("expected generated TURN username not to be empty, got empty") } - if credentials.Password == "" { + if credentials.Signature == "" { t.Errorf("expected generated TURN password not to be empty, got empty") } - validateMAC(t, credentials.Username, credentials.Password, []byte(secret)) + validateMAC(t, credentials.Payload, credentials.Signature, []byte(secret)) } @@ -52,7 +52,7 @@ func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) { CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }) + }, "") tested.SetupRefresh(peer) @@ -100,7 +100,7 @@ func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) { CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }) + }, "") tested.SetupRefresh(peer) if _, ok := tested.cancelMap[peer]; !ok { diff --git a/management/server/turncredentials.go b/management/server/turncredentials.go deleted file mode 100644 index aedcf2ee1..000000000 --- a/management/server/turncredentials.go +++ /dev/null @@ -1,125 +0,0 @@ -package server - -import ( - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "fmt" - "sync" - "time" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/management/proto" -) - -// TURNCredentialsManager used to manage TURN credentials -type TURNCredentialsManager interface { - GenerateCredentials() TURNCredentials - SetupRefresh(peerKey string) - CancelRefresh(peerKey string) -} - -// TimeBasedAuthSecretsManager generates credentials with TTL and using pre-shared secret known to TURN server -type TimeBasedAuthSecretsManager struct { - mux sync.Mutex - config *TURNConfig - updateManager *PeersUpdateManager - cancelMap map[string]chan struct{} -} - -type TURNCredentials struct { - Username string - Password string -} - -func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, config *TURNConfig) *TimeBasedAuthSecretsManager { - return &TimeBasedAuthSecretsManager{ - mux: sync.Mutex{}, - config: config, - updateManager: updateManager, - cancelMap: make(map[string]chan struct{}), - } -} - -// GenerateCredentials generates new time-based secret credentials - basically username is a unix timestamp and password is a HMAC hash of a timestamp with a preshared TURN secret -func (m *TimeBasedAuthSecretsManager) GenerateCredentials() TURNCredentials { - mac := hmac.New(sha1.New, []byte(m.config.Secret)) - - timeAuth := time.Now().Add(m.config.CredentialsTTL.Duration).Unix() - - username := fmt.Sprint(timeAuth) - - _, err := mac.Write([]byte(username)) - if err != nil { - log.Errorln("Generating turn password failed with error: ", err) - } - - bytePassword := mac.Sum(nil) - password := base64.StdEncoding.EncodeToString(bytePassword) - - return TURNCredentials{ - Username: username, - Password: password, - } - -} - -func (m *TimeBasedAuthSecretsManager) cancel(peerID string) { - if channel, ok := m.cancelMap[peerID]; ok { - close(channel) - delete(m.cancelMap, peerID) - } -} - -// CancelRefresh cancels scheduled peer credentials refresh -func (m *TimeBasedAuthSecretsManager) CancelRefresh(peerID string) { - m.mux.Lock() - defer m.mux.Unlock() - m.cancel(peerID) -} - -// SetupRefresh starts peer credentials refresh. Since credentials are expiring (TTL) it is necessary to always generate them and send to the peer. -// A goroutine is created and put into TimeBasedAuthSecretsManager.cancelMap. This routine should be cancelled if peer is gone. -func (m *TimeBasedAuthSecretsManager) SetupRefresh(peerID string) { - m.mux.Lock() - defer m.mux.Unlock() - m.cancel(peerID) - cancel := make(chan struct{}, 1) - m.cancelMap[peerID] = cancel - log.Debugf("starting turn refresh for %s", peerID) - - go func() { - // we don't want to regenerate credentials right on expiration, so we do it slightly before (at 3/4 of TTL) - ticker := time.NewTicker(m.config.CredentialsTTL.Duration / 4 * 3) - - for { - select { - case <-cancel: - log.Debugf("stopping turn refresh for %s", peerID) - return - case <-ticker.C: - c := m.GenerateCredentials() - var turns []*proto.ProtectedHostConfig - for _, host := range m.config.Turns { - turns = append(turns, &proto.ProtectedHostConfig{ - HostConfig: &proto.HostConfig{ - Uri: host.URI, - Protocol: ToResponseProto(host.Proto), - }, - User: c.Username, - Password: c.Password, - }) - } - - update := &proto.SyncResponse{ - WiretrusteeConfig: &proto.WiretrusteeConfig{ - Turns: turns, - }, - } - log.Debugf("sending new TURN credentials to peer %s", peerID) - m.updateManager.SendUpdate(peerID, &UpdateMessage{Update: update}) - } - } - }() -} diff --git a/relay/auth/allow_all.go b/relay/auth/allow_all.go new file mode 100644 index 000000000..653fd0801 --- /dev/null +++ b/relay/auth/allow_all.go @@ -0,0 +1,9 @@ +package auth + +// AllowAllAuth is a Validator that allows all connections. +type AllowAllAuth struct { +} + +func (a *AllowAllAuth) Validate(any) error { + return nil +} diff --git a/relay/auth/hmac/store.go b/relay/auth/hmac/store.go new file mode 100644 index 000000000..c9e8cc278 --- /dev/null +++ b/relay/auth/hmac/store.go @@ -0,0 +1,24 @@ +package hmac + +import ( + "sync" +) + +// Store is a simple in-memory store for token +// With this can update the token in thread safe way +type Store struct { + mu sync.Mutex + token Token +} + +func (a *Store) UpdateToken(token Token) { + a.mu.Lock() + defer a.mu.Unlock() + a.token = token +} + +func (a *Store) Token() ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + return marshalToken(a.token) +} diff --git a/relay/auth/hmac/token.go b/relay/auth/hmac/token.go new file mode 100644 index 000000000..e886bc7ae --- /dev/null +++ b/relay/auth/hmac/token.go @@ -0,0 +1,104 @@ +package hmac + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/gob" + "fmt" + "strconv" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +type Token struct { + Payload string + Signature string +} + +func marshalToken(token Token) ([]byte, error) { + buffer := bytes.NewBuffer([]byte{}) + encoder := gob.NewEncoder(buffer) + err := encoder.Encode(token) + if err != nil { + log.Errorf("failed to marshal token: %s", err) + return nil, err + } + return buffer.Bytes(), nil +} + +func unmarshalToken(payload []byte) (Token, error) { + var creds Token + buffer := bytes.NewBuffer(payload) + decoder := gob.NewDecoder(buffer) + err := decoder.Decode(&creds) + return creds, err +} + +// TimedHMAC generates token with TTL and using pre-shared secret known to TURN server +type TimedHMAC struct { + mux sync.Mutex + secret string + timeToLive time.Duration +} + +func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC { + return &TimedHMAC{ + secret: secret, + timeToLive: timeToLive, + } +} + +// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC hash of a timestamp with a preshared TURN secret +func (m *TimedHMAC) GenerateToken() (*Token, error) { + timeAuth := time.Now().Add(m.timeToLive).Unix() + timeStamp := fmt.Sprint(timeAuth) + + checksum, err := m.generate(timeStamp) + if err != nil { + return nil, err + } + + return &Token{ + Payload: timeStamp, + Signature: base64.StdEncoding.EncodeToString(checksum), + }, nil +} + +func (m *TimedHMAC) Validate(token Token) error { + expectedMAC, err := m.generate(token.Payload) + if err != nil { + return err + } + + expectedSignature := base64.StdEncoding.EncodeToString(expectedMAC) + + if !hmac.Equal([]byte(expectedSignature), []byte(token.Signature)) { + return fmt.Errorf("signature mismatch") + } + + timeAuthInt, err := strconv.ParseInt(token.Payload, 10, 64) + if err != nil { + return fmt.Errorf("invalid payload: %s", err) + } + + if time.Now().Unix() > timeAuthInt { + return fmt.Errorf("expired token") + } + + return nil +} + +func (m *TimedHMAC) generate(payload string) ([]byte, error) { + mac := hmac.New(sha1.New, []byte(m.secret)) + _, err := mac.Write([]byte(payload)) + if err != nil { + log.Errorf("failed to generate token: %s", err) + return nil, err + } + + return mac.Sum(nil), nil +} diff --git a/relay/auth/hmac/token_test.go b/relay/auth/hmac/token_test.go new file mode 100644 index 000000000..cbe36d5a7 --- /dev/null +++ b/relay/auth/hmac/token_test.go @@ -0,0 +1,103 @@ +package hmac + +import ( + "encoding/base64" + "strconv" + "testing" + "time" +) + +func TestGenerateCredentials(t *testing.T) { + secret := "secret" + timeToLive := 1 * time.Hour + v := NewTimedHMAC(secret, timeToLive) + + creds, err := v.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if creds.Payload == "" { + t.Fatalf("expected non-empty payload") + } + + _, err = strconv.ParseInt(creds.Payload, 10, 64) + if err != nil { + t.Fatalf("expected payload to be a valid unix timestamp, got %v", err) + } + + _, err = base64.StdEncoding.DecodeString(creds.Signature) + if err != nil { + t.Fatalf("expected signature to be base64 encoded, got %v", err) + } +} + +func TestValidateCredentials(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + manager := NewTimedHMAC(secret, timeToLive) + + // Test valid token + creds, err := manager.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err := manager.Validate(*creds); err != nil { + t.Fatalf("expected valid token: %s", err) + } +} + +func TestInvalidSignature(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + manager := NewTimedHMAC(secret, timeToLive) + + creds, err := manager.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + invalidCreds := &Token{ + Payload: creds.Payload, + Signature: "invalidsignature", + } + + if err = manager.Validate(*invalidCreds); err == nil { + t.Fatalf("expected invalid token due to signature mismatch") + } +} + +func TestExpired(t *testing.T) { + secret := "supersecret" + v := NewTimedHMAC(secret, -1*time.Hour) + expiredCreds, err := v.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err = v.Validate(*expiredCreds); err == nil { + t.Fatalf("expected invalid token due to expiration") + } +} + +func TestInvalidPayload(t *testing.T) { + secret := "supersecret" + timeToLive := 1 * time.Hour + v := NewTimedHMAC(secret, timeToLive) + + creds, err := v.GenerateToken() + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + // Test invalid payload + invalidPayloadCreds := &Token{ + Payload: "invalidtimestamp", + Signature: creds.Signature, + } + + if err = v.Validate(*invalidPayloadCreds); err == nil { + t.Fatalf("expected invalid token due to invalid payload") + } +} diff --git a/relay/auth/hmac/validator.go b/relay/auth/hmac/validator.go new file mode 100644 index 000000000..92669cce6 --- /dev/null +++ b/relay/auth/hmac/validator.go @@ -0,0 +1,27 @@ +package hmac + +import ( + log "github.com/sirupsen/logrus" + "time" +) + +type TimedHMACValidator struct { + *TimedHMAC +} + +func NewTimedHMACValidator(secret string, duration time.Duration) *TimedHMACValidator { + ta := NewTimedHMAC(secret, duration) + return &TimedHMACValidator{ + ta, + } +} + +func (a *TimedHMACValidator) Validate(credentials any) error { + b := credentials.([]byte) + c, err := unmarshalToken(b) + if err != nil { + log.Errorf("failed to unmarshal token: %s", err) + return err + } + return a.TimedHMAC.Validate(c) +} diff --git a/relay/auth/validator.go b/relay/auth/validator.go new file mode 100644 index 000000000..d76a90c74 --- /dev/null +++ b/relay/auth/validator.go @@ -0,0 +1,5 @@ +package auth + +type Validator interface { + Validate(any) error +} diff --git a/relay/client/client.go b/relay/client/client.go index 9fc5b84b8..1b2bf27c7 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -10,6 +10,7 @@ import ( log "github.com/sirupsen/logrus" + auth "github.com/netbirdio/netbird/relay/auth/hmac" "github.com/netbirdio/netbird/relay/client/dialer/ws" "github.com/netbirdio/netbird/relay/healthcheck" "github.com/netbirdio/netbird/relay/messages" @@ -98,6 +99,7 @@ type Client struct { log *log.Entry parentCtx context.Context connectionURL string + authStore *auth.Store hashedID []byte bufPool *sync.Pool @@ -115,12 +117,13 @@ type Client struct { } // NewClient creates a new client for the relay server. The client is not connected to the server until the Connect -func NewClient(ctx context.Context, serverURL, peerID string) *Client { +func NewClient(ctx context.Context, serverURL string, authStore *auth.Store, peerID string) *Client { hashedID, hashedStringId := messages.HashID(peerID) return &Client{ log: log.WithField("client_id", hashedStringId), parentCtx: ctx, connectionURL: serverURL, + authStore: authStore, hashedID: hashedID, bufPool: &sync.Pool{ New: func() any { @@ -234,7 +237,12 @@ func (c *Client) connect() error { } func (c *Client) handShake() error { - msg, err := messages.MarshalHelloMsg(c.hashedID) + t, err := c.authStore.Token() + if err != nil { + return err + } + + msg, err := messages.MarshalHelloMsg(c.hashedID, t) if err != nil { log.Errorf("failed to marshal hello message: %s", err) return err @@ -262,11 +270,11 @@ func (c *Client) handShake() error { return fmt.Errorf("unexpected message type") } - domain, err := messages.UnmarshalHelloResponse(buf[:n]) + ia, err := messages.UnmarshalHelloResponse(buf[:n]) if err != nil { return err } - c.instanceURL = domain + c.instanceURL = ia return nil } diff --git a/relay/client/manager.go b/relay/client/manager.go index 109d3b139..0e552754e 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -8,6 +8,8 @@ import ( "time" log "github.com/sirupsen/logrus" + + relayAuth "github.com/netbirdio/netbird/relay/auth/hmac" ) var ( @@ -35,9 +37,10 @@ func NewRelayTrack() *RelayTrack { // relay servers will be closed if there is no active connection. Periodically the manager will check if there is any // unused relay connection and close it. type Manager struct { - ctx context.Context - serverURL string - peerID string + ctx context.Context + serverURL string + peerID string + tokenStore *relayAuth.Store relayClient *Client reconnectGuard *Guard @@ -54,6 +57,7 @@ func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { ctx: ctx, serverURL: serverURL, peerID: peerID, + tokenStore: &relayAuth.Store{}, relayClients: make(map[string]*RelayTrack), onDisconnectedListeners: make(map[string]map[*func()]struct{}), } @@ -65,7 +69,7 @@ func (m *Manager) Serve() error { return fmt.Errorf("manager already serving") } - m.relayClient = NewClient(m.ctx, m.serverURL, m.peerID) + m.relayClient = NewClient(m.ctx, m.serverURL, m.tokenStore, m.peerID) err := m.relayClient.Connect() if err != nil { log.Errorf("failed to connect to relay server: %s", err) @@ -158,7 +162,7 @@ func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { m.relayClients[serverAddress] = rt m.relayClientsMutex.Unlock() - relayClient := NewClient(m.ctx, serverAddress, m.peerID) + relayClient := NewClient(m.ctx, serverAddress, m.tokenStore, m.peerID) err := relayClient.Connect() if err != nil { rt.Unlock() @@ -260,3 +264,7 @@ func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { m.listenerLock.Unlock() } + +func (m *Manager) UpdateToken(token relayAuth.Token) { + m.tokenStore.UpdateToken(token) +} diff --git a/relay/cmd/main.go b/relay/cmd/main.go index a7cd5efa8..7315bd005 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -24,6 +24,7 @@ var ( letsencryptDomains []string tlsCertFile string tlsKeyFile string + authSecret string rootCmd = &cobra.Command{ Use: "relay", @@ -41,7 +42,7 @@ func init() { rootCmd.PersistentFlags().StringArrayVarP(&letsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") rootCmd.PersistentFlags().StringVarP(&tlsCertFile, "tls-cert-file", "c", "", "") rootCmd.PersistentFlags().StringVarP(&tlsKeyFile, "tls-key-file", "k", "", "") - + rootCmd.PersistentFlags().StringVarP(&authSecret, "auth-secret", "s", "", "log level") } func waitForExitSignal() { @@ -56,6 +57,11 @@ func execute(cmd *cobra.Command, args []string) { os.Exit(1) } + if authSecret == "" { + log.Errorf("auth secret is required") + os.Exit(1) + } + srvListenerCfg := server.ListenerConfig{ Address: listenAddress, } @@ -76,7 +82,7 @@ func execute(cmd *cobra.Command, args []string) { } tlsSupport := srvListenerCfg.TLSConfig != nil - srv := server.NewServer(exposedAddress, tlsSupport) + srv := server.NewServer(exposedAddress, tlsSupport, authSecret) log.Infof("server will be available on: %s", srv.InstanceURL()) err := srv.Listen(srvListenerCfg) if err != nil { diff --git a/relay/messages/message.go b/relay/messages/message.go index 991bc36d0..dc2e9660c 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -15,8 +15,10 @@ const ( MsgTypeClose MsgType = 3 MsgTypeHealthCheck MsgType = 4 - headerSizeTransport = 1 + IDSize // 1 byte for msg type, IDSize for peerID - headerSizeHello = 1 + 4 + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID + sizeOfMsgType = 1 + sizeOfMagicBye = 4 + headerSizeTransport = sizeOfMsgType + IDSize // 1 byte for msg type, IDSize for peerID + headerSizeHello = sizeOfMsgType + sizeOfMagicBye + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID MaxHandshakeSize = 90 ) @@ -47,7 +49,7 @@ func (m MsgType) String() string { } type HelloResponse struct { - DomainAddress string + InstanceAddress string } func DetermineClientMsgType(msg []byte) (MsgType, error) { @@ -83,28 +85,29 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { } // MarshalHelloMsg initial hello message -func MarshalHelloMsg(peerID []byte) ([]byte, error) { +func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { if len(peerID) != IDSize { return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) } - msg := make([]byte, 5, headerSizeHello) + msg := make([]byte, 5, headerSizeHello+len(additions)) msg[0] = byte(MsgTypeHello) copy(msg[1:5], magicHeader) msg = append(msg, peerID...) + msg = append(msg, additions...) return msg, nil } -func UnmarshalHelloMsg(msg []byte) ([]byte, error) { +func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { if len(msg) < headerSizeHello { - return nil, fmt.Errorf("invalid 'hello' messge") + return nil, nil, fmt.Errorf("invalid 'hello' messge") } bytes.Equal(msg[1:5], magicHeader) - return msg[5:], nil + return msg[5:], msg[headerSizeHello:], nil } func MarshalHelloResponse(DomainAddress string) ([]byte, error) { payload := HelloResponse{ - DomainAddress: DomainAddress, + InstanceAddress: DomainAddress, } buf := new(bytes.Buffer) @@ -135,7 +138,7 @@ func UnmarshalHelloResponse(msg []byte) (string, error) { log.Errorf("failed to gob decode hello response: %s", err) return "", err } - return payload.DomainAddress, nil + return payload.InstanceAddress, nil } // Close message diff --git a/relay/messages/message_test.go b/relay/messages/message_test.go index d40963d8b..b0546a77c 100644 --- a/relay/messages/message_test.go +++ b/relay/messages/message_test.go @@ -6,12 +6,12 @@ import ( func TestMarshalHelloMsg(t *testing.T) { peerID := []byte("abdFAaBcawquEiCMzAabYosuUaGLtSNhKxz+") - bHello, err := MarshalHelloMsg(peerID) + bHello, err := MarshalHelloMsg(peerID, nil) if err != nil { t.Fatalf("error: %v", err) } - receivedPeerID, err := UnmarshalHelloMsg(bHello) + receivedPeerID, _, err := UnmarshalHelloMsg(bHello) if err != nil { t.Fatalf("error: %v", err) } diff --git a/relay/server/relay.go b/relay/server/relay.go index aff15a5fb..928c52322 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -8,20 +8,24 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/relay/auth" "github.com/netbirdio/netbird/relay/messages" ) type Relay struct { + validator auth.Validator + store *Store - instaceURL string // domain:port + instaceURL string closed bool closeMu sync.RWMutex } -func NewRelay(exposedAddress string, tlsSupport bool) *Relay { +func NewRelay(exposedAddress string, tlsSupport bool, validator auth.Validator) *Relay { r := &Relay{ - store: NewStore(), + validator: validator, + store: NewStore(), } if tlsSupport { @@ -29,6 +33,7 @@ func NewRelay(exposedAddress string, tlsSupport bool) *Relay { } else { r.instaceURL = fmt.Sprintf("rel://%s", exposedAddress) } + return r } @@ -94,12 +99,17 @@ func (r *Relay) handShake(conn net.Conn) ([]byte, error) { return nil, tErr } - peerID, err := messages.UnmarshalHelloMsg(buf[:n]) + peerID, authPayload, err := messages.UnmarshalHelloMsg(buf[:n]) if err != nil { log.Errorf("failed to handshake: %s", err) return nil, err } + if err := r.validator.Validate(authPayload); err != nil { + log.Errorf("failed to authenticate peer with id: %s, %s", peerID, err) + return nil, fmt.Errorf("failed to authenticate peer") + } + msg, _ := messages.MarshalHelloResponse(r.instaceURL) _, err = conn.Write(msg) if err != nil { diff --git a/relay/server/server.go b/relay/server/server.go index 5341cad2e..6ab994a6a 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -9,6 +9,7 @@ import ( log "github.com/sirupsen/logrus" + auth "github.com/netbirdio/netbird/relay/auth/hmac" "github.com/netbirdio/netbird/relay/server/listener" "github.com/netbirdio/netbird/relay/server/listener/udp" "github.com/netbirdio/netbird/relay/server/listener/ws" @@ -25,9 +26,12 @@ type Server struct { wSListener listener.Listener } -func NewServer(exposedAddress string, tlsSupport bool) *Server { +func NewServer(exposedAddress string, tlsSupport bool, authSecret string) *Server { return &Server{ - relay: NewRelay(exposedAddress, tlsSupport), + relay: NewRelay( + exposedAddress, + tlsSupport, + auth.NewTimedHMACValidator(authSecret, 24*time.Hour)), } } From 48a2f6e69da6316eb468b0655916c5fad37c2ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 16:12:08 +0200 Subject: [PATCH 083/199] Mock wginterface --- client/internal/engine.go | 2 +- client/internal/engine_test.go | 63 ++++++++---- client/internal/peer/conn.go | 13 ++- client/internal/routemanager/client.go | 4 +- client/internal/routemanager/manager.go | 4 +- .../routemanager/server_nonandroid.go | 4 +- .../routemanager/sysctl/sysctl_linux.go | 2 +- .../routemanager/systemops/systemops.go | 4 +- .../systemops/systemops_generic.go | 2 +- iface/iface.go | 20 ++++ iface/iface_moc.go | 98 +++++++++++++++++++ 11 files changed, 182 insertions(+), 34 deletions(-) create mode 100644 iface/iface_moc.go diff --git a/client/internal/engine.go b/client/internal/engine.go index ec513391a..fbd8ee6d8 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -140,7 +140,7 @@ type Engine struct { ctx context.Context cancel context.CancelFunc - wgInterface *iface.WGIface + wgInterface iface.IWGIface wgProxyFactory *wgproxy.Factory udpMux *bind.UniversalUDPMuxDefault diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 79b3cd498..b8a85b071 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -58,6 +58,12 @@ var ( } ) +func TestMain(m *testing.M) { + _ = util.InitLog("debug", "console") + code := m.Run() + os.Exit(code) +} + func TestEngine_SSH(t *testing.T) { // todo resolve test execution on freebsd if runtime.GOOS == "windows" || runtime.GOOS == "freebsd" { @@ -74,14 +80,22 @@ func TestEngine_SSH(t *testing.T) { defer cancel() relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ - WgIfaceName: "utun101", - WgAddr: "100.64.0.1/24", - WgPrivateKey: key, - WgPort: 33100, - ServerSSHAllowed: true, - }, - MobileDependency{}, peer.NewRecorder("https://mgm"), nil) + engine := NewEngine( + ctx, cancel, + &signal.MockClient{}, + &mgmt.MockClient{}, + relayMgr, + &EngineConfig{ + WgIfaceName: "utun101", + WgAddr: "100.64.0.1/24", + WgPrivateKey: key, + WgPort: 33100, + ServerSSHAllowed: true, + }, + MobileDependency{}, + peer.NewRecorder("https://mgm"), + nil, + ) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, @@ -211,20 +225,27 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { defer cancel() relayMgr := relayClient.NewManager(ctx, "", key.PublicKey().String()) - engine := NewEngine(ctx, cancel, &signal.MockClient{}, &mgmt.MockClient{}, relayMgr, &EngineConfig{ - WgIfaceName: "utun102", - WgAddr: "100.64.0.1/24", - WgPrivateKey: key, - WgPort: 33100, - }, MobileDependency{}, peer.NewRecorder("https://mgm"), nil) - newNet, err := stdnet.NewNet() - if err != nil { - t.Fatal(err) - } - engine.wgInterface, err = iface.NewWGIFace("utun102", "100.64.0.1/24", engine.config.WgPort, key.String(), iface.DefaultMTU, newNet, nil, nil) - if err != nil { - t.Fatal(err) + engine := NewEngine( + ctx, cancel, + &signal.MockClient{}, + &mgmt.MockClient{}, + relayMgr, + &EngineConfig{ + WgIfaceName: "utun102", + WgAddr: "100.64.0.1/24", + WgPrivateKey: key, + WgPort: 33100, + }, + MobileDependency{}, + peer.NewRecorder("https://mgm"), + nil) + + wgIface := &iface.MockWGIface{ + RemovePeerFunc: func(peerKey string) error { + return nil + }, } + engine.wgInterface = wgIface engine.routeManager = routemanager.NewManager(ctx, key.PublicKey().String(), time.Minute, engine.wgInterface, engine.statusRecorder, nil) engine.dnsServer = &dns.MockServer{ UpdateDNSServerFunc: func(serial uint64, update nbdns.Config) error { return nil }, diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 3901709ef..759b9e999 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -35,7 +35,7 @@ const ( type WgConfig struct { WgListenPort int RemoteKey string - WgInterface *iface.WGIface + WgInterface iface.IWGIface AllowedIps string PreSharedKey *wgtypes.Key } @@ -91,6 +91,7 @@ type Conn struct { statusRelay ConnStatus statusICE ConnStatus currentConnType ConnPriority + opened bool // this flag is used to prevent close in case of not opened connection workerICE *WorkerICE workerRelay *WorkerRelay @@ -167,6 +168,9 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu // be used. func (conn *Conn) Open() { conn.log.Debugf("open connection to peer") + conn.mu.Lock() + defer conn.mu.Unlock() + conn.opened = true peerState := State{ PubKey: conn.config.Key, @@ -197,6 +201,11 @@ func (conn *Conn) Close() { conn.ctxCancel() + if !conn.opened { + log.Infof("IGNORE close connection to peer") + return + } + if conn.wgProxyRelay != nil { err := conn.wgProxyRelay.CloseConn() if err != nil { @@ -302,7 +311,7 @@ func (conn *Conn) GetKey() string { } func (conn *Conn) reconnectLoop() { - ticker := time.NewTicker(conn.config.Timeout) // todo use the interval from config + ticker := time.NewTicker(conn.config.Timeout) if !conn.workerRelay.IsController() { ticker.Stop() } else { diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index 3c230df21..57ed9ed1e 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -42,7 +42,7 @@ type clientNetwork struct { ctx context.Context cancel context.CancelFunc statusRecorder *peer.Status - wgInterface *iface.WGIface + wgInterface iface.IWGIface routes map[route.ID]*route.Route routeUpdate chan routesUpdate peerStateUpdate chan struct{} @@ -52,7 +52,7 @@ type clientNetwork struct { updateSerial uint64 } -func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface *iface.WGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork { +func newClientNetworkWatcher(ctx context.Context, dnsRouteInterval time.Duration, wgInterface iface.IWGIface, statusRecorder *peer.Status, rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter) *clientNetwork { ctx, cancel := context.WithCancel(ctx) client := &clientNetwork{ diff --git a/client/internal/routemanager/manager.go b/client/internal/routemanager/manager.go index 0673ea6c3..2e1378414 100644 --- a/client/internal/routemanager/manager.go +++ b/client/internal/routemanager/manager.go @@ -48,7 +48,7 @@ type DefaultManager struct { serverRouter serverRouter sysOps *systemops.SysOps statusRecorder *peer.Status - wgInterface *iface.WGIface + wgInterface iface.IWGIface pubKey string notifier *notifier routeRefCounter *refcounter.RouteRefCounter @@ -60,7 +60,7 @@ func NewManager( ctx context.Context, pubKey string, dnsRouteInterval time.Duration, - wgInterface *iface.WGIface, + wgInterface iface.IWGIface, statusRecorder *peer.Status, initialRoutes []*route.Route, ) *DefaultManager { diff --git a/client/internal/routemanager/server_nonandroid.go b/client/internal/routemanager/server_nonandroid.go index 8470934c2..43a266cd2 100644 --- a/client/internal/routemanager/server_nonandroid.go +++ b/client/internal/routemanager/server_nonandroid.go @@ -22,11 +22,11 @@ type defaultServerRouter struct { ctx context.Context routes map[route.ID]*route.Route firewall firewall.Manager - wgInterface *iface.WGIface + wgInterface iface.IWGIface statusRecorder *peer.Status } -func newServerRouter(ctx context.Context, wgInterface *iface.WGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) { +func newServerRouter(ctx context.Context, wgInterface iface.IWGIface, firewall firewall.Manager, statusRecorder *peer.Status) (serverRouter, error) { return &defaultServerRouter{ ctx: ctx, routes: make(map[route.ID]*route.Route), diff --git a/client/internal/routemanager/sysctl/sysctl_linux.go b/client/internal/routemanager/sysctl/sysctl_linux.go index 3f2937c89..3e176860f 100644 --- a/client/internal/routemanager/sysctl/sysctl_linux.go +++ b/client/internal/routemanager/sysctl/sysctl_linux.go @@ -22,7 +22,7 @@ const ( ) // Setup configures sysctl settings for RP filtering and source validation. -func Setup(wgIface *iface.WGIface) (map[string]int, error) { +func Setup(wgIface iface.IWGIface) (map[string]int, error) { keys := map[string]int{} var result *multierror.Error diff --git a/client/internal/routemanager/systemops/systemops.go b/client/internal/routemanager/systemops/systemops.go index 9ee51538b..fa7ab0290 100644 --- a/client/internal/routemanager/systemops/systemops.go +++ b/client/internal/routemanager/systemops/systemops.go @@ -17,10 +17,10 @@ type ExclusionCounter = refcounter.Counter[any, Nexthop] type SysOps struct { refCounter *ExclusionCounter - wgInterface *iface.WGIface + wgInterface iface.IWGIface } -func NewSysOps(wgInterface *iface.WGIface) *SysOps { +func NewSysOps(wgInterface iface.IWGIface) *SysOps { return &SysOps{ wgInterface: wgInterface, } diff --git a/client/internal/routemanager/systemops/systemops_generic.go b/client/internal/routemanager/systemops/systemops_generic.go index 0d1c16ca1..0c152d233 100644 --- a/client/internal/routemanager/systemops/systemops_generic.go +++ b/client/internal/routemanager/systemops/systemops_generic.go @@ -122,7 +122,7 @@ func (r *SysOps) addRouteForCurrentDefaultGateway(prefix netip.Prefix) error { // addRouteToNonVPNIntf adds a new route to the routing table for the given prefix and returns the next hop and interface. // If the next hop or interface is pointing to the VPN interface, it will return the initial values. -func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf *iface.WGIface, initialNextHop Nexthop) (Nexthop, error) { +func (r *SysOps) addRouteToNonVPNIntf(prefix netip.Prefix, vpnIntf iface.IWGIface, initialNextHop Nexthop) (Nexthop, error) { addr := prefix.Addr() switch { case addr.IsLoopback(), diff --git a/iface/iface.go b/iface/iface.go index 928077a3d..448a47763 100644 --- a/iface/iface.go +++ b/iface/iface.go @@ -17,6 +17,26 @@ const ( DefaultWgPort = 51820 ) +type IWGIface interface { + Create() error + CreateOnAndroid(routeRange []string, ip string, domains []string) error + IsUserspaceBind() bool + Name() string + Address() WGAddress + ToInterface() *net.Interface + Up() (*bind.UniversalUDPMuxDefault, error) + UpdateAddr(newAddr string) error + UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeer(peerKey string) error + AddAllowedIP(peerKey string, allowedIP string) error + RemoveAllowedIP(peerKey string, allowedIP string) error + Close() error + SetFilter(filter PacketFilter) error + GetFilter() PacketFilter + GetDevice() *DeviceWrapper + GetStats(peerKey string) (WGStats, error) +} + // WGIface represents a interface instance type WGIface struct { tun wgTunDevice diff --git a/iface/iface_moc.go b/iface/iface_moc.go new file mode 100644 index 000000000..179c5fae4 --- /dev/null +++ b/iface/iface_moc.go @@ -0,0 +1,98 @@ +package iface + +import ( + "net" + "time" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/iface/bind" +) + +type MockWGIface struct { + IsUserspaceBindFunc func() bool + NameFunc func() string + AddressFunc func() WGAddress + ToInterfaceFunc func() *net.Interface + UpFunc func() (*bind.UniversalUDPMuxDefault, error) + UpdateAddrFunc func(newAddr string) error + UpdatePeerFunc func(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeerFunc func(peerKey string) error + AddAllowedIPFunc func(peerKey string, allowedIP string) error + RemoveAllowedIPFunc func(peerKey string, allowedIP string) error + CloseFunc func() error + SetFilterFunc func(filter PacketFilter) error + GetFilterFunc func() PacketFilter + GetDeviceFunc func() *DeviceWrapper + GetStatsFunc func(peerKey string) (WGStats, error) +} + +func (m *MockWGIface) Create() error { + //TODO implement me + panic("implement me") +} + +func (m *MockWGIface) CreateOnAndroid(routeRange []string, ip string, domains []string) error { + //TODO implement me + panic("implement me") +} + +func (m *MockWGIface) IsUserspaceBind() bool { + return m.IsUserspaceBindFunc() +} + +func (m *MockWGIface) Name() string { + return m.NameFunc() +} + +func (m *MockWGIface) Address() WGAddress { + return m.AddressFunc() +} + +func (m *MockWGIface) ToInterface() *net.Interface { + return m.ToInterfaceFunc() +} + +func (m *MockWGIface) Up() (*bind.UniversalUDPMuxDefault, error) { + return m.UpFunc() +} + +func (m *MockWGIface) UpdateAddr(newAddr string) error { + return m.UpdateAddrFunc(newAddr) +} + +func (m *MockWGIface) UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error { + return m.UpdatePeerFunc(peerKey, allowedIps, keepAlive, endpoint, preSharedKey) +} + +func (m *MockWGIface) RemovePeer(peerKey string) error { + return m.RemovePeerFunc(peerKey) +} + +func (m *MockWGIface) AddAllowedIP(peerKey string, allowedIP string) error { + return m.AddAllowedIPFunc(peerKey, allowedIP) +} + +func (m *MockWGIface) RemoveAllowedIP(peerKey string, allowedIP string) error { + return m.RemoveAllowedIPFunc(peerKey, allowedIP) +} + +func (m *MockWGIface) Close() error { + return m.CloseFunc() +} + +func (m *MockWGIface) SetFilter(filter PacketFilter) error { + return m.SetFilterFunc(filter) +} + +func (m *MockWGIface) GetFilter() PacketFilter { + return m.GetFilterFunc() +} + +func (m *MockWGIface) GetDevice() *DeviceWrapper { + return m.GetDeviceFunc() +} + +func (m *MockWGIface) GetStats(peerKey string) (WGStats, error) { + return m.GetStatsFunc(peerKey) +} From 75f5b75bc4a23f25a9269faba17b8078d9b7ec06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 16:15:04 +0200 Subject: [PATCH 084/199] Mock wginterface --- client/internal/routemanager/server_android.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/routemanager/server_android.go b/client/internal/routemanager/server_android.go index b4065bca6..2057b9cc8 100644 --- a/client/internal/routemanager/server_android.go +++ b/client/internal/routemanager/server_android.go @@ -11,6 +11,6 @@ import ( "github.com/netbirdio/netbird/iface" ) -func newServerRouter(context.Context, *iface.WGIface, firewall.Manager, *peer.Status) (serverRouter, error) { +func newServerRouter(context.Context, iface.IWGIface, firewall.Manager, *peer.Status) (serverRouter, error) { return nil, fmt.Errorf("server route not supported on this os") } From 1f949f8cee3d40c37db6d9e54109e5b8cd9bb69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 17:01:11 +0200 Subject: [PATCH 085/199] Fix parameters of tests --- relay/auth/hmac/store.go | 8 ++-- relay/client/client.go | 24 ++++++------ relay/client/client_test.go | 75 ++++++++++++++++++++---------------- relay/client/manager.go | 4 +- relay/client/manager_test.go | 14 +++---- relay/cmd/main.go | 6 ++- relay/server/server.go | 7 ++-- 7 files changed, 76 insertions(+), 62 deletions(-) diff --git a/relay/auth/hmac/store.go b/relay/auth/hmac/store.go index c9e8cc278..6c0541efc 100644 --- a/relay/auth/hmac/store.go +++ b/relay/auth/hmac/store.go @@ -4,20 +4,20 @@ import ( "sync" ) -// Store is a simple in-memory store for token +// TokenStore is a simple in-memory store for token // With this can update the token in thread safe way -type Store struct { +type TokenStore struct { mu sync.Mutex token Token } -func (a *Store) UpdateToken(token Token) { +func (a *TokenStore) UpdateToken(token Token) { a.mu.Lock() defer a.mu.Unlock() a.token = token } -func (a *Store) Token() ([]byte, error) { +func (a *TokenStore) Token() ([]byte, error) { a.mu.Lock() defer a.mu.Unlock() return marshalToken(a.token) diff --git a/relay/client/client.go b/relay/client/client.go index 1b2bf27c7..9b0539b92 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -96,11 +96,11 @@ func (cc *connContainer) close() { // the client can be reused by calling Connect again. When the client is closed, all connections are closed too. // While the Connect is in progress, the OpenConn function will block until the connection is established. type Client struct { - log *log.Entry - parentCtx context.Context - connectionURL string - authStore *auth.Store - hashedID []byte + log *log.Entry + parentCtx context.Context + connectionURL string + authTokenStore *auth.TokenStore + hashedID []byte bufPool *sync.Pool @@ -117,14 +117,14 @@ type Client struct { } // NewClient creates a new client for the relay server. The client is not connected to the server until the Connect -func NewClient(ctx context.Context, serverURL string, authStore *auth.Store, peerID string) *Client { +func NewClient(ctx context.Context, serverURL string, authTokenStore *auth.TokenStore, peerID string) *Client { hashedID, hashedStringId := messages.HashID(peerID) return &Client{ - log: log.WithField("client_id", hashedStringId), - parentCtx: ctx, - connectionURL: serverURL, - authStore: authStore, - hashedID: hashedID, + log: log.WithField("client_id", hashedStringId), + parentCtx: ctx, + connectionURL: serverURL, + authTokenStore: authTokenStore, + hashedID: hashedID, bufPool: &sync.Pool{ New: func() any { buf := make([]byte, bufferSize) @@ -237,7 +237,7 @@ func (c *Client) connect() error { } func (c *Client) handShake() error { - t, err := c.authStore.Token() + t, err := c.authTokenStore.Token() if err != nil { return err } diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 7b1ee5c62..739c745fb 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -9,11 +9,20 @@ import ( log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/auth/hmac" "github.com/netbirdio/netbird/util" "github.com/netbirdio/netbird/relay/server" ) +var ( + av = &auth.AllowAllAuth{} + hmacTokenStore = &hmac.TokenStore{} + serverListenAddr = "localhost:1234" + serverURL = "rel://localhost:1234" +) + func TestMain(m *testing.M) { _ = util.InitLog("trace", "console") code := m.Run() @@ -23,8 +32,8 @@ func TestMain(m *testing.M) { func TestClient(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -39,21 +48,21 @@ func TestClient(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, "alice") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientAlice.Close() - clientPlaceHolder := NewClient(ctx, srvCfg.Address, "clientPlaceHolder") + clientPlaceHolder := NewClient(ctx, serverURL, hmacTokenStore, "clientPlaceHolder") err = clientPlaceHolder.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } defer clientPlaceHolder.Close() - clientBob := NewClient(ctx, srvCfg.Address, "bob") + clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -91,8 +100,8 @@ func TestClient(t *testing.T) { func TestRegistration(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -100,7 +109,7 @@ func TestRegistration(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, "alice") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { _ = srv.Close() @@ -140,7 +149,7 @@ func TestRegistrationTimeout(t *testing.T) { _ = fakeTCPListener.Close() }(fakeTCPListener) - clientAlice := NewClient(ctx, "127.0.0.1:1234", "alice") + clientAlice := NewClient(ctx, "127.0.0.1:1234", hmacTokenStore, "alice") err = clientAlice.Connect() if err == nil { t.Errorf("failed to connect to server: %s", err) @@ -156,8 +165,8 @@ func TestEcho(t *testing.T) { ctx := context.Background() idAlice := "alice" idBob := "bob" - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -172,7 +181,7 @@ func TestEcho(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, idAlice) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice) err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -184,7 +193,7 @@ func TestEcho(t *testing.T) { } }() - clientBob := NewClient(ctx, srvCfg.Address, idBob) + clientBob := NewClient(ctx, serverURL, hmacTokenStore, idBob) err = clientBob.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -236,8 +245,8 @@ func TestEcho(t *testing.T) { func TestBindToUnavailabePeer(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -253,7 +262,7 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, "alice") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -273,8 +282,8 @@ func TestBindToUnavailabePeer(t *testing.T) { func TestBindReconnect(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -290,7 +299,7 @@ func TestBindReconnect(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, "alice") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -301,7 +310,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to bind channel: %s", err) } - clientBob := NewClient(ctx, srvCfg.Address, "bob") + clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") err = clientBob.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -318,7 +327,7 @@ func TestBindReconnect(t *testing.T) { t.Errorf("failed to close client: %s", err) } - clientAlice = NewClient(ctx, srvCfg.Address, "alice") + clientAlice = NewClient(ctx, serverURL, hmacTokenStore, "alice") err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -355,8 +364,8 @@ func TestBindReconnect(t *testing.T) { func TestCloseConn(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -372,7 +381,7 @@ func TestCloseConn(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, "alice") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) @@ -403,8 +412,8 @@ func TestCloseConn(t *testing.T) { func TestCloseRelayConn(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -419,7 +428,7 @@ func TestCloseRelayConn(t *testing.T) { } }() - clientAlice := NewClient(ctx, srvCfg.Address, "alice") + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -446,8 +455,8 @@ func TestCloseRelayConn(t *testing.T) { func TestCloseByServer(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv1 := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv1 := server.NewServer(serverURL, false, av) go func() { err := srv1.Listen(srvCfg) if err != nil { @@ -457,7 +466,7 @@ func TestCloseByServer(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") - relayClient := NewClient(ctx, srvCfg.Address, idAlice) + relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) err := relayClient.Connect() if err != nil { log.Fatalf("failed to connect to server: %s", err) @@ -489,8 +498,8 @@ func TestCloseByServer(t *testing.T) { func TestCloseByClient(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: "localhost:1234"} - srv := server.NewServer(srvCfg.Address, false) + srvCfg := server.ListenerConfig{Address: serverListenAddr} + srv := server.NewServer(serverURL, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { @@ -500,7 +509,7 @@ func TestCloseByClient(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") - relayClient := NewClient(ctx, srvCfg.Address, idAlice) + relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) err := relayClient.Connect() if err != nil { log.Fatalf("failed to connect to server: %s", err) diff --git a/relay/client/manager.go b/relay/client/manager.go index 0e552754e..d1a26d705 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -40,7 +40,7 @@ type Manager struct { ctx context.Context serverURL string peerID string - tokenStore *relayAuth.Store + tokenStore *relayAuth.TokenStore relayClient *Client reconnectGuard *Guard @@ -57,7 +57,7 @@ func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { ctx: ctx, serverURL: serverURL, peerID: peerID, - tokenStore: &relayAuth.Store{}, + tokenStore: &relayAuth.TokenStore{}, relayClients: make(map[string]*RelayTrack), onDisconnectedListeners: make(map[string]map[*func()]struct{}), } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 3192c1a09..be0b9ad99 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -16,7 +16,7 @@ func TestForeignConn(t *testing.T) { srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer(srvCfg1.Address, false) + srv1 := server.NewServer(srvCfg1.Address, false, av) go func() { err := srv1.Listen(srvCfg1) if err != nil { @@ -34,7 +34,7 @@ func TestForeignConn(t *testing.T) { srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer(srvCfg2.Address, false) + srv2 := server.NewServer(srvCfg2.Address, false, av) go func() { err := srv2.Listen(srvCfg2) if err != nil { @@ -107,7 +107,7 @@ func TestForeginConnClose(t *testing.T) { srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer(srvCfg1.Address, false) + srv1 := server.NewServer(srvCfg1.Address, false, av) go func() { err := srv1.Listen(srvCfg1) if err != nil { @@ -125,7 +125,7 @@ func TestForeginConnClose(t *testing.T) { srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer(srvCfg2.Address, false) + srv2 := server.NewServer(srvCfg2.Address, false, av) go func() { err := srv2.Listen(srvCfg2) if err != nil { @@ -164,7 +164,7 @@ func TestForeginAutoClose(t *testing.T) { srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer(srvCfg1.Address, false) + srv1 := server.NewServer(srvCfg1.Address, false, av) go func() { t.Log("binding server 1.") err := srv1.Listen(srvCfg1) @@ -185,7 +185,7 @@ func TestForeginAutoClose(t *testing.T) { srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer(srvCfg2.Address, false) + srv2 := server.NewServer(srvCfg2.Address, false, av) go func() { t.Log("binding server 2.") err := srv2.Listen(srvCfg2) @@ -237,7 +237,7 @@ func TestAutoReconnect(t *testing.T) { srvCfg := server.ListenerConfig{ Address: "localhost:1234", } - srv := server.NewServer(srvCfg.Address, false) + srv := server.NewServer(srvCfg.Address, false, av) go func() { err := srv.Listen(srvCfg) if err != nil { diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 7315bd005..5e454b000 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -6,11 +6,13 @@ import ( "os" "os/signal" "syscall" + "time" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/netbirdio/netbird/encryption" + auth "github.com/netbirdio/netbird/relay/auth/hmac" "github.com/netbirdio/netbird/relay/server" "github.com/netbirdio/netbird/util" ) @@ -82,7 +84,9 @@ func execute(cmd *cobra.Command, args []string) { } tlsSupport := srvListenerCfg.TLSConfig != nil - srv := server.NewServer(exposedAddress, tlsSupport, authSecret) + + authenticator := auth.NewTimedHMACValidator(authSecret, 24*time.Hour) + srv := server.NewServer(exposedAddress, tlsSupport, authenticator) log.Infof("server will be available on: %s", srv.InstanceURL()) err := srv.Listen(srvListenerCfg) if err != nil { diff --git a/relay/server/server.go b/relay/server/server.go index 6ab994a6a..cac884372 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -9,7 +9,7 @@ import ( log "github.com/sirupsen/logrus" - auth "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/auth" "github.com/netbirdio/netbird/relay/server/listener" "github.com/netbirdio/netbird/relay/server/listener/udp" "github.com/netbirdio/netbird/relay/server/listener/ws" @@ -26,12 +26,13 @@ type Server struct { wSListener listener.Listener } -func NewServer(exposedAddress string, tlsSupport bool, authSecret string) *Server { +func NewServer(exposedAddress string, tlsSupport bool, authValidator auth.Validator) *Server { return &Server{ relay: NewRelay( exposedAddress, tlsSupport, - auth.NewTimedHMACValidator(authSecret, 24*time.Hour)), + authValidator, + ), } } From f28a657a1d8798df533f61e498f4f697a86193a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 17:08:54 +0200 Subject: [PATCH 086/199] extend wginterface func with windows related things --- iface/iface.go | 20 -------------------- iface/iface_moc.go | 31 ++++++++++++++++--------------- iface/iwginterface.go | 32 ++++++++++++++++++++++++++++++++ iface/iwginterface_windows.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 35 deletions(-) create mode 100644 iface/iwginterface.go create mode 100644 iface/iwginterface_windows.go diff --git a/iface/iface.go b/iface/iface.go index 448a47763..928077a3d 100644 --- a/iface/iface.go +++ b/iface/iface.go @@ -17,26 +17,6 @@ const ( DefaultWgPort = 51820 ) -type IWGIface interface { - Create() error - CreateOnAndroid(routeRange []string, ip string, domains []string) error - IsUserspaceBind() bool - Name() string - Address() WGAddress - ToInterface() *net.Interface - Up() (*bind.UniversalUDPMuxDefault, error) - UpdateAddr(newAddr string) error - UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error - RemovePeer(peerKey string) error - AddAllowedIP(peerKey string, allowedIP string) error - RemoveAllowedIP(peerKey string, allowedIP string) error - Close() error - SetFilter(filter PacketFilter) error - GetFilter() PacketFilter - GetDevice() *DeviceWrapper - GetStats(peerKey string) (WGStats, error) -} - // WGIface represents a interface instance type WGIface struct { tun wgTunDevice diff --git a/iface/iface_moc.go b/iface/iface_moc.go index 179c5fae4..e56a69c69 100644 --- a/iface/iface_moc.go +++ b/iface/iface_moc.go @@ -10,21 +10,22 @@ import ( ) type MockWGIface struct { - IsUserspaceBindFunc func() bool - NameFunc func() string - AddressFunc func() WGAddress - ToInterfaceFunc func() *net.Interface - UpFunc func() (*bind.UniversalUDPMuxDefault, error) - UpdateAddrFunc func(newAddr string) error - UpdatePeerFunc func(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error - RemovePeerFunc func(peerKey string) error - AddAllowedIPFunc func(peerKey string, allowedIP string) error - RemoveAllowedIPFunc func(peerKey string, allowedIP string) error - CloseFunc func() error - SetFilterFunc func(filter PacketFilter) error - GetFilterFunc func() PacketFilter - GetDeviceFunc func() *DeviceWrapper - GetStatsFunc func(peerKey string) (WGStats, error) + IsUserspaceBindFunc func() bool + NameFunc func() string + AddressFunc func() WGAddress + ToInterfaceFunc func() *net.Interface + UpFunc func() (*bind.UniversalUDPMuxDefault, error) + UpdateAddrFunc func(newAddr string) error + UpdatePeerFunc func(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeerFunc func(peerKey string) error + AddAllowedIPFunc func(peerKey string, allowedIP string) error + RemoveAllowedIPFunc func(peerKey string, allowedIP string) error + CloseFunc func() error + SetFilterFunc func(filter PacketFilter) error + GetFilterFunc func() PacketFilter + GetDeviceFunc func() *DeviceWrapper + GetStatsFunc func(peerKey string) (WGStats, error) + GetInterfaceGUIDStringFunc func() (string, error) } func (m *MockWGIface) Create() error { diff --git a/iface/iwginterface.go b/iface/iwginterface.go new file mode 100644 index 000000000..501f51d2b --- /dev/null +++ b/iface/iwginterface.go @@ -0,0 +1,32 @@ +//go:build !windows + +package iface + +import ( + "net" + "time" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/iface/bind" +) + +type IWGIface interface { + Create() error + CreateOnAndroid(routeRange []string, ip string, domains []string) error + IsUserspaceBind() bool + Name() string + Address() WGAddress + ToInterface() *net.Interface + Up() (*bind.UniversalUDPMuxDefault, error) + UpdateAddr(newAddr string) error + UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeer(peerKey string) error + AddAllowedIP(peerKey string, allowedIP string) error + RemoveAllowedIP(peerKey string, allowedIP string) error + Close() error + SetFilter(filter PacketFilter) error + GetFilter() PacketFilter + GetDevice() *DeviceWrapper + GetStats(peerKey string) (WGStats, error) +} diff --git a/iface/iwginterface_windows.go b/iface/iwginterface_windows.go new file mode 100644 index 000000000..b5053474e --- /dev/null +++ b/iface/iwginterface_windows.go @@ -0,0 +1,31 @@ +package iface + +import ( + "net" + "time" + + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" + + "github.com/netbirdio/netbird/iface/bind" +) + +type IWGIface interface { + Create() error + CreateOnAndroid(routeRange []string, ip string, domains []string) error + IsUserspaceBind() bool + Name() string + Address() WGAddress + ToInterface() *net.Interface + Up() (*bind.UniversalUDPMuxDefault, error) + UpdateAddr(newAddr string) error + UpdatePeer(peerKey string, allowedIps string, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error + RemovePeer(peerKey string) error + AddAllowedIP(peerKey string, allowedIP string) error + RemoveAllowedIP(peerKey string, allowedIP string) error + Close() error + SetFilter(filter PacketFilter) error + GetFilter() PacketFilter + GetDevice() *DeviceWrapper + GetStats(peerKey string) (WGStats, error) + GetInterfaceGUIDString() (string, error) +} From 2803e1669be1227134c733061b86431f7368b668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 17:24:49 +0200 Subject: [PATCH 087/199] Remove meaningless tests --- client/internal/peer/conn_test.go | 49 -------------------------- relay/test/manager_test.go | 57 ------------------------------- 2 files changed, 106 deletions(-) delete mode 100644 relay/test/manager_test.go diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index 7caef01bf..59f249b82 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -8,12 +8,10 @@ import ( "time" "github.com/magiconair/properties/assert" - log "github.com/sirupsen/logrus" "github.com/netbirdio/netbird/client/internal/stdnet" "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" - relayClient "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/util" ) @@ -168,50 +166,3 @@ func TestConn_Status(t *testing.T) { }) } } - -func TestConn_Switch(t *testing.T) { - ctx := context.Background() - - wgProxyFactory := wgproxy.NewFactory(ctx, false, connConf.LocalWgPort) - connConfAlice := ConnConfig{ - Key: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - LocalKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - Timeout: time.Second, - LocalWgPort: 51820, - ICEConfig: ICEConfig{ - InterfaceBlackList: nil, - }, - WgConfig: WgConfig{ - WgListenPort: 51820, - RemoteKey: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - AllowedIps: "172.16.254.0/16", - }, - } - relayManagerAlice := relayClient.NewManager(ctx, "127.0.0.1:1234", connConf.LocalKey) - connAlice, err := NewConn(ctx, connConfAlice, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, relayManagerAlice) - if err != nil { - log.Fatalf("failed to create conn: %v", err) - } - connAlice.Open() - - connConfbob := ConnConfig{ - Key: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - LocalKey: "LLHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - Timeout: time.Second, - LocalWgPort: 51820, - ICEConfig: ICEConfig{ - InterfaceBlackList: nil, - }, - WgConfig: WgConfig{ - WgListenPort: 51820, - RemoteKey: "RRHf3Ma6z6mdLbriAJbqhX7+nM/B71lgw2+91q3LfhU=", - AllowedIps: "172.16.254.0/16", - }, - } - relayManagerBob := relayClient.NewManager(ctx, "127.0.0.1:1234", connConf.LocalKey) - connBob, err := NewConn(ctx, connConfbob, NewRecorder("https://mgm"), wgProxyFactory, nil, nil, relayManagerBob) - if err != nil { - log.Fatalf("failed to create conn: %v", err) - } - connBob.Open() -} diff --git a/relay/test/manager_test.go b/relay/test/manager_test.go deleted file mode 100644 index 09edeff84..000000000 --- a/relay/test/manager_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package test - -import ( - "context" - "testing" - "time" - - "github.com/netbirdio/netbird/relay/client" - "github.com/netbirdio/netbird/relay/server" -) - -func TestManager(t *testing.T) { - addr := "localhost:1239" - - srv := server.NewServer() - go func() { - err := srv.Listen(addr) - if err != nil { - t.Fatalf("failed to bind server: %s", err) - } - }() - - defer func() { - err := srv.Close() - if err != nil { - t.Errorf("failed to close server: %s", err) - } - }() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cm := client.NewManager(ctx, addr, "me") - cm.Serve() - - // wait for the relay handshake to complete - time.Sleep(1 * time.Second) - conn, err := cm.OpenConn("remotepeer") - if err != nil { - t.Errorf("failed to open connection: %s", err) - } - - readCtx, readCancel := context.WithCancel(context.Background()) - defer readCancel() - go func() { - _, _ = conn.Read(make([]byte, 1)) - readCancel() - }() - - cancel() - - select { - case <-time.After(2 * time.Second): - t.Errorf("client peer conn did not close automatically") - case <-readCtx.Done(): - // conn exited well - } -} From 931f165c9a48b20a06aeaf4130bdf0ff4f57621a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 17:38:23 +0200 Subject: [PATCH 088/199] Remove garbage --- relay/messages/id_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/relay/messages/id_test.go b/relay/messages/id_test.go index 25c64955b..a384e8662 100644 --- a/relay/messages/id_test.go +++ b/relay/messages/id_test.go @@ -1,10 +1,7 @@ package messages import ( - "encoding/binary" "testing" - - log "github.com/sirupsen/logrus" ) func TestHashID(t *testing.T) { @@ -13,14 +10,4 @@ func TestHashID(t *testing.T) { if enc != hashedStringId { t.Errorf("expected %s, got %s", hashedStringId, enc) } - - var magicHeader uint32 = 0x2112A442 // size 4 byte - - msg := make([]byte, 4) - binary.BigEndian.PutUint32(msg, magicHeader) - - magicHeader2 := []byte{0x21, 0x12, 0xA4, 0x42} - - log.Infof("msg: %v, %v", msg, magicHeader2) - } From a9e6742d9a5fe254602fceb0e3b75001b9daf1bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 8 Jul 2024 17:55:48 +0200 Subject: [PATCH 089/199] - Remove heartbeat logs - Fix relay client tests - Fix auth ID unmarshalling - Add magic header check --- relay/client/client.go | 1 - relay/client/client_test.go | 9 +++++++-- relay/messages/id.go | 2 +- relay/messages/id_test.go | 2 +- relay/messages/message.go | 8 ++++++-- relay/server/peer.go | 1 - 6 files changed, 15 insertions(+), 8 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 9b0539b92..9c8bd0365 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -308,7 +308,6 @@ func (c *Client) readLoop(relayConn net.Conn) { switch msgType { case messages.MsgTypeHealthCheck: - log.Debugf("on new heartbeat") msg := messages.MarshalHealthcheck() _, wErr := c.relayConn.Write(msg) if wErr != nil { diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 739c745fb..471667a71 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -32,10 +32,10 @@ func TestMain(m *testing.M) { func TestClient(t *testing.T) { ctx := context.Background() - srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) go func() { - err := srv.Listen(srvCfg) + listenCfg := server.ListenerConfig{Address: serverListenAddr} + err := srv.Listen(listenCfg) if err != nil { t.Fatalf("failed to bind server: %s", err) } @@ -48,6 +48,7 @@ func TestClient(t *testing.T) { } }() + t.Log("alice connecting to server") clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { @@ -55,6 +56,7 @@ func TestClient(t *testing.T) { } defer clientAlice.Close() + t.Log("placeholder connecting to server") clientPlaceHolder := NewClient(ctx, serverURL, hmacTokenStore, "clientPlaceHolder") err = clientPlaceHolder.Connect() if err != nil { @@ -62,6 +64,7 @@ func TestClient(t *testing.T) { } defer clientPlaceHolder.Close() + t.Log("Bob connecting to server") clientBob := NewClient(ctx, serverURL, hmacTokenStore, "bob") err = clientBob.Connect() if err != nil { @@ -69,11 +72,13 @@ func TestClient(t *testing.T) { } defer clientBob.Close() + t.Log("Alice open connection to Bob") connAliceToBob, err := clientAlice.OpenConn("bob") if err != nil { t.Fatalf("failed to bind channel: %s", err) } + t.Log("Bob open connection to Alice") connBobToAlice, err := clientBob.OpenConn("alice") if err != nil { t.Fatalf("failed to bind channel: %s", err) diff --git a/relay/messages/id.go b/relay/messages/id.go index dbb49cc58..d80c09414 100644 --- a/relay/messages/id.go +++ b/relay/messages/id.go @@ -8,7 +8,7 @@ import ( const ( prefixLength = 4 - IDSize = sha256.Size + 4 // 4 is equal with len(prefix) + IDSize = prefixLength + sha256.Size ) var ( diff --git a/relay/messages/id_test.go b/relay/messages/id_test.go index a384e8662..271a8f90d 100644 --- a/relay/messages/id_test.go +++ b/relay/messages/id_test.go @@ -5,7 +5,7 @@ import ( ) func TestHashID(t *testing.T) { - hashedID, hashedStringId := HashID("abc") + hashedID, hashedStringId := HashID("alice") enc := HashIDToString(hashedID) if enc != hashedStringId { t.Errorf("expected %s, got %s", hashedStringId, enc) diff --git a/relay/messages/message.go b/relay/messages/message.go index dc2e9660c..95b8ed7eb 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -89,6 +89,8 @@ func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { if len(peerID) != IDSize { return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) } + + // 5 = 1 byte for msg type, 4 byte for magic header msg := make([]byte, 5, headerSizeHello+len(additions)) msg[0] = byte(MsgTypeHello) copy(msg[1:5], magicHeader) @@ -101,8 +103,10 @@ func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { if len(msg) < headerSizeHello { return nil, nil, fmt.Errorf("invalid 'hello' messge") } - bytes.Equal(msg[1:5], magicHeader) - return msg[5:], msg[headerSizeHello:], nil + if !bytes.Equal(msg[1:5], magicHeader) { + return nil, nil, fmt.Errorf("invalid magic header") + } + return msg[5 : 5+IDSize], msg[headerSizeHello:], nil } func MarshalHelloResponse(DomainAddress string) ([]byte, error) { diff --git a/relay/server/peer.go b/relay/server/peer.go index 2869340a9..c49b8feca 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -147,7 +147,6 @@ func (p *Peer) healthcheck(ctx context.Context, hc *healthcheck.Sender) { for { select { case <-hc.HealthCheck: - p.log.Debugf("sending healthcheck message") _, err := p.Write(messages.MarshalHealthcheck()) if err != nil { p.log.Errorf("failed to send healthcheck message: %s", err) From d9dfae625b6eb12496aed68aaea291e733db3d40 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 21:18:19 +0200 Subject: [PATCH 090/199] Fix manager_test --- relay/client/manager_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index be0b9ad99..ca4416766 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -53,12 +53,12 @@ func TestForeignConn(t *testing.T) { log.Debugf("connect by alice") mCtx, cancel := context.WithCancel(ctx) defer cancel() - clientAlice := NewManager(mCtx, srvCfg1.Address, idAlice) + clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) clientAlice.Serve() idBob := "bob" log.Debugf("connect by bob") - clientBob := NewManager(mCtx, srvCfg2.Address, idBob) + clientBob := NewManager(mCtx, toURL(srvCfg2), idBob) clientBob.Serve() bobsSrvAddr, err := clientBob.RelayInstanceAddress() @@ -144,10 +144,10 @@ func TestForeginConnClose(t *testing.T) { log.Debugf("connect by alice") mCtx, cancel := context.WithCancel(ctx) defer cancel() - mgr := NewManager(mCtx, srvCfg1.Address, idAlice) + mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) mgr.Serve() - conn, err := mgr.OpenConn(srvCfg2.Address, "anotherpeer", nil) + conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -206,11 +206,11 @@ func TestForeginAutoClose(t *testing.T) { t.Log("connect to server 1.") mCtx, cancel := context.WithCancel(ctx) defer cancel() - mgr := NewManager(mCtx, srvCfg1.Address, idAlice) + mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) mgr.Serve() t.Log("open connection to another peer") - conn, err := mgr.OpenConn(srvCfg2.Address, "anotherpeer", nil) + conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -254,7 +254,7 @@ func TestAutoReconnect(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() - clientAlice := NewManager(mCtx, srvCfg.Address, "alice") + clientAlice := NewManager(mCtx, toURL(srvCfg), "alice") clientAlice.Serve() ra, err := clientAlice.RelayInstanceAddress() if err != nil { @@ -283,3 +283,7 @@ func TestAutoReconnect(t *testing.T) { t.Errorf("failed to open channel: %s", err) } } + +func toURL(address server.ListenerConfig) string { + return "rel://" + address.Address +} From cfac8c47623e1928849035992ce4e1dddf01dbcb Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 21:34:39 +0200 Subject: [PATCH 091/199] fix test timing --- relay/healthcheck/sender_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/relay/healthcheck/sender_test.go b/relay/healthcheck/sender_test.go index a571437ef..220bb8ac0 100644 --- a/relay/healthcheck/sender_test.go +++ b/relay/healthcheck/sender_test.go @@ -6,11 +6,14 @@ import ( "time" ) -func TestNewHealthPeriod(t *testing.T) { +func TestMain(m *testing.M) { // override the health check interval to speed up the test healthCheckInterval = 1 * time.Second healthCheckTimeout = 100 * time.Millisecond + m.Run() +} +func TestNewHealthPeriod(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() hc := NewSender(ctx) @@ -30,10 +33,6 @@ func TestNewHealthPeriod(t *testing.T) { } func TestNewHealthFailed(t *testing.T) { - // override the health check interval to speed up the test - healthCheckInterval = 1 * time.Second - healthCheckTimeout = 500 * time.Millisecond - ctx, cancel := context.WithCancel(context.Background()) defer cancel() hc := NewSender(ctx) @@ -46,18 +45,17 @@ func TestNewHealthFailed(t *testing.T) { } func TestNewHealthcheckStop(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) hc := NewSender(ctx) - time.Sleep(1 * time.Second) + time.Sleep(300 * time.Millisecond) cancel() select { case <-hc.HealthCheck: - t.Fatalf("is not closed") + t.Fatalf("health check on received") case <-hc.Timeout: - t.Fatalf("is not closed") + t.Fatalf("health check timedout") case <-ctx.Done(): // expected case <-time.After(1 * time.Second): From c3e8187a47263dce3bf3ffbc9440f6a13fde1064 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 21:53:20 +0200 Subject: [PATCH 092/199] Fix lint issues --- relay/auth/hmac/token.go | 2 -- relay/client/dialer/ws/ws.go | 3 ++- relay/client/manager.go | 2 +- relay/messages/id.go | 4 +++- relay/server/listener/quic/conn.go | 10 +++++----- relay/server/listener/quic/listener.go | 5 ++--- relay/server/listener/tcp/listener.go | 2 +- relay/server/listener/udp/conn.go | 24 ++++++++++++------------ relay/server/listener/udp/listener.go | 4 ++-- relay/server/relay.go | 2 +- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/relay/auth/hmac/token.go b/relay/auth/hmac/token.go index e886bc7ae..a5fa30a14 100644 --- a/relay/auth/hmac/token.go +++ b/relay/auth/hmac/token.go @@ -8,7 +8,6 @@ import ( "encoding/gob" "fmt" "strconv" - "sync" "time" log "github.com/sirupsen/logrus" @@ -40,7 +39,6 @@ func unmarshalToken(payload []byte) (Token, error) { // TimedHMAC generates token with TTL and using pre-shared secret known to TURN server type TimedHMAC struct { - mux sync.Mutex secret string timeToLive time.Duration } diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index 3a2bf48c8..d53ced58e 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -23,11 +23,12 @@ func Dial(address string) (net.Conn, error) { HTTPClient: httpClientNbDialer(), } - wsConn, _, err := websocket.Dial(context.Background(), wsURL, opts) + wsConn, resp, err := websocket.Dial(context.Background(), wsURL, opts) if err != nil { log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err) return nil, err } + _ = resp.Body.Close() conn := NewConn(wsConn) return conn, nil diff --git a/relay/client/manager.go b/relay/client/manager.go index d1a26d705..a71cdf2c3 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -18,7 +18,7 @@ var ( errRelayClientNotConnected = fmt.Errorf("relay client not connected") ) -// RelayTrack hold the relay clients for the foregin relay servers. +// RelayTrack hold the relay clients for the foreign relay servers. // With the mutex can ensure we can open new connection in case the relay connection has been established with // the relay server. type RelayTrack struct { diff --git a/relay/messages/id.go b/relay/messages/id.go index d80c09414..531eefa25 100644 --- a/relay/messages/id.go +++ b/relay/messages/id.go @@ -18,7 +18,9 @@ var ( func HashID(peerID string) ([]byte, string) { idHash := sha256.Sum256([]byte(peerID)) idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:]) - prefixedHash := append(prefix, idHash[:]...) + var prefixedHash []byte + prefixedHash = append(prefixedHash, prefix...) + prefixedHash = append(prefixedHash, idHash[:]...) return prefixedHash, idHashString } diff --git a/relay/server/listener/quic/conn.go b/relay/server/listener/quic/conn.go index 12d22bb5e..5414f9eee 100644 --- a/relay/server/listener/quic/conn.go +++ b/relay/server/listener/quic/conn.go @@ -7,19 +7,19 @@ import ( log "github.com/sirupsen/logrus" ) -type QuicConn struct { +type Conn struct { quic.Stream qConn quic.Connection } func NewConn(stream quic.Stream, qConn quic.Connection) net.Conn { - return &QuicConn{ + return &Conn{ Stream: stream, qConn: qConn, } } -func (q QuicConn) Write(b []byte) (n int, err error) { +func (q Conn) Write(b []byte) (n int, err error) { n, err = q.Stream.Write(b) if n != len(b) { log.Errorf("failed to write out the full message") @@ -27,10 +27,10 @@ func (q QuicConn) Write(b []byte) (n int, err error) { return } -func (q QuicConn) LocalAddr() net.Addr { +func (q Conn) LocalAddr() net.Addr { return q.qConn.LocalAddr() } -func (q QuicConn) RemoteAddr() net.Addr { +func (q Conn) RemoteAddr() net.Addr { return q.qConn.RemoteAddr() } diff --git a/relay/server/listener/quic/listener.go b/relay/server/listener/quic/listener.go index 8244bb2e2..55107e7ba 100644 --- a/relay/server/listener/quic/listener.go +++ b/relay/server/listener/quic/listener.go @@ -18,8 +18,7 @@ import ( ) type Listener struct { - address string - onAcceptFn func(conn net.Conn) + address string listener *quic.Listener quit chan struct{} @@ -88,7 +87,7 @@ func (l *Listener) acceptLoop(acceptFn func(conn net.Conn)) { // Setup a bare-bones TLS config for the server func generateTLSConfig() *tls.Config { - key, err := rsa.GenerateKey(rand.Reader, 1024) + key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { panic(err) } diff --git a/relay/server/listener/tcp/listener.go b/relay/server/listener/tcp/listener.go index bd96e3b30..fb26b0fc4 100644 --- a/relay/server/listener/tcp/listener.go +++ b/relay/server/listener/tcp/listener.go @@ -10,7 +10,7 @@ import ( ) // Listener -// Is it just demo code. It does not work in real life environment because the TCP is a streaming protocol, adn +// Is it just demo code. It does not work in real life environment because the TCP is a streaming protocol, and // it does not handle framing. type Listener struct { address string diff --git a/relay/server/listener/udp/conn.go b/relay/server/listener/udp/conn.go index 24b6d4640..46c7580d5 100644 --- a/relay/server/listener/udp/conn.go +++ b/relay/server/listener/udp/conn.go @@ -6,21 +6,21 @@ import ( "time" ) -type UDPConn struct { +type Conn struct { *net.UDPConn addr *net.UDPAddr msgChannel chan []byte } -func NewConn(conn *net.UDPConn, addr *net.UDPAddr) *UDPConn { - return &UDPConn{ +func NewConn(conn *net.UDPConn, addr *net.UDPAddr) *Conn { + return &Conn{ UDPConn: conn, addr: addr, msgChannel: make(chan []byte), } } -func (u *UDPConn) Read(b []byte) (n int, err error) { +func (u *Conn) Read(b []byte) (n int, err error) { msg, ok := <-u.msgChannel if !ok { return 0, io.EOF @@ -30,39 +30,39 @@ func (u *UDPConn) Read(b []byte) (n int, err error) { return n, nil } -func (u *UDPConn) Write(b []byte) (n int, err error) { +func (u *Conn) Write(b []byte) (n int, err error) { return u.UDPConn.WriteTo(b, u.addr) } -func (u *UDPConn) Close() error { +func (u *Conn) Close() error { //TODO implement me //panic("implement me") return nil } -func (u *UDPConn) LocalAddr() net.Addr { +func (u *Conn) LocalAddr() net.Addr { return u.UDPConn.LocalAddr() } -func (u *UDPConn) RemoteAddr() net.Addr { +func (u *Conn) RemoteAddr() net.Addr { return u.addr } -func (u *UDPConn) SetDeadline(t time.Time) error { +func (u *Conn) SetDeadline(t time.Time) error { //TODO implement me panic("implement me") } -func (u *UDPConn) SetReadDeadline(t time.Time) error { +func (u *Conn) SetReadDeadline(t time.Time) error { //TODO implement me panic("implement me") } -func (u *UDPConn) SetWriteDeadline(t time.Time) error { +func (u *Conn) SetWriteDeadline(t time.Time) error { //TODO implement me panic("implement me") } -func (u *UDPConn) onNewMsg(b []byte) { +func (u *Conn) onNewMsg(b []byte) { u.msgChannel <- b } diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go index 31b6098e5..a895cdc32 100644 --- a/relay/server/listener/udp/listener.go +++ b/relay/server/listener/udp/listener.go @@ -11,7 +11,7 @@ import ( type Listener struct { address string - conns map[string]*UDPConn + conns map[string]*Conn onAcceptFn func(conn net.Conn) listener *net.UDPConn @@ -24,7 +24,7 @@ type Listener struct { func NewListener(address string) listener.Listener { return &Listener{ address: address, - conns: make(map[string]*UDPConn), + conns: make(map[string]*Conn), } } diff --git a/relay/server/relay.go b/relay/server/relay.go index e3aefb06a..cb35c6e17 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -66,7 +66,7 @@ func (r *Relay) Accept(conn net.Conn) { } func (r *Relay) Close(ctx context.Context) { - log.Infof("closeing connection with all peers") + log.Infof("close connection with all peers") r.closeMu.Lock() wg := sync.WaitGroup{} peers := r.store.Peers() From 2d7e797e08700bfb0476a660bb350c97edd48e16 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 21:55:03 +0200 Subject: [PATCH 093/199] Fix body close --- relay/client/dialer/ws/ws.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index d53ced58e..ccdfe2d9c 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -28,8 +28,9 @@ func Dial(address string) (net.Conn, error) { log.Errorf("failed to dial to Relay server '%s': %s", wsURL, err) return nil, err } - _ = resp.Body.Close() - + if resp.Body != nil { + _ = resp.Body.Close() + } conn := NewConn(wsConn) return conn, nil } From dab50f35d791fa44d61e8411c163d65138e3f393 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 21:56:15 +0200 Subject: [PATCH 094/199] Fix ipv6 issue on tests --- relay/client/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 471667a71..c98524748 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -19,8 +19,8 @@ import ( var ( av = &auth.AllowAllAuth{} hmacTokenStore = &hmac.TokenStore{} - serverListenAddr = "localhost:1234" - serverURL = "rel://localhost:1234" + serverListenAddr = "127.0.0.1:1234" + serverURL = "rel://127.0.0.1:1234" ) func TestMain(m *testing.M) { From e4ec1fd757eabbfca1b1701ab0a54b4d5aaa7a91 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 22:13:31 +0200 Subject: [PATCH 095/199] Add sleep time after server started --- relay/client/client_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index c98524748..a14b3f6d0 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -48,6 +48,9 @@ func TestClient(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + t.Log("alice connecting to server") clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -114,6 +117,9 @@ func TestRegistration(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { @@ -186,6 +192,9 @@ func TestEcho(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice) err := clientAlice.Connect() if err != nil { @@ -267,6 +276,9 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { @@ -304,6 +316,9 @@ func TestBindReconnect(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { @@ -386,6 +401,9 @@ func TestCloseConn(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { @@ -433,6 +451,9 @@ func TestCloseRelayConn(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() if err != nil { @@ -469,6 +490,9 @@ func TestCloseByServer(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + idAlice := "alice" log.Debugf("connect by alice") relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) @@ -512,6 +536,9 @@ func TestCloseByClient(t *testing.T) { } }() + // wait for server to start + time.Sleep(300 * time.Millisecond) + idAlice := "alice" log.Debugf("connect by alice") relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) From 7bf0d04bedca10d46aa3531d55ff73cf460ad258 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 22:19:18 +0200 Subject: [PATCH 096/199] Remove unused function --- client/internal/engine.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index fbd8ee6d8..10c3a9edd 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -910,13 +910,6 @@ func (e *Engine) addNewPeer(peerConfig *mgmProto.RemotePeerConfig) error { return nil } -func (e *Engine) peerExists(peerKey string) bool { - e.syncMsgMux.Lock() - defer e.syncMsgMux.Unlock() - _, ok := e.peerConns[peerKey] - return ok -} - func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, error) { log.Debugf("creating peer connection %s", pubKey) From defdcb631ef3030c67a8c94304793b1bc4449503 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 22:42:30 +0200 Subject: [PATCH 097/199] Add sleep time for tests --- relay/client/manager_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index ca4416766..360cc62f5 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -202,6 +202,8 @@ func TestForeginAutoClose(t *testing.T) { t.Logf("server 2 closed.") }() + // wait for servers to start + time.Sleep(300 * time.Second) idAlice := "alice" t.Log("connect to server 1.") mCtx, cancel := context.WithCancel(ctx) From 63f2f51614b170fa64df704c5e5279c60f58d2ca Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Mon, 8 Jul 2024 23:14:09 +0200 Subject: [PATCH 098/199] Fix typo --- relay/client/manager_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 360cc62f5..e32170b92 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -145,8 +145,10 @@ func TestForeginConnClose(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) - mgr.Serve() - + err := mgr.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer", nil) if err != nil { t.Fatalf("failed to bind channel: %s", err) @@ -203,13 +205,16 @@ func TestForeginAutoClose(t *testing.T) { }() // wait for servers to start - time.Sleep(300 * time.Second) + time.Sleep(300 * time.Millisecond) idAlice := "alice" t.Log("connect to server 1.") mCtx, cancel := context.WithCancel(ctx) defer cancel() mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) - mgr.Serve() + err := mgr.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } t.Log("open connection to another peer") conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer", nil) From 5625d83c3fbda35b02488a6af55f56a3f89995cd Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 9 Jul 2024 09:44:23 +0200 Subject: [PATCH 099/199] Fix lint --- relay/client/client_test.go | 91 +++++++++++++++++++++++++----------- relay/client/manager_test.go | 67 ++++++++++++++++++++------ 2 files changed, 119 insertions(+), 39 deletions(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index a14b3f6d0..cfb62189f 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -33,11 +33,12 @@ func TestClient(t *testing.T) { ctx := context.Background() srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { listenCfg := server.ListenerConfig{Address: serverListenAddr} err := srv.Listen(listenCfg) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() @@ -49,8 +50,9 @@ func TestClient(t *testing.T) { }() // wait for server to start - time.Sleep(300 * time.Millisecond) - + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } t.Log("alice connecting to server") clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -110,15 +112,18 @@ func TestRegistration(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() // wait for server to start - time.Sleep(300 * time.Millisecond) + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -178,10 +183,11 @@ func TestEcho(t *testing.T) { idBob := "bob" srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() @@ -192,8 +198,10 @@ func TestEcho(t *testing.T) { } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice) err := clientAlice.Connect() @@ -261,10 +269,11 @@ func TestBindToUnavailabePeer(t *testing.T) { srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() @@ -276,8 +285,10 @@ func TestBindToUnavailabePeer(t *testing.T) { } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -301,10 +312,11 @@ func TestBindReconnect(t *testing.T) { srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Errorf("failed to bind server: %s", err) + errChan <- err } }() @@ -316,8 +328,10 @@ func TestBindReconnect(t *testing.T) { } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -386,10 +400,11 @@ func TestCloseConn(t *testing.T) { srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Errorf("failed to bind server: %s", err) + errChan <- err } }() @@ -401,8 +416,10 @@ func TestCloseConn(t *testing.T) { } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -437,10 +454,11 @@ func TestCloseRelayConn(t *testing.T) { srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Errorf("failed to bind server: %s", err) + errChan <- err } }() @@ -451,8 +469,10 @@ func TestCloseRelayConn(t *testing.T) { } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err := clientAlice.Connect() @@ -483,15 +503,19 @@ func TestCloseByServer(t *testing.T) { srvCfg := server.ListenerConfig{Address: serverListenAddr} srv1 := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) + go func() { err := srv1.Listen(srvCfg) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } idAlice := "alice" log.Debugf("connect by alice") @@ -529,15 +553,18 @@ func TestCloseByClient(t *testing.T) { srvCfg := server.ListenerConfig{Address: serverListenAddr} srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() - // wait for server to start - time.Sleep(300 * time.Millisecond) + // wait for servers to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } idAlice := "alice" log.Debugf("connect by alice") @@ -562,3 +589,15 @@ func TestCloseByClient(t *testing.T) { t.Fatalf("failed to close server: %s", err) } } + +func waitForServerToStart(errChan chan error) error { + select { + case err := <-errChan: + if err != nil { + return err + } + case <-time.After(300 * time.Second): + return nil + } + return nil +} diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index e32170b92..8da1ef20f 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -17,10 +17,11 @@ func TestForeignConn(t *testing.T) { Address: "localhost:1234", } srv1 := server.NewServer(srvCfg1.Address, false, av) + errChan := make(chan error, 1) go func() { err := srv1.Listen(srvCfg1) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() @@ -31,14 +32,19 @@ func TestForeignConn(t *testing.T) { } }() + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } srv2 := server.NewServer(srvCfg2.Address, false, av) + errChan2 := make(chan error, 1) go func() { err := srv2.Listen(srvCfg2) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan2 <- err } }() @@ -49,18 +55,27 @@ func TestForeignConn(t *testing.T) { } }() + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + idAlice := "alice" log.Debugf("connect by alice") mCtx, cancel := context.WithCancel(ctx) defer cancel() clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) - clientAlice.Serve() + err := clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } idBob := "bob" log.Debugf("connect by bob") clientBob := NewManager(mCtx, toURL(srvCfg2), idBob) - clientBob.Serve() - + err = clientBob.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } bobsSrvAddr, err := clientBob.RelayInstanceAddress() if err != nil { t.Fatalf("failed to get relay address: %s", err) @@ -108,10 +123,11 @@ func TestForeginConnClose(t *testing.T) { Address: "localhost:1234", } srv1 := server.NewServer(srvCfg1.Address, false, av) + errChan := make(chan error, 1) go func() { err := srv1.Listen(srvCfg1) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() @@ -122,14 +138,19 @@ func TestForeginConnClose(t *testing.T) { } }() + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } srv2 := server.NewServer(srvCfg2.Address, false, av) + errChan2 := make(chan error, 1) go func() { err := srv2.Listen(srvCfg2) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan2 <- err } }() @@ -140,6 +161,10 @@ func TestForeginConnClose(t *testing.T) { } }() + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + idAlice := "alice" log.Debugf("connect by alice") mCtx, cancel := context.WithCancel(ctx) @@ -167,11 +192,12 @@ func TestForeginAutoClose(t *testing.T) { Address: "localhost:1234", } srv1 := server.NewServer(srvCfg1.Address, false, av) + errChan := make(chan error, 1) go func() { t.Log("binding server 1.") err := srv1.Listen(srvCfg1) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan <- err } }() @@ -184,15 +210,20 @@ func TestForeginAutoClose(t *testing.T) { t.Logf("server 1. closed") }() + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } srv2 := server.NewServer(srvCfg2.Address, false, av) + errChan2 := make(chan error, 1) go func() { t.Log("binding server 2.") err := srv2.Listen(srvCfg2) if err != nil { - t.Fatalf("failed to bind server: %s", err) + errChan2 <- err } }() defer func() { @@ -204,8 +235,10 @@ func TestForeginAutoClose(t *testing.T) { t.Logf("server 2 closed.") }() - // wait for servers to start - time.Sleep(300 * time.Millisecond) + if err := waitForServerToStart(errChan2); err != nil { + t.Fatalf("failed to start server: %s", err) + } + idAlice := "alice" t.Log("connect to server 1.") mCtx, cancel := context.WithCancel(ctx) @@ -245,10 +278,11 @@ func TestAutoReconnect(t *testing.T) { Address: "localhost:1234", } srv := server.NewServer(srvCfg.Address, false, av) + errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) if err != nil { - t.Errorf("failed to bind server: %s", err) + errChan <- err } }() @@ -259,10 +293,17 @@ func TestAutoReconnect(t *testing.T) { } }() + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + mCtx, cancel := context.WithCancel(ctx) defer cancel() clientAlice := NewManager(mCtx, toURL(srvCfg), "alice") - clientAlice.Serve() + err := clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } ra, err := clientAlice.RelayInstanceAddress() if err != nil { t.Errorf("failed to get relay address: %s", err) From d4ff55e6feec51bfc6d89cace879bcbd4745fda6 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 9 Jul 2024 10:09:09 +0200 Subject: [PATCH 100/199] Fix typo --- relay/client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index cfb62189f..b9068ccd9 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -596,7 +596,7 @@ func waitForServerToStart(errChan chan error) error { if err != nil { return err } - case <-time.After(300 * time.Second): + case <-time.After(300 * time.Millisecond): return nil } return nil From 1065e0a6c571d9fc94021e8c9bcb7882742c11ba Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 9 Jul 2024 10:22:38 +0200 Subject: [PATCH 101/199] Fix moc interface --- iface/iface_moc.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/iface/iface_moc.go b/iface/iface_moc.go index e56a69c69..e0d9451fd 100644 --- a/iface/iface_moc.go +++ b/iface/iface_moc.go @@ -28,14 +28,16 @@ type MockWGIface struct { GetInterfaceGUIDStringFunc func() (string, error) } +func (m *MockWGIface) GetInterfaceGUIDString() (string, error) { + return m.GetInterfaceGUIDStringFunc() +} + func (m *MockWGIface) Create() error { - //TODO implement me - panic("implement me") + return m.Create() } func (m *MockWGIface) CreateOnAndroid(routeRange []string, ip string, domains []string) error { - //TODO implement me - panic("implement me") + return m.CreateOnAndroid(routeRange, ip, domains) } func (m *MockWGIface) IsUserspaceBind() bool { From 3bd15dd1c48a4363a3d7e6b1de19f3f9cf2efbd5 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 9 Jul 2024 10:34:13 +0200 Subject: [PATCH 102/199] Fix moc interface --- iface/iface_moc.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/iface/iface_moc.go b/iface/iface_moc.go index e0d9451fd..fab3054a0 100644 --- a/iface/iface_moc.go +++ b/iface/iface_moc.go @@ -10,6 +10,8 @@ import ( ) type MockWGIface struct { + CreateFunc func() error + CreateOnAndroidFunc func(routeRange []string, ip string, domains []string) error IsUserspaceBindFunc func() bool NameFunc func() string AddressFunc func() WGAddress @@ -33,11 +35,11 @@ func (m *MockWGIface) GetInterfaceGUIDString() (string, error) { } func (m *MockWGIface) Create() error { - return m.Create() + return m.CreateFunc() } func (m *MockWGIface) CreateOnAndroid(routeRange []string, ip string, domains []string) error { - return m.CreateOnAndroid(routeRange, ip, domains) + return m.CreateOnAndroidFunc(routeRange, ip, domains) } func (m *MockWGIface) IsUserspaceBind() bool { From 7ef191903e93621e100e55245b2682b1dcd0d494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:06:36 +0200 Subject: [PATCH 103/199] Fix logging in handshaker --- client/internal/peer/handshaker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 0ec615291..825d6a7e8 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -80,14 +80,14 @@ func (h *Handshaker) AddOnNewOfferListener(offer func(remoteOfferAnswer *OfferAn func (h *Handshaker) Listen() { for { - log.Debugf("wait for remote offer confirmation") + h.log.Debugf("wait for remote offer confirmation") remoteOfferAnswer, err := h.waitForRemoteOfferConfirmation() if err != nil { if _, ok := err.(*ConnectionClosedError); ok { - log.Tracef("stop handshaker") + h.log.Tracef("stop handshaker") return } - log.Errorf("failed to received remote offer confirmation: %s", err) + h.log.Errorf("failed to received remote offer confirmation: %s", err) continue } From 57b85f4f8dae611d346f6ca6059d5d08de49d8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:15:25 +0200 Subject: [PATCH 104/199] Optimisation for sonar --- relay/client/client.go | 101 ++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 9c8bd0365..35c338e34 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -297,7 +297,7 @@ func (c *Client) readLoop(relayConn net.Conn) { c.log.Debugf("failed to read message from relay server: %s", errExit) } c.mu.Unlock() - goto Exit + break } msgType, err := messages.DetermineServerMsgType(buf[:n]) @@ -306,55 +306,74 @@ func (c *Client) readLoop(relayConn net.Conn) { continue } - switch msgType { - case messages.MsgTypeHealthCheck: - msg := messages.MarshalHealthcheck() - _, wErr := c.relayConn.Write(msg) - if wErr != nil { - if c.serviceIsRunning && !internallyStoppedFlag.isSet() { - c.log.Errorf("failed to send heartbeat: %s", wErr) - } - } - hc.Heartbeat() - case messages.MsgTypeTransport: - peerID, payload, err := messages.UnmarshalTransportMsg(buf[:n]) - if err != nil { - if c.serviceIsRunning && !internallyStoppedFlag.isSet() { - c.log.Errorf("failed to parse transport message: %v", err) - } - continue - } - stringID := messages.HashIDToString(peerID) - - c.mu.Lock() - if !c.serviceIsRunning { - c.mu.Unlock() - goto Exit - } - container, ok := c.conns[stringID] - c.mu.Unlock() - if !ok { - c.log.Errorf("peer not found: %s", stringID) - continue - } - - container.writeMsg(Msg{ - bufPool: c.bufPool, - bufPtr: bufPtr, - Payload: payload}) - case messages.MsgTypeClose: - log.Debugf("relay connection close by server") - goto Exit + if !c.handleMsg(msgType, buf[:n], bufPtr, hc, internallyStoppedFlag) { + break } } -Exit: hc.Stop() c.notifyDisconnected() c.wgReadLoop.Done() _ = c.close(false) } +func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) { + switch msgType { + case messages.MsgTypeHealthCheck: + msg := messages.MarshalHealthcheck() + _, wErr := c.relayConn.Write(msg) + if wErr != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to send heartbeat: %s", wErr) + } + } + hc.Heartbeat() + c.bufPool.Put(bufPtr) + case messages.MsgTypeTransport: + return c.handleTrasnportMsg(buf, bufPtr, internallyStoppedFlag) + case messages.MsgTypeClose: + log.Debugf("relay connection close by server") + c.bufPool.Put(bufPtr) + return false + } + + return true +} + +func (c *Client) handleTrasnportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool { + peerID, payload, err := messages.UnmarshalTransportMsg(buf) + if err != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to parse transport message: %v", err) + } + + c.bufPool.Put(bufPtr) + return true + } + stringID := messages.HashIDToString(peerID) + + c.mu.Lock() + if !c.serviceIsRunning { + c.mu.Unlock() + c.bufPool.Put(bufPtr) + return false + } + container, ok := c.conns[stringID] + c.mu.Unlock() + if !ok { + c.log.Errorf("peer not found: %s", stringID) + c.bufPool.Put(bufPtr) + return true + } + msg := Msg{ + bufPool: c.bufPool, + bufPtr: bufPtr, + Payload: payload, + } + container.writeMsg(msg) + return true +} + // todo check by reference too, the id is not enought because the id come from the outer conn func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { c.mu.Lock() From d9d275a7ce128bd3d69099cd7a89d65f6d83d603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:27:20 +0200 Subject: [PATCH 105/199] Optimisation for sonar --- relay/client/client.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 35c338e34..1897e521b 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -320,15 +320,7 @@ func (c *Client) readLoop(relayConn net.Conn) { func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) { switch msgType { case messages.MsgTypeHealthCheck: - msg := messages.MarshalHealthcheck() - _, wErr := c.relayConn.Write(msg) - if wErr != nil { - if c.serviceIsRunning && !internallyStoppedFlag.isSet() { - c.log.Errorf("failed to send heartbeat: %s", wErr) - } - } - hc.Heartbeat() - c.bufPool.Put(bufPtr) + c.handleHealthCheck(buf, bufPtr, hc, internallyStoppedFlag) case messages.MsgTypeTransport: return c.handleTrasnportMsg(buf, bufPtr, internallyStoppedFlag) case messages.MsgTypeClose: @@ -340,6 +332,19 @@ func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, return true } +func (c *Client) handleHealthCheck(buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) { + c.handleHealthCheck(buf, bufPtr, hc, internallyStoppedFlag) + msg := messages.MarshalHealthcheck() + _, wErr := c.relayConn.Write(msg) + if wErr != nil { + if c.serviceIsRunning && !internallyStoppedFlag.isSet() { + c.log.Errorf("failed to send heartbeat: %s", wErr) + } + } + hc.Heartbeat() + c.bufPool.Put(bufPtr) +} + func (c *Client) handleTrasnportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool { peerID, payload, err := messages.UnmarshalTransportMsg(buf) if err != nil { From 7369f4bc388abc582277794e34915baab36a6fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:29:38 +0200 Subject: [PATCH 106/199] Optimisation for sonar --- relay/client/dialer/ws/conn.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go index 1632fcb45..fc07fc3f3 100644 --- a/relay/client/dialer/ws/conn.go +++ b/relay/client/dialer/ws/conn.go @@ -40,23 +40,23 @@ func (c *Conn) Write(b []byte) (n int, err error) { } func (c *Conn) RemoteAddr() net.Addr { - panic("not implemented") + panic("RemoteAddr is not implemented") } func (c *Conn) LocalAddr() net.Addr { - panic("not implemented") + panic("LocalAddr is not implemented") } func (c *Conn) SetReadDeadline(t time.Time) error { - return fmt.Errorf("not implemented") + return fmt.Errorf("SetReadDeadline is not implemented") } func (c *Conn) SetWriteDeadline(t time.Time) error { - return fmt.Errorf("not implemented") + return fmt.Errorf("SetWriteDeadline is not implemented") } func (c *Conn) SetDeadline(t time.Time) error { - return fmt.Errorf("not implemented") + return fmt.Errorf("SetDeadline is not implemented") } func (c *Conn) Close() error { From 19103031eec3545a95efa5c9ea39275ecd95fbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:38:50 +0200 Subject: [PATCH 107/199] Optimisation for sonar --- relay/client/client.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 1897e521b..efc3fbdae 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -320,9 +320,10 @@ func (c *Client) readLoop(relayConn net.Conn) { func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) (continueLoop bool) { switch msgType { case messages.MsgTypeHealthCheck: - c.handleHealthCheck(buf, bufPtr, hc, internallyStoppedFlag) + c.handleHealthCheck(hc, internallyStoppedFlag) + c.bufPool.Put(bufPtr) case messages.MsgTypeTransport: - return c.handleTrasnportMsg(buf, bufPtr, internallyStoppedFlag) + return c.handleTransportMsg(buf, bufPtr, internallyStoppedFlag) case messages.MsgTypeClose: log.Debugf("relay connection close by server") c.bufPool.Put(bufPtr) @@ -332,8 +333,7 @@ func (c *Client) handleMsg(msgType messages.MsgType, buf []byte, bufPtr *[]byte, return true } -func (c *Client) handleHealthCheck(buf []byte, bufPtr *[]byte, hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) { - c.handleHealthCheck(buf, bufPtr, hc, internallyStoppedFlag) +func (c *Client) handleHealthCheck(hc *healthcheck.Receiver, internallyStoppedFlag *internalStopFlag) { msg := messages.MarshalHealthcheck() _, wErr := c.relayConn.Write(msg) if wErr != nil { @@ -342,10 +342,9 @@ func (c *Client) handleHealthCheck(buf []byte, bufPtr *[]byte, hc *healthcheck.R } } hc.Heartbeat() - c.bufPool.Put(bufPtr) } -func (c *Client) handleTrasnportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool { +func (c *Client) handleTransportMsg(buf []byte, bufPtr *[]byte, internallyStoppedFlag *internalStopFlag) bool { peerID, payload, err := messages.UnmarshalTransportMsg(buf) if err != nil { if c.serviceIsRunning && !internallyStoppedFlag.isSet() { From 6cefcbfe5dc78f0fd93858f13078d317e5c46a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:44:12 +0200 Subject: [PATCH 108/199] Add comment --- relay/client/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/client/client.go b/relay/client/client.go index efc3fbdae..4105331e5 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -205,6 +205,7 @@ func (c *Client) SetOnDisconnectListener(fn func()) { c.onDisconnectListener = fn } +// HasConns returns true if there are connections. func (c *Client) HasConns() bool { c.mu.Lock() defer c.mu.Unlock() From 86f14523e4aaabd74215c703c8580794b06ac9ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:46:43 +0200 Subject: [PATCH 109/199] Add comment --- relay/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/client/client.go b/relay/client/client.go index 4105331e5..7846505bb 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -94,7 +94,7 @@ func (cc *connContainer) close() { // Client is a client for the relay server. It is responsible for establishing a connection to the relay server and // managing connections to other peers. All exported functions are safe to call concurrently. After close the connection, // the client can be reused by calling Connect again. When the client is closed, all connections are closed too. -// While the Connect is in progress, the OpenConn function will block until the connection is established. +// While the Connect is in progress, the OpenConn function will block until the connection is established with relay server. type Client struct { log *log.Entry parentCtx context.Context From f0c829afacaf121b65c0ba617e6ddefa9585c978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:48:50 +0200 Subject: [PATCH 110/199] Sonar fix --- relay/client/conn.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relay/client/conn.go b/relay/client/conn.go index 19f3c8df4..e6c32dfa9 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -53,15 +53,15 @@ func (c *Conn) RemoteAddr() net.Addr { func (c *Conn) SetDeadline(t time.Time) error { //TODO implement me - panic("implement me") + panic("SetDeadline is not implemented") } func (c *Conn) SetReadDeadline(t time.Time) error { //TODO implement me - panic("implement me") + panic("SetReadDeadline is not implemented") } func (c *Conn) SetWriteDeadline(t time.Time) error { //TODO implement me - panic("implement me") + panic("SetReadDeadline is not implemented") } From 6553d8ce0338583579f34b95ff5ac69fd7a08d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:50:29 +0200 Subject: [PATCH 111/199] Sonar fix --- relay/server/peer.go | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/relay/server/peer.go b/relay/server/peer.go index c49b8feca..6509f7bd3 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -65,26 +65,7 @@ func (p *Peer) Work() { case messages.MsgTypeHealthCheck: hc.OnHCResponse() case messages.MsgTypeTransport: - peerID, err := messages.UnmarshalTransportID(msg) - if err != nil { - p.log.Errorf("failed to unmarshal transport message: %s", err) - continue - } - stringPeerID := messages.HashIDToString(peerID) - dp, ok := p.store.Peer(stringPeerID) - if !ok { - p.log.Errorf("peer not found: %s", stringPeerID) - continue - } - err = messages.UpdateTransportMsg(msg, p.idB) - if err != nil { - p.log.Errorf("failed to update transport message: %s", err) - continue - } - _, err = dp.Write(msg) - if err != nil { - p.log.Errorf("failed to write transport message to: %s", dp.String()) - } + p.handleTransportMsg(msg) case messages.MsgTypeClose: p.log.Infof("peer exited gracefully") _ = p.conn.Close() @@ -161,3 +142,26 @@ func (p *Peer) healthcheck(ctx context.Context, hc *healthcheck.Sender) { } } } + +func (p *Peer) handleTransportMsg(msg []byte) { + peerID, err := messages.UnmarshalTransportID(msg) + if err != nil { + p.log.Errorf("failed to unmarshal transport message: %s", err) + return + } + stringPeerID := messages.HashIDToString(peerID) + dp, ok := p.store.Peer(stringPeerID) + if !ok { + p.log.Errorf("peer not found: %s", stringPeerID) + return + } + err = messages.UpdateTransportMsg(msg, p.idB) + if err != nil { + p.log.Errorf("failed to update transport message: %s", err) + return + } + _, err = dp.Write(msg) + if err != nil { + p.log.Errorf("failed to write transport message to: %s", dp.String()) + } +} From 1f95467b02eef59c968d7c5f4097f517982c52a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 9 Jul 2024 16:51:40 +0200 Subject: [PATCH 112/199] Sonar fix --- relay/server/listener/ws/conn.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relay/server/listener/ws/conn.go b/relay/server/listener/ws/conn.go index c43546b96..c248963b9 100644 --- a/relay/server/listener/ws/conn.go +++ b/relay/server/listener/ws/conn.go @@ -74,15 +74,15 @@ func (c *Conn) RemoteAddr() net.Addr { } func (c *Conn) SetReadDeadline(t time.Time) error { - return fmt.Errorf("not implemented") + return fmt.Errorf("SetReadDeadline is not implemented") } func (c *Conn) SetWriteDeadline(t time.Time) error { - return fmt.Errorf("not implemented") + return fmt.Errorf("SetWriteDeadline is not implemented") } func (c *Conn) SetDeadline(t time.Time) error { - return fmt.Errorf("not implemented") + return fmt.Errorf("SetDeadline is not implemented") } func (c *Conn) Close() error { From e0d086a8a80b95d82eb12cfabdf41783193bf833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 10:12:49 +0200 Subject: [PATCH 113/199] Implement dummy RemoteAddr on client conn --- relay/client/dialer/ws/addr.go | 12 ++++++++++++ relay/client/dialer/ws/conn.go | 4 ++-- relay/client/dialer/ws/ws.go | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 relay/client/dialer/ws/addr.go diff --git a/relay/client/dialer/ws/addr.go b/relay/client/dialer/ws/addr.go new file mode 100644 index 000000000..2bdffb864 --- /dev/null +++ b/relay/client/dialer/ws/addr.go @@ -0,0 +1,12 @@ +package ws + +type WebsocketAddr struct { +} + +func (a WebsocketAddr) Network() string { + return "websocket" +} + +func (a WebsocketAddr) String() string { + return "websocket/unknown-addr" +} diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go index fc07fc3f3..bc8675e8e 100644 --- a/relay/client/dialer/ws/conn.go +++ b/relay/client/dialer/ws/conn.go @@ -40,11 +40,11 @@ func (c *Conn) Write(b []byte) (n int, err error) { } func (c *Conn) RemoteAddr() net.Addr { - panic("RemoteAddr is not implemented") + return WebsocketAddr{} } func (c *Conn) LocalAddr() net.Addr { - panic("LocalAddr is not implemented") + return WebsocketAddr{} } func (c *Conn) SetReadDeadline(t time.Time) error { diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index ccdfe2d9c..19ec44a1e 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -31,6 +31,7 @@ func Dial(address string) (net.Conn, error) { if resp.Body != nil { _ = resp.Body.Close() } + conn := NewConn(wsConn) return conn, nil } From 820e2feec9a74e5c670720f5b562588cbad96c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 11:30:02 +0200 Subject: [PATCH 114/199] Move to relay address config to object Add test for mgm config parser --- management/cmd/management.go | 6 +++- management/cmd/management_test.go | 54 +++++++++++++++++++++++++++++++ management/server/config.go | 12 ++++--- management/server/grpcserver.go | 4 +-- management/server/token_mgr.go | 9 ++++-- 5 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 management/cmd/management_test.go diff --git a/management/cmd/management.go b/management/cmd/management.go index e15c9aa2c..ab8f9b805 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -195,7 +195,7 @@ var ( return fmt.Errorf("failed to build default manager: %v", err) } - turnRelayTokenManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.RelayAddress) + turnRelayTokenManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.RelayConfig) trustedPeers := config.ReverseProxy.TrustedPeers defaultTrustedPeers := []netip.Prefix{netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0")} @@ -538,6 +538,10 @@ func loadMgmtConfig(ctx context.Context, mgmtConfigPath string) (*server.Config, } } + if loadedConfig.RelayConfig != nil { + log.Infof("Relay address: %v", loadedConfig.RelayConfig.Address) + } + return loadedConfig, err } diff --git a/management/cmd/management_test.go b/management/cmd/management_test.go new file mode 100644 index 000000000..ae6ac978f --- /dev/null +++ b/management/cmd/management_test.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "context" + "os" + "testing" +) + +const ( + exampleConfig = `{ + "RelayConfig": { + "Address": "rels://relay.stage.npeer.io" + }, + "HttpConfig": { + "AuthAudience": "https://stageapp/", + "AuthIssuer": "https://something.eu.auth0.com/", + "OIDCConfigEndpoint": "https://something.eu.auth0.com/.well-known/openid-configuration" + } + }` +) + +func Test_loadMgmtConfig(t *testing.T) { + tmpFile, err := createConfig() + if err != nil { + t.Fatalf("failed to create config: %s", err) + } + + cfg, err := loadMgmtConfig(context.Background(), tmpFile) + if err != nil { + t.Fatalf("failed to load management config: %s", err) + } + if cfg.RelayConfig == nil { + t.Fatalf("config is nil") + } + if cfg.RelayConfig.Address == "" { + t.Fatalf("relay address is empty") + } +} + +func createConfig() (string, error) { + tmpfile, err := os.CreateTemp("", "config.json") + if err != nil { + return "", err + } + _, err = tmpfile.Write([]byte(exampleConfig)) + if err != nil { + return "", err + } + + if err := tmpfile.Close(); err != nil { + return "", err + } + return tmpfile.Name(), nil +} diff --git a/management/server/config.go b/management/server/config.go index cbc8a4e72..beba239e6 100644 --- a/management/server/config.go +++ b/management/server/config.go @@ -32,10 +32,10 @@ const ( // Config of the Management service type Config struct { - Stuns []*Host - TURNConfig *TURNConfig - RelayAddress string - Signal *Host + Stuns []*Host + TURNConfig *TURNConfig + RelayConfig *RelayConfig + Signal *Host Datadir string DataStoreEncryptionKey string @@ -72,6 +72,10 @@ type TURNConfig struct { Turns []*Host } +type RelayConfig struct { + Address string +} + // HttpServerConfig is a config of the HTTP Management service server type HttpServerConfig struct { LetsEncryptDomain string diff --git a/management/server/grpcserver.go b/management/server/grpcserver.go index 3abcd1ccd..0e42b9f20 100644 --- a/management/server/grpcserver.go +++ b/management/server/grpcserver.go @@ -518,9 +518,9 @@ func toWiretrusteeConfig(config *Config, turnCredentials *TURNRelayToken, relayT } var relayCfg *proto.RelayConfig - if config.RelayAddress != "" { + if config.RelayConfig != nil && config.RelayConfig.Address != "" { relayCfg = &proto.RelayConfig{ - Urls: []string{config.RelayAddress}, + Urls: []string{config.RelayConfig.Address}, } if relayToken != nil { diff --git a/management/server/token_mgr.go b/management/server/token_mgr.go index f5003004b..c84f815e1 100644 --- a/management/server/token_mgr.go +++ b/management/server/token_mgr.go @@ -31,12 +31,17 @@ type TimeBasedAuthSecretsManager struct { type TURNRelayToken auth.Token -func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayAddress string) *TimeBasedAuthSecretsManager { +func NewTimeBasedAuthSecretsManager(updateManager *PeersUpdateManager, turnCfg *TURNConfig, relayConfig *RelayConfig) *TimeBasedAuthSecretsManager { + + var relayAddr string + if relayConfig != nil { + relayAddr = relayConfig.Address + } return &TimeBasedAuthSecretsManager{ mux: sync.Mutex{}, updateManager: updateManager, turnCfg: turnCfg, - relayAddr: relayAddress, + relayAddr: relayAddr, hmacToken: auth.NewTimedHMAC(turnCfg.Secret, turnCfg.CredentialsTTL.Duration), cancelMap: make(map[string]chan struct{}), } From d1b6387803e8b127fab7f57cc344a43900cbb9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 13:21:50 +0200 Subject: [PATCH 115/199] Fix token sending --- client/internal/connect.go | 24 ++++++++++++++++-------- client/internal/engine.go | 2 +- relay/auth/hmac/store.go | 20 +++++++++++++++----- relay/client/client.go | 7 ++----- relay/client/manager.go | 2 +- relay/messages/message.go | 2 +- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index a20557f89..c8479e409 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -249,7 +249,9 @@ func (c *ConnectClient) run( relayURL, token := parseRelayInfo(loginResp) relayManager := relayClient.NewManager(engineCtx, relayURL, myPrivateKey.PublicKey().String()) if relayURL != "" { - relayManager.UpdateToken(token) + if token != nil { + relayManager.UpdateToken(token) + } if err = relayManager.Serve(); err != nil { log.Error(err) return wrapErr(err) @@ -311,15 +313,18 @@ func (c *ConnectClient) run( return nil } -func parseRelayInfo(resp *mgmProto.LoginResponse) (string, hmac.Token) { +func parseRelayInfo(resp *mgmProto.LoginResponse) (string, *hmac.Token) { // todo remove this - if ra := peer.ForcedRelayAddress(); ra != "" { - return ra, hmac.Token{} - } + ra := peer.ForcedRelayAddress() + /* + if ra := peer.ForcedRelayAddress(); ra != "" { + return ra, nil + } + */ msg := resp.GetWiretrusteeConfig().GetRelay() if msg == nil { - return "", hmac.Token{} + return "", nil } var url string @@ -327,11 +332,14 @@ func parseRelayInfo(resp *mgmProto.LoginResponse) (string, hmac.Token) { url = msg.GetUrls()[0] } - token := hmac.Token{ + token := &hmac.Token{ Payload: msg.GetTokenPayload(), Signature: msg.GetTokenSignature(), } - return url, token + + log.Tracef("Relay URL: %s", url) + + return ra, token } func (c *ConnectClient) Engine() *Engine { diff --git a/client/internal/engine.go b/client/internal/engine.go index 10c3a9edd..9da385949 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -502,7 +502,7 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error { relayMsg := wCfg.GetRelay() if relayMsg != nil { - c := auth.Token{ + c := &auth.Token{ Payload: relayMsg.GetTokenPayload(), Signature: relayMsg.GetTokenSignature(), } diff --git a/relay/auth/hmac/store.go b/relay/auth/hmac/store.go index 6c0541efc..cba5b4f30 100644 --- a/relay/auth/hmac/store.go +++ b/relay/auth/hmac/store.go @@ -2,23 +2,33 @@ package hmac import ( "sync" + + log "github.com/sirupsen/logrus" ) // TokenStore is a simple in-memory store for token // With this can update the token in thread safe way type TokenStore struct { mu sync.Mutex - token Token + token []byte } -func (a *TokenStore) UpdateToken(token Token) { +func (a *TokenStore) UpdateToken(token *Token) { a.mu.Lock() defer a.mu.Unlock() - a.token = token + if token == nil { + return + } + + t, err := marshalToken(*token) + if err != nil { + log.Errorf("failed to marshal token: %s", err) + } + a.token = t } -func (a *TokenStore) Token() ([]byte, error) { +func (a *TokenStore) TokenBinary() []byte { a.mu.Lock() defer a.mu.Unlock() - return marshalToken(a.token) + return a.token } diff --git a/relay/client/client.go b/relay/client/client.go index 7846505bb..630f49708 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -238,12 +238,9 @@ func (c *Client) connect() error { } func (c *Client) handShake() error { - t, err := c.authTokenStore.Token() - if err != nil { - return err - } + tb := c.authTokenStore.TokenBinary() - msg, err := messages.MarshalHelloMsg(c.hashedID, t) + msg, err := messages.MarshalHelloMsg(c.hashedID, tb) if err != nil { log.Errorf("failed to marshal hello message: %s", err) return err diff --git a/relay/client/manager.go b/relay/client/manager.go index a71cdf2c3..24d564833 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -265,6 +265,6 @@ func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { } -func (m *Manager) UpdateToken(token relayAuth.Token) { +func (m *Manager) UpdateToken(token *relayAuth.Token) { m.tokenStore.UpdateToken(token) } diff --git a/relay/messages/message.go b/relay/messages/message.go index 95b8ed7eb..564c82e03 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -20,7 +20,7 @@ const ( headerSizeTransport = sizeOfMsgType + IDSize // 1 byte for msg type, IDSize for peerID headerSizeHello = sizeOfMsgType + sizeOfMagicBye + IDSize // 1 byte for msg type, 4 byte for magic header, IDSize for peerID - MaxHandshakeSize = 90 + MaxHandshakeSize = 8192 ) var ( From 469be3442d80ed118c5e1ba38edc07c4b0fab937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 14:17:50 +0200 Subject: [PATCH 116/199] Remove hardcoded debug lines --- client/internal/connect.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index c8479e409..eb0fc0ae4 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -252,6 +252,7 @@ func (c *ConnectClient) run( if token != nil { relayManager.UpdateToken(token) } + log.Infof("connecting to the Relay service %s", relayURL) if err = relayManager.Serve(); err != nil { log.Error(err) return wrapErr(err) @@ -315,12 +316,9 @@ func (c *ConnectClient) run( func parseRelayInfo(resp *mgmProto.LoginResponse) (string, *hmac.Token) { // todo remove this - ra := peer.ForcedRelayAddress() - /* - if ra := peer.ForcedRelayAddress(); ra != "" { - return ra, nil - } - */ + if ra := peer.ForcedRelayAddress(); ra != "" { + return ra, nil + } msg := resp.GetWiretrusteeConfig().GetRelay() if msg == nil { @@ -337,9 +335,7 @@ func parseRelayInfo(resp *mgmProto.LoginResponse) (string, *hmac.Token) { Signature: msg.GetTokenSignature(), } - log.Tracef("Relay URL: %s", url) - - return ra, token + return url, token } func (c *ConnectClient) Engine() *Engine { From 7acaef1152f5df208d11d8de16687dff3dd334c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 16:51:38 +0200 Subject: [PATCH 117/199] Try to fix wgproxy reference --- client/internal/peer/conn.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 759b9e999..a543e1161 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -375,9 +375,13 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon if iceConnInfo.RelayedOnLocal { conn.log.Debugf("setup ice turn connection") wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - ep, err := conn.wgProxyICE.AddTurnConn(iceConnInfo.RemoteConn) + ep, err := wgProxy.AddTurnConn(iceConnInfo.RemoteConn) if err != nil { conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) + err = wgProxy.CloseConn() + if err != nil { + conn.log.Warnf("failed to close turn proxy connection: %v", err) + } return } endpoint = ep From 06afe64aff7c883918dcb3cecccc971838c4a57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 18:34:04 +0200 Subject: [PATCH 118/199] Fix deadlock --- client/internal/peer/worker_ice.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 75892ca52..0d4a6b59b 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -139,6 +139,7 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { agent, err := w.reCreateAgent(agentCancel, preferredCandidateTypes) if err != nil { w.log.Errorf("failed to recreate ICE Agent: %s", err) + w.muxAgent.Unlock() return } w.agent = agent From 4e75e15ea1c5c7d025f2d15689933638e4562c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 18:39:24 +0200 Subject: [PATCH 119/199] Add relay address to signal OFFER --- client/internal/engine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/client/internal/engine.go b/client/internal/engine.go index 9da385949..57a2acb14 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -1009,6 +1009,7 @@ func (e *Engine) receiveSignalEvents() { Version: msg.GetBody().GetNetBirdVersion(), RosenpassPubKey: rosenpassPubKey, RosenpassAddr: rosenpassAddr, + RelaySrvAddress: msg.GetBody().GetRelayServerAddress(), }) case sProto.Body_ANSWER: remoteCred, err := signal.UnMarshalCredential(msg) From 83b83ccfd2f86fb21f96884502619a824c02f084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 22:17:54 +0200 Subject: [PATCH 120/199] Add relay server address for the status --- client/cmd/up.go | 5 ++++- client/internal/connect.go | 1 + client/internal/peer/status.go | 32 +++++++++++++++++++++++++++++++- client/internal/relay/relay.go | 4 ++-- client/server/debug.go | 4 ++-- client/server/server.go | 2 +- relay/client/manager.go | 13 +++++++++---- 7 files changed, 50 insertions(+), 11 deletions(-) diff --git a/client/cmd/up.go b/client/cmd/up.go index f69e9eb27..56ddbb6ed 100644 --- a/client/cmd/up.go +++ b/client/cmd/up.go @@ -163,7 +163,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command) error { ctx, cancel = context.WithCancel(ctx) SetupCloseHandler(ctx, cancel) - connectClient := internal.NewConnectClient(ctx, config, peer.NewRecorder(config.ManagementURL.String())) + r := peer.NewRecorder(config.ManagementURL.String()) + r.GetFullStatus() + + connectClient := internal.NewConnectClient(ctx, config, r) return connectClient.Run() } diff --git a/client/internal/connect.go b/client/internal/connect.go index eb0fc0ae4..1dc515203 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -257,6 +257,7 @@ func (c *ConnectClient) run( log.Error(err) return wrapErr(err) } + c.statusRecorder.SetRelayMgr(relayManager) } peerConfig := loginResp.GetPeerConfig() diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index a7cfb95c4..ff673f197 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -13,6 +13,7 @@ import ( "github.com/netbirdio/netbird/client/internal/relay" "github.com/netbirdio/netbird/iface" "github.com/netbirdio/netbird/management/domain" + relayClient "github.com/netbirdio/netbird/relay/client" ) // State contains the latest state of a peer @@ -142,6 +143,8 @@ type Status struct { // Some Peer actions mostly used by in a batch when the network map has been synchronized. In these type of events // set to true this variable and at the end of the processing we will reset it by the FinishPeerListModifications() peerListChangedForNotification bool + + relayMgr *relayClient.Manager } // NewRecorder returns a new Status instance @@ -156,6 +159,12 @@ func NewRecorder(mgmAddress string) *Status { } } +func (d *Status) SetRelayMgr(manager *relayClient.Manager) { + d.mux.Lock() + defer d.mux.Unlock() + d.relayMgr = manager +} + // ReplaceOfflinePeers replaces func (d *Status) ReplaceOfflinePeers(replacement []State) { d.mux.Lock() @@ -503,7 +512,28 @@ func (d *Status) GetSignalState() SignalState { } func (d *Status) GetRelayStates() []relay.ProbeResult { - return d.relayStates + if d.relayMgr == nil { + return d.relayStates + } + + // extend the list of stun, turn servers with relay address + relaysState := make([]relay.ProbeResult, len(d.relayStates), len(d.relayStates)+1) + copy(relaysState, d.relayStates) + + relayState := relay.ProbeResult{} + + // if the server connection is not established then we will use the general address + // in case of connection we will use the instance specific address + instanceAddr, err := d.relayMgr.RelayInstanceAddress() + if err != nil { + relayState.URI = d.relayMgr.ServerURL() + relayState.Err = err + } else { + relayState.URI = instanceAddr + } + + relaysState = append(relaysState, relayState) + return relaysState } func (d *Status) GetDNSStates() []NSGroupState { diff --git a/client/internal/relay/relay.go b/client/internal/relay/relay.go index 4542a37fe..7d98a6060 100644 --- a/client/internal/relay/relay.go +++ b/client/internal/relay/relay.go @@ -17,7 +17,7 @@ import ( // ProbeResult holds the info about the result of a relay probe request type ProbeResult struct { - URI *stun.URI + URI string Err error Addr string } @@ -176,7 +176,7 @@ func ProbeAll( wg.Add(1) go func(res *ProbeResult, stunURI *stun.URI) { defer wg.Done() - res.URI = stunURI + res.URI = stunURI.String() res.Addr, res.Err = fn(ctx, stunURI) }(&results[i], uri) } diff --git a/client/server/debug.go b/client/server/debug.go index 9b6a52659..dcefb66ca 100644 --- a/client/server/debug.go +++ b/client/server/debug.go @@ -174,8 +174,8 @@ func seedFromStatus(a *anonymize.Anonymizer, status *peer.FullStatus) { } for _, relay := range status.Relays { - if relay.URI != nil { - a.AnonymizeURI(relay.URI.String()) + if relay.URI != "" { + a.AnonymizeURI(relay.URI) } } } diff --git a/client/server/server.go b/client/server/server.go index 2805c10f4..e1b4cc394 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -745,7 +745,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { for _, relayState := range fullStatus.Relays { pbRelayState := &proto.RelayState{ - URI: relayState.URI.String(), + URI: relayState.URI, Available: relayState.Err == nil, } if err := relayState.Err; err != nil { diff --git a/relay/client/manager.go b/relay/client/manager.go index 24d564833..ef4588f85 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -129,10 +129,19 @@ func (m *Manager) RelayInstanceAddress() (string, error) { return m.relayClient.ServerInstanceURL() } +// ServerURL returns the address of the permanent relay server. +func (m *Manager) ServerURL() string { + return m.serverURL +} + func (m *Manager) HasRelayAddress() bool { return m.serverURL != "" } +func (m *Manager) UpdateToken(token *relayAuth.Token) { + m.tokenStore.UpdateToken(token) +} + func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { // check if already has a connection to the desired relay server m.relayClientsMutex.RLock() @@ -264,7 +273,3 @@ func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { m.listenerLock.Unlock() } - -func (m *Manager) UpdateToken(token *relayAuth.Token) { - m.tokenStore.UpdateToken(token) -} From cb77ff466163988d2a994a5ac3e4bd0f5fe13cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 10 Jul 2024 22:33:15 +0200 Subject: [PATCH 121/199] Fix relay instance address indication --- relay/client/client.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 630f49708..8358c21f7 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -111,6 +111,7 @@ type Client struct { readLoopMutex sync.Mutex wgReadLoop sync.WaitGroup instanceURL string + muInstanceURL sync.Mutex onDisconnectListener func() listenerMutex sync.Mutex @@ -190,8 +191,8 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { // ServerInstanceURL returns the address of the relay server. It could change after the close and reopen the connection. func (c *Client) ServerInstanceURL() (string, error) { - c.mu.Lock() - defer c.mu.Unlock() + c.muInstanceURL.Lock() + defer c.muInstanceURL.Unlock() if c.instanceURL == "" { return "", fmt.Errorf("relay connection is not established") } @@ -272,7 +273,9 @@ func (c *Client) handShake() error { if err != nil { return err } + c.muInstanceURL.Lock() c.instanceURL = ia + c.muInstanceURL.Unlock() return nil } @@ -310,6 +313,11 @@ func (c *Client) readLoop(relayConn net.Conn) { } hc.Stop() + + c.muInstanceURL.Lock() + c.instanceURL = "" + c.muInstanceURL.Unlock() + c.notifyDisconnected() c.wgReadLoop.Done() _ = c.close(false) From ea93a5edd38d30ef0d5187839376b4575388f4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 11 Jul 2024 14:37:22 +0200 Subject: [PATCH 122/199] Add reconnect logic --- client/internal/peer/conn.go | 103 +++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 24 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index a543e1161..b6519d278 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/cenkalti/backoff/v4" "github.com/pion/ice/v3" log "github.com/sirupsen/logrus" "golang.zx2c4.com/wireguard/wgctrl/wgtypes" @@ -102,8 +103,9 @@ type Conn struct { endpointRelay *net.UDPAddr - iCEDisconnected chan struct{} - relayDisconnected chan struct{} + // for reconnection operations + iCEDisconnected chan bool + relayDisconnected chan bool } // NewConn creates a new not opened Conn to the remote peer. @@ -130,8 +132,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu allowedIPsIP: allowedIPsIP.String(), statusRelay: StatusDisconnected, statusICE: StatusDisconnected, - iCEDisconnected: make(chan struct{}), - relayDisconnected: make(chan struct{}), + iCEDisconnected: make(chan bool, 1), + relayDisconnected: make(chan bool, 1), } rFns := WorkerRelayCallbacks{ @@ -184,14 +186,18 @@ func (conn *Conn) Open() { conn.log.Warnf("error while updating the state err: %v", err) } - conn.waitRandomSleepTime() + conn.waitInitialRandomSleepTime() err = conn.doHandshake() if err != nil { conn.log.Errorf("failed to send offer: %v", err) } - go conn.reconnectLoop() + if conn.workerRelay.IsController() { + go conn.reconnectLoopWithRetry() + } else { + go conn.reconnectLoopForOnDisconnectedEvent() + } } // Close closes this peer Conn issuing a close event to the Conn closeCh @@ -310,30 +316,75 @@ func (conn *Conn) GetKey() string { return conn.config.Key } -func (conn *Conn) reconnectLoop() { - ticker := time.NewTicker(conn.config.Timeout) - if !conn.workerRelay.IsController() { - ticker.Stop() - } else { - defer ticker.Stop() +func (conn *Conn) reconnectLoopWithRetry() { + // wait for the initial connection to be established + select { + case <-conn.ctx.Done(): + case <-time.After(3 * time.Second): } + bo := backoff.WithContext(&backoff.ExponentialBackOff{ + InitialInterval: 800 * time.Millisecond, + RandomizationFactor: 0, + Multiplier: 1.7, + MaxInterval: conn.config.Timeout * time.Second, + MaxElapsedTime: 0, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + }, conn.ctx) + + ticker := backoff.NewTicker(bo) + defer ticker.Stop() + + no := time.Now() for { select { case <-ticker.C: // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer // todo check wg handshake + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s, elapsed time: %s", conn.statusRelay, conn.statusICE, time.Since(no)) + no = time.Now() + if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { continue } - case <-conn.relayDisconnected: - conn.log.Debugf("Relay connection is disconnected, start to send new offer") - ticker.Reset(10 * time.Second) - conn.waitRandomSleepTime() - case <-conn.iCEDisconnected: - conn.log.Debugf("ICE connection is disconnected, start to send new offer") - ticker.Reset(10 * time.Second) - conn.waitRandomSleepTime() + + log.Debugf("ticker timed out, retry to do handshake") + err := conn.doHandshake() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, reset reconnect timer") + bo.Reset() + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, reset reconnect timer") + bo.Reset() + case <-conn.ctx.Done(): + return + } + } +} + +func (conn *Conn) reconnectLoopForOnDisconnectedEvent() { + for { + select { + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, try to send new offer") + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, try to send new offer") case <-conn.ctx.Done(): return } @@ -438,10 +489,12 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { conn.log.Tracef("ICE connection state changed to %s", newState) defer func() { + + changed := conn.statusICE != newState && newState != StatusConnecting conn.statusICE = newState select { - case conn.iCEDisconnected <- struct{}{}: + case conn.iCEDisconnected <- changed: default: } }() @@ -542,10 +595,11 @@ func (conn *Conn) onWorkerRelayStateDisconnected() { conn.mu.Lock() defer conn.mu.Unlock() defer func() { + changed := conn.statusRelay != StatusDisconnected conn.statusRelay = StatusDisconnected select { - case conn.relayDisconnected <- struct{}{}: + case conn.relayDisconnected <- changed: default: } }() @@ -619,11 +673,12 @@ func (conn *Conn) doHandshake() error { if err == nil { ha.RelayAddr = addr } - conn.log.Tracef("send new offer: %#v", ha) + + conn.log.Tracef("do handshake with args: %v", ha) return conn.handshaker.SendOffer(ha) } -func (conn *Conn) waitRandomSleepTime() { +func (conn *Conn) waitInitialRandomSleepTime() { minWait := 500 maxWait := 2000 duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond From 30331e8f62d9597f3858c2ac7a26a6f2d6beb14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 11 Jul 2024 15:37:07 +0200 Subject: [PATCH 123/199] Change random wait time --- client/internal/peer/conn.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b6519d278..39c4b9af3 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -372,6 +372,10 @@ func (conn *Conn) reconnectLoopWithRetry() { } } +// reconnectLoopForOnDisconnectedEvent is used when the peer is not a controller and it should reconnect to the peer +// when the connection is lost. It will try to establish a connection only once time if before the connection was established +// It track separately the ice and relay connection status. Just because a lover priority connection reestablished it does not +// mean that to switch to it. We always force to use the higher priority connection. func (conn *Conn) reconnectLoopForOnDisconnectedEvent() { for { select { @@ -679,8 +683,8 @@ func (conn *Conn) doHandshake() error { } func (conn *Conn) waitInitialRandomSleepTime() { - minWait := 500 - maxWait := 2000 + minWait := 100 + maxWait := 800 duration := time.Duration(rand.Intn(maxWait-minWait)+minWait) * time.Millisecond timeout := time.NewTimer(duration) From b5c9af9e9c8ca97d948baa42722bb41ee76701b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 11 Jul 2024 15:37:34 +0200 Subject: [PATCH 124/199] Add comment --- client/internal/peer/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 39c4b9af3..2f755a043 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -317,7 +317,7 @@ func (conn *Conn) GetKey() string { } func (conn *Conn) reconnectLoopWithRetry() { - // wait for the initial connection to be established + // give chance to the peer to establish the initial connection select { case <-conn.ctx.Done(): case <-time.After(3 * time.Second): From 807b8306630bc96bfdb56bce3f08e599217ee4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 11 Jul 2024 15:46:07 +0200 Subject: [PATCH 125/199] Fix backoff ticker --- client/internal/peer/conn.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 2f755a043..470a322be 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -326,7 +326,7 @@ func (conn *Conn) reconnectLoopWithRetry() { bo := backoff.WithContext(&backoff.ExponentialBackOff{ InitialInterval: 800 * time.Millisecond, RandomizationFactor: 0, - Multiplier: 1.7, + Multiplier: 1.99, MaxInterval: conn.config.Timeout * time.Second, MaxElapsedTime: 0, Stop: backoff.Stop, @@ -336,10 +336,16 @@ func (conn *Conn) reconnectLoopWithRetry() { ticker := backoff.NewTicker(bo) defer ticker.Stop() + <-ticker.C // consume the initial tick what is happening right after the ticker has been created + no := time.Now() for { select { - case <-ticker.C: + case t := <-ticker.C: + if t.IsZero() { + // in case if the ticker has been canceled by context then avoid the temporary loop + return + } // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer // todo check wg handshake conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s, elapsed time: %s", conn.statusRelay, conn.statusICE, time.Since(no)) From 4ea55bfe3c6379ae652ee4b251db960a3aad7e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 16 Jul 2024 11:02:32 +0200 Subject: [PATCH 126/199] - Implement remote addr for conn - Eliminate cached offeranswer arguments - Fix exponent reset in conn reconnect loop - Fix on disconnected callback for permanent server - Add peer relay status for status details command --- client/cmd/status.go | 6 + client/internal/peer/conn.go | 265 ++++++++++---------- client/internal/peer/handshaker.go | 52 ++-- client/internal/peer/status.go | 160 +++++++++++- client/internal/wgproxy/proxy_ebpf.go | 2 +- client/proto/daemon.pb.go | 346 +++++++++++++------------- client/proto/daemon.proto | 1 + client/server/server.go | 1 + relay/client/addr.go | 13 + relay/client/client.go | 12 +- relay/client/conn.go | 6 +- relay/client/dialer/ws/addr.go | 3 +- relay/client/dialer/ws/conn.go | 12 +- relay/client/dialer/ws/ws.go | 2 +- relay/client/guard.go | 1 + relay/client/manager.go | 19 +- 16 files changed, 544 insertions(+), 357 deletions(-) create mode 100644 relay/client/addr.go diff --git a/client/cmd/status.go b/client/cmd/status.go index e6c7b8be8..f99d5674a 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -34,6 +34,7 @@ type peerStateDetailOutput struct { Direct bool `json:"direct" yaml:"direct"` IceCandidateType iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"` IceCandidateEndpoint iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"` + RelayAddress string `json:"relayAddress" yaml:"relayAddress"` LastWireguardHandshake time.Time `json:"lastWireguardHandshake" yaml:"lastWireguardHandshake"` TransferReceived int64 `json:"transferReceived" yaml:"transferReceived"` TransferSent int64 `json:"transferSent" yaml:"transferSent"` @@ -339,6 +340,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { remoteICE := "" localICEEndpoint := "" remoteICEEndpoint := "" + relayServerAddress := "" connType := "" peersConnected := 0 lastHandshake := time.Time{} @@ -360,6 +362,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { if pbPeerState.Relayed { connType = "Relayed" } + relayServerAddress = pbPeerState.GetRelayAddress() lastHandshake = pbPeerState.GetLastWireguardHandshake().AsTime().Local() transferReceived = pbPeerState.GetBytesRx() transferSent = pbPeerState.GetBytesTx() @@ -381,6 +384,7 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { Local: localICEEndpoint, Remote: remoteICEEndpoint, }, + RelayAddress: relayServerAddress, FQDN: pbPeerState.GetFqdn(), LastWireguardHandshake: lastHandshake, TransferReceived: transferReceived, @@ -644,6 +648,7 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo " Direct: %t\n"+ " ICE candidate (Local/Remote): %s/%s\n"+ " ICE candidate endpoints (Local/Remote): %s/%s\n"+ + " Relay server address: %s\n"+ " Last connection update: %s\n"+ " Last WireGuard handshake: %s\n"+ " Transfer status (received/sent) %s/%s\n"+ @@ -660,6 +665,7 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo remoteICE, localICEEndpoint, remoteICEEndpoint, + peerState.RelayAddress, timeAgo(peerState.LastStatusUpdate), timeAgo(peerState.LastWireguardHandshake), toIEC(peerState.TransferReceived), diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 470a322be..f2914b38e 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -146,7 +146,6 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu OnStatusChanged: conn.onWorkerICEStateDisconnected, } - conn.handshaker = NewHandshaker(ctx, connLog, config, signaler) conn.workerRelay = NewWorkerRelay(ctx, connLog, config, relayManager, rFns) relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() @@ -155,6 +154,8 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu return nil, err } + conn.handshaker = NewHandshaker(ctx, connLog, config, signaler, conn.workerICE, conn.workerRelay) + conn.handshaker.AddOnNewOfferListener(conn.workerRelay.OnNewOffer) if os.Getenv("NB_FORCE_RELAY") != "true" { conn.handshaker.AddOnNewOfferListener(conn.workerICE.OnNewOffer) @@ -188,7 +189,7 @@ func (conn *Conn) Open() { conn.waitInitialRandomSleepTime() - err = conn.doHandshake() + err = conn.handshaker.sendOffer() if err != nil { conn.log.Errorf("failed to send offer: %v", err) } @@ -323,57 +324,62 @@ func (conn *Conn) reconnectLoopWithRetry() { case <-time.After(3 * time.Second): } - bo := backoff.WithContext(&backoff.ExponentialBackOff{ - InitialInterval: 800 * time.Millisecond, - RandomizationFactor: 0, - Multiplier: 1.99, - MaxInterval: conn.config.Timeout * time.Second, - MaxElapsedTime: 0, - Stop: backoff.Stop, - Clock: backoff.SystemClock, - }, conn.ctx) - - ticker := backoff.NewTicker(bo) - defer ticker.Stop() - - <-ticker.C // consume the initial tick what is happening right after the ticker has been created - - no := time.Now() for { - select { - case t := <-ticker.C: - if t.IsZero() { - // in case if the ticker has been canceled by context then avoid the temporary loop + bo := backoff.WithContext(&backoff.ExponentialBackOff{ + InitialInterval: 800 * time.Millisecond, + RandomizationFactor: 0, // todo: add randomisation factor + Multiplier: 1.99, + MaxInterval: conn.config.Timeout * time.Second, + MaxElapsedTime: 0, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + }, conn.ctx) + + ticker := backoff.NewTicker(bo) + defer ticker.Stop() + + <-ticker.C // consume the initial tick what is happening right after the ticker has been created + + no := time.Now() + L: + for { + select { + case t := <-ticker.C: + if t.IsZero() { + // in case if the ticker has been canceled by context then avoid the temporary loop + return + } + // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer + // todo check wg handshake + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s, elapsed time: %s", conn.statusRelay, conn.statusICE, time.Since(no)) + no = time.Now() + + if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { + continue + } + + conn.log.Debugf("ticker timed out, retry to do handshake") + err := conn.handshaker.sendOffer() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, reset reconnect timer") + ticker.Stop() + break L + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, reset reconnect timer") + ticker.Stop() + break L + case <-conn.ctx.Done(): return } - // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer - // todo check wg handshake - conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s, elapsed time: %s", conn.statusRelay, conn.statusICE, time.Since(no)) - no = time.Now() - - if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { - continue - } - - log.Debugf("ticker timed out, retry to do handshake") - err := conn.doHandshake() - if err != nil { - conn.log.Errorf("failed to do handshake: %v", err) - } - case changed := <-conn.relayDisconnected: - if !changed { - continue - } - conn.log.Debugf("Relay state changed, reset reconnect timer") - bo.Reset() - case changed := <-conn.iCEDisconnected: - if !changed { - continue - } - conn.log.Debugf("ICE state changed, reset reconnect timer") - bo.Reset() - case <-conn.ctx.Done(): - return } } } @@ -399,7 +405,7 @@ func (conn *Conn) reconnectLoopForOnDisconnectedEvent() { return } - err := conn.doHandshake() + err := conn.handshaker.SendOffer() if err != nil { conn.log.Errorf("failed to do handshake: %v", err) } @@ -419,6 +425,8 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.statusICE = StatusConnected + defer conn.updateIceState(iceConnInfo) + if conn.currentConnType > priority { return } @@ -480,16 +488,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.currentConnType = priority - peerState := State{ - LocalIceCandidateType: iceConnInfo.LocalIceCandidateType, - RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, - LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, - RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, - Direct: iceConnInfo.Direct, - Relayed: iceConnInfo.Relayed, - } - - conn.updateStatus(peerState, iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) + conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) } // todo review to make sense to handle connection and disconnected status also? @@ -498,16 +497,6 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { defer conn.mu.Unlock() conn.log.Tracef("ICE connection state changed to %s", newState) - defer func() { - - changed := conn.statusICE != newState && newState != StatusConnecting - conn.statusICE = newState - - select { - case conn.iCEDisconnected <- changed: - default: - } - }() // switch back to relay connection if conn.endpointRelay != nil { @@ -516,26 +505,27 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { if err != nil { conn.log.Errorf("failed to switch to relay conn: %v", err) } - // todo update status to relay related things - return } - if conn.statusRelay == StatusConnected { - return + changed := conn.statusICE != newState && newState != StatusConnecting + conn.statusICE = newState + + select { + case conn.iCEDisconnected <- changed: + default: } - if conn.evalStatus() == newState { - return + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.evalStatus(), + Direct: false, // todo fix it + Relayed: true, // todo fix it + ConnStatusUpdate: time.Now(), } - if newState > conn.statusICE { - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: newState, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - _ = conn.statusRecorder.UpdatePeerState(peerState) + err := conn.statusRecorder.UpdatePeerICEStateToDisconnected(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's state, got error: %v", err) } } @@ -561,6 +551,8 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.endpointRelay = endpointUdpAddr conn.log.Debugf("conn resolved IP for %s: %s", endpoint, endpointUdpAddr.IP) + defer conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey) + if conn.currentConnType > connPriorityRelay { if conn.statusICE == StatusConnected { log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnType) @@ -592,27 +584,13 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.wgProxyRelay = wgProxy conn.currentConnType = connPriorityRelay - peerState := State{ - Direct: false, - Relayed: true, - } - conn.log.Infof("start to communicate with peer via relay") - conn.updateStatus(peerState, rci.rosenpassPubKey, rci.rosenpassAddr) + conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr) } func (conn *Conn) onWorkerRelayStateDisconnected() { conn.mu.Lock() defer conn.mu.Unlock() - defer func() { - changed := conn.statusRelay != StatusDisconnected - conn.statusRelay = StatusDisconnected - - select { - case conn.relayDisconnected <- changed: - default: - } - }() if conn.wgProxyRelay != nil { conn.endpointRelay = nil @@ -620,22 +598,25 @@ func (conn *Conn) onWorkerRelayStateDisconnected() { conn.wgProxyRelay = nil } - if conn.statusICE == StatusConnected { - return + changed := conn.statusRelay != StatusDisconnected + conn.statusRelay = StatusDisconnected + + select { + case conn.relayDisconnected <- changed: + default: } - if conn.evalStatus() == StatusDisconnected { - return + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.evalStatus(), + Direct: false, // todo fix it + Relayed: true, // todo fix it + ConnStatusUpdate: time.Now(), } - if StatusDisconnected > conn.statusRelay { - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: StatusDisconnected, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - _ = conn.statusRecorder.UpdatePeerState(peerState) + err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's state, got error: %v", err) } } @@ -648,18 +629,45 @@ func (conn *Conn) configureWGEndpoint(addr *net.UDPAddr) error { conn.config.WgConfig.PreSharedKey, ) } -func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { - peerState.PubKey = conn.config.Key - peerState.ConnStatus = StatusConnected - peerState.ConnStatusUpdate = time.Now() - peerState.RosenpassEnabled = isRosenpassEnabled(remoteRosenpassPubKey) - peerState.Mux = new(sync.RWMutex) - err := conn.statusRecorder.UpdatePeerState(peerState) +func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []byte) { + peerState := State{ + PubKey: conn.config.Key, + ConnStatusUpdate: time.Now(), + ConnStatus: conn.evalStatus(), // todo fix it + Direct: false, // todo fix it + Relayed: true, // todo fix it + RelayServerAddress: relayServerAddr, + RosenpassEnabled: isRosenpassEnabled(rosenpassPubKey), + } + + err := conn.statusRecorder.UpdatePeerRelayedState(peerState) if err != nil { conn.log.Warnf("unable to save peer's state, got error: %v", err) } +} +func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) { + peerState := State{ + PubKey: conn.config.Key, + ConnStatusUpdate: time.Now(), + ConnStatus: conn.evalStatus(), + Direct: iceConnInfo.Direct, // todo fix it + Relayed: iceConnInfo.Relayed, // todo fix it + LocalIceCandidateType: iceConnInfo.LocalIceCandidateType, + RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, + LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, + RemoteIceCandidateEndpoint: iceConnInfo.RemoteIceCandidateEndpoint, + RosenpassEnabled: isRosenpassEnabled(iceConnInfo.RosenpassPubKey), + } + + err := conn.statusRecorder.UpdatePeerICEState(peerState) + if err != nil { + conn.log.Warnf("unable to save peer's state, got error: %v", err) + } +} + +func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { if runtime.GOOS == "ios" { runtime.GC() } @@ -669,25 +677,6 @@ func (conn *Conn) updateStatus(peerState State, remoteRosenpassPubKey []byte, re } } -func (conn *Conn) doHandshake() error { - if !conn.signaler.Ready() { - return ErrSignalIsNotReady - } - - var ( - ha HandshakeArgs - err error - ) - ha.IceUFrag, ha.IcePwd = conn.workerICE.GetLocalUserCredentials() - addr, err := conn.workerRelay.RelayInstanceAddress() - if err == nil { - ha.RelayAddr = addr - } - - conn.log.Tracef("do handshake with args: %v", ha) - return conn.handshaker.SendOffer(ha) -} - func (conn *Conn) waitInitialRandomSleepTime() { minWait := 100 maxWait := 800 diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index 825d6a7e8..bff025cf8 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -41,34 +41,30 @@ type OfferAnswer struct { RelaySrvAddress string } -type HandshakeArgs struct { - IceUFrag string - IcePwd string - RelayAddr string -} - type Handshaker struct { mu sync.Mutex ctx context.Context log *log.Entry config ConnConfig signaler *Signaler + ice *WorkerICE + relay *WorkerRelay onNewOfferListeners []func(*OfferAnswer) // remoteOffersCh is a channel used to wait for remote credentials to proceed with the connection remoteOffersCh chan OfferAnswer // remoteAnswerCh is a channel used to wait for remote credentials answer (confirmation of our offer) to proceed with the connection remoteAnswerCh chan OfferAnswer - - lastOfferArgs HandshakeArgs } -func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler) *Handshaker { +func NewHandshaker(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay) *Handshaker { return &Handshaker{ ctx: ctx, log: log, config: config, signaler: signaler, + ice: ice, + relay: relay, remoteOffersCh: make(chan OfferAnswer), remoteAnswerCh: make(chan OfferAnswer), } @@ -98,17 +94,10 @@ func (h *Handshaker) Listen() { } } -func (h *Handshaker) SendOffer(args HandshakeArgs) error { +func (h *Handshaker) SendOffer() error { h.mu.Lock() defer h.mu.Unlock() - - err := h.sendOffer(args) - if err != nil { - return err - } - - h.lastOfferArgs = args - return nil + return h.sendOffer() } // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise @@ -163,14 +152,23 @@ func (h *Handshaker) waitForRemoteOfferConfirmation() (*OfferAnswer, error) { } // sendOffer prepares local user credentials and signals them to the remote peer -func (h *Handshaker) sendOffer(args HandshakeArgs) error { +func (h *Handshaker) sendOffer() error { + if !h.signaler.Ready() { + return ErrSignalIsNotReady + } + + iceUFrag, icePwd := h.ice.GetLocalUserCredentials() offer := OfferAnswer{ - IceCredentials: IceCredentials{args.IceUFrag, args.IcePwd}, + IceCredentials: IceCredentials{iceUFrag, icePwd}, WgListenPort: h.config.LocalWgPort, Version: version.NetbirdVersion(), RosenpassPubKey: h.config.RosenpassPubKey, RosenpassAddr: h.config.RosenpassAddr, - RelaySrvAddress: args.RelayAddr, + } + + addr, err := h.relay.RelayInstanceAddress() + if err == nil { + offer.RelaySrvAddress = addr } return h.signaler.SignalOffer(offer, h.config.Key) @@ -178,15 +176,21 @@ func (h *Handshaker) sendOffer(args HandshakeArgs) error { func (h *Handshaker) sendAnswer() error { h.log.Debugf("sending answer") + uFrag, pwd := h.ice.GetLocalUserCredentials() + answer := OfferAnswer{ - IceCredentials: IceCredentials{h.lastOfferArgs.IceUFrag, h.lastOfferArgs.IcePwd}, + IceCredentials: IceCredentials{uFrag, pwd}, WgListenPort: h.config.LocalWgPort, Version: version.NetbirdVersion(), RosenpassPubKey: h.config.RosenpassPubKey, RosenpassAddr: h.config.RosenpassAddr, - RelaySrvAddress: h.lastOfferArgs.RelayAddr, } - err := h.signaler.SignalAnswer(answer, h.config.Key) + addr, err := h.relay.RelayInstanceAddress() + if err == nil { + answer.RelaySrvAddress = addr + } + + err = h.signaler.SignalAnswer(answer, h.config.Key) if err != nil { return err } diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index ff673f197..bf1e34312 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -23,6 +23,8 @@ type State struct { PubKey string FQDN string ConnStatus ConnStatus + connStatusRelay ConnStatus + connStatusICE ConnStatus ConnStatusUpdate time.Time Relayed bool Direct bool @@ -30,6 +32,7 @@ type State struct { RemoteIceCandidateType string LocalIceCandidateEndpoint string RemoteIceCandidateEndpoint string + RelayServerAddress string LastWireguardHandshake time.Time BytesTx int64 BytesRx int64 @@ -240,7 +243,7 @@ func (d *Status) UpdatePeerState(receivedState State) error { peerState.SetRoutes(receivedState.GetRoutes()) } - skipNotification := shouldSkipNotify(receivedState, peerState) + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) if receivedState.ConnStatus != peerState.ConnStatus { peerState.ConnStatus = receivedState.ConnStatus @@ -251,6 +254,7 @@ func (d *Status) UpdatePeerState(receivedState State) error { peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + peerState.RelayServerAddress = receivedState.RelayServerAddress peerState.RosenpassEnabled = receivedState.RosenpassEnabled } @@ -270,6 +274,152 @@ func (d *Status) UpdatePeerState(receivedState State) error { return nil } +func (d *Status) UpdatePeerICEState(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + if receivedState.IP != "" { + peerState.IP = receivedState.IP + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.Direct = receivedState.Direct + peerState.Relayed = receivedState.Relayed + peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType + peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType + peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint + peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + peerState.RosenpassEnabled = receivedState.RosenpassEnabled + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerRelayedState(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.Direct = receivedState.Direct + peerState.Relayed = receivedState.Relayed + peerState.RelayServerAddress = receivedState.RelayServerAddress + peerState.RosenpassEnabled = receivedState.RosenpassEnabled + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.Direct = receivedState.Direct + peerState.Relayed = receivedState.Relayed + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.RelayServerAddress = "" + //peerState.RosenpassEnabled = receivedState.RosenpassEnabled // todo: check this variable + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + +func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error { + d.mux.Lock() + defer d.mux.Unlock() + + peerState, ok := d.peers[receivedState.PubKey] + if !ok { + return errors.New("peer doesn't exist") + } + + skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) + + peerState.ConnStatus = receivedState.ConnStatus + peerState.Direct = receivedState.Direct + peerState.Relayed = receivedState.Relayed + peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate + peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType + peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType + peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint + peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint + //peerState.RosenpassEnabled = receivedState.RosenpassEnabled // todo: check this variable + + d.peers[receivedState.PubKey] = peerState + + if skipNotification { + return nil + } + + ch, found := d.changeNotify[receivedState.PubKey] + if found && ch != nil { + close(ch) + d.changeNotify[receivedState.PubKey] = nil + } + + d.notifyPeerListChanged() + return nil +} + // UpdateWireGuardPeerState updates the WireGuard bits of the peer state func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) error { d.mux.Lock() @@ -289,13 +439,13 @@ func (d *Status) UpdateWireGuardPeerState(pubKey string, wgStats iface.WGStats) return nil } -func shouldSkipNotify(received, curr State) bool { +func shouldSkipNotify(receivedConnStatus ConnStatus, curr State) bool { switch { - case received.ConnStatus == StatusConnecting: + case receivedConnStatus == StatusConnecting: return true - case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting: + case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusConnecting: return true - case received.ConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected: + case receivedConnStatus == StatusDisconnected && curr.ConnStatus == StatusDisconnected: return curr.IP != "" default: return false diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index bbd00d6e2..9d08a0d24 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -182,7 +182,7 @@ func (p *WGEBPFProxy) proxyToRemote() { p.turnConnMutex.Unlock() if !ok { log.Infof("turn conn not found by port: %d", addr.Port) - continue + return // todo replace it to return. For debug troubleshooting keep it } _, err = conn.Write(buf[:n]) diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index 813540246..1b037a496 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 -// protoc v4.23.4 +// protoc v3.21.12 // source: daemon.proto package proto @@ -911,6 +911,7 @@ type PeerState struct { RosenpassEnabled bool `protobuf:"varint,15,opt,name=rosenpassEnabled,proto3" json:"rosenpassEnabled,omitempty"` Routes []string `protobuf:"bytes,16,rep,name=routes,proto3" json:"routes,omitempty"` Latency *durationpb.Duration `protobuf:"bytes,17,opt,name=latency,proto3" json:"latency,omitempty"` + RelayAddress string `protobuf:"bytes,18,opt,name=relayAddress,proto3" json:"relayAddress,omitempty"` } func (x *PeerState) Reset() { @@ -1064,6 +1065,13 @@ func (x *PeerState) GetLatency() *durationpb.Duration { return nil } +func (x *PeerState) GetRelayAddress() string { + if x != nil { + return x.RelayAddress + } + return "" +} + // LocalPeerState contains the latest state of the local peer type LocalPeerState struct { state protoimpl.MessageState @@ -2235,7 +2243,7 @@ var file_daemon_proto_rawDesc = []byte{ 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xce, 0x05, 0x0a, 0x09, 0x50, 0x65, + 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xf2, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, @@ -2280,174 +2288,176 @@ var file_daemon_proto_rawDesc = []byte{ 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, - 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, - 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, - 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, - 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, - 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, - 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, - 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, - 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, - 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, - 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, - 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, - 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, - 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, - 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, - 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, - 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, - 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, - 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, - 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, - 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, - 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, - 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, - 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, - 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, - 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, - 0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, - 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, - 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, - 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, - 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, - 0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, - 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x4a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, - 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, - 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, - 0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, - 0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, - 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, - 0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, - 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, - 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, - 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, - 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, - 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, - 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, - 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, - 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, - 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, - 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, - 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, - 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, - 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, + 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, + 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, + 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, + 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, + 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, + 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, + 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, + 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, + 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, + 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, + 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, + 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, + 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, + 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, + 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, + 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, + 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, + 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, + 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, + 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, + 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, + 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, + 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, + 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, + 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, + 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, + 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, + 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, + 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, + 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, + 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, + 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, + 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, + 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, + 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, + 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, + 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, + 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, + 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, + 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, + 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, + 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, + 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, + 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, + 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, + 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, + 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, + 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, + 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, + 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, + 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, + 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, + 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, + 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, + 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, + 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, - 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, - 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, - 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, - 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, - 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, + 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, + 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, + 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, - 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, - 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, - 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, + 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, + 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index 267eec279..f0dcc7924 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -180,6 +180,7 @@ message PeerState { bool rosenpassEnabled = 15; repeated string routes = 16; google.protobuf.Duration latency = 17; + string relayAddress = 18; } // LocalPeerState contains the latest state of the local peer diff --git a/client/server/server.go b/client/server/server.go index e1b4cc394..9695c49b3 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -732,6 +732,7 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { RemoteIceCandidateType: peerState.RemoteIceCandidateType, LocalIceCandidateEndpoint: peerState.LocalIceCandidateEndpoint, RemoteIceCandidateEndpoint: peerState.RemoteIceCandidateEndpoint, + RelayAddress: peerState.RelayServerAddress, Fqdn: peerState.FQDN, LastWireguardHandshake: timestamppb.New(peerState.LastWireguardHandshake), BytesRx: peerState.BytesRx, diff --git a/relay/client/addr.go b/relay/client/addr.go new file mode 100644 index 000000000..af4f459f8 --- /dev/null +++ b/relay/client/addr.go @@ -0,0 +1,13 @@ +package client + +type RelayAddr struct { + addr string +} + +func (a RelayAddr) Network() string { + return "relay" +} + +func (a RelayAddr) String() string { + return a.addr +} diff --git a/relay/client/client.go b/relay/client/client.go index 8358c21f7..1590cfc72 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -110,7 +110,7 @@ type Client struct { mu sync.Mutex // protect serviceIsRunning and conns readLoopMutex sync.Mutex wgReadLoop sync.WaitGroup - instanceURL string + instanceURL *RelayAddr muInstanceURL sync.Mutex onDisconnectListener func() @@ -183,7 +183,7 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { log.Infof("open connection to peer: %s", hashedStringID) msgChannel := make(chan Msg, 2) - conn := NewConn(c, hashedID, hashedStringID, msgChannel) + conn := NewConn(c, hashedID, hashedStringID, msgChannel, c.instanceURL) c.conns[hashedStringID] = newConnContainer(conn, msgChannel) return conn, nil @@ -193,10 +193,10 @@ func (c *Client) OpenConn(dstPeerID string) (net.Conn, error) { func (c *Client) ServerInstanceURL() (string, error) { c.muInstanceURL.Lock() defer c.muInstanceURL.Unlock() - if c.instanceURL == "" { + if c.instanceURL == nil { return "", fmt.Errorf("relay connection is not established") } - return c.instanceURL, nil + return c.instanceURL.String(), nil } // SetOnDisconnectListener sets a function that will be called when the connection to the relay server is closed. @@ -274,7 +274,7 @@ func (c *Client) handShake() error { return err } c.muInstanceURL.Lock() - c.instanceURL = ia + c.instanceURL = &RelayAddr{addr: ia} c.muInstanceURL.Unlock() return nil } @@ -315,7 +315,7 @@ func (c *Client) readLoop(relayConn net.Conn) { hc.Stop() c.muInstanceURL.Lock() - c.instanceURL = "" + c.instanceURL = nil c.muInstanceURL.Unlock() c.notifyDisconnected() diff --git a/relay/client/conn.go b/relay/client/conn.go index e6c32dfa9..a3a2fdabb 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -11,14 +11,16 @@ type Conn struct { dstID []byte dstStringID string messageChan chan Msg + instanceURL *RelayAddr } -func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg) *Conn { +func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg, instanceURL *RelayAddr) *Conn { c := &Conn{ client: client, dstID: dstID, dstStringID: dstStringID, messageChan: messageChan, + instanceURL: instanceURL, } return c @@ -48,7 +50,7 @@ func (c *Conn) LocalAddr() net.Addr { } func (c *Conn) RemoteAddr() net.Addr { - return c.client.relayConn.RemoteAddr() + return c.instanceURL } func (c *Conn) SetDeadline(t time.Time) error { diff --git a/relay/client/dialer/ws/addr.go b/relay/client/dialer/ws/addr.go index 2bdffb864..43f5dd6af 100644 --- a/relay/client/dialer/ws/addr.go +++ b/relay/client/dialer/ws/addr.go @@ -1,6 +1,7 @@ package ws type WebsocketAddr struct { + addr string } func (a WebsocketAddr) Network() string { @@ -8,5 +9,5 @@ func (a WebsocketAddr) Network() string { } func (a WebsocketAddr) String() string { - return "websocket/unknown-addr" + return a.addr } diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go index bc8675e8e..188582964 100644 --- a/relay/client/dialer/ws/conn.go +++ b/relay/client/dialer/ws/conn.go @@ -12,12 +12,14 @@ import ( type Conn struct { ctx context.Context *websocket.Conn + remoteAddr WebsocketAddr } -func NewConn(wsConn *websocket.Conn) net.Conn { +func NewConn(wsConn *websocket.Conn, serverAddress string) net.Conn { return &Conn{ - ctx: context.Background(), - Conn: wsConn, + ctx: context.Background(), + Conn: wsConn, + remoteAddr: WebsocketAddr{serverAddress}, } } @@ -40,11 +42,11 @@ func (c *Conn) Write(b []byte) (n int, err error) { } func (c *Conn) RemoteAddr() net.Addr { - return WebsocketAddr{} + return c.remoteAddr } func (c *Conn) LocalAddr() net.Addr { - return WebsocketAddr{} + return WebsocketAddr{addr: "unknown"} } func (c *Conn) SetReadDeadline(t time.Time) error { diff --git a/relay/client/dialer/ws/ws.go b/relay/client/dialer/ws/ws.go index 19ec44a1e..bccd6464a 100644 --- a/relay/client/dialer/ws/ws.go +++ b/relay/client/dialer/ws/ws.go @@ -32,7 +32,7 @@ func Dial(address string) (net.Conn, error) { _ = resp.Body.Close() } - conn := NewConn(wsConn) + conn := NewConn(wsConn, address) return conn, nil } diff --git a/relay/client/guard.go b/relay/client/guard.go index 050a9f0e7..5eb68d215 100644 --- a/relay/client/guard.go +++ b/relay/client/guard.go @@ -25,6 +25,7 @@ func NewGuard(context context.Context, relayClient *Client) *Guard { } func (g *Guard) OnDisconnected() { + // todo prevent multiple reconnect ticker := time.NewTicker(reconnectingTimeout) defer ticker.Stop() diff --git a/relay/client/manager.go b/relay/client/manager.go index ef4588f85..ffc8faef0 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -108,13 +108,20 @@ func (m *Manager) OpenConn(serverAddress, peerKey string, onClosedListener func( log.Debugf("open peer connection via foreign server: %s", serverAddress) netConn, err = m.openConnVia(serverAddress, peerKey) } - if err != nil { return nil, err } if onClosedListener != nil { - m.addListener(serverAddress, onClosedListener) + var listenerAddr string + if foreign { + m.addListener(serverAddress, onClosedListener) + listenerAddr = serverAddress + } else { + listenerAddr = m.serverURL + } + m.addListener(listenerAddr, onClosedListener) + } return netConn, err @@ -196,7 +203,7 @@ func (m *Manager) openConnVia(serverAddress, peerKey string) (net.Conn, error) { func (m *Manager) onServerDisconnected(serverAddress string) { if serverAddress == m.serverURL { - m.reconnectGuard.OnDisconnected() + go m.reconnectGuard.OnDisconnected() } m.notifyOnDisconnectListeners(serverAddress) @@ -251,17 +258,19 @@ func (m *Manager) cleanUpUnusedRelays() { func (m *Manager) addListener(serverAddress string, onClosedListener func()) { m.listenerLock.Lock() + defer m.listenerLock.Unlock() l, ok := m.onDisconnectedListeners[serverAddress] if !ok { l = make(map[*func()]struct{}) } l[&onClosedListener] = struct{}{} m.onDisconnectedListeners[serverAddress] = l - m.listenerLock.Unlock() } func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { m.listenerLock.Lock() + defer m.listenerLock.Unlock() + l, ok := m.onDisconnectedListeners[serverAddress] if !ok { return @@ -270,6 +279,4 @@ func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { go (*f)() } delete(m.onDisconnectedListeners, serverAddress) - m.listenerLock.Unlock() - } From 03e8acccde3ba8debd3bfd7606cd051d8adb3905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 16 Jul 2024 14:06:51 +0200 Subject: [PATCH 127/199] Fix peer state indication --- client/cmd/status.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/client/cmd/status.go b/client/cmd/status.go index f99d5674a..ce5d6e04d 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -336,17 +336,18 @@ func mapNSGroups(servers []*proto.NSGroupState) []nsServerGroupStateOutput { func mapPeers(peers []*proto.PeerState) peersStateOutput { var peersStateDetail []peerStateDetailOutput - localICE := "" - remoteICE := "" - localICEEndpoint := "" - remoteICEEndpoint := "" - relayServerAddress := "" - connType := "" peersConnected := 0 - lastHandshake := time.Time{} - transferReceived := int64(0) - transferSent := int64(0) for _, pbPeerState := range peers { + localICE := "" + remoteICE := "" + localICEEndpoint := "" + remoteICEEndpoint := "" + relayServerAddress := "" + connType := "" + lastHandshake := time.Time{} + transferReceived := int64(0) + transferSent := int64(0) + isPeerConnected := pbPeerState.ConnStatus == peer.StatusConnected.String() if skipDetailByFilters(pbPeerState, isPeerConnected) { continue From 7162e0a2ac79e9c85be403a58daf4ba13fe14e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 17 Jul 2024 16:26:41 +0200 Subject: [PATCH 128/199] - remove direct field from status - add randomisation factor for reconnection - fix rosenpass status --- client/cmd/status.go | 4 - client/internal/peer/conn.go | 28 +- client/internal/peer/status.go | 8 - client/internal/peer/worker_ice.go | 2 - client/internal/routemanager/client.go | 6 - client/proto/daemon.pb.go | 409 ++++++++++++------------- client/proto/daemon.proto | 1 - client/server/server.go | 1 - 8 files changed, 218 insertions(+), 241 deletions(-) diff --git a/client/cmd/status.go b/client/cmd/status.go index ce5d6e04d..5ed2da301 100644 --- a/client/cmd/status.go +++ b/client/cmd/status.go @@ -31,7 +31,6 @@ type peerStateDetailOutput struct { Status string `json:"status" yaml:"status"` LastStatusUpdate time.Time `json:"lastStatusUpdate" yaml:"lastStatusUpdate"` ConnType string `json:"connectionType" yaml:"connectionType"` - Direct bool `json:"direct" yaml:"direct"` IceCandidateType iceCandidateType `json:"iceCandidateType" yaml:"iceCandidateType"` IceCandidateEndpoint iceCandidateType `json:"iceCandidateEndpoint" yaml:"iceCandidateEndpoint"` RelayAddress string `json:"relayAddress" yaml:"relayAddress"` @@ -376,7 +375,6 @@ func mapPeers(peers []*proto.PeerState) peersStateOutput { Status: pbPeerState.GetConnStatus(), LastStatusUpdate: timeLocal, ConnType: connType, - Direct: pbPeerState.GetDirect(), IceCandidateType: iceCandidateType{ Local: localICE, Remote: remoteICE, @@ -646,7 +644,6 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo " Status: %s\n"+ " -- detail --\n"+ " Connection type: %s\n"+ - " Direct: %t\n"+ " ICE candidate (Local/Remote): %s/%s\n"+ " ICE candidate endpoints (Local/Remote): %s/%s\n"+ " Relay server address: %s\n"+ @@ -661,7 +658,6 @@ func parsePeers(peers peersStateOutput, rosenpassEnabled, rosenpassPermissive bo peerState.PubKey, peerState.Status, peerState.ConnType, - peerState.Direct, localICE, remoteICE, localICEEndpoint, diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index f2914b38e..d90c41e5f 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -327,7 +327,7 @@ func (conn *Conn) reconnectLoopWithRetry() { for { bo := backoff.WithContext(&backoff.ExponentialBackOff{ InitialInterval: 800 * time.Millisecond, - RandomizationFactor: 0, // todo: add randomisation factor + RandomizationFactor: 1, Multiplier: 1.99, MaxInterval: conn.config.Timeout * time.Second, MaxElapsedTime: 0, @@ -518,8 +518,7 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { peerState := State{ PubKey: conn.config.Key, ConnStatus: conn.evalStatus(), - Direct: false, // todo fix it - Relayed: true, // todo fix it + Relayed: conn.isRelayed(), ConnStatusUpdate: time.Now(), } @@ -609,8 +608,7 @@ func (conn *Conn) onWorkerRelayStateDisconnected() { peerState := State{ PubKey: conn.config.Key, ConnStatus: conn.evalStatus(), - Direct: false, // todo fix it - Relayed: true, // todo fix it + Relayed: conn.isRelayed(), ConnStatusUpdate: time.Now(), } @@ -634,9 +632,8 @@ func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []by peerState := State{ PubKey: conn.config.Key, ConnStatusUpdate: time.Now(), - ConnStatus: conn.evalStatus(), // todo fix it - Direct: false, // todo fix it - Relayed: true, // todo fix it + ConnStatus: conn.evalStatus(), + Relayed: conn.isRelayed(), RelayServerAddress: relayServerAddr, RosenpassEnabled: isRosenpassEnabled(rosenpassPubKey), } @@ -652,8 +649,7 @@ func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) { PubKey: conn.config.Key, ConnStatusUpdate: time.Now(), ConnStatus: conn.evalStatus(), - Direct: iceConnInfo.Direct, // todo fix it - Relayed: iceConnInfo.Relayed, // todo fix it + Relayed: iceConnInfo.Relayed, LocalIceCandidateType: iceConnInfo.LocalIceCandidateType, RemoteIceCandidateType: iceConnInfo.RemoteIceCandidateType, LocalIceCandidateEndpoint: iceConnInfo.LocalIceCandidateEndpoint, @@ -691,6 +687,18 @@ func (conn *Conn) waitInitialRandomSleepTime() { } } +func (conn *Conn) isRelayed() bool { + if conn.statusRelay == StatusDisconnected && (conn.statusICE == StatusDisconnected || conn.statusICE == StatusConnecting) { + return false + } + + if conn.currentConnType == connPriorityICEP2P { + return false + } + + return true +} + func (conn *Conn) evalStatus() ConnStatus { if conn.statusRelay == StatusConnected || conn.statusICE == StatusConnected { return StatusConnected diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index bf1e34312..f5f9a07a3 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -27,7 +27,6 @@ type State struct { connStatusICE ConnStatus ConnStatusUpdate time.Time Relayed bool - Direct bool LocalIceCandidateType string RemoteIceCandidateType string LocalIceCandidateEndpoint string @@ -248,7 +247,6 @@ func (d *Status) UpdatePeerState(receivedState State) error { if receivedState.ConnStatus != peerState.ConnStatus { peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType @@ -291,7 +289,6 @@ func (d *Status) UpdatePeerICEState(receivedState State) error { peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType @@ -328,7 +325,6 @@ func (d *Status) UpdatePeerRelayedState(receivedState State) error { peerState.ConnStatus = receivedState.ConnStatus peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.RelayServerAddress = receivedState.RelayServerAddress peerState.RosenpassEnabled = receivedState.RosenpassEnabled @@ -361,11 +357,9 @@ func (d *Status) UpdatePeerRelayedStateToDisconnected(receivedState State) error skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) peerState.ConnStatus = receivedState.ConnStatus - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate peerState.RelayServerAddress = "" - //peerState.RosenpassEnabled = receivedState.RosenpassEnabled // todo: check this variable d.peers[receivedState.PubKey] = peerState @@ -395,14 +389,12 @@ func (d *Status) UpdatePeerICEStateToDisconnected(receivedState State) error { skipNotification := shouldSkipNotify(receivedState.ConnStatus, peerState) peerState.ConnStatus = receivedState.ConnStatus - peerState.Direct = receivedState.Direct peerState.Relayed = receivedState.Relayed peerState.ConnStatusUpdate = receivedState.ConnStatusUpdate peerState.LocalIceCandidateType = receivedState.LocalIceCandidateType peerState.RemoteIceCandidateType = receivedState.RemoteIceCandidateType peerState.LocalIceCandidateEndpoint = receivedState.LocalIceCandidateEndpoint peerState.RemoteIceCandidateEndpoint = receivedState.RemoteIceCandidateEndpoint - //peerState.RosenpassEnabled = receivedState.RosenpassEnabled // todo: check this variable d.peers[receivedState.PubKey] = peerState diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 0d4a6b59b..55b9b8c4e 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -59,7 +59,6 @@ type ICEConnInfo struct { RemoteIceCandidateType string RemoteIceCandidateEndpoint string LocalIceCandidateEndpoint string - Direct bool Relayed bool RelayedOnLocal bool } @@ -187,7 +186,6 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) { RemoteIceCandidateType: pair.Remote.Type().String(), LocalIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Local.Address(), pair.Local.Port()), RemoteIceCandidateEndpoint: fmt.Sprintf("%s:%d", pair.Remote.Address(), pair.Remote.Port()), - Direct: !isRelayCandidate(pair.Local), Relayed: isRelayed(pair), RelayedOnLocal: isRelayCandidate(pair.Local), } diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index 57ed9ed1e..adf3b6a2f 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -20,7 +20,6 @@ import ( type routerPeerStatus struct { connected bool relayed bool - direct bool latency time.Duration } @@ -80,7 +79,6 @@ func (c *clientNetwork) getRouterPeerStatuses() map[route.ID]routerPeerStatus { routePeerStatuses[r.ID] = routerPeerStatus{ connected: peerStatus.ConnStatus == peer.StatusConnected, relayed: peerStatus.Relayed, - direct: peerStatus.Direct, latency: peerStatus.Latency, } } @@ -135,10 +133,6 @@ func (c *clientNetwork) getBestRouteFromStatuses(routePeerStatuses map[route.ID] tempScore++ } - if peerStatus.direct { - tempScore++ - } - if tempScore > chosenScore || (tempScore == chosenScore && chosen == "") { chosen = r.ID chosenScore = tempScore diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index 1b037a496..f604a885c 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -899,7 +899,6 @@ type PeerState struct { ConnStatus string `protobuf:"bytes,3,opt,name=connStatus,proto3" json:"connStatus,omitempty"` ConnStatusUpdate *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=connStatusUpdate,proto3" json:"connStatusUpdate,omitempty"` Relayed bool `protobuf:"varint,5,opt,name=relayed,proto3" json:"relayed,omitempty"` - Direct bool `protobuf:"varint,6,opt,name=direct,proto3" json:"direct,omitempty"` LocalIceCandidateType string `protobuf:"bytes,7,opt,name=localIceCandidateType,proto3" json:"localIceCandidateType,omitempty"` RemoteIceCandidateType string `protobuf:"bytes,8,opt,name=remoteIceCandidateType,proto3" json:"remoteIceCandidateType,omitempty"` Fqdn string `protobuf:"bytes,9,opt,name=fqdn,proto3" json:"fqdn,omitempty"` @@ -981,13 +980,6 @@ func (x *PeerState) GetRelayed() bool { return false } -func (x *PeerState) GetDirect() bool { - if x != nil { - return x.Direct - } - return false -} - func (x *PeerState) GetLocalIceCandidateType() string { if x != nil { return x.LocalIceCandidateType @@ -2243,7 +2235,7 @@ var file_daemon_proto_rawDesc = []byte{ 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xf2, 0x05, 0x0a, 0x09, 0x50, 0x65, + 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x22, 0xda, 0x05, 0x0a, 0x09, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, @@ -2255,209 +2247,208 @@ var file_daemon_proto_rawDesc = []byte{ 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x65, - 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, - 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, + 0x64, 0x12, 0x34, 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, + 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, + 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x36, 0x0a, 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, - 0x69, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x16, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, - 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, - 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, - 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, - 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, - 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, - 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, - 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, - 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, - 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, - 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, - 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, - 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, - 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, - 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, - 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, - 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, - 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, - 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, - 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, - 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, - 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, - 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, - 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, - 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, - 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, - 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, - 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, - 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, - 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, - 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, - 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, - 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, - 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, - 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, - 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, - 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, - 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, - 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, - 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, - 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, + 0x12, 0x0a, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, + 0x71, 0x64, 0x6e, 0x12, 0x3c, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, 0x43, + 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x3e, 0x0a, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, 0x43, 0x61, + 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x49, 0x63, 0x65, + 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x52, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, + 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x16, 0x6c, + 0x61, 0x73, 0x74, 0x57, 0x69, 0x72, 0x65, 0x67, 0x75, 0x61, 0x72, 0x64, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, + 0x18, 0x0d, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x52, 0x78, 0x12, + 0x18, 0x0a, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x07, 0x62, 0x79, 0x74, 0x65, 0x73, 0x54, 0x78, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, + 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, + 0x10, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x33, 0x0a, + 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0xec, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, + 0x50, 0x65, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x50, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x50, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, + 0x4b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x4b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x0f, 0x6b, 0x65, 0x72, 0x6e, 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, + 0x66, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x6b, 0x65, 0x72, 0x6e, + 0x65, 0x6c, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x66, + 0x71, 0x64, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x71, 0x64, 0x6e, 0x12, + 0x2a, 0x0a, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x6f, 0x73, 0x65, 0x6e, + 0x70, 0x61, 0x73, 0x73, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x13, 0x72, + 0x6f, 0x73, 0x65, 0x6e, 0x70, 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, + 0x76, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x72, 0x6f, 0x73, 0x65, 0x6e, 0x70, + 0x61, 0x73, 0x73, 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x22, 0x53, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x57, 0x0a, 0x0f, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x55, 0x52, 0x4c, 0x12, + 0x1c, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x22, 0x52, 0x0a, 0x0a, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x55, 0x52, 0x49, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x55, 0x52, 0x49, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x72, 0x0a, 0x0c, 0x4e, 0x53, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x73, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0xd2, 0x02, 0x0a, 0x0a, + 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x41, 0x0a, 0x0f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x61, 0x6e, + 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x6d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, + 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0b, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x3e, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x65, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x65, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x70, 0x65, 0x65, 0x72, 0x73, 0x12, 0x2a, 0x0a, + 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x6e, 0x73, + 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4e, 0x53, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x0a, 0x64, 0x6e, 0x73, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3b, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x52, 0x06, 0x72, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x22, 0x5b, 0x0a, 0x13, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, + 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, + 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, + 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a, + 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a, + 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x4a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, + 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, + 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x13, 0x44, + 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, + 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e, + 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12, + 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, + 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09, + 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, + 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, + 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, + 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, + 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, + 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, + 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, + 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, + 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x08, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x49, 0x44, 0x73, 0x12, 0x16, 0x0a, - 0x06, 0x61, 0x70, 0x70, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, - 0x70, 0x70, 0x65, 0x6e, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x03, 0x61, 0x6c, 0x6c, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x1a, 0x0a, 0x06, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x69, 0x70, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x69, 0x70, 0x73, 0x22, 0xf9, 0x01, 0x0a, 0x05, - 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, - 0x1a, 0x0a, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x08, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x6f, - 0x6d, 0x61, 0x69, 0x6e, 0x73, 0x12, 0x40, 0x0a, 0x0b, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, - 0x64, 0x49, 0x50, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, - 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x6f, - 0x6c, 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x1a, 0x4e, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x6f, 0x6c, - 0x76, 0x65, 0x64, 0x49, 0x50, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x4a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, - 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, - 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, - 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, - 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x3d, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, - 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x6c, - 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, - 0x76, 0x65, 0x6c, 0x22, 0x3c, 0x0a, 0x12, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x05, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x10, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, - 0x6c, 0x22, 0x15, 0x0a, 0x13, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x62, 0x0a, 0x08, 0x4c, 0x6f, 0x67, 0x4c, - 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, - 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x41, 0x4e, 0x49, 0x43, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, - 0x46, 0x41, 0x54, 0x41, 0x4c, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, - 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x57, 0x41, 0x52, 0x4e, 0x10, 0x04, 0x12, 0x08, 0x0a, 0x04, - 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x45, 0x42, 0x55, 0x47, 0x10, - 0x06, 0x12, 0x09, 0x0a, 0x05, 0x54, 0x52, 0x41, 0x43, 0x45, 0x10, 0x07, 0x32, 0xb8, 0x06, 0x0a, - 0x0d, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, - 0x0a, 0x05, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x14, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, - 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0c, 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, - 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x12, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, - 0x57, 0x61, 0x69, 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x57, 0x61, 0x69, - 0x74, 0x53, 0x53, 0x4f, 0x4c, 0x6f, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x02, 0x55, 0x70, 0x12, 0x11, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x15, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, - 0x04, 0x44, 0x6f, 0x77, 0x6e, 0x12, 0x13, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, - 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x18, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, - 0x75, 0x74, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, - 0x0c, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, - 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, - 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4d, 0x0a, 0x0e, 0x44, 0x65, - 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x64, 0x61, 0x65, 0x6d, - 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, - 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, - 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, - 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, - 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, - 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, - 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, - 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, - 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, + 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, + 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, + 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x47, + 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, + 0x6d, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x48, 0x0a, 0x0b, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, 0x4c, + 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1a, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, + 0x74, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1b, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x74, 0x4c, 0x6f, 0x67, + 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x08, 0x5a, 0x06, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( diff --git a/client/proto/daemon.proto b/client/proto/daemon.proto index f0dcc7924..1d7388bfd 100644 --- a/client/proto/daemon.proto +++ b/client/proto/daemon.proto @@ -168,7 +168,6 @@ message PeerState { string connStatus = 3; google.protobuf.Timestamp connStatusUpdate = 4; bool relayed = 5; - bool direct = 6; string localIceCandidateType = 7; string remoteIceCandidateType = 8; string fqdn = 9; diff --git a/client/server/server.go b/client/server/server.go index 9695c49b3..502d4168c 100644 --- a/client/server/server.go +++ b/client/server/server.go @@ -727,7 +727,6 @@ func toProtoFullStatus(fullStatus peer.FullStatus) *proto.FullStatus { ConnStatus: peerState.ConnStatus.String(), ConnStatusUpdate: timestamppb.New(peerState.ConnStatusUpdate), Relayed: peerState.Relayed, - Direct: peerState.Direct, LocalIceCandidateType: peerState.LocalIceCandidateType, RemoteIceCandidateType: peerState.RemoteIceCandidateType, LocalIceCandidateEndpoint: peerState.LocalIceCandidateEndpoint, From e75fbd34a7f603b8be9357c1f76768dc58398fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 17 Jul 2024 17:10:33 +0200 Subject: [PATCH 129/199] Add config file handling --- relay/cmd/main.go | 126 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 38 deletions(-) diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 5e454b000..b96482ef8 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -17,18 +17,40 @@ import ( "github.com/netbirdio/netbird/util" ) -var ( - listenAddress string +type Config struct { + ListenAddress string // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection // it is a domain:port or ip:port - exposedAddress string - letsencryptDataDir string - letsencryptDomains []string - tlsCertFile string - tlsKeyFile string - authSecret string + ExposedAddress string + LetsencryptDataDir string + LetsencryptDomains []string + TlsCertFile string + TlsKeyFile string + AuthSecret string +} - rootCmd = &cobra.Command{ +func (c Config) Validate() error { + if c.ExposedAddress == "" { + return fmt.Errorf("exposed address is required") + } + if c.AuthSecret == "" { + return fmt.Errorf("auth secret is required") + } + return nil +} + +func (c Config) HasCertConfig() bool { + return c.TlsCertFile != "" && c.TlsKeyFile != "" +} + +func (c Config) HasLetsEncrypt() bool { + return c.LetsencryptDataDir != "" && c.LetsencryptDomains != nil && len(c.LetsencryptDomains) > 0 +} + +var ( + cobraConfig *Config + cfgFile string + rootCmd = &cobra.Command{ Use: "relay", Short: "Relay service", Long: "Relay service for Netbird agents", @@ -38,13 +60,15 @@ var ( func init() { _ = util.InitLog("trace", "console") - rootCmd.PersistentFlags().StringVarP(&listenAddress, "listen-address", "l", ":443", "listen address") - rootCmd.PersistentFlags().StringVarP(&exposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") - rootCmd.PersistentFlags().StringVarP(&letsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") - rootCmd.PersistentFlags().StringArrayVarP(&letsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") - rootCmd.PersistentFlags().StringVarP(&tlsCertFile, "tls-cert-file", "c", "", "") - rootCmd.PersistentFlags().StringVarP(&tlsKeyFile, "tls-key-file", "k", "", "") - rootCmd.PersistentFlags().StringVarP(&authSecret, "auth-secret", "s", "", "log level") + cobraConfig = &Config{} + rootCmd.PersistentFlags().StringVarP(&cfgFile, "config-file", "f", "/etc/netbird/relay.json", "Relay server config file location") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.ListenAddress, "listen-address", "l", ":443", "listen address") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.ExposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.LetsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") + rootCmd.PersistentFlags().StringArrayVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsCertFile, "tls-cert-file", "c", "", "") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsKeyFile, "tls-key-file", "k", "", "") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "log level") } func waitForExitSignal() { @@ -53,29 +77,64 @@ func waitForExitSignal() { <-osSigs } +func loadConfig(configFile string) (*Config, error) { + log.Infof("loading config from: %s", configFile) + loadedConfig := &Config{} + _, err := util.ReadJson(configFile, loadedConfig) + if err != nil { + return nil, err + } + if cobraConfig.ListenAddress != "" { + loadedConfig.ListenAddress = cobraConfig.ListenAddress + } + + if cobraConfig.ExposedAddress != "" { + loadedConfig.ExposedAddress = cobraConfig.ExposedAddress + } + if cobraConfig.LetsencryptDataDir != "" { + loadedConfig.LetsencryptDataDir = cobraConfig.LetsencryptDataDir + } + if len(cobraConfig.LetsencryptDomains) > 0 { + loadedConfig.LetsencryptDomains = cobraConfig.LetsencryptDomains + } + if cobraConfig.TlsCertFile != "" { + loadedConfig.TlsCertFile = cobraConfig.TlsCertFile + } + if cobraConfig.TlsKeyFile != "" { + loadedConfig.TlsKeyFile = cobraConfig.TlsKeyFile + } + if cobraConfig.AuthSecret != "" { + loadedConfig.AuthSecret = cobraConfig.AuthSecret + } + + return loadedConfig, err +} + func execute(cmd *cobra.Command, args []string) { - if exposedAddress == "" { - log.Errorf("exposed address is required") + cfg, err := loadConfig(cfgFile) + if err != nil { + log.Errorf("failed to load config: %s", err) os.Exit(1) } - if authSecret == "" { - log.Errorf("auth secret is required") + err = cfg.Validate() + if err != nil { + log.Errorf("invalid config: %s", err) os.Exit(1) } srvListenerCfg := server.ListenerConfig{ - Address: listenAddress, + Address: cfg.ListenAddress, } - if hasLetsEncrypt() { - tlsCfg, err := setupTLSCertManager() + if cfg.HasLetsEncrypt() { + tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...) if err != nil { log.Errorf("%s", err) os.Exit(1) } srvListenerCfg.TLSConfig = tlsCfg - } else if hasCertConfig() { - tlsCfg, err := encryption.LoadTLSConfig(tlsCertFile, tlsKeyFile) + } else if cfg.HasCertConfig() { + tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile) if err != nil { log.Errorf("%s", err) os.Exit(1) @@ -85,10 +144,10 @@ func execute(cmd *cobra.Command, args []string) { tlsSupport := srvListenerCfg.TLSConfig != nil - authenticator := auth.NewTimedHMACValidator(authSecret, 24*time.Hour) - srv := server.NewServer(exposedAddress, tlsSupport, authenticator) + authenticator := auth.NewTimedHMACValidator(cfg.AuthSecret, 24*time.Hour) + srv := server.NewServer(cfg.ExposedAddress, tlsSupport, authenticator) log.Infof("server will be available on: %s", srv.InstanceURL()) - err := srv.Listen(srvListenerCfg) + err = srv.Listen(srvListenerCfg) if err != nil { log.Errorf("failed to bind server: %s", err) os.Exit(1) @@ -103,16 +162,7 @@ func execute(cmd *cobra.Command, args []string) { } } -func hasCertConfig() bool { - return tlsCertFile != "" && tlsKeyFile != "" - -} - -func hasLetsEncrypt() bool { - return letsencryptDataDir != "" && letsencryptDomains != nil && len(letsencryptDomains) > 0 -} - -func setupTLSCertManager() (*tls.Config, error) { +func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) { certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...) if err != nil { return nil, fmt.Errorf("failed creating LetsEncrypt cert manager: %v", err) From 233a7b9802ae4af2fa1a034ce1335010d035c3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 18 Jul 2024 10:41:14 +0200 Subject: [PATCH 130/199] Remove env var for debug purpose --- client/internal/connect.go | 5 ----- client/internal/peer/env_config.go | 7 ------- client/internal/peer/handshaker.go | 8 -------- 3 files changed, 20 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index 1dc515203..21423455e 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -316,11 +316,6 @@ func (c *ConnectClient) run( } func parseRelayInfo(resp *mgmProto.LoginResponse) (string, *hmac.Token) { - // todo remove this - if ra := peer.ForcedRelayAddress(); ra != "" { - return ra, nil - } - msg := resp.GetWiretrusteeConfig().GetRelay() if msg == nil { return "", nil diff --git a/client/internal/peer/env_config.go b/client/internal/peer/env_config.go index 21269168f..87b626df7 100644 --- a/client/internal/peer/env_config.go +++ b/client/internal/peer/env_config.go @@ -16,13 +16,6 @@ const ( envICEForceRelayConn = "NB_ICE_FORCE_RELAY_CONN" ) -func ForcedRelayAddress() string { - if envRelay := os.Getenv("NB_RELAY_ADDRESS"); envRelay != "" { - return envRelay - } - return "" -} - func iceKeepAlive() time.Duration { keepAliveEnv := os.Getenv(envICEKeepAliveIntervalSec) if keepAliveEnv == "" { diff --git a/client/internal/peer/handshaker.go b/client/internal/peer/handshaker.go index bff025cf8..d5ed10357 100644 --- a/client/internal/peer/handshaker.go +++ b/client/internal/peer/handshaker.go @@ -103,10 +103,6 @@ func (h *Handshaker) SendOffer() error { // OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise // doesn't block, discards the message if connection wasn't ready func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { - // todo remove this if signaling can support relay - if ForcedRelayAddress() != "" { - offer.RelaySrvAddress = ForcedRelayAddress() - } select { case h.remoteOffersCh <- offer: return true @@ -120,10 +116,6 @@ func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) bool { // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise // doesn't block, discards the message if connection wasn't ready func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) bool { - // todo remove this if signaling can support relay - if ForcedRelayAddress() != "" { - answer.RelaySrvAddress = ForcedRelayAddress() - } select { case h.remoteAnswerCh <- answer: return true From f3282bea80233a2c7dabe9a061714bde25cd1fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 18 Jul 2024 13:11:27 +0200 Subject: [PATCH 131/199] - add ip to log - remove unused timestamp from log --- client/internal/peer/conn.go | 4 +--- relay/server/listener/ws/listener.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index d90c41e5f..721d5d036 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -340,7 +340,6 @@ func (conn *Conn) reconnectLoopWithRetry() { <-ticker.C // consume the initial tick what is happening right after the ticker has been created - no := time.Now() L: for { select { @@ -351,8 +350,7 @@ func (conn *Conn) reconnectLoopWithRetry() { } // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer // todo check wg handshake - conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s, elapsed time: %s", conn.statusRelay, conn.statusICE, time.Since(no)) - no = time.Now() + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { continue diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index 6b2f669f4..51a578dc4 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -67,7 +67,7 @@ func (l *Listener) Close() error { func (l *Listener) onAccept(w http.ResponseWriter, r *http.Request) { wsConn, err := websocket.Accept(w, r, nil) if err != nil { - log.Errorf("failed to accept ws connection: %s", err) + log.Errorf("failed to accept ws connection from %s: %s", r.RemoteAddr, err) return } From 894d68adf22a5405aaae091552f9855bcf40a031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 18 Jul 2024 13:16:50 +0200 Subject: [PATCH 132/199] Fix reference check --- relay/client/client.go | 23 ++++++++++++----------- relay/client/conn.go | 4 ++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index 1590cfc72..a0d5124dc 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -384,20 +384,18 @@ func (c *Client) handleTransportMsg(buf []byte, bufPtr *[]byte, internallyStoppe return true } -// todo check by reference too, the id is not enought because the id come from the outer conn -func (c *Client) writeTo(id string, dstID []byte, payload []byte) (int, error) { +func (c *Client) writeTo(connReference *Conn, id string, dstID []byte, payload []byte) (int, error) { c.mu.Lock() - // conn, ok := c.conns[id] - _, ok := c.conns[id] + conn, ok := c.conns[id] c.mu.Unlock() if !ok { return 0, io.EOF } - /* - if conn != clientRef { - return 0, io.EOF - } - */ + + if conn.conn != connReference { + return 0, io.EOF + } + // todo: use buffer pool instead of create new transport msg. msg, err := messages.MarshalTransportMsg(dstID, payload) if err != nil { @@ -439,8 +437,7 @@ func (c *Client) closeAllConns() { c.conns = make(map[string]*connContainer) } -// todo check by reference too, the id is not enought because the id come from the outer conn -func (c *Client) closeConn(id string) error { +func (c *Client) closeConn(connReference *Conn, id string) error { c.mu.Lock() defer c.mu.Unlock() @@ -448,6 +445,10 @@ func (c *Client) closeConn(id string) error { if !ok { return fmt.Errorf("connection already closed") } + + if container.conn != connReference { + return fmt.Errorf("conn reference mismatch") + } container.close() delete(c.conns, id) diff --git a/relay/client/conn.go b/relay/client/conn.go index a3a2fdabb..783b6a660 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -27,7 +27,7 @@ func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan } func (c *Conn) Write(p []byte) (n int, err error) { - return c.client.writeTo(c.dstStringID, c.dstID, p) + return c.client.writeTo(c, c.dstStringID, c.dstID, p) } func (c *Conn) Read(b []byte) (n int, err error) { @@ -42,7 +42,7 @@ func (c *Conn) Read(b []byte) (n int, err error) { } func (c *Conn) Close() error { - return c.client.closeConn(c.dstStringID) + return c.client.closeConn(c, c.dstStringID) } func (c *Conn) LocalAddr() net.Addr { From f247a7be46689dd21fa72b1e9313b810619b68d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 18 Jul 2024 14:21:32 +0200 Subject: [PATCH 133/199] Fix reference check --- client/internal/peer/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 721d5d036..613007fdf 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -324,6 +324,7 @@ func (conn *Conn) reconnectLoopWithRetry() { case <-time.After(3 * time.Second): } +L: for { bo := backoff.WithContext(&backoff.ExponentialBackOff{ InitialInterval: 800 * time.Millisecond, @@ -340,7 +341,6 @@ func (conn *Conn) reconnectLoopWithRetry() { <-ticker.C // consume the initial tick what is happening right after the ticker has been created - L: for { select { case t := <-ticker.C: From 787c900342ae0df3dbcf97a0eb62647ce4520485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 18 Jul 2024 14:45:36 +0200 Subject: [PATCH 134/199] Revert break --- client/internal/peer/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 613007fdf..721d5d036 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -324,7 +324,6 @@ func (conn *Conn) reconnectLoopWithRetry() { case <-time.After(3 * time.Second): } -L: for { bo := backoff.WithContext(&backoff.ExponentialBackOff{ InitialInterval: 800 * time.Millisecond, @@ -341,6 +340,7 @@ L: <-ticker.C // consume the initial tick what is happening right after the ticker has been created + L: for { select { case t := <-ticker.C: From e10bc658f585381569a27b30bdaf389937caaf00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 19 Jul 2024 12:00:19 +0200 Subject: [PATCH 135/199] Fix reconnect loop --- client/internal/peer/conn.go | 110 ++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 52 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 721d5d036..b7c106fdb 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -318,70 +318,76 @@ func (conn *Conn) GetKey() string { } func (conn *Conn) reconnectLoopWithRetry() { - // give chance to the peer to establish the initial connection + // Give chance to the peer to establish the initial connection. + // With it, we can decrease to send necessary offer select { case <-conn.ctx.Done(): case <-time.After(3 * time.Second): } + ticker := conn.prepareExponentTicker() + defer ticker.Stop() + time.Sleep(1 * time.Second) for { - bo := backoff.WithContext(&backoff.ExponentialBackOff{ - InitialInterval: 800 * time.Millisecond, - RandomizationFactor: 1, - Multiplier: 1.99, - MaxInterval: conn.config.Timeout * time.Second, - MaxElapsedTime: 0, - Stop: backoff.Stop, - Clock: backoff.SystemClock, - }, conn.ctx) - - ticker := backoff.NewTicker(bo) - defer ticker.Stop() - - <-ticker.C // consume the initial tick what is happening right after the ticker has been created - - L: - for { - select { - case t := <-ticker.C: - if t.IsZero() { - // in case if the ticker has been canceled by context then avoid the temporary loop - return - } - // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer - // todo check wg handshake - conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) - - if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { - continue - } - - conn.log.Debugf("ticker timed out, retry to do handshake") - err := conn.handshaker.sendOffer() - if err != nil { - conn.log.Errorf("failed to do handshake: %v", err) - } - case changed := <-conn.relayDisconnected: - if !changed { - continue - } - conn.log.Debugf("Relay state changed, reset reconnect timer") - ticker.Stop() - break L - case changed := <-conn.iCEDisconnected: - if !changed { - continue - } - conn.log.Debugf("ICE state changed, reset reconnect timer") - ticker.Stop() - break L - case <-conn.ctx.Done(): + select { + case t := <-ticker.C: + if t.IsZero() { + // in case if the ticker has been canceled by context then avoid the temporary loop return } + + // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer + // todo check wg handshake + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + conn.mu.Lock() + if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { + conn.mu.Unlock() + continue + } + conn.mu.Unlock() + + conn.log.Debugf("ticker timed out, retry to do handshake") + err := conn.handshaker.sendOffer() + if err != nil { + conn.log.Errorf("failed to do handshake: %v", err) + } + case changed := <-conn.relayDisconnected: + if !changed { + continue + } + conn.log.Debugf("Relay state changed, reset reconnect timer") + ticker.Stop() + ticker = conn.prepareExponentTicker() + case changed := <-conn.iCEDisconnected: + if !changed { + continue + } + conn.log.Debugf("ICE state changed, reset reconnect timer") + ticker.Stop() + ticker = conn.prepareExponentTicker() + case <-conn.ctx.Done(): + return } } } +func (conn *Conn) prepareExponentTicker() *backoff.Ticker { + bo := backoff.WithContext(&backoff.ExponentialBackOff{ + InitialInterval: 800 * time.Millisecond, + RandomizationFactor: 1, + Multiplier: 1.99, + MaxInterval: conn.config.Timeout, + MaxElapsedTime: 0, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + }, conn.ctx) + + ticker := backoff.NewTicker(bo) + <-ticker.C // consume the initial tick what is happening right after the ticker has been created + + return ticker +} + // reconnectLoopForOnDisconnectedEvent is used when the peer is not a controller and it should reconnect to the peer // when the connection is lost. It will try to establish a connection only once time if before the connection was established // It track separately the ice and relay connection status. Just because a lover priority connection reestablished it does not From 8c7215a9f5c5834f6c47d733a3b6dd26a35a8c6c Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sat, 20 Jul 2024 13:00:19 +0200 Subject: [PATCH 136/199] Add data transfer test --- relay/client/client_test.go | 128 ++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index b9068ccd9..42cd5bc67 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -2,8 +2,11 @@ package client import ( "context" + "crypto/rand" + "fmt" "net" "os" + "sync" "testing" "time" @@ -108,6 +111,122 @@ func TestClient(t *testing.T) { } } +func TestDataTransfer(t *testing.T) { + ctx := context.Background() + + testData, err := seedRandomData() + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + srv := server.NewServer(serverURL, false, av) + errChan := make(chan error, 1) + go func() { + listenCfg := server.ListenerConfig{Address: serverListenAddr} + err := srv.Listen(listenCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + peerPairs := 50 + clientsSender := make([]*Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := NewClient(ctx, serverURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + clientsReceiver := make([]*Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := NewClient(ctx, serverURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + + conn, err = clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + + wg := sync.WaitGroup{} + for i := 0; i < len(connsSender); i++ { + wg.Add(2) + go func(i int) { + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, err := connsSender[i].Write(testData[j:end]) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + } + wg.Done() + }(i) + + go func(i int) { + for receivedSize := 0; receivedSize < len(testData); { + buf := make([]byte, 8192) + n, err := connsReceiver[i].Read(buf) + if err != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + receivedSize += n + } + wg.Done() + }(i) + } + + wg.Wait() + + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + err = connsReceiver[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + } +} + func TestRegistration(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} @@ -601,3 +720,12 @@ func waitForServerToStart(errChan chan error) error { } return nil } + +func seedRandomData() ([]byte, error) { + token := make([]byte, 1024*1024*10) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} From 03df0878dc0a58eb65b135db9175c8e1b20848c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Sun, 21 Jul 2024 13:40:23 +0200 Subject: [PATCH 137/199] Add benchmark test --- relay/client/client_test.go | 57 ++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 42cd5bc67..7b768a4cf 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -27,7 +27,7 @@ var ( ) func TestMain(m *testing.M) { - _ = util.InitLog("trace", "console") + _ = util.InitLog("error", "console") code := m.Run() os.Exit(code) } @@ -112,17 +112,31 @@ func TestClient(t *testing.T) { } func TestDataTransfer(t *testing.T) { - ctx := context.Background() + dataSize := 1024 * 1024 * 10 - testData, err := seedRandomData() + testData, err := seedRandomData(dataSize) if err != nil { t.Fatalf("failed to seed random data: %s", err) } - srv := server.NewServer(serverURL, false, av) + for _, peerPairs := range []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + transfer(t, testData, peerPairs) + }) + } +} + +func transfer(t *testing.T, testData []byte, peerPairs int) { + t.Helper() + ctx := context.Background() + port := 35000 + peerPairs + serverAddress := fmt.Sprintf("127.0.0.1:%d", port) + serverConnURL := fmt.Sprintf("rel://%s", serverAddress) + + srv := server.NewServer(serverConnURL, false, av) errChan := make(chan error, 1) go func() { - listenCfg := server.ListenerConfig{Address: serverListenAddr} + listenCfg := server.ListenerConfig{Address: serverAddress} err := srv.Listen(listenCfg) if err != nil { errChan <- err @@ -141,10 +155,9 @@ func TestDataTransfer(t *testing.T) { t.Fatalf("failed to start server: %s", err) } - peerPairs := 50 clientsSender := make([]*Client, peerPairs) for i := 0; i < cap(clientsSender); i++ { - c := NewClient(ctx, serverURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + c := NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) err := c.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -153,8 +166,8 @@ func TestDataTransfer(t *testing.T) { } clientsReceiver := make([]*Client, peerPairs) - for i := 0; i < cap(clientsSender); i++ { - c := NewClient(ctx, serverURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + for i := 0; i < cap(clientsReceiver); i++ { + c := NewClient(ctx, serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) err := c.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) @@ -178,9 +191,11 @@ func TestDataTransfer(t *testing.T) { connsReceiver = append(connsReceiver, conn) } + var transferDuration []time.Duration wg := sync.WaitGroup{} for i := 0; i < len(connsSender); i++ { wg.Add(2) + start := time.Now() go func(i int) { pieceSize := 1024 testDataLen := len(testData) @@ -198,22 +213,36 @@ func TestDataTransfer(t *testing.T) { wg.Done() }(i) - go func(i int) { + go func(i int, start time.Time) { + buf := make([]byte, 8192) + rcv := 0 for receivedSize := 0; receivedSize < len(testData); { - buf := make([]byte, 8192) + n, err := connsReceiver[i].Read(buf) if err != nil { t.Fatalf("failed to read from channel: %s", err) } receivedSize += n + rcv += n } + transferDuration = append(transferDuration, time.Since(start)) wg.Done() - }(i) + }(i, start) } wg.Wait() + // calculate the megabytes per second from the average transferDuration against the dataSize + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) + for i := 0; i < len(connsSender); i++ { err := connsSender[i].Close() if err != nil { @@ -721,8 +750,8 @@ func waitForServerToStart(errChan chan error) error { return nil } -func seedRandomData() ([]byte, error) { - token := make([]byte, 1024*1024*10) +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) _, err := rand.Read(token) if err != nil { return nil, err From 0329c12173bec73c8b85f5bc7876f24418d614a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 22 Jul 2024 13:04:32 +0200 Subject: [PATCH 138/199] Fix relay close message handling --- relay/messages/message.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/relay/messages/message.go b/relay/messages/message.go index 564c82e03..0211e1900 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -43,6 +43,8 @@ func (m MsgType) String() string { return "transport" case MsgTypeClose: return "close" + case MsgTypeHealthCheck: + return "health check" default: return "unknown" } @@ -150,7 +152,7 @@ func UnmarshalHelloResponse(msg []byte) (string, error) { func MarshalCloseMsg() []byte { msg := make([]byte, 1) msg[0] = byte(MsgTypeClose) - return healthCheckMsg + return msg } // Transport message From 1097285d80f3df35cb854362d25151e2ccfef978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 22 Jul 2024 13:13:12 +0200 Subject: [PATCH 139/199] Fix len of write operation --- relay/client/client.go | 6 ++++-- relay/client/dialer/ws/conn.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/relay/client/client.go b/relay/client/client.go index a0d5124dc..4a20a3b00 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -402,11 +402,13 @@ func (c *Client) writeTo(connReference *Conn, id string, dstID []byte, payload [ log.Errorf("failed to marshal transport message: %s", err) return 0, err } - n, err := c.relayConn.Write(msg) + + // the write always return with 0 length because the underling does not support the size feedback. + _, err = c.relayConn.Write(msg) if err != nil { log.Errorf("failed to write transport message: %s", err) } - return n, err + return len(payload), err } func (c *Client) listenForStopEvents(hc *healthcheck.Receiver, conn net.Conn, internalStopFlag *internalStopFlag) { diff --git a/relay/client/dialer/ws/conn.go b/relay/client/dialer/ws/conn.go index 188582964..e7f771b8d 100644 --- a/relay/client/dialer/ws/conn.go +++ b/relay/client/dialer/ws/conn.go @@ -38,7 +38,7 @@ func (c *Conn) Read(b []byte) (n int, err error) { func (c *Conn) Write(b []byte) (n int, err error) { err = c.Conn.Write(c.ctx, websocket.MessageBinary, b) - return len(b), err + return 0, err } func (c *Conn) RemoteAddr() net.Addr { From 2576221315f0a49dac8b738fdd02235ddfbb2973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 22 Jul 2024 15:51:44 +0200 Subject: [PATCH 140/199] Fix isConnected logic in reconnection loop --- client/internal/peer/conn.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b7c106fdb..dbc07d754 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -337,14 +337,10 @@ func (conn *Conn) reconnectLoopWithRetry() { } // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer - // todo check wg handshake conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) - conn.mu.Lock() - if conn.statusRelay == StatusConnected && conn.statusICE == StatusConnected { - conn.mu.Unlock() + if conn.isConnected() { continue } - conn.mu.Unlock() conn.log.Debugf("ticker timed out, retry to do handshake") err := conn.handshaker.sendOffer() @@ -495,7 +491,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) } -// todo review to make sense to handle connection and disconnected status also? +// todo review to make sense to handle connecting and disconnected status also? func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { conn.mu.Lock() defer conn.mu.Unlock() @@ -715,6 +711,31 @@ func (conn *Conn) evalStatus() ConnStatus { return StatusDisconnected } +func (conn *Conn) isConnected() bool { + conn.mu.Lock() + defer conn.mu.Unlock() + + if conn.statusRelay != StatusConnected { + return false + } + + if conn.statusICE != StatusConnected && conn.statusICE != StatusConnecting { + return false + } + + wgStats, err := conn.config.WgConfig.WgInterface.GetStats(conn.config.Key) + if err != nil { + conn.log.Errorf("failed to get wg stats: %v", err) + return false + } + + if time.Since(wgStats.LastHandshake) > 2*time.Minute { + return false + } + return true + +} + func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } From e9e3b8ba105778e70a980880ecc2a0c925e25bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 23 Jul 2024 22:43:20 +0200 Subject: [PATCH 141/199] Check wg handshake status on worker relay --- client/internal/peer/conn.go | 16 +------- client/internal/peer/worker_relay.go | 55 +++++++++++++++++++++++++-- client/internal/wgproxy/proxy_ebpf.go | 2 +- relay/client/manager.go | 9 +++++ 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index dbc07d754..561437f04 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -336,13 +336,12 @@ func (conn *Conn) reconnectLoopWithRetry() { return } - // checks if there is peer connection is established via relay or ice and that it has a wireguard handshake and skip offer - conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + // checks if there is peer connection is established via relay or ice + conn.log.Infof("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) if conn.isConnected() { continue } - conn.log.Debugf("ticker timed out, retry to do handshake") err := conn.handshaker.sendOffer() if err != nil { conn.log.Errorf("failed to do handshake: %v", err) @@ -722,18 +721,7 @@ func (conn *Conn) isConnected() bool { if conn.statusICE != StatusConnected && conn.statusICE != StatusConnecting { return false } - - wgStats, err := conn.config.WgConfig.WgInterface.GetStats(conn.config.Key) - if err != nil { - conn.log.Errorf("failed to get wg stats: %v", err) - return false - } - - if time.Since(wgStats.LastHandshake) > 2*time.Minute { - return false - } return true - } func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index beea912aa..f4acdba6a 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -4,12 +4,19 @@ import ( "context" "errors" "net" + "time" log "github.com/sirupsen/logrus" + "github.com/netbirdio/netbird/iface" relayClient "github.com/netbirdio/netbird/relay/client" ) +var ( + wgHandshakePeriod = 2 * time.Minute + wgHandshakeOvertime = 30000 * time.Millisecond +) + type RelayConnInfo struct { relayedConn net.Conn rosenpassPubKey []byte @@ -25,11 +32,12 @@ type WorkerRelay struct { ctx context.Context log *log.Entry config ConnConfig - relayManager *relayClient.Manager + wgInterface iface.IWGIface + relayManager relayClient.ManagerService conn WorkerRelayCallbacks } -func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager *relayClient.Manager, callbacks WorkerRelayCallbacks) *WorkerRelay { +func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay { return &WorkerRelay{ ctx: ctx, log: log, @@ -48,7 +56,7 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { // the relayManager will return with error in case if the connection has lost with relay server currentRelayAddress, err := w.relayManager.RelayInstanceAddress() if err != nil { - w.log.Infof("local Relay connection is lost, skipping connection attempt") + w.log.Errorf("failed to handle new offer: %s", err) return } @@ -61,10 +69,12 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { w.log.Infof("do not need to reopen relay connection") return } - w.log.Infof("do not need to reopen relay connection: %s", err) + w.log.Errorf("failed to open connection via Relay: %s", err) return } + go w.wgStateCheck(relayedConn) + w.log.Debugf("Relay connection established with %s", srv) go w.conn.OnConnReady(RelayConnInfo{ relayedConn: relayedConn, @@ -85,6 +95,35 @@ func (w *WorkerRelay) RelayIsSupportedLocally() bool { return w.relayManager.HasRelayAddress() } +// wgStateCheck help to check the state of the wireguard handshake and relay connection +func (w *WorkerRelay) wgStateCheck(conn net.Conn) { + timer := time.NewTimer(wgHandshakeOvertime) + defer timer.Stop() + + for { + select { + case <-timer.C: + lastHandshake, err := w.wgState() + if err != nil { + w.log.Errorf("failed to read wg stats: %v", err) + continue + } + log.Infof("last handshake: %v", lastHandshake) + + if time.Since(lastHandshake) > wgHandshakePeriod { + w.log.Infof("Wireguard handshake timed out, closing relay connection") + _ = conn.Close() + w.conn.OnDisconnected() + return + } + resetTime := (lastHandshake.Add(wgHandshakeOvertime + wgHandshakePeriod)).Sub(time.Now()) + timer.Reset(resetTime) + case <-w.ctx.Done(): + return + } + } +} + func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool { if !w.relayManager.HasRelayAddress() { return false @@ -98,3 +137,11 @@ func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress st } return remoteRelayAddress } + +func (w *WorkerRelay) wgState() (time.Time, error) { + wgState, err := w.config.WgConfig.WgInterface.GetStats(w.config.Key) + if err != nil { + return time.Time{}, err + } + return wgState.LastHandshake, nil +} diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index 9d08a0d24..bbaafa756 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -181,7 +181,7 @@ func (p *WGEBPFProxy) proxyToRemote() { conn, ok := p.turnConnStore[uint16(addr.Port)] p.turnConnMutex.Unlock() if !ok { - log.Infof("turn conn not found by port: %d", addr.Port) + log.Infof("turn conn not found by port, exit form proxy: %d", addr.Port) return // todo replace it to return. For debug troubleshooting keep it } diff --git a/relay/client/manager.go b/relay/client/manager.go index ffc8faef0..e75455e47 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -30,6 +30,15 @@ func NewRelayTrack() *RelayTrack { return &RelayTrack{} } +type ManagerService interface { + Serve() error + OpenConn(serverAddress, peerKey string, onClosedListener func()) (net.Conn, error) + RelayInstanceAddress() (string, error) + ServerURL() string + HasRelayAddress() bool + UpdateToken(token *relayAuth.Token) +} + // Manager is a manager for the relay client. It establish one persistent connection to the given relay server. In case // of network error the manager will try to reconnect to the server. // The manager also manage temproary relay connection. If a client wants to communicate with an another client on a From 20eb1f50e3985bf65c39033566018d0219f388d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 23 Jul 2024 23:04:38 +0200 Subject: [PATCH 142/199] Fix loop close --- client/internal/peer/conn.go | 2 +- client/internal/peer/worker_relay.go | 19 ++++++++++++++----- client/internal/wgproxy/proxy_ebpf.go | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 561437f04..4610afba2 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -337,7 +337,7 @@ func (conn *Conn) reconnectLoopWithRetry() { } // checks if there is peer connection is established via relay or ice - conn.log.Infof("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) if conn.isConnected() { continue } diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index f4acdba6a..2b90430f0 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -29,17 +29,20 @@ type WorkerRelayCallbacks struct { } type WorkerRelay struct { - ctx context.Context + parentCtx context.Context log *log.Entry config ConnConfig wgInterface iface.IWGIface relayManager relayClient.ManagerService conn WorkerRelayCallbacks + + ctx context.Context + ctxCancel context.CancelFunc } func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay { return &WorkerRelay{ - ctx: ctx, + parentCtx: ctx, log: log, config: config, relayManager: relayManager, @@ -62,8 +65,10 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress) - relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key, w.conn.OnDisconnected) + w.ctx, w.ctxCancel = context.WithCancel(w.parentCtx) + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key, w.disconnected) if err != nil { + w.ctxCancel() // todo handle all type errors if errors.Is(err, relayClient.ErrConnAlreadyExists) { w.log.Infof("do not need to reopen relay connection") @@ -99,7 +104,6 @@ func (w *WorkerRelay) RelayIsSupportedLocally() bool { func (w *WorkerRelay) wgStateCheck(conn net.Conn) { timer := time.NewTimer(wgHandshakeOvertime) defer timer.Stop() - for { select { case <-timer.C: @@ -108,7 +112,7 @@ func (w *WorkerRelay) wgStateCheck(conn net.Conn) { w.log.Errorf("failed to read wg stats: %v", err) continue } - log.Infof("last handshake: %v", lastHandshake) + w.log.Tracef("last handshake: %v", lastHandshake) if time.Since(lastHandshake) > wgHandshakePeriod { w.log.Infof("Wireguard handshake timed out, closing relay connection") @@ -145,3 +149,8 @@ func (w *WorkerRelay) wgState() (time.Time, error) { } return wgState.LastHandshake, nil } + +func (w *WorkerRelay) disconnected() { + w.ctxCancel() + w.conn.OnDisconnected() +} diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index bbaafa756..969a836ff 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -206,7 +206,7 @@ func (p *WGEBPFProxy) storeTurnConn(turnConn net.Conn) (uint16, error) { } func (p *WGEBPFProxy) removeTurnConn(turnConnID uint16) { - log.Tracef("remove turn conn from store by port: %d", turnConnID) + log.Debugf("remove turn conn from store by port: %d", turnConnID) p.turnConnMutex.Lock() defer p.turnConnMutex.Unlock() delete(p.turnConnStore, turnConnID) From ecb6f0831edb6ebcbd330c2732d84befedb92a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 16:26:26 +0200 Subject: [PATCH 143/199] Add metrics --- relay/client/client_test.go | 69 ++++++++++++++++++++++++++---------- relay/client/manager_test.go | 44 +++++++++++++++++------ relay/cmd/main.go | 44 ++++++++++++++++------- relay/metrics/realy.go | 21 +++++++++++ relay/server/relay.go | 16 +++++++-- relay/server/server.go | 15 ++++---- 6 files changed, 156 insertions(+), 53 deletions(-) create mode 100644 relay/metrics/realy.go diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 7b768a4cf..b8685568b 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -11,6 +11,7 @@ import ( "time" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" "github.com/netbirdio/netbird/relay/auth" "github.com/netbirdio/netbird/relay/auth/hmac" @@ -35,7 +36,10 @@ func TestMain(m *testing.M) { func TestClient(t *testing.T) { ctx := context.Background() - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { listenCfg := server.ListenerConfig{Address: serverListenAddr} @@ -58,7 +62,7 @@ func TestClient(t *testing.T) { } t.Log("alice connecting to server") clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } @@ -133,7 +137,10 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { serverAddress := fmt.Sprintf("127.0.0.1:%d", port) serverConnURL := fmt.Sprintf("rel://%s", serverAddress) - srv := server.NewServer(serverConnURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverConnURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { listenCfg := server.ListenerConfig{Address: serverAddress} @@ -259,7 +266,10 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { func TestRegistration(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -274,7 +284,7 @@ func TestRegistration(t *testing.T) { } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { _ = srv.Close() t.Fatalf("failed to connect to server: %s", err) @@ -330,7 +340,10 @@ func TestEcho(t *testing.T) { idAlice := "alice" idBob := "bob" srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -352,7 +365,7 @@ func TestEcho(t *testing.T) { } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, idAlice) - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } @@ -416,7 +429,10 @@ func TestBindToUnavailabePeer(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -439,7 +455,7 @@ func TestBindToUnavailabePeer(t *testing.T) { } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) } @@ -459,7 +475,10 @@ func TestBindReconnect(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -482,7 +501,7 @@ func TestBindReconnect(t *testing.T) { } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) } @@ -547,7 +566,10 @@ func TestCloseConn(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -570,7 +592,7 @@ func TestCloseConn(t *testing.T) { } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { t.Errorf("failed to connect to server: %s", err) } @@ -601,7 +623,10 @@ func TestCloseRelayConn(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -623,7 +648,7 @@ func TestCloseRelayConn(t *testing.T) { } clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") - err := clientAlice.Connect() + err = clientAlice.Connect() if err != nil { t.Fatalf("failed to connect to server: %s", err) } @@ -650,7 +675,10 @@ func TestCloseByServer(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv1 := server.NewServer(serverURL, false, av) + srv1, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { @@ -668,7 +696,7 @@ func TestCloseByServer(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) - err := relayClient.Connect() + err = relayClient.Connect() if err != nil { log.Fatalf("failed to connect to server: %s", err) } @@ -700,7 +728,10 @@ func TestCloseByClient(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} - srv := server.NewServer(serverURL, false, av) + srv, err := server.NewServer(otel.Meter(""), serverURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -717,7 +748,7 @@ func TestCloseByClient(t *testing.T) { idAlice := "alice" log.Debugf("connect by alice") relayClient := NewClient(ctx, serverURL, hmacTokenStore, idAlice) - err := relayClient.Connect() + err = relayClient.Connect() if err != nil { log.Fatalf("failed to connect to server: %s", err) } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 8da1ef20f..928171175 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -6,6 +6,7 @@ import ( "time" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" "github.com/netbirdio/netbird/relay/server" ) @@ -16,7 +17,10 @@ func TestForeignConn(t *testing.T) { srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer(srvCfg1.Address, false, av) + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv1.Listen(srvCfg1) @@ -39,7 +43,10 @@ func TestForeignConn(t *testing.T) { srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer(srvCfg2.Address, false, av) + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan2 := make(chan error, 1) go func() { err := srv2.Listen(srvCfg2) @@ -64,7 +71,7 @@ func TestForeignConn(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) - err := clientAlice.Serve() + err = clientAlice.Serve() if err != nil { t.Fatalf("failed to serve manager: %s", err) } @@ -122,7 +129,10 @@ func TestForeginConnClose(t *testing.T) { srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer(srvCfg1.Address, false, av) + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv1.Listen(srvCfg1) @@ -145,7 +155,10 @@ func TestForeginConnClose(t *testing.T) { srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer(srvCfg2.Address, false, av) + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan2 := make(chan error, 1) go func() { err := srv2.Listen(srvCfg2) @@ -170,7 +183,7 @@ func TestForeginConnClose(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) - err := mgr.Serve() + err = mgr.Serve() if err != nil { t.Fatalf("failed to serve manager: %s", err) } @@ -191,7 +204,10 @@ func TestForeginAutoClose(t *testing.T) { srvCfg1 := server.ListenerConfig{ Address: "localhost:1234", } - srv1 := server.NewServer(srvCfg1.Address, false, av) + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { t.Log("binding server 1.") @@ -217,7 +233,10 @@ func TestForeginAutoClose(t *testing.T) { srvCfg2 := server.ListenerConfig{ Address: "localhost:2234", } - srv2 := server.NewServer(srvCfg2.Address, false, av) + srv2, err := server.NewServer(otel.Meter(""), srvCfg2.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan2 := make(chan error, 1) go func() { t.Log("binding server 2.") @@ -244,7 +263,7 @@ func TestForeginAutoClose(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() mgr := NewManager(mCtx, toURL(srvCfg1), idAlice) - err := mgr.Serve() + err = mgr.Serve() if err != nil { t.Fatalf("failed to serve manager: %s", err) } @@ -277,7 +296,10 @@ func TestAutoReconnect(t *testing.T) { srvCfg := server.ListenerConfig{ Address: "localhost:1234", } - srv := server.NewServer(srvCfg.Address, false, av) + srv, err := server.NewServer(otel.Meter(""), srvCfg.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } errChan := make(chan error, 1) go func() { err := srv.Listen(srvCfg) @@ -300,7 +322,7 @@ func TestAutoReconnect(t *testing.T) { mCtx, cancel := context.WithCancel(ctx) defer cancel() clientAlice := NewManager(mCtx, toURL(srvCfg), "alice") - err := clientAlice.Serve() + err = clientAlice.Serve() if err != nil { t.Fatalf("failed to serve manager: %s", err) } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index b96482ef8..22dc025a8 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -2,7 +2,9 @@ package main import ( "crypto/tls" + "errors" "fmt" + "net/http" "os" "os/signal" "syscall" @@ -14,9 +16,14 @@ import ( "github.com/netbirdio/netbird/encryption" auth "github.com/netbirdio/netbird/relay/auth/hmac" "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/signal/metrics" "github.com/netbirdio/netbird/util" ) +const ( + metricsPort = 9090 +) + type Config struct { ListenAddress string // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection @@ -54,7 +61,7 @@ var ( Use: "relay", Short: "Relay service", Long: "Relay service for Netbird agents", - Run: execute, + RunE: execute, } ) @@ -110,11 +117,10 @@ func loadConfig(configFile string) (*Config, error) { return loadedConfig, err } -func execute(cmd *cobra.Command, args []string) { +func execute(cmd *cobra.Command, args []string) error { cfg, err := loadConfig(cfgFile) if err != nil { - log.Errorf("failed to load config: %s", err) - os.Exit(1) + return fmt.Errorf("failed to load config: %s", err) } err = cfg.Validate() @@ -123,21 +129,31 @@ func execute(cmd *cobra.Command, args []string) { os.Exit(1) } + metricsServer, err := metrics.NewServer(metricsPort, "") + if err != nil { + return fmt.Errorf("setup metrics: %v", err) + } + + go func() { + log.Infof("running metrics server: %s%s", metricsServer.Addr, metricsServer.Endpoint) + if err := metricsServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("Failed to start metrics server: %v", err) + } + }() + srvListenerCfg := server.ListenerConfig{ Address: cfg.ListenAddress, } if cfg.HasLetsEncrypt() { tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...) if err != nil { - log.Errorf("%s", err) - os.Exit(1) + return fmt.Errorf("%s", err) } srvListenerCfg.TLSConfig = tlsCfg } else if cfg.HasCertConfig() { tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile) if err != nil { - log.Errorf("%s", err) - os.Exit(1) + return fmt.Errorf("%s", err) } srvListenerCfg.TLSConfig = tlsCfg } @@ -145,21 +161,23 @@ func execute(cmd *cobra.Command, args []string) { tlsSupport := srvListenerCfg.TLSConfig != nil authenticator := auth.NewTimedHMACValidator(cfg.AuthSecret, 24*time.Hour) - srv := server.NewServer(cfg.ExposedAddress, tlsSupport, authenticator) + srv, err := server.NewServer(metricsServer.Meter, cfg.ExposedAddress, tlsSupport, authenticator) + if err != nil { + return fmt.Errorf("failed to create relay server: %v", err) + } log.Infof("server will be available on: %s", srv.InstanceURL()) err = srv.Listen(srvListenerCfg) if err != nil { - log.Errorf("failed to bind server: %s", err) - os.Exit(1) + return fmt.Errorf("failed to bind server: %s", err) } waitForExitSignal() err = srv.Close() if err != nil { - log.Errorf("failed to close server: %s", err) - os.Exit(1) + return fmt.Errorf("failed to close server: %s", err) } + return nil } func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) { diff --git a/relay/metrics/realy.go b/relay/metrics/realy.go new file mode 100644 index 000000000..29dc6557b --- /dev/null +++ b/relay/metrics/realy.go @@ -0,0 +1,21 @@ +package metrics + +import "go.opentelemetry.io/otel/metric" + +type Metrics struct { + metric.Meter + + Peers metric.Int64UpDownCounter +} + +func NewMetrics(meter metric.Meter) (*Metrics, error) { + peers, err := meter.Int64UpDownCounter("peers") + if err != nil { + return nil, err + } + + return &Metrics{ + Meter: meter, + Peers: peers, + }, nil +} diff --git a/relay/server/relay.go b/relay/server/relay.go index cb35c6e17..6dca3e62c 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -7,12 +7,15 @@ import ( "sync" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" "github.com/netbirdio/netbird/relay/auth" "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/metrics" ) type Relay struct { + metrics *metrics.Metrics validator auth.Validator store *Store @@ -22,8 +25,14 @@ type Relay struct { closeMu sync.RWMutex } -func NewRelay(exposedAddress string, tlsSupport bool, validator auth.Validator) *Relay { +func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, validator auth.Validator) (*Relay, error) { + m, err := metrics.NewMetrics(meter) + if err != nil { + return nil, fmt.Errorf("creating app metrics: %v", err) + } + r := &Relay{ + metrics: m, validator: validator, store: NewStore(), } @@ -34,7 +43,7 @@ func NewRelay(exposedAddress string, tlsSupport bool, validator auth.Validator) r.instaceURL = fmt.Sprintf("rel://%s", exposedAddress) } - return r + return r, nil } func (r *Relay) Accept(conn net.Conn) { @@ -57,11 +66,12 @@ func (r *Relay) Accept(conn net.Conn) { peer := NewPeer(peerID, conn, r.store) peer.log.Infof("peer connected from: %s", conn.RemoteAddr()) r.store.AddPeer(peer) - + r.metrics.Peers.Add(context.Background(), 1) go func() { peer.Work() r.store.DeletePeer(peer) peer.log.Debugf("relay connection closed") + r.metrics.Peers.Add(context.Background(), -1) }() } diff --git a/relay/server/server.go b/relay/server/server.go index cac884372..0eff59eb2 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -8,6 +8,7 @@ import ( "time" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" "github.com/netbirdio/netbird/relay/auth" "github.com/netbirdio/netbird/relay/server/listener" @@ -26,14 +27,14 @@ type Server struct { wSListener listener.Listener } -func NewServer(exposedAddress string, tlsSupport bool, authValidator auth.Validator) *Server { - return &Server{ - relay: NewRelay( - exposedAddress, - tlsSupport, - authValidator, - ), +func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authValidator auth.Validator) (*Server, error) { + relay, err := NewRelay(meter, exposedAddress, tlsSupport, authValidator) + if err != nil { + return nil, err } + return &Server{ + relay: relay, + }, nil } func (r *Server) Listen(cfg ListenerConfig) error { From 57f8c620c0d4f8ac484bd0117c88c5f6bae86c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 16:34:47 +0200 Subject: [PATCH 144/199] Typo fix --- relay/server/relay.go | 12 ++++++------ relay/server/server.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/relay/server/relay.go b/relay/server/relay.go index 6dca3e62c..c0d61ad03 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -18,8 +18,8 @@ type Relay struct { metrics *metrics.Metrics validator auth.Validator - store *Store - instaceURL string + store *Store + instanceURL string closed bool closeMu sync.RWMutex @@ -38,9 +38,9 @@ func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, valida } if tlsSupport { - r.instaceURL = fmt.Sprintf("rels://%s", exposedAddress) + r.instanceURL = fmt.Sprintf("rels://%s", exposedAddress) } else { - r.instaceURL = fmt.Sprintf("rel://%s", exposedAddress) + r.instanceURL = fmt.Sprintf("rel://%s", exposedAddress) } return r, nil @@ -120,7 +120,7 @@ func (r *Relay) handShake(conn net.Conn) ([]byte, error) { return nil, err } - msg, _ := messages.MarshalHelloResponse(r.instaceURL) + msg, _ := messages.MarshalHelloResponse(r.instanceURL) _, err = conn.Write(msg) if err != nil { return nil, err @@ -129,5 +129,5 @@ func (r *Relay) handShake(conn net.Conn) ([]byte, error) { } func (r *Relay) InstanceURL() string { - return r.instaceURL + return r.instanceURL } diff --git a/relay/server/server.go b/relay/server/server.go index 0eff59eb2..80d1ba00d 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -91,5 +91,5 @@ func (r *Server) Close() error { } func (r *Server) InstanceURL() string { - return r.relay.instaceURL + return r.relay.instanceURL } From 7ae9cffccc6e8d9e322ec52f8c6a6becf1052b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 16:51:33 +0200 Subject: [PATCH 145/199] Fix missing method after merge --- client/internal/engine.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/client/internal/engine.go b/client/internal/engine.go index 57a2acb14..0f66f5a4e 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -1428,3 +1428,20 @@ func isChecksEqual(checks []*mgmProto.Checks, oChecks []*mgmProto.Checks) bool { return slices.Equal(checks.Files, oChecks.Files) }) } + +func (e *Engine) IsWGIfaceUp() bool { + if e == nil || e.wgInterface == nil { + return false + } + iface, err := net.InterfaceByName(e.wgInterface.Name()) + if err != nil { + log.Debugf("failed to get interface by name %s: %v", e.wgInterface.Name(), err) + return false + } + + if iface.Flags&net.FlagUp != 0 { + return true + } + + return false +} From 42ea9611d537e4f75eb5ba17b59002ac69a33a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 17:36:46 +0200 Subject: [PATCH 146/199] Fix test --- client/server/server_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/server/server_test.go b/client/server/server_test.go index 47738cb84..469e54eb5 100644 --- a/client/server/server_test.go +++ b/client/server/server_test.go @@ -124,7 +124,10 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") + rc := &server.RelayConfig{ + Address: "localhost:0", + } + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { return nil, "", err From 7bfc5059622458a1939eabb026ee5bada1d6ea46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 17:42:51 +0200 Subject: [PATCH 147/199] Fix test --- client/cmd/status_test.go | 4 ---- client/cmd/testutil_test.go | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/cmd/status_test.go b/client/cmd/status_test.go index 46620a956..b5fbce9eb 100644 --- a/client/cmd/status_test.go +++ b/client/cmd/status_test.go @@ -37,7 +37,6 @@ var resp = &proto.StatusResponse{ ConnStatus: "Connected", ConnStatusUpdate: timestamppb.New(time.Date(2001, time.Month(1), 1, 1, 1, 1, 0, time.UTC)), Relayed: false, - Direct: true, LocalIceCandidateType: "", RemoteIceCandidateType: "", LocalIceCandidateEndpoint: "", @@ -57,7 +56,6 @@ var resp = &proto.StatusResponse{ ConnStatus: "Connected", ConnStatusUpdate: timestamppb.New(time.Date(2002, time.Month(2), 2, 2, 2, 2, 0, time.UTC)), Relayed: true, - Direct: false, LocalIceCandidateType: "relay", RemoteIceCandidateType: "prflx", LocalIceCandidateEndpoint: "10.0.0.1:10001", @@ -137,7 +135,6 @@ var overview = statusOutputOverview{ Status: "Connected", LastStatusUpdate: time.Date(2001, 1, 1, 1, 1, 1, 0, time.UTC), ConnType: "P2P", - Direct: true, IceCandidateType: iceCandidateType{ Local: "", Remote: "", @@ -161,7 +158,6 @@ var overview = statusOutputOverview{ Status: "Connected", LastStatusUpdate: time.Date(2002, 2, 2, 2, 2, 2, 0, time.UTC), ConnType: "Relayed", - Direct: false, IceCandidateType: iceCandidateType{ Local: "relay", Remote: "prflx", diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index 63d90cc63..889431777 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -92,7 +92,11 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") + + rc := &mgmt.RelayConfig{ + Address: "localhost:0", + } + turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { t.Fatal(err) From a4ba8bd3b8ebf852d4b83b326b226ad68a73a3a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 17:45:42 +0200 Subject: [PATCH 148/199] Fix test --- client/internal/routemanager/client_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/client/internal/routemanager/client_test.go b/client/internal/routemanager/client_test.go index 0ae10e568..c4783d5f3 100644 --- a/client/internal/routemanager/client_test.go +++ b/client/internal/routemanager/client_test.go @@ -24,7 +24,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -43,7 +42,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: true, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -62,7 +60,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: true, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -81,7 +78,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: false, relayed: false, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -100,12 +96,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: false, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -129,12 +123,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: true, - direct: true, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -158,12 +150,10 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, }, "route2": { connected: true, relayed: false, - direct: false, }, }, existingRoutes: map[route.ID]*route.Route{ @@ -241,13 +231,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 15 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, @@ -272,13 +260,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 200 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, @@ -303,13 +289,11 @@ func TestGetBestrouteFromStatuses(t *testing.T) { "route1": { connected: true, relayed: false, - direct: true, latency: 20 * time.Millisecond, }, "route2": { connected: true, relayed: false, - direct: true, latency: 10 * time.Millisecond, }, }, From c42f7628d7869245621cf21ccb3ce7ff0f3628ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 17:47:32 +0200 Subject: [PATCH 149/199] Fix test --- management/server/management_proto_test.go | 6 +++++- management/server/management_test.go | 6 +++++- management/server/token_mgr_test.go | 15 ++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index 7976e76e4..5aaf95344 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -419,7 +419,11 @@ func startManagement(t *testing.T, config *Config) (*grpc.Server, string, error) if err != nil { return nil, "", err } - turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") + + rc := &RelayConfig{ + Address: "localhost:0", + } + turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) ephemeralMgr := NewEphemeralManager(store, accountManager) mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, ephemeralMgr) diff --git a/management/server/management_test.go b/management/server/management_test.go index 1ef97f73b..af9a7eae4 100644 --- a/management/server/management_test.go +++ b/management/server/management_test.go @@ -546,7 +546,11 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) { if err != nil { log.Fatalf("failed creating a manager: %v", err) } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") + + rc := &server.RelayConfig{ + Address: "localhost:0", + } + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) Expect(err).NotTo(HaveOccurred()) mgmtProto.RegisterManagementServiceServer(s, mgmtServer) diff --git a/management/server/token_mgr_test.go b/management/server/token_mgr_test.go index 6ac3571bb..debd798d0 100644 --- a/management/server/token_mgr_test.go +++ b/management/server/token_mgr_test.go @@ -23,11 +23,14 @@ func TestTimeBasedAuthSecretsManager_GenerateCredentials(t *testing.T) { secret := "some_secret" peersManager := NewPeersUpdateManager(nil) + rc := &RelayConfig{ + Address: "localhost:0", + } tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }, "") + }, rc) credentials, _ := tested.Generate() @@ -49,11 +52,14 @@ func TestTimeBasedAuthSecretsManager_SetupRefresh(t *testing.T) { peer := "some_peer" updateChannel := peersManager.CreateChannel(context.Background(), peer) + rc := &RelayConfig{ + Address: "localhost:0", + } tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }, "") + }, rc) tested.SetupRefresh(context.Background(), peer) @@ -97,11 +103,14 @@ func TestTimeBasedAuthSecretsManager_CancelRefresh(t *testing.T) { peersManager := NewPeersUpdateManager(nil) peer := "some_peer" + rc := &RelayConfig{ + Address: "localhost:0", + } tested := NewTimeBasedAuthSecretsManager(peersManager, &TURNConfig{ CredentialsTTL: ttl, Secret: secret, Turns: []*Host{TurnTestHost}, - }, "") + }, rc) tested.SetupRefresh(context.Background(), peer) if _, ok := tested.cancelMap[peer]; !ok { From 1d833113ce2e108950701711f316d7660b87f255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 17:51:00 +0200 Subject: [PATCH 150/199] Fix test --- client/internal/engine_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index b8a85b071..bdb0fbe62 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -1101,7 +1101,10 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error) if err != nil { return nil, "", err } - turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") + rc := &server.RelayConfig{ + Address: "127.0.0.1:1234", + } + turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := server.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { return nil, "", err From fdf97568086384bfb39c22f07094a19512a9c490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 17:52:19 +0200 Subject: [PATCH 151/199] Fix test --- client/ios/NetBirdSDK/client.go | 1 - management/client/client_test.go | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/ios/NetBirdSDK/client.go b/client/ios/NetBirdSDK/client.go index d80072c78..c0c7447c2 100644 --- a/client/ios/NetBirdSDK/client.go +++ b/client/ios/NetBirdSDK/client.go @@ -168,7 +168,6 @@ func (c *Client) GetStatusDetails() *StatusDetails { BytesTx: p.BytesTx, ConnStatus: p.ConnStatus.String(), ConnStatusUpdate: p.ConnStatusUpdate.Format("2006-01-02 15:04:05"), - Direct: p.Direct, LastWireguardHandshake: p.LastWireguardHandshake.String(), Relayed: p.Relayed, RosenpassEnabled: p.RosenpassEnabled, diff --git a/management/client/client_test.go b/management/client/client_test.go index da6b574b0..35bba5bb9 100644 --- a/management/client/client_test.go +++ b/management/client/client_test.go @@ -75,7 +75,10 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) { if err != nil { t.Fatal(err) } - turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, "") + rc := &mgmt.RelayConfig{ + Address: "localhost:0", + } + turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, rc) mgmtServer, err := mgmt.NewServer(context.Background(), config, accountManager, peersUpdateManager, turnManager, nil, nil) if err != nil { t.Fatal(err) From 8568fbffddb0fc09d1f73a336eaa00c656ef9862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 24 Jul 2024 18:01:43 +0200 Subject: [PATCH 152/199] Fix test --- client/internal/peer/status.go | 2 -- client/internal/peer/worker_relay.go | 4 +--- relay/client/client_test.go | 28 ++++++++++++++++++++-------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index f5f9a07a3..57accd28d 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -23,8 +23,6 @@ type State struct { PubKey string FQDN string ConnStatus ConnStatus - connStatusRelay ConnStatus - connStatusICE ConnStatus ConnStatusUpdate time.Time Relayed bool LocalIceCandidateType string diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 2b90430f0..138f4aeae 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -8,7 +8,6 @@ import ( log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/iface" relayClient "github.com/netbirdio/netbird/relay/client" ) @@ -32,7 +31,6 @@ type WorkerRelay struct { parentCtx context.Context log *log.Entry config ConnConfig - wgInterface iface.IWGIface relayManager relayClient.ManagerService conn WorkerRelayCallbacks @@ -120,7 +118,7 @@ func (w *WorkerRelay) wgStateCheck(conn net.Conn) { w.conn.OnDisconnected() return } - resetTime := (lastHandshake.Add(wgHandshakeOvertime + wgHandshakePeriod)).Sub(time.Now()) + resetTime := time.Until(lastHandshake.Add(wgHandshakeOvertime + wgHandshakePeriod)) timer.Reset(resetTime) case <-w.ctx.Done(): return diff --git a/relay/client/client_test.go b/relay/client/client_test.go index b8685568b..7e511d302 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -200,10 +200,13 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { var transferDuration []time.Duration wg := sync.WaitGroup{} + var writeErr error + var readErr error for i := 0; i < len(connsSender); i++ { wg.Add(2) start := time.Now() go func(i int) { + defer wg.Done() pieceSize := 1024 testDataLen := len(testData) @@ -212,34 +215,43 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { if end > testDataLen { end = testDataLen } - _, err := connsSender[i].Write(testData[j:end]) - if err != nil { - t.Fatalf("failed to write to channel: %s", err) + _, writeErr = connsSender[i].Write(testData[j:end]) + if writeErr != nil { + return } } - wg.Done() + }(i) go func(i int, start time.Time) { + defer wg.Done() buf := make([]byte, 8192) rcv := 0 + var n int for receivedSize := 0; receivedSize < len(testData); { - n, err := connsReceiver[i].Read(buf) - if err != nil { - t.Fatalf("failed to read from channel: %s", err) + n, readErr = connsReceiver[i].Read(buf) + if readErr != nil { + return } receivedSize += n rcv += n } transferDuration = append(transferDuration, time.Since(start)) - wg.Done() }(i, start) } wg.Wait() + if writeErr != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + if readErr != nil { + t.Fatalf("failed to read from channel: %s", err) + } + // calculate the megabytes per second from the average transferDuration against the dataSize var totalDuration time.Duration for _, d := range transferDuration { From 4339b6528f026e97d9d464e0f4e39c21abb40cfb Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 24 Jul 2024 23:11:41 +0200 Subject: [PATCH 153/199] Skip benchmark test --- relay/client/client_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 7e511d302..cc2bc71d9 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -116,6 +116,7 @@ func TestClient(t *testing.T) { } func TestDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test dataSize := 1024 * 1024 * 10 testData, err := seedRandomData(dataSize) From 334926ce90bf59ee145b1cfe7f20b434a2bfa2fc Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 24 Jul 2024 23:27:15 +0200 Subject: [PATCH 154/199] Fix status test --- client/cmd/status_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/cmd/status_test.go b/client/cmd/status_test.go index b5fbce9eb..ca43df8a5 100644 --- a/client/cmd/status_test.go +++ b/client/cmd/status_test.go @@ -279,7 +279,6 @@ func TestParsingToJSON(t *testing.T) { "status": "Connected", "lastStatusUpdate": "2001-01-01T01:01:01Z", "connectionType": "P2P", - "direct": true, "iceCandidateType": { "local": "", "remote": "" @@ -288,6 +287,7 @@ func TestParsingToJSON(t *testing.T) { "local": "", "remote": "" }, + "relayAddress": "", "lastWireguardHandshake": "2001-01-01T01:01:02Z", "transferReceived": 200, "transferSent": 100, @@ -304,7 +304,6 @@ func TestParsingToJSON(t *testing.T) { "status": "Connected", "lastStatusUpdate": "2002-02-02T02:02:02Z", "connectionType": "Relayed", - "direct": false, "iceCandidateType": { "local": "relay", "remote": "prflx" @@ -313,6 +312,7 @@ func TestParsingToJSON(t *testing.T) { "local": "10.0.0.1:10001", "remote": "10.0.10.1:10002" }, + "relayAddress": "", "lastWireguardHandshake": "2002-02-02T02:02:03Z", "transferReceived": 2000, "transferSent": 1000, @@ -404,13 +404,13 @@ func TestParsingToYAML(t *testing.T) { status: Connected lastStatusUpdate: 2001-01-01T01:01:01Z connectionType: P2P - direct: true iceCandidateType: local: "" remote: "" iceCandidateEndpoint: local: "" remote: "" + relayAddress: "" lastWireguardHandshake: 2001-01-01T01:01:02Z transferReceived: 200 transferSent: 100 @@ -424,13 +424,13 @@ func TestParsingToYAML(t *testing.T) { status: Connected lastStatusUpdate: 2002-02-02T02:02:02Z connectionType: Relayed - direct: false iceCandidateType: local: relay remote: prflx iceCandidateEndpoint: local: 10.0.0.1:10001 remote: 10.0.10.1:10002 + relayAddress: "" lastWireguardHandshake: 2002-02-02T02:02:03Z transferReceived: 2000 transferSent: 1000 @@ -501,9 +501,9 @@ func TestParsingToDetail(t *testing.T) { Status: Connected -- detail -- Connection type: P2P - Direct: true ICE candidate (Local/Remote): -/- ICE candidate endpoints (Local/Remote): -/- + Relay server address: Last connection update: %s Last WireGuard handshake: %s Transfer status (received/sent) 200 B/100 B @@ -517,9 +517,9 @@ func TestParsingToDetail(t *testing.T) { Status: Connected -- detail -- Connection type: Relayed - Direct: false ICE candidate (Local/Remote): relay/prflx ICE candidate endpoints (Local/Remote): 10.0.0.1:10001/10.0.10.1:10002 + Relay server address: Last connection update: %s Last WireGuard handshake: %s Transfer status (received/sent) 2.0 KiB/1000 B From 3d2ef17364ec7fae9e6637fe139676ffd48bf0b7 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 24 Jul 2024 23:46:05 +0200 Subject: [PATCH 155/199] Fix sonar issue --- client/internal/peer/conn.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 4610afba2..65c9bf7b7 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -523,7 +523,7 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { err := conn.statusRecorder.UpdatePeerICEStateToDisconnected(peerState) if err != nil { - conn.log.Warnf("unable to save peer's state, got error: %v", err) + conn.log.Warnf("unable to set peer's state to disconnected ice, got error: %v", err) } } @@ -613,7 +613,7 @@ func (conn *Conn) onWorkerRelayStateDisconnected() { err := conn.statusRecorder.UpdatePeerRelayedStateToDisconnected(peerState) if err != nil { - conn.log.Warnf("unable to save peer's state, got error: %v", err) + conn.log.Warnf("unable to save peer's state to Relay disconnected, got error: %v", err) } } @@ -639,7 +639,7 @@ func (conn *Conn) updateRelayStatus(relayServerAddr string, rosenpassPubKey []by err := conn.statusRecorder.UpdatePeerRelayedState(peerState) if err != nil { - conn.log.Warnf("unable to save peer's state, got error: %v", err) + conn.log.Warnf("unable to save peer's Relay state, got error: %v", err) } } @@ -658,7 +658,7 @@ func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) { err := conn.statusRecorder.UpdatePeerICEState(peerState) if err != nil { - conn.log.Warnf("unable to save peer's state, got error: %v", err) + conn.log.Warnf("unable to save peer's ICE state, got error: %v", err) } } From 5dbe5d0d493f3e83e242f44d6135bce27045fc2c Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 24 Jul 2024 23:50:25 +0200 Subject: [PATCH 156/199] Fix sonar issue --- client/internal/peer/conn.go | 42 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 65c9bf7b7..092356b3d 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -209,7 +209,7 @@ func (conn *Conn) Close() { conn.ctxCancel() if !conn.opened { - log.Infof("IGNORE close connection to peer") + log.Debugf("ignore close connection to peer") return } @@ -247,24 +247,7 @@ func (conn *Conn) Close() { conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) } - conn.statusRelay = StatusDisconnected - conn.statusICE = StatusDisconnected - - peerState := State{ - PubKey: conn.config.Key, - ConnStatus: StatusDisconnected, - ConnStatusUpdate: time.Now(), - Mux: new(sync.RWMutex), - } - err = conn.statusRecorder.UpdatePeerState(peerState) - if err != nil { - // pretty common error because by that time Engine can already remove the peer and status won't be available. - // todo rethink status updates - conn.log.Debugf("error while updating peer's state, err: %v", err) - } - if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { - conn.log.Debugf("failed to reset wireguard stats for peer: %s", err) - } + conn.setStatusToDisconnected() } // OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise @@ -662,6 +645,27 @@ func (conn *Conn) updateIceState(iceConnInfo ICEConnInfo) { } } +func (conn *Conn) setStatusToDisconnected() { + conn.statusRelay = StatusDisconnected + conn.statusICE = StatusDisconnected + + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: StatusDisconnected, + ConnStatusUpdate: time.Now(), + Mux: new(sync.RWMutex), + } + err := conn.statusRecorder.UpdatePeerState(peerState) + if err != nil { + // pretty common error because by that time Engine can already remove the peer and status won't be available. + // todo rethink status updates + conn.log.Debugf("error while updating peer's state, err: %v", err) + } + if err := conn.statusRecorder.UpdateWireGuardPeerState(conn.config.Key, iface.WGStats{}); err != nil { + conn.log.Debugf("failed to reset wireguard stats for peer: %s", err) + } +} + func (conn *Conn) doOnConnected(remoteRosenpassPubKey []byte, remoteRosenpassAddr string) { if runtime.GOOS == "ios" { runtime.GC() From 08022dca10911122f5e06a24c72f2194776457ca Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 24 Jul 2024 23:57:33 +0200 Subject: [PATCH 157/199] Fix sonar issue and fix conn id handling --- client/internal/peer/conn.go | 40 ++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 092356b3d..bc965de13 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -97,7 +97,8 @@ type Conn struct { workerICE *WorkerICE workerRelay *WorkerRelay - connID nbnet.ConnectionID + connIDRelay nbnet.ConnectionID + connIDICE nbnet.ConnectionID beforeAddPeerHooks []nbnet.AddHookFunc afterRemovePeerHooks []nbnet.RemoveHookFunc @@ -234,14 +235,7 @@ func (conn *Conn) Close() { conn.log.Errorf("failed to remove wg endpoint: %v", err) } - if conn.connID != "" { - for _, hook := range conn.afterRemovePeerHooks { - if err := hook(conn.connID); err != nil { - conn.log.Errorf("After remove peer hook failed: %v", err) - } - } - conn.connID = "" - } + conn.freeUpConnID() if conn.evalStatus() == StatusConnected && conn.onDisconnected != nil { conn.onDisconnected(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps) @@ -443,9 +437,9 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) conn.log.Debugf("Conn resolved IP is %s for endopint %s", endpoint, endpointUdpAddr.IP) - conn.connID = nbnet.GenerateConnID() + conn.connIDICE = nbnet.GenerateConnID() for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { + if err := hook(conn.connIDICE, endpointUdpAddr.IP); err != nil { conn.log.Errorf("Before add peer hook failed: %v", err) } } @@ -541,9 +535,9 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { } } - conn.connID = nbnet.GenerateConnID() + conn.connIDRelay = nbnet.GenerateConnID() for _, hook := range conn.beforeAddPeerHooks { - if err := hook(conn.connID, endpointUdpAddr.IP); err != nil { + if err := hook(conn.connIDRelay, endpointUdpAddr.IP); err != nil { conn.log.Errorf("Before add peer hook failed: %v", err) } } @@ -728,6 +722,26 @@ func (conn *Conn) isConnected() bool { return true } +func (conn *Conn) freeUpConnID() { + if conn.connIDRelay != "" { + for _, hook := range conn.afterRemovePeerHooks { + if err := hook(conn.connIDRelay); err != nil { + conn.log.Errorf("After remove peer hook failed: %v", err) + } + } + conn.connIDRelay = "" + } + + if conn.connIDICE != "" { + for _, hook := range conn.afterRemovePeerHooks { + if err := hook(conn.connIDICE); err != nil { + conn.log.Errorf("After remove peer hook failed: %v", err) + } + } + conn.connIDICE = "" + } +} + func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } From ff167e2907b8585faf1f1cb249573292623a007e Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:04:27 +0200 Subject: [PATCH 158/199] Fix sonar issue --- client/internal/peer/conn.go | 42 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index bc965de13..b236ef78f 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -413,25 +413,9 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.log.Infof("set ICE to active connection") } - var ( - endpoint net.Addr - wgProxy wgproxy.Proxy - ) - if iceConnInfo.RelayedOnLocal { - conn.log.Debugf("setup ice turn connection") - wgProxy = conn.wgProxyFactory.GetProxy(conn.ctx) - ep, err := wgProxy.AddTurnConn(iceConnInfo.RemoteConn) - if err != nil { - conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) - err = wgProxy.CloseConn() - if err != nil { - conn.log.Warnf("failed to close turn proxy connection: %v", err) - } - return - } - endpoint = ep - } else { - endpoint = iceConnInfo.RemoteConn.RemoteAddr() + endpoint, wgProxy, err := conn.getEndpointForICEConnInfo(iceConnInfo) + if err != nil { + return } endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) @@ -444,7 +428,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon } } - err := conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) + err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) if err != nil { if wgProxy != nil { if err := wgProxy.CloseConn(); err != nil { @@ -742,6 +726,24 @@ func (conn *Conn) freeUpConnID() { } } +func (conn *Conn) getEndpointForICEConnInfo(iceConnInfo ICEConnInfo) (net.Addr, wgproxy.Proxy, error) { + if !iceConnInfo.RelayedOnLocal { + return iceConnInfo.RemoteConn.RemoteAddr(), nil, nil + } + conn.log.Debugf("setup ice turn connection") + wgProxy := conn.wgProxyFactory.GetProxy(conn.ctx) + ep, err := wgProxy.AddTurnConn(iceConnInfo.RemoteConn) + if err != nil { + conn.log.Errorf("failed to add turn net.Conn to local proxy: %v", err) + err = wgProxy.CloseConn() + if err != nil { + conn.log.Warnf("failed to close turn proxy connection: %v", err) + } + return nil, nil, err + } + return ep, wgProxy, nil +} + func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } From 3e82fcbdd0d1a912a00a624841386c0103875692 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:09:44 +0200 Subject: [PATCH 159/199] Fix sonar issue --- client/internal/connect.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index 21423455e..56fad2734 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -164,10 +164,8 @@ func (c *ConnectClient) run( defer c.statusRecorder.ClientStop() operation := func() error { // if context cancelled we not start new backoff cycle - select { - case <-c.ctx.Done(): + if c.isContextCancelled() { return nil - default: } state.Set(StatusConnecting) @@ -189,8 +187,7 @@ func (c *ConnectClient) run( log.Debugf("connected to the Management service %s", c.config.ManagementURL.Host) defer func() { - err = mgmClient.Close() - if err != nil { + if err = mgmClient.Close(); err != nil { log.Warnf("failed to close the Management service client %v", err) } }() @@ -213,7 +210,6 @@ func (c *ConnectClient) run( KernelInterface: iface.WireGuardModuleIsLoaded(), FQDN: loginResp.GetPeerConfig().GetFqdn(), } - c.statusRecorder.UpdateLocalPeerState(localPeerState) signalURL := fmt.Sprintf("%s://%s", @@ -274,8 +270,7 @@ func (c *ConnectClient) run( c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, mgmProbe, signalProbe, relayProbe, wgProbe, checks) c.engineMutex.Unlock() - err = c.engine.Start() - if err != nil { + if err := c.engine.Start(); err != nil { log.Errorf("error while starting Netbird Connection Engine: %s", err) return wrapErr(err) } @@ -342,6 +337,15 @@ func (c *ConnectClient) Engine() *Engine { return e } +func (c *ConnectClient) isContextCancelled() bool { + select { + case <-c.ctx.Done(): + return true + default: + return false + } +} + // createEngineConfig converts configuration received from Management Service to EngineConfig func createEngineConfig(key wgtypes.Key, config *Config, peerConfig *mgmProto.PeerConfig) (*EngineConfig, error) { nm := false From 6f36ec7a321fa3d91df78a027fe845ec389877ad Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:13:05 +0200 Subject: [PATCH 160/199] Fix sonar issue --- client/internal/peer/worker_ice.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 55b9b8c4e..26f3d04ab 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -72,7 +72,6 @@ type WorkerICE struct { ctx context.Context log *log.Entry config ConnConfig - configICE ICEConfig signaler *Signaler iFaceDiscover stdnet.ExternalIFaceDiscover statusRecorder *Status @@ -92,12 +91,11 @@ type WorkerICE struct { localPwd string } -func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, configICE ICEConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) { +func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool, callBacks WorkerICECallbacks) (*WorkerICE, error) { w := &WorkerICE{ ctx: ctx, log: log, config: config, - configICE: configICE, signaler: signaler, iFaceDiscover: ifaceDiscover, statusRecorder: statusRecorder, @@ -233,12 +231,12 @@ func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport [ agentConfig := &ice.AgentConfig{ MulticastDNSMode: ice.MulticastDNSModeDisabled, NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: w.configICE.StunTurn.Load().([]*stun.URI), + Urls: w.config.ICEConfig.StunTurn.Load().([]*stun.URI), CandidateTypes: relaySupport, - InterfaceFilter: stdnet.InterfaceFilter(w.configICE.InterfaceBlackList), - UDPMux: w.configICE.UDPMux, - UDPMuxSrflx: w.configICE.UDPMuxSrflx, - NAT1To1IPs: w.configICE.NATExternalIPs, + InterfaceFilter: stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList), + UDPMux: w.config.ICEConfig.UDPMux, + UDPMuxSrflx: w.config.ICEConfig.UDPMuxSrflx, + NAT1To1IPs: w.config.ICEConfig.NATExternalIPs, Net: transportNet, FailedTimeout: &failedTimeout, DisconnectedTimeout: &iceDisconnectedTimeout, @@ -248,7 +246,7 @@ func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport [ LocalPwd: w.localPwd, } - if w.configICE.DisableIPv6Discovery { + if w.config.ICEConfig.DisableIPv6Discovery { agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} } @@ -308,7 +306,7 @@ func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) return } - mux, ok := w.configICE.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) + mux, ok := w.config.ICEConfig.UDPMuxSrflx.(*bind.UniversalUDPMuxDefault) if !ok { w.log.Warn("invalid udp mux conversion") return From 856c97b9df8b68fa68a3bccbca84bd2ff19d4c72 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:14:19 +0200 Subject: [PATCH 161/199] Fix sonar issue --- relay/server/listener/udp/conn.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relay/server/listener/udp/conn.go b/relay/server/listener/udp/conn.go index 46c7580d5..1c811064c 100644 --- a/relay/server/listener/udp/conn.go +++ b/relay/server/listener/udp/conn.go @@ -50,17 +50,17 @@ func (u *Conn) RemoteAddr() net.Addr { func (u *Conn) SetDeadline(t time.Time) error { //TODO implement me - panic("implement me") + panic("implement SetDeadline") } func (u *Conn) SetReadDeadline(t time.Time) error { //TODO implement me - panic("implement me") + panic("implement SetReadDeadline") } func (u *Conn) SetWriteDeadline(t time.Time) error { //TODO implement me - panic("implement me") + panic("implement SetWriteDeadline") } func (u *Conn) onNewMsg(b []byte) { From 7b677f8db2d506d06322cb97dfcf090c757e5329 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:19:07 +0200 Subject: [PATCH 162/199] Fix sonar issue --- client/internal/peer/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index b236ef78f..0fca5250e 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -150,7 +150,7 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu conn.workerRelay = NewWorkerRelay(ctx, connLog, config, relayManager, rFns) relayIsSupportedLocally := conn.workerRelay.RelayIsSupportedLocally() - conn.workerICE, err = NewWorkerICE(ctx, connLog, config, config.ICEConfig, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns) + conn.workerICE, err = NewWorkerICE(ctx, connLog, config, signaler, iFaceDiscover, statusRecorder, relayIsSupportedLocally, wFns) if err != nil { return nil, err } From 7279d58110148208f30724e33cae3977c5f59184 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:20:03 +0200 Subject: [PATCH 163/199] Fix sonar issue --- client/internal/peer/stdnet_android.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/stdnet_android.go b/client/internal/peer/stdnet_android.go index 27f49be9f..b411405bb 100644 --- a/client/internal/peer/stdnet_android.go +++ b/client/internal/peer/stdnet_android.go @@ -3,5 +3,5 @@ package peer import "github.com/netbirdio/netbird/client/internal/stdnet" func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.configICE.InterfaceBlackList) + return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList) } From 163a80d53e942da719bec655150191f4c62a4598 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 25 Jul 2024 00:23:34 +0200 Subject: [PATCH 164/199] Fix sonar issue --- client/internal/peer/stdnet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/stdnet.go b/client/internal/peer/stdnet.go index 7882ff5da..ae31ebbf0 100644 --- a/client/internal/peer/stdnet.go +++ b/client/internal/peer/stdnet.go @@ -7,5 +7,5 @@ import ( ) func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNet(w.configICE.InterfaceBlackList) + return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList) } From e1ee73500a8de2763510b2e51fd49f5fcda1b514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 25 Jul 2024 11:03:54 +0200 Subject: [PATCH 165/199] Remove unused codes --- relay/client/dialer/quic/conn.go | 52 ------------ relay/client/dialer/quic/quic.go | 32 ------- relay/client/dialer/tcp/tcp.go | 7 -- relay/client/dialer/udp/udp.go | 14 ---- relay/server/listener/quic/conn.go | 36 -------- relay/server/listener/quic/listener.go | 110 ------------------------- relay/server/listener/tcp/listener.go | 80 ------------------ relay/server/listener/udp/conn.go | 68 --------------- relay/server/listener/udp/listener.go | 104 ----------------------- relay/server/server.go | 39 ++------- 10 files changed, 6 insertions(+), 536 deletions(-) delete mode 100644 relay/client/dialer/quic/conn.go delete mode 100644 relay/client/dialer/quic/quic.go delete mode 100644 relay/client/dialer/tcp/tcp.go delete mode 100644 relay/client/dialer/udp/udp.go delete mode 100644 relay/server/listener/quic/conn.go delete mode 100644 relay/server/listener/quic/listener.go delete mode 100644 relay/server/listener/tcp/listener.go delete mode 100644 relay/server/listener/udp/conn.go delete mode 100644 relay/server/listener/udp/listener.go diff --git a/relay/client/dialer/quic/conn.go b/relay/client/dialer/quic/conn.go deleted file mode 100644 index 2c8b29a3c..000000000 --- a/relay/client/dialer/quic/conn.go +++ /dev/null @@ -1,52 +0,0 @@ -package quic - -import ( - "net" - - "github.com/quic-go/quic-go" - log "github.com/sirupsen/logrus" -) - -type Conn struct { - quic.Stream - qConn quic.Connection -} - -func NewConn(stream quic.Stream, qConn quic.Connection) net.Conn { - return &Conn{ - Stream: stream, - qConn: qConn, - } -} - -func (q *Conn) Write(b []byte) (n int, err error) { - log.Debugf("writing: %d, %x\n", len(b), b) - n, err = q.Stream.Write(b) - if n != len(b) { - log.Errorf("failed to write out the full message") - } - return -} - -func (q *Conn) Close() error { - err := q.Stream.Close() - if err != nil { - log.Errorf("failed to close stream: %s", err) - return err - } - err = q.qConn.CloseWithError(0, "") - if err != nil { - log.Errorf("failed to close connection: %s", err) - return err - - } - return err -} - -func (c *Conn) LocalAddr() net.Addr { - return c.qConn.LocalAddr() -} - -func (c *Conn) RemoteAddr() net.Addr { - return c.qConn.RemoteAddr() -} diff --git a/relay/client/dialer/quic/quic.go b/relay/client/dialer/quic/quic.go deleted file mode 100644 index 4863ed7bd..000000000 --- a/relay/client/dialer/quic/quic.go +++ /dev/null @@ -1,32 +0,0 @@ -package quic - -import ( - "context" - "crypto/tls" - "net" - - "github.com/quic-go/quic-go" - log "github.com/sirupsen/logrus" -) - -func Dial(address string) (net.Conn, error) { - tlsConf := &tls.Config{ - InsecureSkipVerify: true, - NextProtos: []string{"quic-echo-example"}, - } - qConn, err := quic.DialAddr(context.Background(), address, tlsConf, &quic.Config{ - EnableDatagrams: true, - }) - if err != nil { - log.Errorf("dial quic address %s failed: %s", address, err) - return nil, err - } - - stream, err := qConn.OpenStreamSync(context.Background()) - if err != nil { - return nil, err - } - - conn := NewConn(stream, qConn) - return conn, nil -} diff --git a/relay/client/dialer/tcp/tcp.go b/relay/client/dialer/tcp/tcp.go deleted file mode 100644 index 47b0a31d5..000000000 --- a/relay/client/dialer/tcp/tcp.go +++ /dev/null @@ -1,7 +0,0 @@ -package tcp - -import "net" - -func Dial(address string) (net.Conn, error) { - return net.Dial("tcp", address) -} diff --git a/relay/client/dialer/udp/udp.go b/relay/client/dialer/udp/udp.go deleted file mode 100644 index ff0fa9c83..000000000 --- a/relay/client/dialer/udp/udp.go +++ /dev/null @@ -1,14 +0,0 @@ -package udp - -import ( - "net" -) - -func Dial(address string) (net.Conn, error) { - udpAddr, err := net.ResolveUDPAddr("udp", address) - if err != nil { - return nil, err - } - - return net.DialUDP("udp", nil, udpAddr) -} diff --git a/relay/server/listener/quic/conn.go b/relay/server/listener/quic/conn.go deleted file mode 100644 index 5414f9eee..000000000 --- a/relay/server/listener/quic/conn.go +++ /dev/null @@ -1,36 +0,0 @@ -package quic - -import ( - "net" - - "github.com/quic-go/quic-go" - log "github.com/sirupsen/logrus" -) - -type Conn struct { - quic.Stream - qConn quic.Connection -} - -func NewConn(stream quic.Stream, qConn quic.Connection) net.Conn { - return &Conn{ - Stream: stream, - qConn: qConn, - } -} - -func (q Conn) Write(b []byte) (n int, err error) { - n, err = q.Stream.Write(b) - if n != len(b) { - log.Errorf("failed to write out the full message") - } - return -} - -func (q Conn) LocalAddr() net.Addr { - return q.qConn.LocalAddr() -} - -func (q Conn) RemoteAddr() net.Addr { - return q.qConn.RemoteAddr() -} diff --git a/relay/server/listener/quic/listener.go b/relay/server/listener/quic/listener.go deleted file mode 100644 index 55107e7ba..000000000 --- a/relay/server/listener/quic/listener.go +++ /dev/null @@ -1,110 +0,0 @@ -package quic - -import ( - "context" - "crypto/rand" - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "math/big" - "net" - "sync" - - "github.com/quic-go/quic-go" - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/relay/server/listener" -) - -type Listener struct { - address string - - listener *quic.Listener - quit chan struct{} - wg sync.WaitGroup -} - -func NewListener(address string) listener.Listener { - return &Listener{ - address: address, - } -} - -func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { - ql, err := quic.ListenAddr(l.address, generateTLSConfig(), &quic.Config{ - EnableDatagrams: true, - }) - if err != nil { - return err - } - l.listener = ql - l.quit = make(chan struct{}) - - log.Infof("quic server is listening on address: %s", l.address) - l.wg.Add(1) - go l.acceptLoop(onAcceptFn) - - <-l.quit - return nil -} - -func (l *Listener) Close() error { - close(l.quit) - err := l.listener.Close() - l.wg.Wait() - return err -} - -func (l *Listener) acceptLoop(acceptFn func(conn net.Conn)) { - defer l.wg.Done() - - for { - qConn, err := l.listener.Accept(context.Background()) - if err != nil { - select { - case <-l.quit: - return - default: - log.Errorf("failed to accept connection: %s", err) - continue - } - } - - log.Infof("new connection from: %s", qConn.RemoteAddr()) - - stream, err := qConn.AcceptStream(context.Background()) - if err != nil { - log.Errorf("failed to open stream: %s", err) - continue - } - - conn := NewConn(stream, qConn) - - go acceptFn(conn) - } -} - -// Setup a bare-bones TLS config for the server -func generateTLSConfig() *tls.Config { - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - panic(err) - } - template := x509.Certificate{SerialNumber: big.NewInt(1)} - certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) - if err != nil { - panic(err) - } - keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) - certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - panic(err) - } - return &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - NextProtos: []string{"quic-echo-example"}, - } -} diff --git a/relay/server/listener/tcp/listener.go b/relay/server/listener/tcp/listener.go deleted file mode 100644 index fb26b0fc4..000000000 --- a/relay/server/listener/tcp/listener.go +++ /dev/null @@ -1,80 +0,0 @@ -package tcp - -import ( - "net" - "sync" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/relay/server/listener" -) - -// Listener -// Is it just demo code. It does not work in real life environment because the TCP is a streaming protocol, and -// it does not handle framing. -type Listener struct { - address string - - onAcceptFn func(conn net.Conn) - wg sync.WaitGroup - quit chan struct{} - listener net.Listener - lock sync.Mutex -} - -func NewListener(address string) listener.Listener { - return &Listener{ - address: address, - } -} - -func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { - l.lock.Lock() - - l.onAcceptFn = onAcceptFn - l.quit = make(chan struct{}) - - li, err := net.Listen("tcp", l.address) - if err != nil { - log.Errorf("failed to listen on address: %s, %s", l.address, err) - l.lock.Unlock() - return err - } - log.Debugf("TCP server is listening on address: %s", l.address) - l.listener = li - l.wg.Add(1) - go l.acceptLoop() - - l.lock.Unlock() - <-l.quit - return nil -} - -// Close todo: prevent multiple call (do not close two times the channel) -func (l *Listener) Close() error { - l.lock.Lock() - defer l.lock.Unlock() - - close(l.quit) - err := l.listener.Close() - l.wg.Wait() - return err -} - -func (l *Listener) acceptLoop() { - defer l.wg.Done() - - for { - conn, err := l.listener.Accept() - if err != nil { - select { - case <-l.quit: - return - default: - log.Errorf("failed to accept connection: %s", err) - continue - } - } - go l.onAcceptFn(conn) - } -} diff --git a/relay/server/listener/udp/conn.go b/relay/server/listener/udp/conn.go deleted file mode 100644 index 1c811064c..000000000 --- a/relay/server/listener/udp/conn.go +++ /dev/null @@ -1,68 +0,0 @@ -package udp - -import ( - "io" - "net" - "time" -) - -type Conn struct { - *net.UDPConn - addr *net.UDPAddr - msgChannel chan []byte -} - -func NewConn(conn *net.UDPConn, addr *net.UDPAddr) *Conn { - return &Conn{ - UDPConn: conn, - addr: addr, - msgChannel: make(chan []byte), - } -} - -func (u *Conn) Read(b []byte) (n int, err error) { - msg, ok := <-u.msgChannel - if !ok { - return 0, io.EOF - } - - n = copy(b, msg) - return n, nil -} - -func (u *Conn) Write(b []byte) (n int, err error) { - return u.UDPConn.WriteTo(b, u.addr) -} - -func (u *Conn) Close() error { - //TODO implement me - //panic("implement me") - return nil -} - -func (u *Conn) LocalAddr() net.Addr { - return u.UDPConn.LocalAddr() -} - -func (u *Conn) RemoteAddr() net.Addr { - return u.addr -} - -func (u *Conn) SetDeadline(t time.Time) error { - //TODO implement me - panic("implement SetDeadline") -} - -func (u *Conn) SetReadDeadline(t time.Time) error { - //TODO implement me - panic("implement SetReadDeadline") -} - -func (u *Conn) SetWriteDeadline(t time.Time) error { - //TODO implement me - panic("implement SetWriteDeadline") -} - -func (u *Conn) onNewMsg(b []byte) { - u.msgChannel <- b -} diff --git a/relay/server/listener/udp/listener.go b/relay/server/listener/udp/listener.go deleted file mode 100644 index a895cdc32..000000000 --- a/relay/server/listener/udp/listener.go +++ /dev/null @@ -1,104 +0,0 @@ -package udp - -import ( - "net" - "sync" - - log "github.com/sirupsen/logrus" - - "github.com/netbirdio/netbird/relay/server/listener" -) - -type Listener struct { - address string - conns map[string]*Conn - onAcceptFn func(conn net.Conn) - - listener *net.UDPConn - - wg sync.WaitGroup - quit chan struct{} - lock sync.Mutex -} - -func NewListener(address string) listener.Listener { - return &Listener{ - address: address, - conns: make(map[string]*Conn), - } -} - -func (l *Listener) Listen(onAcceptFn func(conn net.Conn)) error { - l.lock.Lock() - - l.onAcceptFn = onAcceptFn - l.quit = make(chan struct{}) - - addr, err := net.ResolveUDPAddr("udp", l.address) - if err != nil { - log.Errorf("invalid listen address '%s': %s", l.address, err) - l.lock.Unlock() - return err - } - - li, err := net.ListenUDP("udp", addr) - if err != nil { - log.Fatalf("%s", err) - l.lock.Unlock() - return err - } - log.Debugf("udp server is listening on address: %s", addr.String()) - l.listener = li - l.wg.Add(1) - go l.readLoop() - - l.lock.Unlock() - <-l.quit - return nil -} - -func (l *Listener) Close() error { - l.lock.Lock() - defer l.lock.Unlock() - - if l.listener == nil { - return nil - } - - log.Infof("closing UDP listener") - close(l.quit) - err := l.listener.Close() - l.wg.Wait() - l.listener = nil - return err -} - -func (l *Listener) readLoop() { - defer l.wg.Done() - - for { - buf := make([]byte, 1500) - n, addr, err := l.listener.ReadFromUDP(buf) - if err != nil { - select { - case <-l.quit: - return - default: - log.Errorf("failed to accept connection: %s", err) - continue - } - } - - pConn, ok := l.conns[addr.String()] - if ok { - pConn.onNewMsg(buf[:n]) - continue - } - - pConn = NewConn(l.listener, addr) - log.Infof("new connection from: %s", pConn.RemoteAddr()) - l.conns[addr.String()] = pConn - go l.onAcceptFn(pConn) - pConn.onNewMsg(buf[:n]) - } -} diff --git a/relay/server/server.go b/relay/server/server.go index 80d1ba00d..c34151c9d 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -3,8 +3,6 @@ package server import ( "context" "crypto/tls" - "errors" - "sync" "time" log "github.com/sirupsen/logrus" @@ -12,7 +10,6 @@ import ( "github.com/netbirdio/netbird/relay/auth" "github.com/netbirdio/netbird/relay/server/listener" - "github.com/netbirdio/netbird/relay/server/listener/udp" "github.com/netbirdio/netbird/relay/server/listener/ws" ) @@ -22,9 +19,8 @@ type ListenerConfig struct { } type Server struct { - relay *Relay - uDPListener listener.Listener - wSListener listener.Listener + relay *Relay + wSListener listener.Listener } func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authValidator auth.Validator) (*Server, error) { @@ -38,9 +34,6 @@ func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authV } func (r *Server) Listen(cfg ListenerConfig) error { - wg := sync.WaitGroup{} - wg.Add(2) - r.wSListener = &ws.Listener{ Address: cfg.Address, TLSConfig: cfg.TLSConfig, @@ -48,46 +41,26 @@ func (r *Server) Listen(cfg ListenerConfig) error { var wslErr error go func() { - defer wg.Done() wslErr = r.wSListener.Listen(r.relay.Accept) if wslErr != nil { log.Errorf("failed to bind ws server: %s", wslErr) } }() - r.uDPListener = udp.NewListener(cfg.Address) - var udpLErr error - go func() { - defer wg.Done() - udpLErr = r.uDPListener.Listen(r.relay.Accept) - if udpLErr != nil { - log.Errorf("failed to bind ws server: %s", udpLErr) - } - }() - - err := errors.Join(wslErr, udpLErr) - return err + return wslErr } -func (r *Server) Close() error { - var wErr error +func (r *Server) Close() (err error) { // stop service new connections if r.wSListener != nil { - wErr = r.wSListener.Close() - } - - var uErr error - if r.uDPListener != nil { - uErr = r.uDPListener.Close() + err = r.wSListener.Close() } // close accepted connections gracefully ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() r.relay.Close(ctx) - - err := errors.Join(wErr, uErr) - return err + return } func (r *Server) InstanceURL() string { From 8861e89ab0bba225ab917a6aacc3790d8cabc591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 25 Jul 2024 11:05:38 +0200 Subject: [PATCH 166/199] Remove unused dependencies --- go.mod | 6 ------ go.sum | 12 ------------ 2 files changed, 18 deletions(-) diff --git a/go.mod b/go.mod index 251eaf777..c0a0dc3c7 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,6 @@ require ( github.com/pion/transport/v3 v3.0.1 github.com/pion/turn/v3 v3.0.1 github.com/prometheus/client_golang v1.19.1 - github.com/quic-go/quic-go v0.45.0 github.com/rs/xid v1.3.0 github.com/shirou/gopsutil/v3 v3.24.4 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 @@ -136,12 +135,10 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-stack/stack v1.8.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.0.1 // indirect - github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect @@ -193,12 +190,9 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect - go.uber.org/mock v0.4.0 // indirect golang.org/x/image v0.18.0 // indirect - golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240509183442-62759503f434 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect diff --git a/go.sum b/go.sum index 9f6ae4a76..3f780eed5 100644 --- a/go.sum +++ b/go.sum @@ -50,9 +50,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.15.0 h1:7NxJhNiBT3NG8pZJ3c+yfrVdHY8ScgKD27sScgjLMMk= github.com/cilium/ebpf v0.15.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -252,7 +249,6 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -421,8 +417,6 @@ github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+a github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= -github.com/quic-go/quic-go v0.45.0 h1:OHmkQGM37luZITyTSu6ff03HP/2IrwDX1ZFiNEhSFUE= -github.com/quic-go/quic-go v0.45.0/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= @@ -470,7 +464,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -533,8 +526,6 @@ go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2L go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4= goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -567,8 +558,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -622,7 +611,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From efa0bbdf3dcddf720ef0c8580bd4276cb62f5093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 25 Jul 2024 11:09:45 +0200 Subject: [PATCH 167/199] Remove unused logs --- client/internal/peer/conn.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 0fca5250e..0b7be7cf8 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -407,11 +407,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon return } - if conn.currentConnType != 0 { - conn.log.Infof("update connection to ICE type") - } else { - conn.log.Infof("set ICE to active connection") - } + conn.log.Infof("set ICE to active connection") endpoint, wgProxy, err := conn.getEndpointForICEConnInfo(iceConnInfo) if err != nil { From b62ad97e59717e5994aa18bf9b5ba1601f7934d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 25 Jul 2024 12:37:59 +0200 Subject: [PATCH 168/199] Fix wg state check exit logic --- client/internal/peer/worker_relay.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 138f4aeae..0e341ed14 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -63,10 +63,8 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress) - w.ctx, w.ctxCancel = context.WithCancel(w.parentCtx) relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key, w.disconnected) if err != nil { - w.ctxCancel() // todo handle all type errors if errors.Is(err, relayClient.ErrConnAlreadyExists) { w.log.Infof("do not need to reopen relay connection") @@ -76,6 +74,8 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { return } + w.ctx, w.ctxCancel = context.WithCancel(w.parentCtx) + go w.wgStateCheck(relayedConn) w.log.Debugf("Relay connection established with %s", srv) From d70df99f7beebc2018c19d7a7e6a0fed6b720a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Thu, 25 Jul 2024 17:21:27 +0200 Subject: [PATCH 169/199] Fix memory leak Avoid to add listeners to multiple times --- client/internal/peer/worker_relay.go | 29 +++++++---- relay/client/manager.go | 52 ++++++++++++------- relay/client/manager_test.go | 77 +++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 34 deletions(-) diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index 0e341ed14..cf72e30e6 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -34,18 +34,18 @@ type WorkerRelay struct { relayManager relayClient.ManagerService conn WorkerRelayCallbacks - ctx context.Context ctxCancel context.CancelFunc } func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay { - return &WorkerRelay{ + r := &WorkerRelay{ parentCtx: ctx, log: log, config: config, relayManager: relayManager, conn: callbacks, } + return r } func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { @@ -63,7 +63,7 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress) - relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key, w.disconnected) + relayedConn, err := w.relayManager.OpenConn(srv, w.config.Key) if err != nil { // todo handle all type errors if errors.Is(err, relayClient.ErrConnAlreadyExists) { @@ -74,11 +74,20 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { return } - w.ctx, w.ctxCancel = context.WithCancel(w.parentCtx) + ctx, ctxCancel := context.WithCancel(w.parentCtx) + w.ctxCancel = ctxCancel - go w.wgStateCheck(relayedConn) + err = w.relayManager.AddCloseListener(srv, w.disconnected) + if err != nil { + log.Errorf("failed to add close listener: %s", err) + _ = relayedConn.Close() + ctxCancel() + return + } - w.log.Debugf("Relay connection established with %s", srv) + go w.wgStateCheck(ctx, relayedConn) + + w.log.Debugf("peer conn opened via Relay: %s", srv) go w.conn.OnConnReady(RelayConnInfo{ relayedConn: relayedConn, rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey, @@ -99,7 +108,7 @@ func (w *WorkerRelay) RelayIsSupportedLocally() bool { } // wgStateCheck help to check the state of the wireguard handshake and relay connection -func (w *WorkerRelay) wgStateCheck(conn net.Conn) { +func (w *WorkerRelay) wgStateCheck(ctx context.Context, conn net.Conn) { timer := time.NewTimer(wgHandshakeOvertime) defer timer.Stop() for { @@ -120,7 +129,7 @@ func (w *WorkerRelay) wgStateCheck(conn net.Conn) { } resetTime := time.Until(lastHandshake.Add(wgHandshakeOvertime + wgHandshakePeriod)) timer.Reset(resetTime) - case <-w.ctx.Done(): + case <-ctx.Done(): return } } @@ -149,6 +158,8 @@ func (w *WorkerRelay) wgState() (time.Time, error) { } func (w *WorkerRelay) disconnected() { - w.ctxCancel() + if w.ctxCancel != nil { + w.ctxCancel() + } w.conn.OnDisconnected() } diff --git a/relay/client/manager.go b/relay/client/manager.go index e75455e47..bb523822a 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -1,9 +1,11 @@ package client import ( + "container/list" "context" "fmt" "net" + "reflect" "sync" "time" @@ -30,9 +32,12 @@ func NewRelayTrack() *RelayTrack { return &RelayTrack{} } +type OnServerCloseListener func() + type ManagerService interface { Serve() error - OpenConn(serverAddress, peerKey string, onClosedListener func()) (net.Conn, error) + OpenConn(serverAddress, peerKey string) (net.Conn, error) + AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error RelayInstanceAddress() (string, error) ServerURL() string HasRelayAddress() bool @@ -57,7 +62,7 @@ type Manager struct { relayClients map[string]*RelayTrack relayClientsMutex sync.RWMutex - onDisconnectedListeners map[string]map[*func()]struct{} + onDisconnectedListeners map[string]*list.List listenerLock sync.Mutex } @@ -68,7 +73,7 @@ func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { peerID: peerID, tokenStore: &relayAuth.TokenStore{}, relayClients: make(map[string]*RelayTrack), - onDisconnectedListeners: make(map[string]map[*func()]struct{}), + onDisconnectedListeners: make(map[string]*list.List), } } @@ -97,7 +102,7 @@ func (m *Manager) Serve() error { // OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be // established via the relay server. If the peer is on a different relay server, the manager will establish a new // connection to the relay server. -func (m *Manager) OpenConn(serverAddress, peerKey string, onClosedListener func()) (net.Conn, error) { +func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { if m.relayClient == nil { return nil, errRelayClientNotConnected } @@ -121,19 +126,23 @@ func (m *Manager) OpenConn(serverAddress, peerKey string, onClosedListener func( return nil, err } - if onClosedListener != nil { - var listenerAddr string - if foreign { - m.addListener(serverAddress, onClosedListener) - listenerAddr = serverAddress - } else { - listenerAddr = m.serverURL - } - m.addListener(listenerAddr, onClosedListener) + return netConn, err +} +func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error { + foreign, err := m.isForeignServer(serverAddress) + if err != nil { + return err } - return netConn, err + var listenerAddr string + if foreign { + listenerAddr = serverAddress + } else { + listenerAddr = m.serverURL + } + m.addListener(listenerAddr, onClosedListener) + return nil } // RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is lost. @@ -265,14 +274,19 @@ func (m *Manager) cleanUpUnusedRelays() { } } -func (m *Manager) addListener(serverAddress string, onClosedListener func()) { +func (m *Manager) addListener(serverAddress string, onClosedListener OnServerCloseListener) { m.listenerLock.Lock() defer m.listenerLock.Unlock() l, ok := m.onDisconnectedListeners[serverAddress] if !ok { - l = make(map[*func()]struct{}) + l = list.New() } - l[&onClosedListener] = struct{}{} + for e := l.Front(); e != nil; e = e.Next() { + if reflect.ValueOf(e.Value).Pointer() == reflect.ValueOf(onClosedListener).Pointer() { + return + } + } + l.PushBack(onClosedListener) m.onDisconnectedListeners[serverAddress] = l } @@ -284,8 +298,8 @@ func (m *Manager) notifyOnDisconnectListeners(serverAddress string) { if !ok { return } - for f := range l { - go (*f)() + for e := l.Front(); e != nil; e = e.Next() { + go e.Value.(OnServerCloseListener)() } delete(m.onDisconnectedListeners, serverAddress) } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 928171175..cfec3f54e 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -87,11 +87,11 @@ func TestForeignConn(t *testing.T) { if err != nil { t.Fatalf("failed to get relay address: %s", err) } - connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob, nil) + connAliceToBob, err := clientAlice.OpenConn(bobsSrvAddr, idBob) if err != nil { t.Fatalf("failed to bind channel: %s", err) } - connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice, nil) + connBobToAlice, err := clientBob.OpenConn(bobsSrvAddr, idAlice) if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -187,7 +187,7 @@ func TestForeginConnClose(t *testing.T) { if err != nil { t.Fatalf("failed to serve manager: %s", err) } - conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer", nil) + conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer") if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -269,7 +269,7 @@ func TestForeginAutoClose(t *testing.T) { } t.Log("open connection to another peer") - conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer", nil) + conn, err := mgr.OpenConn(toURL(srvCfg2), "anotherpeer") if err != nil { t.Fatalf("failed to bind channel: %s", err) } @@ -330,7 +330,7 @@ func TestAutoReconnect(t *testing.T) { if err != nil { t.Errorf("failed to get relay address: %s", err) } - conn, err := clientAlice.OpenConn(ra, "bob", nil) + conn, err := clientAlice.OpenConn(ra, "bob") if err != nil { t.Errorf("failed to bind channel: %s", err) } @@ -348,12 +348,77 @@ func TestAutoReconnect(t *testing.T) { time.Sleep(reconnectingTimeout + 1*time.Second) log.Infof("reopent the connection") - _, err = clientAlice.OpenConn(ra, "bob", nil) + _, err = clientAlice.OpenConn(ra, "bob") if err != nil { t.Errorf("failed to open channel: %s", err) } } +func TestNotifierDoubleAdd(t *testing.T) { + ctx := context.Background() + + srvCfg1 := server.ListenerConfig{ + Address: "localhost:1234", + } + srv1, err := server.NewServer(otel.Meter(""), srvCfg1.Address, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + err := srv1.Listen(srvCfg1) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv1.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + idAlice := "alice" + log.Debugf("connect by alice") + mCtx, cancel := context.WithCancel(ctx) + defer cancel() + clientAlice := NewManager(mCtx, toURL(srvCfg1), idAlice) + err = clientAlice.Serve() + if err != nil { + t.Fatalf("failed to serve manager: %s", err) + } + + conn1, err := clientAlice.OpenConn(clientAlice.ServerURL(), "idBob") + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + + fnCloseListener := OnServerCloseListener(func() { + log.Infof("close listener") + }) + + err = clientAlice.AddCloseListener(clientAlice.ServerURL(), fnCloseListener) + if err != nil { + t.Fatalf("failed to add close listener: %s", err) + } + + err = clientAlice.AddCloseListener(clientAlice.ServerURL(), fnCloseListener) + if err != nil { + t.Fatalf("failed to add close listener: %s", err) + } + + err = conn1.Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + +} + func toURL(address server.ListenerConfig) string { return "rel://" + address.Address } From 9ee062b4d19425a6852229f1955786faac12eb77 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 26 Jul 2024 13:37:53 +0200 Subject: [PATCH 170/199] Change log --- client/internal/peer/conn.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 0b7be7cf8..d6726589f 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -313,8 +313,11 @@ func (conn *Conn) reconnectLoopWithRetry() { return } + if conn.statusRelay == StatusDisconnected || conn.statusICE == StatusDisconnected { + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + } + // checks if there is peer connection is established via relay or ice - conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) if conn.isConnected() { continue } From a31d43a14c40561bbeb7a00de87a637d644597ba Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 26 Jul 2024 13:38:32 +0200 Subject: [PATCH 171/199] Rename variable --- client/internal/peer/conn.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index d6726589f..abb5fc458 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -89,10 +89,10 @@ type Conn struct { onConnected func(remoteWireGuardKey string, remoteRosenpassPubKey []byte, wireGuardIP string, remoteRosenpassAddr string) onDisconnected func(remotePeer string, wgIP string) - statusRelay ConnStatus - statusICE ConnStatus - currentConnType ConnPriority - opened bool // this flag is used to prevent close in case of not opened connection + statusRelay ConnStatus + statusICE ConnStatus + currentConnPriority ConnPriority + opened bool // this flag is used to prevent close in case of not opened connection workerICE *WorkerICE workerRelay *WorkerRelay @@ -406,7 +406,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon defer conn.updateIceState(iceConnInfo) - if conn.currentConnType > priority { + if conn.currentConnPriority > priority { return } @@ -445,7 +445,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon } conn.wgProxyICE = wgProxy - conn.currentConnType = priority + conn.currentConnPriority = priority conn.doOnConnected(iceConnInfo.RosenpassPubKey, iceConnInfo.RosenpassAddr) } @@ -511,9 +511,9 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { defer conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey) - if conn.currentConnType > connPriorityRelay { + if conn.currentConnPriority > connPriorityRelay { if conn.statusICE == StatusConnected { - log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnType) + log.Debugf("do not switch to relay because current priority is: %v", conn.currentConnPriority) return } } @@ -540,7 +540,7 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { } } conn.wgProxyRelay = wgProxy - conn.currentConnType = connPriorityRelay + conn.currentConnPriority = connPriorityRelay conn.log.Infof("start to communicate with peer via relay") conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr) @@ -672,7 +672,7 @@ func (conn *Conn) isRelayed() bool { return false } - if conn.currentConnType == connPriorityICEP2P { + if conn.currentConnPriority == connPriorityICEP2P { return false } From 61c06c7dd20d0014b3488817b55b91040291c790 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 26 Jul 2024 13:40:43 +0200 Subject: [PATCH 172/199] Avoid unnecessary wg reconfiguration --- client/internal/peer/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index abb5fc458..a02ab7d9c 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -458,7 +458,7 @@ func (conn *Conn) onWorkerICEStateDisconnected(newState ConnStatus) { conn.log.Tracef("ICE connection state changed to %s", newState) // switch back to relay connection - if conn.endpointRelay != nil { + if conn.endpointRelay != nil && conn.currentConnPriority != connPriorityRelay { conn.log.Debugf("ICE disconnected, set Relay to active connection") err := conn.configureWGEndpoint(conn.endpointRelay) if err != nil { From a8b58a182efe127d494eba5127fa4b05f7231ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 26 Jul 2024 13:42:33 +0200 Subject: [PATCH 173/199] Change exponent settings --- client/internal/peer/conn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index a02ab7d9c..4fbc0c8dc 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -349,8 +349,8 @@ func (conn *Conn) reconnectLoopWithRetry() { func (conn *Conn) prepareExponentTicker() *backoff.Ticker { bo := backoff.WithContext(&backoff.ExponentialBackOff{ InitialInterval: 800 * time.Millisecond, - RandomizationFactor: 1, - Multiplier: 1.99, + RandomizationFactor: 0.01, + Multiplier: 2, MaxInterval: conn.config.Timeout, MaxElapsedTime: 0, Stop: backoff.Stop, From b946088a909e015a0878d4d938c6c6e18e5e9970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 26 Jul 2024 14:13:29 +0200 Subject: [PATCH 174/199] Add benchmark test --- relay/client/client_test.go | 173 ---------------- relay/test/benchmark_test.go | 379 +++++++++++++++++++++++++++++++++++ 2 files changed, 379 insertions(+), 173 deletions(-) create mode 100644 relay/test/benchmark_test.go diff --git a/relay/client/client_test.go b/relay/client/client_test.go index cc2bc71d9..592d5bb4d 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -2,11 +2,8 @@ package client import ( "context" - "crypto/rand" - "fmt" "net" "os" - "sync" "testing" "time" @@ -115,167 +112,6 @@ func TestClient(t *testing.T) { } } -func TestDataTransfer(t *testing.T) { - t.SkipNow() // skip this test on CI because it is a benchmark test - dataSize := 1024 * 1024 * 10 - - testData, err := seedRandomData(dataSize) - if err != nil { - t.Fatalf("failed to seed random data: %s", err) - } - - for _, peerPairs := range []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} { - t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { - transfer(t, testData, peerPairs) - }) - } -} - -func transfer(t *testing.T, testData []byte, peerPairs int) { - t.Helper() - ctx := context.Background() - port := 35000 + peerPairs - serverAddress := fmt.Sprintf("127.0.0.1:%d", port) - serverConnURL := fmt.Sprintf("rel://%s", serverAddress) - - srv, err := server.NewServer(otel.Meter(""), serverConnURL, false, av) - if err != nil { - t.Fatalf("failed to create server: %s", err) - } - errChan := make(chan error, 1) - go func() { - listenCfg := server.ListenerConfig{Address: serverAddress} - err := srv.Listen(listenCfg) - if err != nil { - errChan <- err - } - }() - - defer func() { - err := srv.Close() - if err != nil { - t.Errorf("failed to close server: %s", err) - } - }() - - // wait for server to start - if err := waitForServerToStart(errChan); err != nil { - t.Fatalf("failed to start server: %s", err) - } - - clientsSender := make([]*Client, peerPairs) - for i := 0; i < cap(clientsSender); i++ { - c := NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) - err := c.Connect() - if err != nil { - t.Fatalf("failed to connect to server: %s", err) - } - clientsSender[i] = c - } - - clientsReceiver := make([]*Client, peerPairs) - for i := 0; i < cap(clientsReceiver); i++ { - c := NewClient(ctx, serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) - err := c.Connect() - if err != nil { - t.Fatalf("failed to connect to server: %s", err) - } - clientsReceiver[i] = c - } - - connsSender := make([]net.Conn, 0, peerPairs) - connsReceiver := make([]net.Conn, 0, peerPairs) - for i := 0; i < len(clientsSender); i++ { - conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) - if err != nil { - t.Fatalf("failed to bind channel: %s", err) - } - connsSender = append(connsSender, conn) - - conn, err = clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) - if err != nil { - t.Fatalf("failed to bind channel: %s", err) - } - connsReceiver = append(connsReceiver, conn) - } - - var transferDuration []time.Duration - wg := sync.WaitGroup{} - var writeErr error - var readErr error - for i := 0; i < len(connsSender); i++ { - wg.Add(2) - start := time.Now() - go func(i int) { - defer wg.Done() - pieceSize := 1024 - testDataLen := len(testData) - - for j := 0; j < testDataLen; j += pieceSize { - end := j + pieceSize - if end > testDataLen { - end = testDataLen - } - _, writeErr = connsSender[i].Write(testData[j:end]) - if writeErr != nil { - return - } - } - - }(i) - - go func(i int, start time.Time) { - defer wg.Done() - buf := make([]byte, 8192) - rcv := 0 - var n int - for receivedSize := 0; receivedSize < len(testData); { - - n, readErr = connsReceiver[i].Read(buf) - if readErr != nil { - return - } - - receivedSize += n - rcv += n - } - transferDuration = append(transferDuration, time.Since(start)) - }(i, start) - } - - wg.Wait() - - if writeErr != nil { - t.Fatalf("failed to write to channel: %s", err) - } - - if readErr != nil { - t.Fatalf("failed to read from channel: %s", err) - } - - // calculate the megabytes per second from the average transferDuration against the dataSize - var totalDuration time.Duration - for _, d := range transferDuration { - totalDuration += d - } - avgDuration := totalDuration / time.Duration(len(transferDuration)) - mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 - t.Logf("average transfer duration: %s", avgDuration) - t.Logf("average transfer speed: %.2f MB/s", mbps) - - for i := 0; i < len(connsSender); i++ { - err := connsSender[i].Close() - if err != nil { - t.Errorf("failed to close connection: %s", err) - } - - err = connsReceiver[i].Close() - if err != nil { - t.Errorf("failed to close connection: %s", err) - } - } -} - func TestRegistration(t *testing.T) { ctx := context.Background() srvCfg := server.ListenerConfig{Address: serverListenAddr} @@ -793,12 +629,3 @@ func waitForServerToStart(errChan chan error) error { } return nil } - -func seedRandomData(size int) ([]byte, error) { - token := make([]byte, size) - _, err := rand.Read(token) - if err != nil { - return nil, err - } - return token, nil -} diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go new file mode 100644 index 000000000..21c65778c --- /dev/null +++ b/relay/test/benchmark_test.go @@ -0,0 +1,379 @@ +package test + +import ( + "context" + "crypto/rand" + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/pion/logging" + "github.com/pion/turn/v3" + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/auth" + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client" + "github.com/netbirdio/netbird/relay/server" +) + +var ( + av = &auth.AllowAllAuth{} + hmacTokenStore = &hmac.TokenStore{} + pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} +) + +func TestRelayDataTransfer(t *testing.T) { + //t.SkipNow() // skip this test on CI because it is a benchmark test + dataSize := 1024 * 1024 * 10 + + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + transfer(t, testData, peerPairs) + }) + } +} + +func transfer(t *testing.T, testData []byte, peerPairs int) { + t.Helper() + ctx := context.Background() + port := 35000 + peerPairs + serverAddress := fmt.Sprintf("127.0.0.1:%d", port) + serverConnURL := fmt.Sprintf("rel://%s", serverAddress) + + srv, err := server.NewServer(otel.Meter(""), serverConnURL, false, av) + if err != nil { + t.Fatalf("failed to create server: %s", err) + } + errChan := make(chan error, 1) + go func() { + listenCfg := server.ListenerConfig{Address: serverAddress} + err := srv.Listen(listenCfg) + if err != nil { + errChan <- err + } + }() + + defer func() { + err := srv.Close() + if err != nil { + t.Errorf("failed to close server: %s", err) + } + }() + + // wait for server to start + if err := waitForServerToStart(errChan); err != nil { + t.Fatalf("failed to start server: %s", err) + } + + clientsSender := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + clientsReceiver := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsReceiver); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + t.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + + conn, err = clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + t.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + + var transferDuration []time.Duration + wg := sync.WaitGroup{} + var writeErr error + var readErr error + for i := 0; i < len(connsSender); i++ { + wg.Add(2) + start := time.Now() + go func(i int) { + defer wg.Done() + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr = connsSender[i].Write(testData[j:end]) + if writeErr != nil { + return + } + } + + }(i) + + go func(i int, start time.Time) { + defer wg.Done() + buf := make([]byte, 8192) + rcv := 0 + var n int + for receivedSize := 0; receivedSize < len(testData); { + + n, readErr = connsReceiver[i].Read(buf) + if readErr != nil { + return + } + + receivedSize += n + rcv += n + } + transferDuration = append(transferDuration, time.Since(start)) + }(i, start) + } + + wg.Wait() + + if writeErr != nil { + t.Fatalf("failed to write to channel: %s", err) + } + + if readErr != nil { + t.Fatalf("failed to read from channel: %s", err) + } + + // calculate the megabytes per second from the average transferDuration against the dataSize + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) + + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + + err = connsReceiver[i].Close() + if err != nil { + t.Errorf("failed to close connection: %s", err) + } + } +} + +// TestTurnDataTransfer run turn server: +// docker run --rm --name coturn -d --network=host coturn/coturn --user test:test +func TestTurnDataTransfer(t *testing.T) { + dataSize := 1024 * 1024 * 10 + + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + tunTurnTest(t, testData, peerPairs) + }) + } +} + +func tunTurnTest(t *testing.T, testData []byte, maxPairs int) { + var transferDuration []time.Duration + var wg sync.WaitGroup + + for i := 0; i < maxPairs; i++ { + wg.Add(1) + go func() { + defer wg.Done() + d := runTurnDataTransfer(t, testData) + transferDuration = append(transferDuration, d) + }() + + } + wg.Wait() + + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + mbps := float64(len(testData)) / avgDuration.Seconds() / 1024 / 1024 + t.Logf("average transfer duration: %s", avgDuration) + t.Logf("average transfer speed: %.2f MB/s", mbps) +} + +func runTurnDataTransfer(t *testing.T, testData []byte) time.Duration { + t.Helper() + testDataLen := len(testData) + relayAddress := "192.168.0.10:3478" + conn, err := net.Dial("tcp", relayAddress) + if err != nil { + t.Fatal(err) + } + defer func(conn net.Conn) { + _ = conn.Close() + }(conn) + + turnClient, err := getTurnClient(t, relayAddress, conn) + if err != nil { + t.Fatal(err) + } + defer turnClient.Close() + + relayConn, err := turnClient.Allocate() + if err != nil { + t.Fatal(err) + } + defer func(relayConn net.PacketConn) { + _ = relayConn.Close() + }(relayConn) + + receiverConn, err := net.Dial("udp", relayConn.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer func(receiverConn net.Conn) { + _ = receiverConn.Close() + }(receiverConn) + + var ( + tb int + start time.Time + timerInit bool + readDone = make(chan struct{}) + ack = make([]byte, 1) + ) + go func() { + defer func() { + readDone <- struct{}{} + }() + buff := make([]byte, 8192) + for { + n, e := receiverConn.Read(buff) + if e != nil { + return + } + if !timerInit { + start = time.Now() + timerInit = true + } + tb += n + _, _ = receiverConn.Write(ack) + + if tb >= testDataLen { + return + } + } + }() + + pieceSize := 1024 + ackBuff := make([]byte, 1) + pipelineSize := 10 + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, err := relayConn.WriteTo(testData[j:end], receiverConn.LocalAddr()) + if err != nil { + t.Fatalf("failed to write to channel: %s", err) + } + if pipelineSize == 0 { + _, _, _ = relayConn.ReadFrom(ackBuff) + } else { + pipelineSize-- + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + select { + case <-readDone: + if tb != testDataLen { + t.Fatalf("failed to read all data: %d/%d", tb, testDataLen) + } + case <-ctx.Done(): + t.Fatal("timeout") + } + return time.Since(start) +} + +func getTurnClient(t *testing.T, address string, conn net.Conn) (*turn.Client, error) { + t.Helper() + // Dial TURN Server + addrStr := fmt.Sprintf("%s:%d", address, 443) + + fac := logging.NewDefaultLoggerFactory() + //fac.DefaultLogLevel = logging.LogLevelTrace + + // Start a new TURN Client and wrap our net.Conn in a STUNConn + // This allows us to simulate datagram based communication over a net.Conn + cfg := &turn.ClientConfig{ + TURNServerAddr: address, + Conn: turn.NewSTUNConn(conn), + Username: "test", + Password: "test", + LoggerFactory: fac, + } + + client, err := turn.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err) + } + + // Start listening on the conn provided. + err = client.Listen() + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err) + } + + return client, nil +} + +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} + +func waitForServerToStart(errChan chan error) error { + select { + case err := <-errChan: + if err != nil { + return err + } + case <-time.After(300 * time.Millisecond): + return nil + } + return nil +} From d2c9a44953d194aa7f414bb2711332a3bdf43afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 26 Jul 2024 14:45:34 +0200 Subject: [PATCH 175/199] Fix server listen --- relay/server/server.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/relay/server/server.go b/relay/server/server.go index c34151c9d..6ddc232a1 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -39,13 +39,10 @@ func (r *Server) Listen(cfg ListenerConfig) error { TLSConfig: cfg.TLSConfig, } - var wslErr error - go func() { - wslErr = r.wSListener.Listen(r.relay.Accept) - if wslErr != nil { - log.Errorf("failed to bind ws server: %s", wslErr) - } - }() + wslErr := r.wSListener.Listen(r.relay.Accept) + if wslErr != nil { + log.Errorf("failed to bind ws server: %s", wslErr) + } return wslErr } From 4b37311e54074a185fa607a4e35d1d7254087100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 26 Jul 2024 15:41:40 +0200 Subject: [PATCH 176/199] Code cleaning --- relay/test/benchmark_test.go | 46 ++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go index 21c65778c..3d1062207 100644 --- a/relay/test/benchmark_test.go +++ b/relay/test/benchmark_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "fmt" "net" + "os" "sync" "testing" "time" @@ -17,18 +18,24 @@ import ( "github.com/netbirdio/netbird/relay/auth/hmac" "github.com/netbirdio/netbird/relay/client" "github.com/netbirdio/netbird/relay/server" + "github.com/netbirdio/netbird/util" ) var ( av = &auth.AllowAllAuth{} hmacTokenStore = &hmac.TokenStore{} pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + dataSize = 1024 * 1024 * 10 ) -func TestRelayDataTransfer(t *testing.T) { - //t.SkipNow() // skip this test on CI because it is a benchmark test - dataSize := 1024 * 1024 * 10 +func TestMain(m *testing.M) { + _ = util.InitLog("error", "console") + code := m.Run() + os.Exit(code) +} +func TestRelayDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test testData, err := seedRandomData(dataSize) if err != nil { t.Fatalf("failed to seed random data: %s", err) @@ -41,6 +48,22 @@ func TestRelayDataTransfer(t *testing.T) { } } +// TestTurnDataTransfer run turn server: +// docker run --rm --name coturn -d --network=host coturn/coturn --user test:test +func TestTurnDataTransfer(t *testing.T) { + t.SkipNow() // skip this test on CI because it is a benchmark test + testData, err := seedRandomData(dataSize) + if err != nil { + t.Fatalf("failed to seed random data: %s", err) + } + + for _, peerPairs := range pairs { + t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { + tunTurnTest(t, testData, peerPairs) + }) + } +} + func transfer(t *testing.T, testData []byte, peerPairs int) { t.Helper() ctx := context.Background() @@ -186,23 +209,6 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { } } -// TestTurnDataTransfer run turn server: -// docker run --rm --name coturn -d --network=host coturn/coturn --user test:test -func TestTurnDataTransfer(t *testing.T) { - dataSize := 1024 * 1024 * 10 - - testData, err := seedRandomData(dataSize) - if err != nil { - t.Fatalf("failed to seed random data: %s", err) - } - - for _, peerPairs := range pairs { - t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { - tunTurnTest(t, testData, peerPairs) - }) - } -} - func tunTurnTest(t *testing.T, testData []byte, maxPairs int) { var transferDuration []time.Duration var wg sync.WaitGroup From 100e31276f8224d7eb32abd34c032e23db970ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 29 Jul 2024 15:56:19 +0200 Subject: [PATCH 177/199] Fix slow peer open function --- client/internal/peer/conn.go | 13 +++++++++---- client/internal/peer/status.go | 1 - 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 4fbc0c8dc..07ff850fc 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -188,18 +188,23 @@ func (conn *Conn) Open() { conn.log.Warnf("error while updating the state err: %v", err) } + go conn.startHandshakeAndReconnect() +} + +func (conn *Conn) startHandshakeAndReconnect() { conn.waitInitialRandomSleepTime() - err = conn.handshaker.sendOffer() + err := conn.handshaker.sendOffer() if err != nil { - conn.log.Errorf("failed to send offer: %v", err) + conn.log.Errorf("failed to send initial offer: %v", err) } if conn.workerRelay.IsController() { - go conn.reconnectLoopWithRetry() + conn.reconnectLoopWithRetry() } else { - go conn.reconnectLoopForOnDisconnectedEvent() + conn.reconnectLoopForOnDisconnectedEvent() } + } // Close closes this peer Conn issuing a close event to the Conn closeCh diff --git a/client/internal/peer/status.go b/client/internal/peer/status.go index 57accd28d..44b6083a6 100644 --- a/client/internal/peer/status.go +++ b/client/internal/peer/status.go @@ -705,7 +705,6 @@ func (d *Status) GetFullStatus() FullStatus { } fullStatus.Peers = append(fullStatus.Peers, d.offlinePeers...) - return fullStatus } From 12f472c58c33e7ab29d6d1c0bf7987167198ef5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 29 Jul 2024 21:39:17 +0200 Subject: [PATCH 178/199] Add test --- relay/client/manager_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index cfec3f54e..53ac9bcd7 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -11,6 +11,14 @@ import ( "github.com/netbirdio/netbird/relay/server" ) +func TestEmptyURL(t *testing.T) { + mgr := NewManager(context.Background(), "", "alice") + err := mgr.Serve() + if err == nil { + t.Errorf("expected error, got nil") + } +} + func TestForeignConn(t *testing.T) { ctx := context.Background() From 5be33d668b6f08eb7093132e5d13f78bea8b377c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 29 Jul 2024 21:53:07 +0200 Subject: [PATCH 179/199] Add doc --- relay/auth/allow_all.go | 1 + relay/auth/doc.go | 26 ++++++++++++++++++++++++++ relay/auth/hmac/doc.go | 8 ++++++++ relay/auth/hmac/token.go | 5 ++++- relay/auth/validator.go | 1 + relay/client/conn.go | 7 +++++++ relay/client/doc.go | 12 ++++++++++++ relay/client/guard.go | 5 ++++- relay/client/manager.go | 23 ++++++++++++++++------- relay/doc.go | 8 ++++++++ relay/healthcheck/doc.go | 17 +++++++++++++++++ relay/healthcheck/sender.go | 5 ++++- relay/messages/doc.go | 5 +++++ relay/messages/id.go | 2 ++ relay/messages/message.go | 33 +++++++++++++++++++++++++++------ relay/server/peer.go | 9 ++++++++- relay/server/relay.go | 37 +++++++++++++++++++++++++++++++------ relay/server/server.go | 15 +++++++++++++++ relay/server/store.go | 8 ++++++++ 19 files changed, 204 insertions(+), 23 deletions(-) create mode 100644 relay/auth/doc.go create mode 100644 relay/auth/hmac/doc.go create mode 100644 relay/client/doc.go create mode 100644 relay/doc.go create mode 100644 relay/healthcheck/doc.go create mode 100644 relay/messages/doc.go diff --git a/relay/auth/allow_all.go b/relay/auth/allow_all.go index 653fd0801..9a3f3cebd 100644 --- a/relay/auth/allow_all.go +++ b/relay/auth/allow_all.go @@ -1,6 +1,7 @@ package auth // AllowAllAuth is a Validator that allows all connections. +// Used this for testing purposes only. type AllowAllAuth struct { } diff --git a/relay/auth/doc.go b/relay/auth/doc.go new file mode 100644 index 000000000..6bfb6cdf9 --- /dev/null +++ b/relay/auth/doc.go @@ -0,0 +1,26 @@ +/* +Package auth manages the authentication process with the relay server. + +Key Components: + +Validator: The Validator interface defines the Validate method. Any type that provides this method can be used as a +Validator. + +Methods: + +Validate(any): This method is defined in the Validator interface and is used to validate the authentication. + +Usage: + +To create a new AllowAllAuth validator, simply instantiate it: + + validator := &auth.AllowAllAuth{} + +To validate the authentication, use the Validate method: + + err := validator.Validate(any) + +This package provides a simple and effective way to manage authentication with the relay server, ensuring that the +peers are authenticated properly. +*/ +package auth diff --git a/relay/auth/hmac/doc.go b/relay/auth/hmac/doc.go new file mode 100644 index 000000000..a1b135aa6 --- /dev/null +++ b/relay/auth/hmac/doc.go @@ -0,0 +1,8 @@ +/* +This package uses a similar HMAC method for authentication with the TURN server. The Management server provides the +tokens for the peers. The peers manage these tokens in the token store. The token store is a simple thread safe store +that keeps the tokens in memory. These tokens are used to authenticate the peers with the Relay server in the hello +message. +*/ + +package hmac diff --git a/relay/auth/hmac/token.go b/relay/auth/hmac/token.go index a5fa30a14..b647ae319 100644 --- a/relay/auth/hmac/token.go +++ b/relay/auth/hmac/token.go @@ -43,6 +43,7 @@ type TimedHMAC struct { timeToLive time.Duration } +// NewTimedHMAC creates a new TimedHMAC instance func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC { return &TimedHMAC{ secret: secret, @@ -50,7 +51,8 @@ func NewTimedHMAC(secret string, timeToLive time.Duration) *TimedHMAC { } } -// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC hash of a timestamp with a preshared TURN secret +// GenerateToken generates new time-based secret token - basically Payload is a unix timestamp and Signature is a HMAC +// hash of a timestamp with a preshared TURN secret func (m *TimedHMAC) GenerateToken() (*Token, error) { timeAuth := time.Now().Add(m.timeToLive).Unix() timeStamp := fmt.Sprint(timeAuth) @@ -66,6 +68,7 @@ func (m *TimedHMAC) GenerateToken() (*Token, error) { }, nil } +// Validate checks if the token is valid func (m *TimedHMAC) Validate(token Token) error { expectedMAC, err := m.generate(token.Payload) if err != nil { diff --git a/relay/auth/validator.go b/relay/auth/validator.go index d76a90c74..067a42268 100644 --- a/relay/auth/validator.go +++ b/relay/auth/validator.go @@ -1,5 +1,6 @@ package auth +// Validator is an interface that defines the Validate method. type Validator interface { Validate(any) error } diff --git a/relay/client/conn.go b/relay/client/conn.go index 783b6a660..b4ff903e8 100644 --- a/relay/client/conn.go +++ b/relay/client/conn.go @@ -6,6 +6,7 @@ import ( "time" ) +// Conn represent a connection to a relayed remote peer. type Conn struct { client *Client dstID []byte @@ -14,6 +15,12 @@ type Conn struct { instanceURL *RelayAddr } +// NewConn creates a new connection to a relayed remote peer. +// client: the client instance, it used to send messages to the destination peer +// dstID: the destination peer ID +// dstStringID: the destination peer ID in string format +// messageChan: the channel where the messages will be received +// instanceURL: the relay instance URL, it used to get the proper server instance address for the remote peer func NewConn(client *Client, dstID []byte, dstStringID string, messageChan chan Msg, instanceURL *RelayAddr) *Conn { c := &Conn{ client: client, diff --git a/relay/client/doc.go b/relay/client/doc.go new file mode 100644 index 000000000..1339251d9 --- /dev/null +++ b/relay/client/doc.go @@ -0,0 +1,12 @@ +/* +Package client contains the implementation of the Relay client. + +The Relay client is responsible for establishing a connection with the Relay server and sending and receiving messages, +Keep persistent connection with the Relay server and handle the connection issues. +It uses the WebSocket protocol for communication and optionally supports TLS (Transport Layer Security). + +If a peer wants to communicate with a peer on a different relay server, the manager will establish a new connection to +the relay server. The connection with these relay servers will be closed if there is no active connection. The peers +negotiate the common relay instance via signaling service. +*/ +package client diff --git a/relay/client/guard.go b/relay/client/guard.go index 5eb68d215..f826cf1b6 100644 --- a/relay/client/guard.go +++ b/relay/client/guard.go @@ -11,11 +11,13 @@ var ( reconnectingTimeout = 5 * time.Second ) +// Guard manage the reconnection tries to the Relay server in case of disconnection event. type Guard struct { ctx context.Context relayClient *Client } +// NewGuard creates a new guard for the relay client. func NewGuard(context context.Context, relayClient *Client) *Guard { g := &Guard{ ctx: context, @@ -24,8 +26,9 @@ func NewGuard(context context.Context, relayClient *Client) *Guard { return g } +// OnDisconnected is called when the relay client is disconnected from the relay server. It will trigger the reconnection +// todo prevent multiple reconnection instances. In the current usage it should not happen, but it is better to prevent func (g *Guard) OnDisconnected() { - // todo prevent multiple reconnect ticker := time.NewTicker(reconnectingTimeout) defer ticker.Stop() diff --git a/relay/client/manager.go b/relay/client/manager.go index bb523822a..4867dd04d 100644 --- a/relay/client/manager.go +++ b/relay/client/manager.go @@ -34,6 +34,7 @@ func NewRelayTrack() *RelayTrack { type OnServerCloseListener func() +// ManagerService is the interface for the relay manager. type ManagerService interface { Serve() error OpenConn(serverAddress, peerKey string) (net.Conn, error) @@ -44,9 +45,9 @@ type ManagerService interface { UpdateToken(token *relayAuth.Token) } -// Manager is a manager for the relay client. It establish one persistent connection to the given relay server. In case -// of network error the manager will try to reconnect to the server. -// The manager also manage temproary relay connection. If a client wants to communicate with an another client on a +// Manager is a manager for the relay client instances. It establishes one persistent connection to the given relay URL +// and automatically reconnect to them in case disconnection. +// The manager also manage temporary relay connection. If a client wants to communicate with a client on a // different relay server, the manager will establish a new connection to the relay server. The connection with these // relay servers will be closed if there is no active connection. Periodically the manager will check if there is any // unused relay connection and close it. @@ -66,6 +67,8 @@ type Manager struct { listenerLock sync.Mutex } +// NewManager creates a new manager instance. +// The serverURL address can be empty. In this case, the manager will not serve. func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { return &Manager{ ctx: ctx, @@ -77,7 +80,8 @@ func NewManager(ctx context.Context, serverURL string, peerID string) *Manager { } } -// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop. +// Serve starts the manager. It will establish a connection to the relay server and start the relay cleanup loop for +// the unused relay connections. The manager will automatically reconnect to the relay server in case of disconnection. func (m *Manager) Serve() error { if m.relayClient != nil { return fmt.Errorf("manager already serving") @@ -101,7 +105,7 @@ func (m *Manager) Serve() error { // OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be // established via the relay server. If the peer is on a different relay server, the manager will establish a new -// connection to the relay server. +// connection to the relay server. It returns back with a net.Conn what represent the remote peer connection. func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { if m.relayClient == nil { return nil, errRelayClientNotConnected @@ -129,6 +133,8 @@ func (m *Manager) OpenConn(serverAddress, peerKey string) (net.Conn, error) { return netConn, err } +// AddCloseListener adds a listener to the given server instance address. The listener will be called if the connection +// closed. func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServerCloseListener) error { foreign, err := m.isForeignServer(serverAddress) if err != nil { @@ -145,8 +151,8 @@ func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServ return nil } -// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is lost. -// This address will be sent to the target peer to choose the common relay server for the communication. +// RelayInstanceAddress returns the address of the permanent relay server. It could change if the network connection is +// lost. This address will be sent to the target peer to choose the common relay server for the communication. func (m *Manager) RelayInstanceAddress() (string, error) { if m.relayClient == nil { return "", errRelayClientNotConnected @@ -159,10 +165,13 @@ func (m *Manager) ServerURL() string { return m.serverURL } +// HasRelayAddress returns true if the manager is serving. With this method can check if the peer can communicate with +// Relay service. func (m *Manager) HasRelayAddress() bool { return m.serverURL != "" } +// UpdateToken updates the token in the token store. func (m *Manager) UpdateToken(token *relayAuth.Token) { m.tokenStore.UpdateToken(token) } diff --git a/relay/doc.go b/relay/doc.go new file mode 100644 index 000000000..09d034edd --- /dev/null +++ b/relay/doc.go @@ -0,0 +1,8 @@ +//Package relay +/* +The `relay` package contains the implementation of the Relay server and client. The Relay server can be used to relay +messages between peers on a single network channel. In this implementation the transport layer is the WebSocket +protocol. + +*/ +package relay diff --git a/relay/healthcheck/doc.go b/relay/healthcheck/doc.go new file mode 100644 index 000000000..da9689c6b --- /dev/null +++ b/relay/healthcheck/doc.go @@ -0,0 +1,17 @@ +/* +The `healthcheck` package is responsible for managing the health checks between the client and the relay server. It +ensures that the connection between the client and the server are alive and functioning properly. + +The `Sender` struct is responsible for sending health check signals to the receiver. The receiver listens for these +signals and sends a new signal back to the sender to acknowledge that the signal has been received. If the sender does +not receive an acknowledgment signal within a certain time frame, it will send a timeout signal via timeout channel +and stop working. + +The `Receiver` struct is responsible for receiving the health check signals from the sender. If the receiver does not +receive a signal within a certain time frame, it will send a timeout signal via the OnTimeout channel and stop working. + +In the Relay usage the signal is sent to the peer in message type Healthcheck. In case of timeout the connection is +closed and the peer is removed from the relay. +*/ + +package healthcheck diff --git a/relay/healthcheck/sender.go b/relay/healthcheck/sender.go index 401170ec9..c5d02a4bb 100644 --- a/relay/healthcheck/sender.go +++ b/relay/healthcheck/sender.go @@ -15,8 +15,10 @@ var ( // If the receiver does not receive the signal in a certain time, it will send a timeout signal and stop to work // It will also stop if the context is canceled type Sender struct { + // HealthCheck is a channel to send health check signal to the peer HealthCheck chan struct{} - Timeout chan struct{} + // Timeout is a channel to the health check signal is not received in a certain time + Timeout chan struct{} ctx context.Context ack chan struct{} @@ -35,6 +37,7 @@ func NewSender(ctx context.Context) *Sender { return hc } +// OnHCResponse sends an acknowledgment signal to the sender func (hc *Sender) OnHCResponse() { select { case hc.ack <- struct{}{}: diff --git a/relay/messages/doc.go b/relay/messages/doc.go new file mode 100644 index 000000000..4c719df3a --- /dev/null +++ b/relay/messages/doc.go @@ -0,0 +1,5 @@ +/* +Package messages provides the message types that are used to communicate between the relay and the client. +This package is used to determine the type of message that is being sent and received between the relay and the client. +*/ +package messages diff --git a/relay/messages/id.go b/relay/messages/id.go index 531eefa25..e2162cd3b 100644 --- a/relay/messages/id.go +++ b/relay/messages/id.go @@ -15,6 +15,7 @@ var ( prefix = []byte("sha-") // 4 bytes ) +// HashID generates a sha256 hash from the peerID and returns the hash and the human-readable string func HashID(peerID string) ([]byte, string) { idHash := sha256.Sum256([]byte(peerID)) idHashString := string(prefix) + base64.StdEncoding.EncodeToString(idHash[:]) @@ -24,6 +25,7 @@ func HashID(peerID string) ([]byte, string) { return prefixedHash, idHashString } +// HashIDToString converts a hash to a human-readable string func HashIDToString(idHash []byte) string { return fmt.Sprintf("%s%s", idHash[:prefixLength], base64.StdEncoding.EncodeToString(idHash[prefixLength:])) } diff --git a/relay/messages/message.go b/relay/messages/message.go index 0211e1900..3770f6398 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -54,6 +54,7 @@ type HelloResponse struct { InstanceAddress string } +// DetermineClientMsgType determines the message type from the first byte of the message func DetermineClientMsgType(msg []byte) (MsgType, error) { msgType := MsgType(msg[0]) switch msgType { @@ -70,6 +71,7 @@ func DetermineClientMsgType(msg []byte) (MsgType, error) { } } +// DetermineServerMsgType determines the message type from the first byte of the message func DetermineServerMsgType(msg []byte) (MsgType, error) { msgType := MsgType(msg[0]) switch msgType { @@ -87,6 +89,10 @@ func DetermineServerMsgType(msg []byte) (MsgType, error) { } // MarshalHelloMsg initial hello message +// The Hello message is the first message sent by a client after establishing a connection with the Relay server. This +// message is used to authenticate the client with the server. The authentication is done using an HMAC method. +// The protocol does not limit to use HMAC, it can be any other method. If the authentication failed the server will +// close the network connection without any response. func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { if len(peerID) != IDSize { return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) @@ -101,6 +107,8 @@ func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { return msg, nil } +// UnmarshalHelloMsg extracts the peerID and the additional data from the hello message. The Additional data is used to +// authenticate the client with the server. func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { if len(msg) < headerSizeHello { return nil, nil, fmt.Errorf("invalid 'hello' messge") @@ -111,6 +119,10 @@ func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { return msg[5 : 5+IDSize], msg[headerSizeHello:], nil } +// MarshalHelloResponse creates a response message to the hello message. +// In case of success connection the server response with a Hello Response message. This message contains the server's +// instance URL. This URL will be used by choose the common Relay server in case if the peers are in different Relay +// servers. func MarshalHelloResponse(DomainAddress string) ([]byte, error) { payload := HelloResponse{ InstanceAddress: DomainAddress, @@ -131,6 +143,7 @@ func MarshalHelloResponse(DomainAddress string) ([]byte, error) { return msg, nil } +// UnmarshalHelloResponse extracts the instance address from the hello response message func UnmarshalHelloResponse(msg []byte) (string, error) { if len(msg) < 2 { return "", fmt.Errorf("invalid 'hello response' message") @@ -147,16 +160,18 @@ func UnmarshalHelloResponse(msg []byte) (string, error) { return payload.InstanceAddress, nil } -// Close message - +// MarshalCloseMsg creates a close message. +// The close message is used to close the connection gracefully between the client and the server. The server and the +// client can send this message. After receiving this message, the server or client will close the connection. func MarshalCloseMsg() []byte { msg := make([]byte, 1) msg[0] = byte(MsgTypeClose) return msg } -// Transport message - +// MarshalTransportMsg creates a transport message. +// The transport message is used to exchange data between peers. The message contains the data to be exchanged and the +// destination peer hashed ID. func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) { if len(peerID) != IDSize { return nil, fmt.Errorf("invalid peerID length: %d", len(peerID)) @@ -169,6 +184,7 @@ func MarshalTransportMsg(peerID []byte, payload []byte) ([]byte, error) { return msg, nil } +// UnmarshalTransportMsg extracts the peerID and the payload from the transport message. func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) { if len(buf) < headerSizeTransport { return nil, nil, ErrInvalidMessageLength @@ -177,6 +193,7 @@ func UnmarshalTransportMsg(buf []byte) ([]byte, []byte, error) { return buf[1:headerSizeTransport], buf[headerSizeTransport:], nil } +// UnmarshalTransportID extracts the peerID from the transport message. func UnmarshalTransportID(buf []byte) ([]byte, error) { if len(buf) < headerSizeTransport { log.Debugf("invalid message length: %d, expected: %d, %x", len(buf), headerSizeTransport, buf) @@ -185,6 +202,9 @@ func UnmarshalTransportID(buf []byte) ([]byte, error) { return buf[1:headerSizeTransport], nil } +// UpdateTransportMsg updates the peerID in the transport message. +// With this function the server can reuse the given byte slice to update the peerID in the transport message. So do +// need to allocate a new byte slice. func UpdateTransportMsg(msg []byte, peerID []byte) error { if len(msg) < 1+len(peerID) { return ErrInvalidMessageLength @@ -193,8 +213,9 @@ func UpdateTransportMsg(msg []byte, peerID []byte) error { return nil } -// health check message - +// MarshalHealthcheck creates a health check message. +// Health check message is sent by the server periodically. The client will respond with a health check response +// message. If the client does not respond to the health check message, the server will close the connection. func MarshalHealthcheck() []byte { return healthCheckMsg } diff --git a/relay/server/peer.go b/relay/server/peer.go index 6509f7bd3..52f3b185b 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -18,6 +18,7 @@ const ( bufferSize = 8820 ) +// Peer represents a peer connection type Peer struct { log *log.Entry idS string @@ -27,6 +28,7 @@ type Peer struct { store *Store } +// NewPeer creates a new Peer instance and prepare custom logging func NewPeer(id []byte, conn net.Conn, store *Store) *Peer { stringID := messages.HashIDToString(id) return &Peer{ @@ -38,6 +40,9 @@ func NewPeer(id []byte, conn net.Conn, store *Store) *Peer { } } +// Work reads data from the connection +// It manages the protocol (healthcheck, transport, close). Read the message and determine the message type and handle +// the message accordingly. func (p *Peer) Work() { ctx, cancel := context.WithCancel(context.Background()) hc := healthcheck.NewSender(ctx) @@ -75,13 +80,14 @@ func (p *Peer) Work() { } // Write writes data to the connection -// it has been called by the remote peer func (p *Peer) Write(b []byte) (int, error) { p.connMu.RLock() defer p.connMu.RUnlock() return p.conn.Write(b) } +// CloseGracefully closes the connection with the peer gracefully. Send a close message to the client and close the +// connection. func (p *Peer) CloseGracefully(ctx context.Context) { p.connMu.Lock() _, err := p.writeWithTimeout(ctx, messages.MarshalCloseMsg()) @@ -97,6 +103,7 @@ func (p *Peer) CloseGracefully(ctx context.Context) { defer p.connMu.Unlock() } +// String returns the peer ID func (p *Peer) String() string { return p.idS } diff --git a/relay/server/relay.go b/relay/server/relay.go index c0d61ad03..4621e96b6 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net" + "net/url" "sync" log "github.com/sirupsen/logrus" @@ -14,6 +15,7 @@ import ( "github.com/netbirdio/netbird/relay/metrics" ) +// Relay represents the relay server type Relay struct { metrics *metrics.Metrics validator auth.Validator @@ -25,6 +27,21 @@ type Relay struct { closeMu sync.RWMutex } +// NewRelay creates a new Relay instance +// +// Parameters: +// meter: An instance of metric.Meter from the go.opentelemetry.io/otel/metric package. It is used to create and manage +// metrics for the relay server. +// exposedAddress: A string representing the address that the relay server is exposed on. The client will use this +// address as the relay server's instance URL. +// tlsSupport: A boolean indicating whether the relay server supports TLS (Transport Layer Security) or not. The +// instance URL depends on this value. +// validator: An instance of auth.Validator from the auth package. It is used to validate the authentication of the +// peers. +// +// Returns: +// A pointer to a Relay instance and an error. If the Relay instance is successfully created, the error is nil. +// Otherwise, the error contains the details of what went wrong. func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, validator auth.Validator) (*Relay, error) { m, err := metrics.NewMetrics(meter) if err != nil { @@ -42,10 +59,15 @@ func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, valida } else { r.instanceURL = fmt.Sprintf("rel://%s", exposedAddress) } + _, err = url.ParseRequestURI(r.instanceURL) + if err != nil { + return nil, fmt.Errorf("invalid exposed address: %v", err) + } return r, nil } +// Accept start to handle a new peer connection func (r *Relay) Accept(conn net.Conn) { r.closeMu.RLock() defer r.closeMu.RUnlock() @@ -53,7 +75,7 @@ func (r *Relay) Accept(conn net.Conn) { return } - peerID, err := r.handShake(conn) + peerID, err := r.handshake(conn) if err != nil { log.Errorf("failed to handshake with %s: %s", conn.RemoteAddr(), err) cErr := conn.Close() @@ -75,6 +97,8 @@ func (r *Relay) Accept(conn net.Conn) { }() } +// Close closes the relay server +// It closes the connection with all peers in gracefully and stops accepting new connections. func (r *Relay) Close(ctx context.Context) { log.Infof("close connection with all peers") r.closeMu.Lock() @@ -91,7 +115,12 @@ func (r *Relay) Close(ctx context.Context) { r.closeMu.Unlock() } -func (r *Relay) handShake(conn net.Conn) ([]byte, error) { +// InstanceURL returns the instance URL of the relay server +func (r *Relay) InstanceURL() string { + return r.instanceURL +} + +func (r *Relay) handshake(conn net.Conn) ([]byte, error) { buf := make([]byte, messages.MaxHandshakeSize) n, err := conn.Read(buf) if err != nil { @@ -127,7 +156,3 @@ func (r *Relay) handShake(conn net.Conn) ([]byte, error) { } return peerID, nil } - -func (r *Relay) InstanceURL() string { - return r.instanceURL -} diff --git a/relay/server/server.go b/relay/server/server.go index 6ddc232a1..2d74a5eef 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -13,16 +13,27 @@ import ( "github.com/netbirdio/netbird/relay/server/listener/ws" ) +// ListenerConfig is the configuration for the listener. +// Address: the address to bind the listener to. It could be an address behind a reverse proxy. +// TLSConfig: the TLS configuration for the listener. type ListenerConfig struct { Address string TLSConfig *tls.Config } +// Server is the main entry point for the relay server. +// It is the gate between the WebSocket listener and the Relay server logic. +// In a new HTTP connection, the server will accept the connection and pass it to the Relay server via the Accept method. type Server struct { relay *Relay wSListener listener.Listener } +// NewServer creates a new relay server instance. +// meter: the OpenTelemetry meter +// exposedAddress: this address will be used as the instance URL. It should be a domain:port format. +// tlsSupport: if true, the server will support TLS +// authValidator: the auth validator to use for the server func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authValidator auth.Validator) (*Server, error) { relay, err := NewRelay(meter, exposedAddress, tlsSupport, authValidator) if err != nil { @@ -33,6 +44,7 @@ func NewServer(meter metric.Meter, exposedAddress string, tlsSupport bool, authV }, nil } +// Listen starts the relay server. func (r *Server) Listen(cfg ListenerConfig) error { r.wSListener = &ws.Listener{ Address: cfg.Address, @@ -47,6 +59,8 @@ func (r *Server) Listen(cfg ListenerConfig) error { return wslErr } +// Close stops the relay server. If there are active connections, they will be closed gracefully. In case of a timeout, +// the connections will be forcefully closed. func (r *Server) Close() (err error) { // stop service new connections if r.wSListener != nil { @@ -60,6 +74,7 @@ func (r *Server) Close() (err error) { return } +// InstanceURL returns the instance URL of the relay server. func (r *Server) InstanceURL() string { return r.relay.instanceURL } diff --git a/relay/server/store.go b/relay/server/store.go index 1f0f08600..79b8aeb5d 100644 --- a/relay/server/store.go +++ b/relay/server/store.go @@ -4,29 +4,36 @@ import ( "sync" ) +// Store is a thread-safe store of peers +// It is used to store the peers that are connected to the relay server type Store struct { peers map[string]*Peer // consider to use [32]byte as key. The Peer(id string) would be faster peersLock sync.RWMutex } +// NewStore creates a new Store instance func NewStore() *Store { return &Store{ peers: make(map[string]*Peer), } } +// AddPeer adds a peer to the store +// It distinguishes the peers by their ID func (s *Store) AddPeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() s.peers[peer.String()] = peer } +// DeletePeer deletes a peer from the store func (s *Store) DeletePeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() delete(s.peers, peer.String()) } +// Peer returns a peer by its ID func (s *Store) Peer(id string) (*Peer, bool) { s.peersLock.RLock() defer s.peersLock.RUnlock() @@ -35,6 +42,7 @@ func (s *Store) Peer(id string) (*Peer, bool) { return p, ok } +// Peers returns all the peers in the store func (s *Store) Peers() []*Peer { s.peersLock.RLock() defer s.peersLock.RUnlock() From 7942b0ebae7ebedd0c6f9a35df1b77d6bf93a0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 29 Jul 2024 21:58:27 +0200 Subject: [PATCH 180/199] Add doc --- relay/doc.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/relay/doc.go b/relay/doc.go index 09d034edd..d0306204a 100644 --- a/relay/doc.go +++ b/relay/doc.go @@ -4,5 +4,11 @@ The `relay` package contains the implementation of the Relay server and client. messages between peers on a single network channel. In this implementation the transport layer is the WebSocket protocol. +Between the server and client communication has been design a custom protocol and message format. These messages are +transported over the WebSocket connection. Optionally the server can use TLS to secure the communication. + +The service can support multiple Relay server instances. For this purpose the peers must know the server instance URL. +This URL will be sent to the target peer to choose the common Relay server for the communication via Signal service. + */ package relay From aa1a4826696289ffaa7ae33448d51554d91cd06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 29 Jul 2024 22:03:09 +0200 Subject: [PATCH 181/199] Fix lint --- relay/test/benchmark_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go index 3d1062207..d6d71a765 100644 --- a/relay/test/benchmark_test.go +++ b/relay/test/benchmark_test.go @@ -59,7 +59,7 @@ func TestTurnDataTransfer(t *testing.T) { for _, peerPairs := range pairs { t.Run(fmt.Sprintf("peerPairs-%d", peerPairs), func(t *testing.T) { - tunTurnTest(t, testData, peerPairs) + runTurnTest(t, testData, peerPairs) }) } } @@ -209,7 +209,8 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { } } -func tunTurnTest(t *testing.T, testData []byte, maxPairs int) { +func runTurnTest(t *testing.T, testData []byte, maxPairs int) { + t.Helper() var transferDuration []time.Duration var wg sync.WaitGroup From e5f8ecdeb7bb10c14c5902e28411f1974212f6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 30 Jul 2024 15:36:10 +0200 Subject: [PATCH 182/199] Fix eBPF conn close logic --- client/internal/peer/conn.go | 6 ++++++ client/internal/wgproxy/proxy_ebpf.go | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 07ff850fc..af8e551db 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -556,6 +556,12 @@ func (conn *Conn) onWorkerRelayStateDisconnected() { defer conn.mu.Unlock() if conn.wgProxyRelay != nil { + log.Debugf("relayed connection is closed, clean up WireGuard config") + err := conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if err != nil { + conn.log.Errorf("failed to remove wg endpoint: %v", err) + } + conn.endpointRelay = nil _ = conn.wgProxyRelay.CloseConn() conn.wgProxyRelay = nil diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index 969a836ff..d385cc4ca 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -181,8 +181,8 @@ func (p *WGEBPFProxy) proxyToRemote() { conn, ok := p.turnConnStore[uint16(addr.Port)] p.turnConnMutex.Unlock() if !ok { - log.Infof("turn conn not found by port, exit form proxy: %d", addr.Port) - return // todo replace it to return. For debug troubleshooting keep it + log.Debugf("turn conn not found by port because conn already has been closed: %d", addr.Port) + continue } _, err = conn.Write(buf[:n]) From cbe90b5dd94bf715a9da1b1538a8123d97121eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Fri, 2 Aug 2024 09:50:42 +0200 Subject: [PATCH 183/199] Fix wg update --- client/internal/peer/conn.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index af8e551db..d68766995 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -442,6 +442,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon conn.log.Warnf("Failed to update wg peer configuration: %v", err) return } + wgConfigWorkaround() if conn.wgProxyICE != nil { if err := conn.wgProxyICE.CloseConn(); err != nil { @@ -538,6 +539,7 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { conn.log.Errorf("Failed to update wg peer configuration: %v", err) return } + wgConfigWorkaround() if conn.wgProxyRelay != nil { if err := conn.wgProxyRelay.CloseConn(); err != nil { @@ -757,3 +759,9 @@ func (conn *Conn) getEndpointForICEConnInfo(iceConnInfo ICEConnInfo) (net.Addr, func isRosenpassEnabled(remoteRosenpassPubKey []byte) bool { return remoteRosenpassPubKey != nil } + +// wgConfigWorkaround is a workaround for the issue with WireGuard configuration update +// When update a peer configuration in near to each other time, the second update can be ignored by WireGuard +func wgConfigWorkaround() { + time.Sleep(100 * time.Millisecond) +} From 7da74e707abef94025a6cf01edca3368d9468df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 5 Aug 2024 09:46:01 +0200 Subject: [PATCH 184/199] Fix continuous handshake sending with the agent without relay support. --- client/internal/peer/conn.go | 21 +++++++++++++++------ client/internal/peer/worker_relay.go | 10 +++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index d68766995..6b615f667 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -318,8 +318,14 @@ func (conn *Conn) reconnectLoopWithRetry() { return } - if conn.statusRelay == StatusDisconnected || conn.statusICE == StatusDisconnected { - conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + if conn.workerRelay.IsRelayConnectionSupportedWithPeer() { + if conn.statusRelay == StatusDisconnected || conn.statusICE == StatusDisconnected { + conn.log.Tracef("ticker timedout, relay state: %s, ice state: %s", conn.statusRelay, conn.statusICE) + } + } else { + if conn.statusICE == StatusDisconnected { + conn.log.Tracef("ticker timedout, ice state: %s", conn.statusICE) + } } // checks if there is peer connection is established via relay or ice @@ -708,13 +714,16 @@ func (conn *Conn) isConnected() bool { conn.mu.Lock() defer conn.mu.Unlock() - if conn.statusRelay != StatusConnected { - return false - } - if conn.statusICE != StatusConnected && conn.statusICE != StatusConnecting { return false } + + if conn.workerRelay.IsRelayConnectionSupportedWithPeer() { + if conn.statusRelay != StatusConnected { + return false + } + } + return true } diff --git a/client/internal/peer/worker_relay.go b/client/internal/peer/worker_relay.go index cf72e30e6..f03626e14 100644 --- a/client/internal/peer/worker_relay.go +++ b/client/internal/peer/worker_relay.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net" + "sync/atomic" "time" log "github.com/sirupsen/logrus" @@ -34,7 +35,8 @@ type WorkerRelay struct { relayManager relayClient.ManagerService conn WorkerRelayCallbacks - ctxCancel context.CancelFunc + ctxCancel context.CancelFunc + relaySupportedOnRemotePeer atomic.Bool } func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, relayManager relayClient.ManagerService, callbacks WorkerRelayCallbacks) *WorkerRelay { @@ -51,8 +53,10 @@ func NewWorkerRelay(ctx context.Context, log *log.Entry, config ConnConfig, rela func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) { if !w.isRelaySupported(remoteOfferAnswer) { w.log.Infof("Relay is not supported by remote peer") + w.relaySupportedOnRemotePeer.Store(false) return } + w.relaySupportedOnRemotePeer.Store(true) // the relayManager will return with error in case if the connection has lost with relay server currentRelayAddress, err := w.relayManager.RelayInstanceAddress() @@ -99,6 +103,10 @@ func (w *WorkerRelay) RelayInstanceAddress() (string, error) { return w.relayManager.RelayInstanceAddress() } +func (w *WorkerRelay) IsRelayConnectionSupportedWithPeer() bool { + return w.relaySupportedOnRemotePeer.Load() && w.RelayIsSupportedLocally() +} + func (w *WorkerRelay) IsController() bool { return w.config.LocalKey > w.config.Key } From 5400754954519e883808cb9c99a35aca41b0e697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 5 Aug 2024 12:05:40 +0200 Subject: [PATCH 185/199] Fix changes after merge - regenerate proto file - use interface instead of pointer --- client/internal/routemanager/client.go | 2 +- client/internal/routemanager/dynamic/route.go | 4 ++-- client/proto/daemon.pb.go | 18 ++++++++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/client/internal/routemanager/client.go b/client/internal/routemanager/client.go index 3dea7213c..f49dd115a 100644 --- a/client/internal/routemanager/client.go +++ b/client/internal/routemanager/client.go @@ -378,7 +378,7 @@ func (c *clientNetwork) peersStateAndUpdateWatcher() { } } -func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface *iface.WGIface) RouteHandler { +func handlerFromRoute(rt *route.Route, routeRefCounter *refcounter.RouteRefCounter, allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, dnsRouterInteval time.Duration, statusRecorder *peer.Status, wgInterface iface.IWGIface) RouteHandler { if rt.IsDynamic() { dns := nbdns.NewServiceViaMemory(wgInterface) return dynamic.NewRoute(rt, routeRefCounter, allowedIPsRefCounter, dnsRouterInteval, statusRecorder, wgInterface, fmt.Sprintf("%s:%d", dns.RuntimeIP(), dns.RuntimePort())) diff --git a/client/internal/routemanager/dynamic/route.go b/client/internal/routemanager/dynamic/route.go index 3296f3ddf..5897031e7 100644 --- a/client/internal/routemanager/dynamic/route.go +++ b/client/internal/routemanager/dynamic/route.go @@ -48,7 +48,7 @@ type Route struct { currentPeerKey string cancel context.CancelFunc statusRecorder *peer.Status - wgInterface *iface.WGIface + wgInterface iface.IWGIface resolverAddr string } @@ -58,7 +58,7 @@ func NewRoute( allowedIPsRefCounter *refcounter.AllowedIPsRefCounter, interval time.Duration, statusRecorder *peer.Status, - wgInterface *iface.WGIface, + wgInterface iface.IWGIface, resolverAddr string, ) *Route { return &Route{ diff --git a/client/proto/daemon.pb.go b/client/proto/daemon.pb.go index f604a885c..b942d8b6e 100644 --- a/client/proto/daemon.pb.go +++ b/client/proto/daemon.pb.go @@ -1828,8 +1828,9 @@ type DebugBundleRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Anonymize bool `protobuf:"varint,1,opt,name=anonymize,proto3" json:"anonymize,omitempty"` - Status string `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + Anonymize bool `protobuf:"varint,1,opt,name=anonymize,proto3" json:"anonymize,omitempty"` + Status string `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + SystemInfo bool `protobuf:"varint,3,opt,name=systemInfo,proto3" json:"systemInfo,omitempty"` } func (x *DebugBundleRequest) Reset() { @@ -1878,6 +1879,13 @@ func (x *DebugBundleRequest) GetStatus() string { return "" } +func (x *DebugBundleRequest) GetSystemInfo() bool { + if x != nil { + return x.SystemInfo + } + return false +} + type DebugBundleResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2371,11 +2379,13 @@ var file_daemon_proto_rawDesc = []byte{ 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x49, 0x50, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x4a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, + 0x6a, 0x0a, 0x12, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x6e, 0x6f, 0x6e, 0x79, 0x6d, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x13, 0x44, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, + 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0a, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x29, 0x0a, 0x13, 0x44, 0x65, 0x62, 0x75, 0x67, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x14, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4c, 0x6f, 0x67, From 351db3dd498d6f191b6d92b7771d7824a462d93f Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 7 Aug 2024 17:35:58 +0200 Subject: [PATCH 186/199] Feature/relay integration metrics (#2376) Extend metrics - TransferBytesSent - Active/idle peers - Connection times --- relay/metrics/realy.go | 131 ++++++++++++++++++++++++++++++++++++++--- relay/server/peer.go | 33 +++++++---- relay/server/relay.go | 23 +++++--- 3 files changed, 157 insertions(+), 30 deletions(-) diff --git a/relay/metrics/realy.go b/relay/metrics/realy.go index 29dc6557b..80e12ee6b 100644 --- a/relay/metrics/realy.go +++ b/relay/metrics/realy.go @@ -1,21 +1,136 @@ package metrics -import "go.opentelemetry.io/otel/metric" +import ( + "context" + "sync" + "time" + + log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" +) + +const ( + idleTimeout = 30 * time.Second +) type Metrics struct { metric.Meter - Peers metric.Int64UpDownCounter + TransferBytesSent metric.Int64Counter + TransferBytesRecv metric.Int64Counter + + peers metric.Int64UpDownCounter + peerActivityChan chan string + peerLastActive map[string]time.Time + mutexActivity sync.Mutex + ctx context.Context } -func NewMetrics(meter metric.Meter) (*Metrics, error) { - peers, err := meter.Int64UpDownCounter("peers") +func NewMetrics(ctx context.Context, meter metric.Meter) (*Metrics, error) { + bytesSent, err := meter.Int64Counter("relay_transfer_sent_bytes_total") if err != nil { return nil, err } - return &Metrics{ - Meter: meter, - Peers: peers, - }, nil + bytesRecv, err := meter.Int64Counter("relay_transfer_received_bytes_total") + if err != nil { + return nil, err + } + + peers, err := meter.Int64UpDownCounter("relay_peers") + if err != nil { + return nil, err + } + + peersActive, err := meter.Int64ObservableGauge("relay_peers_active") + if err != nil { + return nil, err + } + + peersIdle, err := meter.Int64ObservableGauge("relay_peers_idle") + if err != nil { + return nil, err + } + + m := &Metrics{ + Meter: meter, + TransferBytesSent: bytesSent, + TransferBytesRecv: bytesRecv, + peers: peers, + + ctx: ctx, + peerActivityChan: make(chan string, 10), + peerLastActive: make(map[string]time.Time), + } + + _, err = meter.RegisterCallback( + func(ctx context.Context, o metric.Observer) error { + active, idle := m.calculateActiveIdleConnections() + o.ObserveInt64(peersActive, active) + o.ObserveInt64(peersIdle, idle) + return nil + }, + peersActive, peersIdle, + ) + if err != nil { + return nil, err + } + + go m.readPeerActivity() + return m, nil +} + +// PeerConnected increments the number of connected peers and increments number of idle connections +func (m *Metrics) PeerConnected(id string) { + m.peers.Add(m.ctx, 1) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + m.peerLastActive[id] = time.Time{} +} + +// PeerDisconnected decrements the number of connected peers and decrements number of idle or active connections +func (m *Metrics) PeerDisconnected(id string) { + m.peers.Add(m.ctx, -1) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + delete(m.peerLastActive, id) +} + +// PeerActivity increases the active connections +func (m *Metrics) PeerActivity(peerID string) { + select { + case m.peerActivityChan <- peerID: + default: + log.Errorf("peer activity channel is full, dropping activity metrics for peer %s", peerID) + } +} + +func (m *Metrics) calculateActiveIdleConnections() (int64, int64) { + active, idle := int64(0), int64(0) + m.mutexActivity.Lock() + defer m.mutexActivity.Unlock() + + for _, lastActive := range m.peerLastActive { + if time.Since(lastActive) > idleTimeout { + idle++ + } else { + active++ + } + } + return active, idle +} + +func (m *Metrics) readPeerActivity() { + for { + select { + case peerID := <-m.peerActivityChan: + m.mutexActivity.Lock() + m.peerLastActive[peerID] = time.Now() + m.mutexActivity.Unlock() + case <-m.ctx.Done(): + return + } + } } diff --git a/relay/server/peer.go b/relay/server/peer.go index 52f3b185b..e3595b2e7 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -12,6 +12,7 @@ import ( "github.com/netbirdio/netbird/relay/healthcheck" "github.com/netbirdio/netbird/relay/messages" + "github.com/netbirdio/netbird/relay/metrics" ) const ( @@ -20,23 +21,25 @@ const ( // Peer represents a peer connection type Peer struct { - log *log.Entry - idS string - idB []byte - conn net.Conn - connMu sync.RWMutex - store *Store + metrics *metrics.Metrics + log *log.Entry + idS string + idB []byte + conn net.Conn + connMu sync.RWMutex + store *Store } // NewPeer creates a new Peer instance and prepare custom logging -func NewPeer(id []byte, conn net.Conn, store *Store) *Peer { +func NewPeer(metrics *metrics.Metrics, id []byte, conn net.Conn, store *Store) *Peer { stringID := messages.HashIDToString(id) return &Peer{ - log: log.WithField("peer_id", stringID), - idS: stringID, - idB: id, - conn: conn, - store: store, + metrics: metrics, + log: log.WithField("peer_id", stringID), + idS: stringID, + idB: id, + conn: conn, + store: store, } } @@ -70,6 +73,8 @@ func (p *Peer) Work() { case messages.MsgTypeHealthCheck: hc.OnHCResponse() case messages.MsgTypeTransport: + p.metrics.TransferBytesRecv.Add(ctx, int64(n)) + p.metrics.PeerActivity(p.String()) p.handleTransportMsg(msg) case messages.MsgTypeClose: p.log.Infof("peer exited gracefully") @@ -167,8 +172,10 @@ func (p *Peer) handleTransportMsg(msg []byte) { p.log.Errorf("failed to update transport message: %s", err) return } - _, err = dp.Write(msg) + n, err := dp.Write(msg) if err != nil { p.log.Errorf("failed to write transport message to: %s", dp.String()) + return } + p.metrics.TransferBytesSent.Add(context.Background(), int64(n)) } diff --git a/relay/server/relay.go b/relay/server/relay.go index 4621e96b6..45f52824a 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -17,8 +17,9 @@ import ( // Relay represents the relay server type Relay struct { - metrics *metrics.Metrics - validator auth.Validator + metrics *metrics.Metrics + metricsCancel context.CancelFunc + validator auth.Validator store *Store instanceURL string @@ -43,15 +44,18 @@ type Relay struct { // A pointer to a Relay instance and an error. If the Relay instance is successfully created, the error is nil. // Otherwise, the error contains the details of what went wrong. func NewRelay(meter metric.Meter, exposedAddress string, tlsSupport bool, validator auth.Validator) (*Relay, error) { - m, err := metrics.NewMetrics(meter) + ctx, metricsCancel := context.WithCancel(context.Background()) + m, err := metrics.NewMetrics(ctx, meter) if err != nil { + metricsCancel() return nil, fmt.Errorf("creating app metrics: %v", err) } r := &Relay{ - metrics: m, - validator: validator, - store: NewStore(), + metrics: m, + metricsCancel: metricsCancel, + validator: validator, + store: NewStore(), } if tlsSupport { @@ -85,15 +89,15 @@ func (r *Relay) Accept(conn net.Conn) { return } - peer := NewPeer(peerID, conn, r.store) + peer := NewPeer(r.metrics, peerID, conn, r.store) peer.log.Infof("peer connected from: %s", conn.RemoteAddr()) r.store.AddPeer(peer) - r.metrics.Peers.Add(context.Background(), 1) + r.metrics.PeerConnected(peer.String()) go func() { peer.Work() r.store.DeletePeer(peer) peer.log.Debugf("relay connection closed") - r.metrics.Peers.Add(context.Background(), -1) + r.metrics.PeerDisconnected(peer.String()) }() } @@ -112,6 +116,7 @@ func (r *Relay) Close(ctx context.Context) { }(peer) } wg.Wait() + r.metricsCancel() r.closeMu.Unlock() } From 4d162f1750ffb65437f5a8a558f3f8e592a0b4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 20 Aug 2024 17:20:30 +0200 Subject: [PATCH 187/199] Fix test after merge conflicts --- management/server/peer_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 918436515..773ce72ef 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -848,9 +848,9 @@ func TestToSyncResponse(t *testing.T) { DNSLabel: "peer1", SSHKey: "peer1-ssh-key", } - turnCredentials := &TURNCredentials{ - Username: "turn-user", - Password: "turn-pass", + turnRelayToken := &TURNRelayToken{ + Payload: "turn-user", + Signature: "turn-pass", } networkMap := &NetworkMap{ Network: &Network{Net: *ipnet, Serial: 1000}, @@ -916,7 +916,7 @@ func TestToSyncResponse(t *testing.T) { } dnsCache := &DNSConfigCache{} - response := toSyncResponse(context.Background(), config, peer, turnCredentials, networkMap, dnsName, checks, dnsCache) + response := toSyncResponse(context.Background(), config, peer, turnRelayToken, turnRelayToken, networkMap, dnsName, checks, dnsCache) assert.NotNil(t, response) // assert peer config From 3bd820f2e4feff174641e6df839c244d4378ef33 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 21 Aug 2024 10:27:02 +0200 Subject: [PATCH 188/199] [client] Remove test scenario (#2447) Fix the flickering test. Because we eliminated the direct property, can flicker a test because any output could be valid. Remove unnecessary test cases. --- client/internal/routemanager/client_test.go | 27 --------------------- relay/healthcheck/sender_test.go | 4 ++- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/client/internal/routemanager/client_test.go b/client/internal/routemanager/client_test.go index c4783d5f3..583156e4d 100644 --- a/client/internal/routemanager/client_test.go +++ b/client/internal/routemanager/client_test.go @@ -144,33 +144,6 @@ func TestGetBestrouteFromStatuses(t *testing.T) { currentRoute: "", expectedRouteID: "route1", }, - { - name: "multiple connected peers with one direct", - statuses: map[route.ID]routerPeerStatus{ - "route1": { - connected: true, - relayed: false, - }, - "route2": { - connected: true, - relayed: false, - }, - }, - existingRoutes: map[route.ID]*route.Route{ - "route1": { - ID: "route1", - Metric: route.MaxMetric, - Peer: "peer1", - }, - "route2": { - ID: "route2", - Metric: route.MaxMetric, - Peer: "peer2", - }, - }, - currentRoute: "", - expectedRouteID: "route1", - }, { name: "multiple connected peers with different latencies", statuses: map[route.ID]routerPeerStatus{ diff --git a/relay/healthcheck/sender_test.go b/relay/healthcheck/sender_test.go index 220bb8ac0..5b6db25f6 100644 --- a/relay/healthcheck/sender_test.go +++ b/relay/healthcheck/sender_test.go @@ -2,6 +2,7 @@ package healthcheck import ( "context" + "os" "testing" "time" ) @@ -10,7 +11,8 @@ func TestMain(m *testing.M) { // override the health check interval to speed up the test healthCheckInterval = 1 * time.Second healthCheckTimeout = 100 * time.Millisecond - m.Run() + code := m.Run() + os.Exit(code) } func TestNewHealthPeriod(t *testing.T) { From 7633cca3b166f88d0f15f763e774476df6b1c56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 21 Aug 2024 15:54:36 +0200 Subject: [PATCH 189/199] Fix exit signal handling --- relay/cmd/main.go | 10 ++++++---- relay/server/listener/ws/listener.go | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 22dc025a8..dc39beaa5 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -166,11 +166,13 @@ func execute(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to create relay server: %v", err) } log.Infof("server will be available on: %s", srv.InstanceURL()) - err = srv.Listen(srvListenerCfg) - if err != nil { - return fmt.Errorf("failed to bind server: %s", err) - } + go func() { + if err := srv.Listen(srvListenerCfg); err != nil { + log.Errorf("failed to bind server: %s", err) + } + }() + // it will block until exit signal waitForExitSignal() err = srv.Close() diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index 51a578dc4..673a5e19a 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -38,7 +38,6 @@ func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { var err error if l.TLSConfig != nil { err = l.server.ListenAndServeTLS("", "") - } else { err = l.server.ListenAndServe() } From a208e7999ce886da584b995559f0534c0814f024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 21 Aug 2024 16:05:04 +0200 Subject: [PATCH 190/199] Pass the ctx to the close function --- relay/client/client_test.go | 20 ++++++++++---------- relay/client/manager_test.go | 16 ++++++++-------- relay/cmd/main.go | 6 +++++- relay/server/listener/listener.go | 7 +++++-- relay/server/listener/ws/listener.go | 6 +----- relay/server/server.go | 8 ++------ relay/test/benchmark_test.go | 2 +- 7 files changed, 32 insertions(+), 33 deletions(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index 592d5bb4d..dc7f70b43 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -47,7 +47,7 @@ func TestClient(t *testing.T) { }() defer func() { - err := srv.Close() + err := srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -135,14 +135,14 @@ func TestRegistration(t *testing.T) { clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err = clientAlice.Connect() if err != nil { - _ = srv.Close() + _ = srv.Close(ctx) t.Fatalf("failed to connect to server: %s", err) } err = clientAlice.Close() if err != nil { t.Errorf("failed to close conn: %s", err) } - err = srv.Close() + err = srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -202,7 +202,7 @@ func TestEcho(t *testing.T) { }() defer func() { - err := srv.Close() + err := srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -292,7 +292,7 @@ func TestBindToUnavailabePeer(t *testing.T) { defer func() { log.Infof("closing server") - err := srv.Close() + err := srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -338,7 +338,7 @@ func TestBindReconnect(t *testing.T) { defer func() { log.Infof("closing server") - err := srv.Close() + err := srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -429,7 +429,7 @@ func TestCloseConn(t *testing.T) { defer func() { log.Infof("closing server") - err := srv.Close() + err := srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -485,7 +485,7 @@ func TestCloseRelayConn(t *testing.T) { }() defer func() { - err := srv.Close() + err := srv.Close(ctx) if err != nil { log.Errorf("failed to close server: %s", err) } @@ -556,7 +556,7 @@ func TestCloseByServer(t *testing.T) { close(disconnected) }) - err = srv1.Close() + err = srv1.Close(ctx) if err != nil { t.Fatalf("failed to close server: %s", err) } @@ -612,7 +612,7 @@ func TestCloseByClient(t *testing.T) { t.Errorf("unexpected opening connection to closed server") } - err = srv.Close() + err = srv.Close(ctx) if err != nil { t.Fatalf("failed to close server: %s", err) } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index 53ac9bcd7..b11e49737 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -38,7 +38,7 @@ func TestForeignConn(t *testing.T) { }() defer func() { - err := srv1.Close() + err := srv1.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -64,7 +64,7 @@ func TestForeignConn(t *testing.T) { }() defer func() { - err := srv2.Close() + err := srv2.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -150,7 +150,7 @@ func TestForeginConnClose(t *testing.T) { }() defer func() { - err := srv1.Close() + err := srv1.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -176,7 +176,7 @@ func TestForeginConnClose(t *testing.T) { }() defer func() { - err := srv2.Close() + err := srv2.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -227,7 +227,7 @@ func TestForeginAutoClose(t *testing.T) { defer func() { t.Logf("closing server 1.") - err := srv1.Close() + err := srv1.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -255,7 +255,7 @@ func TestForeginAutoClose(t *testing.T) { }() defer func() { t.Logf("closing server 2.") - err := srv2.Close() + err := srv2.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -317,7 +317,7 @@ func TestAutoReconnect(t *testing.T) { }() defer func() { - err := srv.Close() + err := srv.Close(ctx) if err != nil { log.Errorf("failed to close server: %s", err) } @@ -381,7 +381,7 @@ func TestNotifierDoubleAdd(t *testing.T) { }() defer func() { - err := srv1.Close() + err := srv1.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index dc39beaa5..c4fcbc994 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/tls" "errors" "fmt" @@ -175,7 +176,10 @@ func execute(cmd *cobra.Command, args []string) error { // it will block until exit signal waitForExitSignal() - err = srv.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err = srv.Close(ctx) if err != nil { return fmt.Errorf("failed to close server: %s", err) } diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go index 66e6d357e..45167ac1d 100644 --- a/relay/server/listener/listener.go +++ b/relay/server/listener/listener.go @@ -1,8 +1,11 @@ package listener -import "net" +import ( + "context" + "net" +) type Listener interface { Listen(func(conn net.Conn)) error - Close() error + Close(ctx context.Context) error } diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index 673a5e19a..c429e4545 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -7,7 +7,6 @@ import ( "fmt" "net" "net/http" - "time" log "github.com/sirupsen/logrus" "nhooyr.io/websocket" @@ -47,14 +46,11 @@ func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { return err } -func (l *Listener) Close() error { +func (l *Listener) Close(ctx context.Context) error { if l.server == nil { return nil } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - log.Infof("stop WS listener") if err := l.server.Shutdown(ctx); err != nil { return fmt.Errorf("server shutdown failed: %v", err) diff --git a/relay/server/server.go b/relay/server/server.go index 2d74a5eef..c96e2c25d 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -3,7 +3,6 @@ package server import ( "context" "crypto/tls" - "time" log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/metric" @@ -61,15 +60,12 @@ func (r *Server) Listen(cfg ListenerConfig) error { // Close stops the relay server. If there are active connections, they will be closed gracefully. In case of a timeout, // the connections will be forcefully closed. -func (r *Server) Close() (err error) { +func (r *Server) Close(ctx context.Context) (err error) { // stop service new connections if r.wSListener != nil { - err = r.wSListener.Close() + err = r.wSListener.Close(ctx) } - // close accepted connections gracefully - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() r.relay.Close(ctx) return } diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go index d6d71a765..620902a2a 100644 --- a/relay/test/benchmark_test.go +++ b/relay/test/benchmark_test.go @@ -85,7 +85,7 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { }() defer func() { - err := srv.Close() + err := srv.Close(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } From c1a54756e89b0d872475c5f5c579e355438a2a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 21 Aug 2024 16:17:39 +0200 Subject: [PATCH 191/199] Rename "Close" to "ShutDown" --- relay/client/client_test.go | 20 ++++++++++---------- relay/client/manager_test.go | 16 ++++++++-------- relay/cmd/main.go | 2 +- relay/server/listener/listener.go | 2 +- relay/server/listener/ws/listener.go | 2 +- relay/server/relay.go | 4 ++-- relay/server/server.go | 8 ++++---- relay/test/benchmark_test.go | 2 +- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/relay/client/client_test.go b/relay/client/client_test.go index dc7f70b43..5ac64f23e 100644 --- a/relay/client/client_test.go +++ b/relay/client/client_test.go @@ -47,7 +47,7 @@ func TestClient(t *testing.T) { }() defer func() { - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -135,14 +135,14 @@ func TestRegistration(t *testing.T) { clientAlice := NewClient(ctx, serverURL, hmacTokenStore, "alice") err = clientAlice.Connect() if err != nil { - _ = srv.Close(ctx) + _ = srv.Shutdown(ctx) t.Fatalf("failed to connect to server: %s", err) } err = clientAlice.Close() if err != nil { t.Errorf("failed to close conn: %s", err) } - err = srv.Close(ctx) + err = srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -202,7 +202,7 @@ func TestEcho(t *testing.T) { }() defer func() { - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -292,7 +292,7 @@ func TestBindToUnavailabePeer(t *testing.T) { defer func() { log.Infof("closing server") - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -338,7 +338,7 @@ func TestBindReconnect(t *testing.T) { defer func() { log.Infof("closing server") - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -429,7 +429,7 @@ func TestCloseConn(t *testing.T) { defer func() { log.Infof("closing server") - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -485,7 +485,7 @@ func TestCloseRelayConn(t *testing.T) { }() defer func() { - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { log.Errorf("failed to close server: %s", err) } @@ -556,7 +556,7 @@ func TestCloseByServer(t *testing.T) { close(disconnected) }) - err = srv1.Close(ctx) + err = srv1.Shutdown(ctx) if err != nil { t.Fatalf("failed to close server: %s", err) } @@ -612,7 +612,7 @@ func TestCloseByClient(t *testing.T) { t.Errorf("unexpected opening connection to closed server") } - err = srv.Close(ctx) + err = srv.Shutdown(ctx) if err != nil { t.Fatalf("failed to close server: %s", err) } diff --git a/relay/client/manager_test.go b/relay/client/manager_test.go index b11e49737..ce94d62fe 100644 --- a/relay/client/manager_test.go +++ b/relay/client/manager_test.go @@ -38,7 +38,7 @@ func TestForeignConn(t *testing.T) { }() defer func() { - err := srv1.Close(ctx) + err := srv1.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -64,7 +64,7 @@ func TestForeignConn(t *testing.T) { }() defer func() { - err := srv2.Close(ctx) + err := srv2.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -150,7 +150,7 @@ func TestForeginConnClose(t *testing.T) { }() defer func() { - err := srv1.Close(ctx) + err := srv1.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -176,7 +176,7 @@ func TestForeginConnClose(t *testing.T) { }() defer func() { - err := srv2.Close(ctx) + err := srv2.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -227,7 +227,7 @@ func TestForeginAutoClose(t *testing.T) { defer func() { t.Logf("closing server 1.") - err := srv1.Close(ctx) + err := srv1.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -255,7 +255,7 @@ func TestForeginAutoClose(t *testing.T) { }() defer func() { t.Logf("closing server 2.") - err := srv2.Close(ctx) + err := srv2.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } @@ -317,7 +317,7 @@ func TestAutoReconnect(t *testing.T) { }() defer func() { - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { log.Errorf("failed to close server: %s", err) } @@ -381,7 +381,7 @@ func TestNotifierDoubleAdd(t *testing.T) { }() defer func() { - err := srv1.Close(ctx) + err := srv1.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } diff --git a/relay/cmd/main.go b/relay/cmd/main.go index c4fcbc994..239976422 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -179,7 +179,7 @@ func execute(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - err = srv.Close(ctx) + err = srv.Shutdown(ctx) if err != nil { return fmt.Errorf("failed to close server: %s", err) } diff --git a/relay/server/listener/listener.go b/relay/server/listener/listener.go index 45167ac1d..535c8bcd9 100644 --- a/relay/server/listener/listener.go +++ b/relay/server/listener/listener.go @@ -7,5 +7,5 @@ import ( type Listener interface { Listen(func(conn net.Conn)) error - Close(ctx context.Context) error + Shutdown(ctx context.Context) error } diff --git a/relay/server/listener/ws/listener.go b/relay/server/listener/ws/listener.go index c429e4545..2034b709a 100644 --- a/relay/server/listener/ws/listener.go +++ b/relay/server/listener/ws/listener.go @@ -46,7 +46,7 @@ func (l *Listener) Listen(acceptFn func(conn net.Conn)) error { return err } -func (l *Listener) Close(ctx context.Context) error { +func (l *Listener) Shutdown(ctx context.Context) error { if l.server == nil { return nil } diff --git a/relay/server/relay.go b/relay/server/relay.go index 45f52824a..31b7db22c 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -101,9 +101,9 @@ func (r *Relay) Accept(conn net.Conn) { }() } -// Close closes the relay server +// Shutdown closes the relay server // It closes the connection with all peers in gracefully and stops accepting new connections. -func (r *Relay) Close(ctx context.Context) { +func (r *Relay) Shutdown(ctx context.Context) { log.Infof("close connection with all peers") r.closeMu.Lock() wg := sync.WaitGroup{} diff --git a/relay/server/server.go b/relay/server/server.go index c96e2c25d..0036e2390 100644 --- a/relay/server/server.go +++ b/relay/server/server.go @@ -58,15 +58,15 @@ func (r *Server) Listen(cfg ListenerConfig) error { return wslErr } -// Close stops the relay server. If there are active connections, they will be closed gracefully. In case of a timeout, +// Shutdown stops the relay server. If there are active connections, they will be closed gracefully. In case of a context, // the connections will be forcefully closed. -func (r *Server) Close(ctx context.Context) (err error) { +func (r *Server) Shutdown(ctx context.Context) (err error) { // stop service new connections if r.wSListener != nil { - err = r.wSListener.Close(ctx) + err = r.wSListener.Shutdown(ctx) } - r.relay.Close(ctx) + r.relay.Shutdown(ctx) return } diff --git a/relay/test/benchmark_test.go b/relay/test/benchmark_test.go index 620902a2a..75560705c 100644 --- a/relay/test/benchmark_test.go +++ b/relay/test/benchmark_test.go @@ -85,7 +85,7 @@ func transfer(t *testing.T, testData []byte, peerPairs int) { }() defer func() { - err := srv.Close(ctx) + err := srv.Shutdown(ctx) if err != nil { t.Errorf("failed to close server: %s", err) } From c057fbd4f40562d67db9acd5b2a58b8d9a279d91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 21 Aug 2024 16:39:52 +0200 Subject: [PATCH 192/199] Close metrics service --- relay/cmd/main.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 239976422..2216bb57f 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -11,6 +11,7 @@ import ( "syscall" "time" + "github.com/hashicorp/go-multierror" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -179,11 +180,15 @@ func execute(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - err = srv.Shutdown(ctx) - if err != nil { - return fmt.Errorf("failed to close server: %s", err) + var shutDownErrors error + if err := srv.Shutdown(ctx); err != nil { + shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close server: %s", err)) } - return nil + + if err := metricsServer.Close(); err != nil { + shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err)) + } + return shutDownErrors } func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) { From 86f1724efa3d292e9fb38889b05a7b2c50030e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 21 Aug 2024 16:46:59 +0200 Subject: [PATCH 193/199] Print out errors on exit from cmd --- relay/cmd/main.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 2216bb57f..e7576fe97 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -60,10 +60,12 @@ var ( cobraConfig *Config cfgFile string rootCmd = &cobra.Command{ - Use: "relay", - Short: "Relay service", - Long: "Relay service for Netbird agents", - RunE: execute, + Use: "relay", + Short: "Relay service", + Long: "Relay service for Netbird agents", + RunE: execute, + SilenceUsage: true, + SilenceErrors: true, } ) @@ -128,7 +130,7 @@ func execute(cmd *cobra.Command, args []string) error { err = cfg.Validate() if err != nil { log.Errorf("invalid config: %s", err) - os.Exit(1) + return err } metricsServer, err := metrics.NewServer(metricsPort, "") @@ -170,7 +172,7 @@ func execute(cmd *cobra.Command, args []string) error { log.Infof("server will be available on: %s", srv.InstanceURL()) go func() { if err := srv.Listen(srvListenerCfg); err != nil { - log.Errorf("failed to bind server: %s", err) + log.Fatalf("failed to bind server: %s", err) } }() @@ -185,6 +187,7 @@ func execute(cmd *cobra.Command, args []string) error { shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close server: %s", err)) } + log.Infof("shutting down metrics server") if err := metricsServer.Close(); err != nil { shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err)) } @@ -200,8 +203,7 @@ func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string } func main() { - err := rootCmd.Execute() - if err != nil { - os.Exit(1) + if err := rootCmd.Execute(); err != nil { + log.Fatalf("%v", err) } } From 82584f20891e1e4f8878bf210c42a2460d0797f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 21 Aug 2024 17:06:05 +0200 Subject: [PATCH 194/199] Gracefully shutdown metrics --- relay/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/cmd/main.go b/relay/cmd/main.go index e7576fe97..7bb7d700a 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -188,7 +188,7 @@ func execute(cmd *cobra.Command, args []string) error { } log.Infof("shutting down metrics server") - if err := metricsServer.Close(); err != nil { + if err := metricsServer.Shutdown(ctx); err != nil { shutDownErrors = multierror.Append(shutDownErrors, fmt.Errorf("failed to close metrics server: %v", err)) } return shutDownErrors From d9f43e8871465df892cb31ee19ddbcc8c12ee942 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 22 Aug 2024 11:26:21 +0200 Subject: [PATCH 195/199] Extend the error messages with IP (#2465) Add IP address for the error messages in the Hello section --- relay/server/relay.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/relay/server/relay.go b/relay/server/relay.go index 31b7db22c..b47b29cfc 100644 --- a/relay/server/relay.go +++ b/relay/server/relay.go @@ -129,7 +129,7 @@ func (r *Relay) handshake(conn net.Conn) ([]byte, error) { buf := make([]byte, messages.MaxHandshakeSize) n, err := conn.Read(buf) if err != nil { - log.Errorf("failed to read message: %s", err) + log.Debugf("failed to read message from: %s, %s", conn.RemoteAddr(), err) return nil, err } msgType, err := messages.DetermineClientMsgType(buf[:n]) @@ -139,18 +139,18 @@ func (r *Relay) handshake(conn net.Conn) ([]byte, error) { if msgType != messages.MsgTypeHello { tErr := fmt.Errorf("invalid message type") - log.Errorf("failed to handshake: %s", tErr) + log.Debugf("failed to handshake with: %s, %s", conn.RemoteAddr(), tErr) return nil, tErr } peerID, authPayload, err := messages.UnmarshalHelloMsg(buf[:n]) if err != nil { - log.Errorf("failed to handshake: %s", err) + log.Debugf("failed to handshake with: %s, %s", conn.RemoteAddr(), err) return nil, err } if err := r.validator.Validate(authPayload); err != nil { - log.Errorf("failed to authenticate connection: %s", err) + log.Debugf("failed to authenticate connection with: %s, %s", conn.RemoteAddr(), err) return nil, err } From 01e7caf64001277b7c0878add0e9d92e4fa03f30 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 22 Aug 2024 14:31:36 +0200 Subject: [PATCH 196/199] [relay] Feature/relay integration r53 (#2399) Support route53 for TLS handling --- client/internal/peer/conn.go | 4 +- encryption/route53.go | 87 +++++++++++ encryption/route53_test.go | 84 +++++++++++ go.mod | 29 +++- go.sum | 67 ++++++++- relay/client/client.go | 2 +- relay/cmd/env.go | 35 +++++ relay/cmd/main.go | 126 ++++++++-------- relay/messages/message.go | 2 +- relay/server/store.go | 11 +- relay/server/store_test.go | 40 +++++ relay/testec2/main.go | 258 ++++++++++++++++++++++++++++++++ relay/testec2/relay.go | 176 ++++++++++++++++++++++ relay/testec2/signal.go | 91 +++++++++++ relay/testec2/start_msg.go | 39 +++++ relay/testec2/tun/proxy.go | 72 +++++++++ relay/testec2/tun/tun.go | 110 ++++++++++++++ relay/testec2/turn.go | 181 ++++++++++++++++++++++ relay/testec2/turn_allocator.go | 83 ++++++++++ 19 files changed, 1424 insertions(+), 73 deletions(-) create mode 100644 encryption/route53.go create mode 100644 encryption/route53_test.go create mode 100644 relay/cmd/env.go create mode 100644 relay/server/store_test.go create mode 100644 relay/testec2/main.go create mode 100644 relay/testec2/relay.go create mode 100644 relay/testec2/signal.go create mode 100644 relay/testec2/start_msg.go create mode 100644 relay/testec2/tun/proxy.go create mode 100644 relay/testec2/tun/tun.go create mode 100644 relay/testec2/turn.go create mode 100644 relay/testec2/turn_allocator.go diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 6b615f667..9f17a087d 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -452,7 +452,7 @@ func (conn *Conn) iCEConnectionIsReady(priority ConnPriority, iceConnInfo ICECon if conn.wgProxyICE != nil { if err := conn.wgProxyICE.CloseConn(); err != nil { - conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) + conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err) } } conn.wgProxyICE = wgProxy @@ -549,7 +549,7 @@ func (conn *Conn) relayConnectionIsReady(rci RelayConnInfo) { if conn.wgProxyRelay != nil { if err := conn.wgProxyRelay.CloseConn(); err != nil { - conn.log.Warnf("failed to close depracated wg proxy conn: %v", err) + conn.log.Warnf("failed to close deprecated wg proxy conn: %v", err) } } conn.wgProxyRelay = wgProxy diff --git a/encryption/route53.go b/encryption/route53.go new file mode 100644 index 000000000..3c81ab103 --- /dev/null +++ b/encryption/route53.go @@ -0,0 +1,87 @@ +package encryption + +import ( + "context" + "crypto/tls" + "fmt" + "os" + "strings" + + "github.com/caddyserver/certmagic" + "github.com/libdns/route53" + log "github.com/sirupsen/logrus" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "golang.org/x/crypto/acme" +) + +// Route53TLS by default, loads the AWS configuration from the environment. +// env variables: AWS_REGION, AWS_PROFILE, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN +type Route53TLS struct { + DataDir string + Email string + Domains []string + CA string +} + +func (r *Route53TLS) GetCertificate() (*tls.Config, error) { + if len(r.Domains) == 0 { + return nil, fmt.Errorf("no domains provided") + } + + certmagic.Default.Logger = logger() + certmagic.Default.Storage = &certmagic.FileStorage{Path: r.DataDir} + certmagic.DefaultACME.Agreed = true + if r.Email != "" { + certmagic.DefaultACME.Email = r.Email + } else { + certmagic.DefaultACME.Email = emailFromDomain(r.Domains[0]) + } + + if r.CA == "" { + certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA + } else { + certmagic.DefaultACME.CA = r.CA + } + + certmagic.DefaultACME.DNS01Solver = &certmagic.DNS01Solver{ + DNSManager: certmagic.DNSManager{ + DNSProvider: &route53.Provider{}, + }, + } + cm := certmagic.NewDefault() + if err := cm.ManageSync(context.Background(), r.Domains); err != nil { + log.Errorf("failed to manage certificate: %v", err) + return nil, err + } + + tlsConfig := &tls.Config{ + GetCertificate: cm.GetCertificate, + NextProtos: []string{"h2", "http/1.1", acme.ALPNProto}, + } + + return tlsConfig, nil +} + +func emailFromDomain(domain string) string { + if domain == "" { + return "" + } + + parts := strings.Split(domain, ".") + if len(parts) < 2 { + return "" + } + if parts[0] == "" { + return "" + } + return fmt.Sprintf("admin@%s.%s", parts[len(parts)-2], parts[len(parts)-1]) +} + +func logger() *zap.Logger { + return zap.New(zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewProductionEncoderConfig()), + os.Stderr, + zap.ErrorLevel, + )) +} diff --git a/encryption/route53_test.go b/encryption/route53_test.go new file mode 100644 index 000000000..765b60f84 --- /dev/null +++ b/encryption/route53_test.go @@ -0,0 +1,84 @@ +package encryption + +import ( + "context" + "io" + "net/http" + "os" + "testing" + "time" +) + +func TestRoute53TLSConfig(t *testing.T) { + t.SkipNow() // This test requires AWS credentials + exampleString := "Hello, world!" + rtls := &Route53TLS{ + DataDir: t.TempDir(), + Email: os.Getenv("LE_EMAIL_ROUTE53"), + Domains: []string{os.Getenv("DOMAIN")}, + } + tlsConfig, err := rtls.GetCertificate() + if err != nil { + t.Errorf("Route53TLSConfig failed: %v", err) + } + + server := &http.Server{ + Addr: ":8443", + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(exampleString)) + }), + TLSConfig: tlsConfig, + } + + go func() { + err := server.ListenAndServeTLS("", "") + if err != http.ErrServerClosed { + t.Errorf("Failed to start server: %v", err) + } + }() + defer func() { + if err := server.Shutdown(context.Background()); err != nil { + t.Errorf("Failed to shutdown server: %v", err) + } + }() + + time.Sleep(1 * time.Second) + resp, err := http.Get("https://relay.godevltd.com:8443") + if err != nil { + t.Errorf("Failed to get response: %v", err) + return + } + defer func() { + _ = resp.Body.Close() + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("Failed to read response body: %v", err) + } + if string(body) != exampleString { + t.Errorf("Unexpected response: %s", body) + } +} + +func Test_emailFromDomain(t *testing.T) { + tests := []struct { + input string + want string + }{ + {"example.com", "admin@example.com"}, + {"x.example.com", "admin@example.com"}, + {"x.x.example.com", "admin@example.com"}, + {"*.example.com", "admin@example.com"}, + {"example", ""}, + {"", ""}, + {".com", ""}, + } + for _, tt := range tests { + t.Run("domain test", func(t *testing.T) { + if got := emailFromDomain(tt.input); got != tt.want { + t.Errorf("emailFromDomain() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/go.mod b/go.mod index b02e19a73..eade75ea5 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( fyne.io/systray v1.11.0 github.com/TheJumpCloud/jcapi-go v3.0.0+incompatible github.com/c-robinson/iplib v1.0.3 + github.com/caddyserver/certmagic v0.21.3 github.com/cilium/ebpf v0.15.0 github.com/coreos/go-iptables v0.7.0 github.com/creack/pty v1.1.18 @@ -50,11 +51,12 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-secure-stdlib/base62 v0.1.2 github.com/hashicorp/go-version v1.6.0 + github.com/libdns/route53 v1.5.0 github.com/libp2p/go-netroute v0.2.1 github.com/magiconair/properties v1.8.7 github.com/mattn/go-sqlite3 v1.14.19 github.com/mdlayher/socket v0.4.1 - github.com/miekg/dns v1.1.43 + github.com/miekg/dns v1.1.59 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/nadoo/ipset v0.5.0 github.com/netbirdio/management-integrations/integrations v0.0.0-20240703085513-32605f7ffd8e @@ -70,6 +72,7 @@ require ( github.com/rs/xid v1.3.0 github.com/shirou/gopsutil/v3 v3.24.4 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/stretchr/testify v1.9.0 github.com/testcontainers/testcontainers-go v0.31.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0 @@ -81,6 +84,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.48.0 go.opentelemetry.io/otel/metric v1.26.0 go.opentelemetry.io/otel/sdk/metric v1.26.0 + go.uber.org/zap v1.27.0 goauthentik.io/api/v3 v3.2023051.3 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/mobile v0.0.0-20231127183840-76ac6878050a @@ -107,8 +111,23 @@ require ( github.com/Microsoft/hcsshim v0.12.3 // indirect github.com/XiaoMi/pegasus-go-client v0.0.0-20210427083443-f3b6b08bc4c2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect + github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect + github.com/aws/smithy-go v1.20.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect + github.com/caddyserver/zerossl v0.1.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/containerd v1.7.16 // indirect github.com/containerd/log v0.1.0 // indirect @@ -141,7 +160,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -150,13 +169,17 @@ require ( github.com/jeandeaual/go-locale v0.0.0-20240223122105-ce5225dcaa49 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/jsummers/gobmp v0.0.0-20151104160322-e2ba15ffa76e // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/libdns/libdns v0.2.2 // indirect github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect github.com/mdlayher/genetlink v1.3.2 // indirect github.com/mdlayher/netlink v1.7.2 // indirect + github.com/mholt/acmez/v2 v2.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect @@ -187,10 +210,12 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/yuin/goldmark v1.7.1 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index c5fc08bfd..8ab365ac8 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,34 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= +github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90= +github.com/aws/aws-sdk-go-v2/config v1.27.27/go.mod h1:MVYamCg76dFNINkZFu4n4RjDixhVr51HLj4ErWzrVwg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27 h1:2raNba6gr2IfA0eqqiP2XiQ0UVOpGPgDSi0I9iAP+UI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.27/go.mod h1:gniiwbGahQByxan6YjQUMcW4Aov6bLC3m+evgcoN4r4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk/UqI9iwMrOx/87PBNIKqw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15/go.mod h1:U9ke74k1n2bf+RIgoX1SXFed1HLs51OgUSs+Ph0KJP8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 h1:C6WHdGnTDIYETAm5iErQUiVNsclNx9qbJVPIt03B6bI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15/go.mod h1:ZQLZqhcu+JhSrA9/NXRm8SkDvsycE+JkV3WGY41e+IM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 h1:HGErhhrxZlQ044RiM+WdoZxp0p+EGM62y3L6pwA4olE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17/go.mod h1:RkZEx4l0EHYDJpWppMJ3nD9wZJAa8/0lq9aVC+r2UII= +github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3 h1:MmLCRqP4U4Cw9gJ4bNrCG0mWqEtBlmAVleyelcHARMU= +github.com/aws/aws-sdk-go-v2/service/route53 v1.42.3/go.mod h1:AMPjK2YnRh0YgOID3PqhJA1BRNfXDfGOnSsKHtAe8yA= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.4/go.mod h1:ooyCOXjvJEsUw7x+ZDHeISPMhtwI3ZCB7ggFMcFfWLU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 h1:yiwVzJW2ZxZTurVbYWA7QOrAaCYQR72t0wrSBfoesUE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4/go.mod h1:0oxfLkpz3rQ/CHlx5hB7H69YUpFiI1tql6Q6Ne+1bCw= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudrvuKpDKgMVRlepGE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -87,6 +115,10 @@ github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwel github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU= github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo= +github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0= +github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI= +github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA= +github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -352,8 +384,9 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -384,6 +417,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -403,6 +440,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -415,6 +455,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s= +github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= +github.com/libdns/route53 v1.5.0 h1:2SKdpPFl/qgWsXQvsLNJJAoX7rSxlk7zgoL4jnWdXVA= +github.com/libdns/route53 v1.5.0/go.mod h1:joT4hKmaTNKHEwb7GmZ65eoDz1whTu7KKYPS8ZqIh6Q= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae h1:dIZY4ULFcto4tAFlj1FYZl8ztUZ13bdq+PLY+NOfbyI= github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= @@ -433,9 +477,11 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k= +github.com/mholt/acmez/v2 v2.0.1/go.mod h1:fX4c9r5jYwMyMsC+7tkYRxHibkOTgta5DIFGoe67e1U= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws= github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -592,6 +638,8 @@ github.com/smartystreets/assertions v1.13.0/go.mod h1:wDmR7qL282YbGsPy6H/yAsesrx github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -660,6 +708,12 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -695,8 +749,14 @@ go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZu go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= goauthentik.io/api/v3 v3.2023051.3 h1:NebAhD/TeTWNo/9X3/Uj+rM5fG1HaiLOlKTNLQv9Qq4= goauthentik.io/api/v3 v3.2023051.3/go.mod h1:nYECml4jGbp/541hj8GcylKQG1gVBsKppHy4+7G8u4U= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -890,7 +950,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/relay/client/client.go b/relay/client/client.go index 4a20a3b00..aba940b41 100644 --- a/relay/client/client.go +++ b/relay/client/client.go @@ -46,7 +46,7 @@ func (isf *internalStopFlag) isSet() bool { return isf.stop } -// Msg carry the payload from the server to the client. With this sturct, the net.Conn can free the buffer. +// Msg carry the payload from the server to the client. With this struct, the net.Conn can free the buffer. type Msg struct { Payload []byte diff --git a/relay/cmd/env.go b/relay/cmd/env.go new file mode 100644 index 000000000..85d3e922b --- /dev/null +++ b/relay/cmd/env.go @@ -0,0 +1,35 @@ +package main + +import ( + "os" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// setFlagsFromEnvVars reads and updates flag values from environment variables with prefix NB_ +func setFlagsFromEnvVars(cmd *cobra.Command) { + flags := cmd.PersistentFlags() + flags.VisitAll(func(f *pflag.Flag) { + newEnvVar := flagNameToEnvVar(f.Name, "NB_") + value, present := os.LookupEnv(newEnvVar) + if !present { + return + } + + err := flags.Set(f.Name, value) + if err != nil { + log.Infof("unable to configure flag %s using variable %s, err: %v", f.Name, newEnvVar, err) + } + }) +} + +// flagNameToEnvVar converts flag name to environment var name adding a prefix, +// replacing dashes and making all uppercase (e.g. setup-keys is converted to NB_SETUP_KEYS according to the input prefix) +func flagNameToEnvVar(cmdFlag string, prefix string) string { + parsed := strings.ReplaceAll(cmdFlag, "-", "_") + upper := strings.ToUpper(parsed) + return prefix + upper +} diff --git a/relay/cmd/main.go b/relay/cmd/main.go index 7bb7d700a..b61f61d1a 100644 --- a/relay/cmd/main.go +++ b/relay/cmd/main.go @@ -31,11 +31,17 @@ type Config struct { // in HA every peer connect to a common domain, the instance domain has been distributed during the p2p connection // it is a domain:port or ip:port ExposedAddress string + LetsencryptEmail string LetsencryptDataDir string LetsencryptDomains []string - TlsCertFile string - TlsKeyFile string - AuthSecret string + // in case of using Route 53 for DNS challenge the credentials should be provided in the environment variables or + // in the AWS credentials file + LetsencryptAWSRoute53 bool + TlsCertFile string + TlsKeyFile string + AuthSecret string + LogLevel string + LogFile string } func (c Config) Validate() error { @@ -58,7 +64,6 @@ func (c Config) HasLetsEncrypt() bool { var ( cobraConfig *Config - cfgFile string rootCmd = &cobra.Command{ Use: "relay", Short: "Relay service", @@ -72,14 +77,19 @@ var ( func init() { _ = util.InitLog("trace", "console") cobraConfig = &Config{} - rootCmd.PersistentFlags().StringVarP(&cfgFile, "config-file", "f", "/etc/netbird/relay.json", "Relay server config file location") rootCmd.PersistentFlags().StringVarP(&cobraConfig.ListenAddress, "listen-address", "l", ":443", "listen address") rootCmd.PersistentFlags().StringVarP(&cobraConfig.ExposedAddress, "exposed-address", "e", "", "instance domain address (or ip) and port, it will be distributes between peers") rootCmd.PersistentFlags().StringVarP(&cobraConfig.LetsencryptDataDir, "letsencrypt-data-dir", "d", "", "a directory to store Let's Encrypt data. Required if Let's Encrypt is enabled.") - rootCmd.PersistentFlags().StringArrayVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringSliceVarP(&cobraConfig.LetsencryptDomains, "letsencrypt-domains", "a", nil, "list of domains to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LetsencryptEmail, "letsencrypt-email", "", "email address to use for Let's Encrypt certificate registration") + rootCmd.PersistentFlags().BoolVar(&cobraConfig.LetsencryptAWSRoute53, "letsencrypt-aws-route53", false, "use AWS Route 53 for Let's Encrypt DNS challenge") rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsCertFile, "tls-cert-file", "c", "", "") rootCmd.PersistentFlags().StringVarP(&cobraConfig.TlsKeyFile, "tls-key-file", "k", "", "") - rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "log level") + rootCmd.PersistentFlags().StringVarP(&cobraConfig.AuthSecret, "auth-secret", "s", "", "auth secret") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LogLevel, "log-level", "info", "log level") + rootCmd.PersistentFlags().StringVar(&cobraConfig.LogFile, "log-file", "console", "log file") + + setFlagsFromEnvVars(rootCmd) } func waitForExitSignal() { @@ -88,49 +98,15 @@ func waitForExitSignal() { <-osSigs } -func loadConfig(configFile string) (*Config, error) { - log.Infof("loading config from: %s", configFile) - loadedConfig := &Config{} - _, err := util.ReadJson(configFile, loadedConfig) - if err != nil { - return nil, err - } - if cobraConfig.ListenAddress != "" { - loadedConfig.ListenAddress = cobraConfig.ListenAddress - } - - if cobraConfig.ExposedAddress != "" { - loadedConfig.ExposedAddress = cobraConfig.ExposedAddress - } - if cobraConfig.LetsencryptDataDir != "" { - loadedConfig.LetsencryptDataDir = cobraConfig.LetsencryptDataDir - } - if len(cobraConfig.LetsencryptDomains) > 0 { - loadedConfig.LetsencryptDomains = cobraConfig.LetsencryptDomains - } - if cobraConfig.TlsCertFile != "" { - loadedConfig.TlsCertFile = cobraConfig.TlsCertFile - } - if cobraConfig.TlsKeyFile != "" { - loadedConfig.TlsKeyFile = cobraConfig.TlsKeyFile - } - if cobraConfig.AuthSecret != "" { - loadedConfig.AuthSecret = cobraConfig.AuthSecret - } - - return loadedConfig, err -} - func execute(cmd *cobra.Command, args []string) error { - cfg, err := loadConfig(cfgFile) + err := cobraConfig.Validate() if err != nil { - return fmt.Errorf("failed to load config: %s", err) + return fmt.Errorf("invalid config: %s", err) } - err = cfg.Validate() + err = util.InitLog(cobraConfig.LogLevel, cobraConfig.LogFile) if err != nil { - log.Errorf("invalid config: %s", err) - return err + return fmt.Errorf("failed to initialize log: %s", err) } metricsServer, err := metrics.NewServer(metricsPort, "") @@ -146,26 +122,17 @@ func execute(cmd *cobra.Command, args []string) error { }() srvListenerCfg := server.ListenerConfig{ - Address: cfg.ListenAddress, - } - if cfg.HasLetsEncrypt() { - tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...) - if err != nil { - return fmt.Errorf("%s", err) - } - srvListenerCfg.TLSConfig = tlsCfg - } else if cfg.HasCertConfig() { - tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile) - if err != nil { - return fmt.Errorf("%s", err) - } - srvListenerCfg.TLSConfig = tlsCfg + Address: cobraConfig.ListenAddress, } - tlsSupport := srvListenerCfg.TLSConfig != nil + tlsConfig, tlsSupport, err := handleTLSConfig(cobraConfig) + if err != nil { + return fmt.Errorf("failed to setup TLS config: %s", err) + } + srvListenerCfg.TLSConfig = tlsConfig - authenticator := auth.NewTimedHMACValidator(cfg.AuthSecret, 24*time.Hour) - srv, err := server.NewServer(metricsServer.Meter, cfg.ExposedAddress, tlsSupport, authenticator) + authenticator := auth.NewTimedHMACValidator(cobraConfig.AuthSecret, 24*time.Hour) + srv, err := server.NewServer(metricsServer.Meter, cobraConfig.ExposedAddress, tlsSupport, authenticator) if err != nil { return fmt.Errorf("failed to create relay server: %v", err) } @@ -194,6 +161,41 @@ func execute(cmd *cobra.Command, args []string) error { return shutDownErrors } +func handleTLSConfig(cfg *Config) (*tls.Config, bool, error) { + if cfg.LetsencryptAWSRoute53 { + log.Debugf("using Let's Encrypt DNS resolver with Route 53 support") + r53 := encryption.Route53TLS{ + DataDir: cfg.LetsencryptDataDir, + Email: cfg.LetsencryptEmail, + Domains: cfg.LetsencryptDomains, + } + tlsCfg, err := r53.GetCertificate() + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + + if cfg.HasLetsEncrypt() { + log.Infof("setting up TLS with Let's Encrypt.") + tlsCfg, err := setupTLSCertManager(cfg.LetsencryptDataDir, cfg.LetsencryptDomains...) + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + + if cfg.HasCertConfig() { + log.Debugf("using file based TLS config") + tlsCfg, err := encryption.LoadTLSConfig(cfg.TlsCertFile, cfg.TlsKeyFile) + if err != nil { + return nil, false, fmt.Errorf("%s", err) + } + return tlsCfg, true, nil + } + return nil, false, nil +} + func setupTLSCertManager(letsencryptDataDir string, letsencryptDomains ...string) (*tls.Config, error) { certManager, err := encryption.CreateCertManager(letsencryptDataDir, letsencryptDomains...) if err != nil { diff --git a/relay/messages/message.go b/relay/messages/message.go index 3770f6398..387d87a94 100644 --- a/relay/messages/message.go +++ b/relay/messages/message.go @@ -111,7 +111,7 @@ func MarshalHelloMsg(peerID []byte, additions []byte) ([]byte, error) { // authenticate the client with the server. func UnmarshalHelloMsg(msg []byte) ([]byte, []byte, error) { if len(msg) < headerSizeHello { - return nil, nil, fmt.Errorf("invalid 'hello' messge") + return nil, nil, fmt.Errorf("invalid 'hello' message") } if !bytes.Equal(msg[1:5], magicHeader) { return nil, nil, fmt.Errorf("invalid magic header") diff --git a/relay/server/store.go b/relay/server/store.go index 79b8aeb5d..96879dae1 100644 --- a/relay/server/store.go +++ b/relay/server/store.go @@ -19,7 +19,7 @@ func NewStore() *Store { } // AddPeer adds a peer to the store -// It distinguishes the peers by their ID +// todo: consider to close peer conn if the peer already exists func (s *Store) AddPeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() @@ -30,6 +30,15 @@ func (s *Store) AddPeer(peer *Peer) { func (s *Store) DeletePeer(peer *Peer) { s.peersLock.Lock() defer s.peersLock.Unlock() + + dp, ok := s.peers[peer.String()] + if !ok { + return + } + if dp != peer { + return + } + delete(s.peers, peer.String()) } diff --git a/relay/server/store_test.go b/relay/server/store_test.go new file mode 100644 index 000000000..4a30bc131 --- /dev/null +++ b/relay/server/store_test.go @@ -0,0 +1,40 @@ +package server + +import ( + "context" + "testing" + + "go.opentelemetry.io/otel" + + "github.com/netbirdio/netbird/relay/metrics" +) + +func TestStore_DeletePeer(t *testing.T) { + s := NewStore() + + m, _ := metrics.NewMetrics(context.Background(), otel.Meter("")) + + p := NewPeer(m, []byte("peer_one"), nil, nil) + s.AddPeer(p) + s.DeletePeer(p) + if _, ok := s.Peer(p.String()); ok { + t.Errorf("peer was not deleted") + } +} + +func TestStore_DeleteDeprecatedPeer(t *testing.T) { + s := NewStore() + + m, _ := metrics.NewMetrics(context.Background(), otel.Meter("")) + + p1 := NewPeer(m, []byte("peer_id"), nil, nil) + p2 := NewPeer(m, []byte("peer_id"), nil, nil) + + s.AddPeer(p1) + s.AddPeer(p2) + s.DeletePeer(p1) + + if _, ok := s.Peer(p2.String()); !ok { + t.Errorf("second peer was deleted") + } +} diff --git a/relay/testec2/main.go b/relay/testec2/main.go new file mode 100644 index 000000000..0c8099a5e --- /dev/null +++ b/relay/testec2/main.go @@ -0,0 +1,258 @@ +//go:build linux || darwin + +package main + +import ( + "crypto/rand" + "flag" + "fmt" + "net" + "os" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/util" +) + +const ( + errMsgFailedReadTCP = "failed to read from tcp: %s" +) + +var ( + dataSize = 1024 * 1024 * 50 // 50MB + pairs = []int{1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + signalListenAddress = ":8081" + + relaySrvAddress string + turnSrvAddress string + signalURL string + udpListener string // used for TURN test +) + +type testResult struct { + numOfPairs int + duration time.Duration + speed float64 +} + +func (tr testResult) Speed() string { + speed := tr.speed + var unit string + + switch { + case speed < 1024: + unit = "B/s" + case speed < 1048576: + speed /= 1024 + unit = "KB/s" + case speed < 1073741824: + speed /= 1048576 + unit = "MB/s" + default: + speed /= 1073741824 + unit = "GB/s" + } + + return fmt.Sprintf("%.2f %s", speed, unit) +} + +func seedRandomData(size int) ([]byte, error) { + token := make([]byte, size) + _, err := rand.Read(token) + if err != nil { + return nil, err + } + return token, nil +} + +func avg(transferDuration []time.Duration) (time.Duration, float64) { + var totalDuration time.Duration + for _, d := range transferDuration { + totalDuration += d + } + avgDuration := totalDuration / time.Duration(len(transferDuration)) + bps := float64(dataSize) / avgDuration.Seconds() + return avgDuration, bps +} + +func RelayReceiverMain() []testResult { + testResults := make([]testResult, 0, len(pairs)) + for _, p := range pairs { + tr := testResult{numOfPairs: p} + td := relayReceive(relaySrvAddress, p) + tr.duration, tr.speed = avg(td) + + testResults = append(testResults, tr) + } + + return testResults +} + +func RelaySenderMain() { + log.Infof("starting sender") + log.Infof("starting seed phase") + + testData, err := seedRandomData(dataSize) + if err != nil { + log.Fatalf("failed to seed random data: %s", err) + } + + log.Infof("data size: %d", len(testData)) + + for n, p := range pairs { + log.Infof("running test with %d pairs", p) + relayTransfer(relaySrvAddress, testData, p) + + // grant time to prepare new receivers + if n < len(pairs)-1 { + time.Sleep(3 * time.Second) + } + } + +} + +// TRUNSenderMain is the sender +// - allocate turn clients +// - send relayed addresses to signal server in batch +// - wait for signal server to send back addresses in a map +// - send test data to each address in parallel +func TRUNSenderMain() { + log.Infof("starting TURN sender test") + + log.Infof("starting seed random data: %d", dataSize) + testData, err := seedRandomData(dataSize) + if err != nil { + log.Fatalf("failed to seed random data: %s", err) + } + + ss := SignalClient{signalURL} + + for _, p := range pairs { + log.Infof("running test with %d pairs", p) + turnSender := &TurnSender{} + + createTurnConns(p, turnSender) + + log.Infof("send addresses via signal server: %d", len(turnSender.addresses)) + clientAddresses, err := ss.SendAddress(turnSender.addresses) + if err != nil { + log.Fatalf("failed to send address: %s", err) + } + log.Infof("received addresses: %v", clientAddresses.Address) + + createSenderDevices(turnSender, clientAddresses) + + log.Infof("waiting for tcpListeners to be ready") + time.Sleep(2 * time.Second) + + tcpConns := make([]net.Conn, 0, len(turnSender.devices)) + for i := range turnSender.devices { + addr := fmt.Sprintf("10.0.%d.2:9999", i) + log.Infof("dialing: %s", addr) + tcpConn, err := net.Dial("tcp", addr) + if err != nil { + log.Fatalf("failed to dial tcp: %s", err) + } + tcpConns = append(tcpConns, tcpConn) + } + + log.Infof("start test data transfer for %d pairs", p) + testDataLen := len(testData) + wg := sync.WaitGroup{} + wg.Add(len(tcpConns)) + for i, tcpConn := range tcpConns { + log.Infof("sending test data to device: %d", i) + go runTurnWriting(tcpConn, testData, testDataLen, &wg) + } + wg.Wait() + + for _, d := range turnSender.devices { + _ = d.Close() + } + + log.Infof("test finished with %d pairs", p) + } +} + +func TURNReaderMain() []testResult { + log.Infof("starting TURN receiver test") + si := NewSignalService() + go func() { + log.Infof("starting signal server") + err := si.Listen(signalListenAddress) + if err != nil { + log.Errorf("failed to listen: %s", err) + } + }() + + testResults := make([]testResult, 0, len(pairs)) + for range pairs { + addresses := <-si.AddressesChan + instanceNumber := len(addresses) + log.Infof("received addresses: %d", instanceNumber) + + turnReceiver := &TurnReceiver{} + err := createDevices(addresses, turnReceiver) + if err != nil { + log.Fatalf("%s", err) + } + + // send client addresses back via signal server + si.ClientAddressChan <- turnReceiver.clientAddresses + + durations := make(chan time.Duration, instanceNumber) + for _, device := range turnReceiver.devices { + go runTurnReading(device, durations) + } + + durationsList := make([]time.Duration, 0, instanceNumber) + for d := range durations { + durationsList = append(durationsList, d) + if len(durationsList) == instanceNumber { + close(durations) + } + } + + avgDuration, avgSpeed := avg(durationsList) + ts := testResult{ + numOfPairs: len(durationsList), + duration: avgDuration, + speed: avgSpeed, + } + testResults = append(testResults, ts) + + for _, d := range turnReceiver.devices { + _ = d.Close() + } + } + return testResults +} + +func main() { + var mode string + + _ = util.InitLog("debug", "console") + flag.StringVar(&mode, "mode", "sender", "sender or receiver mode") + flag.Parse() + + relaySrvAddress = os.Getenv("TEST_RELAY_SERVER") // rel://ip:port + turnSrvAddress = os.Getenv("TEST_TURN_SERVER") // ip:3478 + signalURL = os.Getenv("TEST_SIGNAL_URL") // http://receiver_ip:8081 + udpListener = os.Getenv("TEST_UDP_LISTENER") // IP:0 + + if mode == "receiver" { + relayResult := RelayReceiverMain() + turnResults := TURNReaderMain() + for i := 0; i < len(turnResults); i++ { + log.Infof("pairs: %d,\tRelay speed:\t%s,\trelay duration:\t%s", relayResult[i].numOfPairs, relayResult[i].Speed(), relayResult[i].duration) + log.Infof("pairs: %d,\tTURN speed:\t%s,\tturn duration:\t%s", turnResults[i].numOfPairs, turnResults[i].Speed(), turnResults[i].duration) + } + } else { + RelaySenderMain() + // grant time for receiver to start + time.Sleep(3 * time.Second) + TRUNSenderMain() + } +} diff --git a/relay/testec2/relay.go b/relay/testec2/relay.go new file mode 100644 index 000000000..93d084387 --- /dev/null +++ b/relay/testec2/relay.go @@ -0,0 +1,176 @@ +//go:build linux || darwin + +package main + +import ( + "context" + "fmt" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/relay/auth/hmac" + "github.com/netbirdio/netbird/relay/client" +) + +var ( + hmacTokenStore = &hmac.TokenStore{} +) + +func relayTransfer(serverConnURL string, testData []byte, peerPairs int) { + connsSender := prepareConnsSender(serverConnURL, peerPairs) + defer func() { + for i := 0; i < len(connsSender); i++ { + err := connsSender[i].Close() + if err != nil { + log.Errorf("failed to close connection: %s", err) + } + } + }() + + wg := sync.WaitGroup{} + wg.Add(len(connsSender)) + for _, conn := range connsSender { + go func(conn net.Conn) { + defer wg.Done() + runWriter(conn, testData) + }(conn) + } + wg.Wait() +} + +func runWriter(conn net.Conn, testData []byte) { + si := NewStartInidication(time.Now(), len(testData)) + _, err := conn.Write(si) + if err != nil { + log.Errorf("failed to write to channel: %s", err) + return + } + log.Infof("sent start indication") + + pieceSize := 1024 + testDataLen := len(testData) + + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr := conn.Write(testData[j:end]) + if writeErr != nil { + log.Errorf("failed to write to channel: %s", writeErr) + return + } + } +} + +func prepareConnsSender(serverConnURL string, peerPairs int) []net.Conn { + ctx := context.Background() + clientsSender := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsSender); i++ { + c := client.NewClient(ctx, serverConnURL, hmacTokenStore, "sender-"+fmt.Sprint(i)) + if err := c.Connect(); err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + clientsSender[i] = c + } + + connsSender := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsSender); i++ { + conn, err := clientsSender[i].OpenConn("receiver-" + fmt.Sprint(i)) + if err != nil { + log.Fatalf("failed to bind channel: %s", err) + } + connsSender = append(connsSender, conn) + } + return connsSender +} + +func relayReceive(serverConnURL string, peerPairs int) []time.Duration { + connsReceiver := prepareConnsReceiver(serverConnURL, peerPairs) + defer func() { + for i := 0; i < len(connsReceiver); i++ { + if err := connsReceiver[i].Close(); err != nil { + log.Errorf("failed to close connection: %s", err) + } + } + }() + + durations := make(chan time.Duration, len(connsReceiver)) + wg := sync.WaitGroup{} + for _, conn := range connsReceiver { + wg.Add(1) + go func(conn net.Conn) { + defer wg.Done() + duration := runReader(conn) + durations <- duration + }(conn) + } + wg.Wait() + + durationsList := make([]time.Duration, 0, len(connsReceiver)) + for d := range durations { + durationsList = append(durationsList, d) + if len(durationsList) == len(connsReceiver) { + close(durations) + } + } + + return durationsList +} + +func runReader(conn net.Conn) time.Duration { + buf := make([]byte, 8192) + + n, readErr := conn.Read(buf) + if readErr != nil { + log.Errorf("failed to read from channel: %s", readErr) + return 0 + } + + si := DecodeStartIndication(buf[:n]) + log.Infof("received start indication: %v", si) + + receivedSize, err := conn.Read(buf) + if err != nil { + log.Fatalf("failed to read from relay: %s", err) + } + now := time.Now() + + rcv := 0 + for receivedSize < si.TransferSize { + n, readErr = conn.Read(buf) + if readErr != nil { + log.Errorf("failed to read from channel: %s", readErr) + return 0 + } + + receivedSize += n + rcv += n + } + return time.Since(now) +} + +func prepareConnsReceiver(serverConnURL string, peerPairs int) []net.Conn { + clientsReceiver := make([]*client.Client, peerPairs) + for i := 0; i < cap(clientsReceiver); i++ { + c := client.NewClient(context.Background(), serverConnURL, hmacTokenStore, "receiver-"+fmt.Sprint(i)) + err := c.Connect() + if err != nil { + log.Fatalf("failed to connect to server: %s", err) + } + clientsReceiver[i] = c + } + + connsReceiver := make([]net.Conn, 0, peerPairs) + for i := 0; i < len(clientsReceiver); i++ { + conn, err := clientsReceiver[i].OpenConn("sender-" + fmt.Sprint(i)) + if err != nil { + log.Fatalf("failed to bind channel: %s", err) + } + connsReceiver = append(connsReceiver, conn) + } + return connsReceiver +} diff --git a/relay/testec2/signal.go b/relay/testec2/signal.go new file mode 100644 index 000000000..fe93a2fe2 --- /dev/null +++ b/relay/testec2/signal.go @@ -0,0 +1,91 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "encoding/json" + "net/http" + + log "github.com/sirupsen/logrus" +) + +type PeerAddr struct { + Address []string +} + +type ClientPeerAddr struct { + Address map[string]string +} + +type Signal struct { + AddressesChan chan []string + ClientAddressChan chan map[string]string +} + +func NewSignalService() *Signal { + return &Signal{ + AddressesChan: make(chan []string), + ClientAddressChan: make(chan map[string]string), + } +} + +func (rs *Signal) Listen(listenAddr string) error { + http.HandleFunc("/", rs.onNewAddresses) + return http.ListenAndServe(listenAddr, nil) +} + +func (rs *Signal) onNewAddresses(w http.ResponseWriter, r *http.Request) { + var msg PeerAddr + err := json.NewDecoder(r.Body).Decode(&msg) + if err != nil { + log.Errorf("Error decoding message: %v", err) + } + + log.Infof("received addresses: %d", len(msg.Address)) + rs.AddressesChan <- msg.Address + clientAddresses := <-rs.ClientAddressChan + + respMsg := ClientPeerAddr{ + Address: clientAddresses, + } + data, err := json.Marshal(respMsg) + if err != nil { + log.Errorf("Error marshalling message: %v", err) + return + } + + _, err = w.Write(data) + if err != nil { + log.Errorf("Error writing response: %v", err) + } +} + +type SignalClient struct { + SignalURL string +} + +func (ss SignalClient) SendAddress(addresses []string) (*ClientPeerAddr, error) { + msg := PeerAddr{ + Address: addresses, + } + data, err := json.Marshal(msg) + if err != nil { + return nil, err + } + + response, err := http.Post(ss.SignalURL, "application/json", bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + defer response.Body.Close() + + log.Debugf("wait for signal response") + var respPeerAddress ClientPeerAddr + err = json.NewDecoder(response.Body).Decode(&respPeerAddress) + if err != nil { + return nil, err + } + return &respPeerAddress, nil +} diff --git a/relay/testec2/start_msg.go b/relay/testec2/start_msg.go new file mode 100644 index 000000000..19b65380b --- /dev/null +++ b/relay/testec2/start_msg.go @@ -0,0 +1,39 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "encoding/gob" + "time" + + log "github.com/sirupsen/logrus" +) + +type StartIndication struct { + Started time.Time + TransferSize int +} + +func NewStartInidication(started time.Time, transferSize int) []byte { + si := StartIndication{ + Started: started, + TransferSize: transferSize, + } + + var data bytes.Buffer + err := gob.NewEncoder(&data).Encode(si) + if err != nil { + log.Fatal("encode error:", err) + } + return data.Bytes() +} + +func DecodeStartIndication(data []byte) StartIndication { + var si StartIndication + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&si) + if err != nil { + log.Fatal("decode error:", err) + } + return si +} diff --git a/relay/testec2/tun/proxy.go b/relay/testec2/tun/proxy.go new file mode 100644 index 000000000..7d84bece7 --- /dev/null +++ b/relay/testec2/tun/proxy.go @@ -0,0 +1,72 @@ +//go:build linux || darwin + +package tun + +import ( + "net" + "sync/atomic" + + log "github.com/sirupsen/logrus" +) + +type Proxy struct { + Device *Device + PConn net.PacketConn + DstAddr net.Addr + shutdownFlag atomic.Bool +} + +func (p *Proxy) Start() { + go p.readFromDevice() + go p.readFromConn() +} + +func (p *Proxy) Close() { + p.shutdownFlag.Store(true) +} + +func (p *Proxy) readFromDevice() { + buf := make([]byte, 1500) + for { + n, err := p.Device.Read(buf) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to read from device: %s", err) + return + } + + _, err = p.PConn.WriteTo(buf[:n], p.DstAddr) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to write to conn: %s", err) + return + } + } +} + +func (p *Proxy) readFromConn() { + buf := make([]byte, 1500) + for { + n, _, err := p.PConn.ReadFrom(buf) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to read from conn: %s", err) + return + } + + _, err = p.Device.Write(buf[:n]) + if err != nil { + if p.shutdownFlag.Load() { + return + } + log.Errorf("failed to write to device: %s", err) + return + } + } +} diff --git a/relay/testec2/tun/tun.go b/relay/testec2/tun/tun.go new file mode 100644 index 000000000..5580785ce --- /dev/null +++ b/relay/testec2/tun/tun.go @@ -0,0 +1,110 @@ +//go:build linux || darwin + +package tun + +import ( + "net" + + log "github.com/sirupsen/logrus" + "github.com/songgao/water" + "github.com/vishvananda/netlink" +) + +type Device struct { + Name string + IP string + PConn net.PacketConn + DstAddr net.Addr + + iFace *water.Interface + proxy *Proxy +} + +func (d *Device) Up() error { + cfg := water.Config{ + DeviceType: water.TUN, + PlatformSpecificParams: water.PlatformSpecificParams{ + Name: d.Name, + }, + } + iFace, err := water.New(cfg) + if err != nil { + return err + } + d.iFace = iFace + + err = d.assignIP() + if err != nil { + return err + } + + err = d.bringUp() + if err != nil { + return err + } + + d.proxy = &Proxy{ + Device: d, + PConn: d.PConn, + DstAddr: d.DstAddr, + } + d.proxy.Start() + return nil +} + +func (d *Device) Close() error { + if d.proxy != nil { + d.proxy.Close() + } + if d.iFace != nil { + return d.iFace.Close() + } + return nil +} + +func (d *Device) Read(b []byte) (int, error) { + return d.iFace.Read(b) +} + +func (d *Device) Write(b []byte) (int, error) { + return d.iFace.Write(b) +} + +func (d *Device) assignIP() error { + iface, err := netlink.LinkByName(d.Name) + if err != nil { + log.Errorf("failed to get TUN device: %v", err) + return err + } + + ip := net.IPNet{ + IP: net.ParseIP(d.IP), + Mask: net.CIDRMask(24, 32), + } + + addr := &netlink.Addr{ + IPNet: &ip, + } + err = netlink.AddrAdd(iface, addr) + if err != nil { + log.Errorf("failed to add IP address: %v", err) + return err + } + return nil +} + +func (d *Device) bringUp() error { + iface, err := netlink.LinkByName(d.Name) + if err != nil { + log.Errorf("failed to get device: %v", err) + return err + } + + // Bring the interface up + err = netlink.LinkSetUp(iface) + if err != nil { + log.Errorf("failed to set device up: %v", err) + return err + } + return nil +} diff --git a/relay/testec2/turn.go b/relay/testec2/turn.go new file mode 100644 index 000000000..8beb40423 --- /dev/null +++ b/relay/testec2/turn.go @@ -0,0 +1,181 @@ +//go:build linux || darwin + +package main + +import ( + "fmt" + "net" + "sync" + "time" + + "github.com/netbirdio/netbird/relay/testec2/tun" + + log "github.com/sirupsen/logrus" +) + +type TurnReceiver struct { + conns []*net.UDPConn + clientAddresses map[string]string + devices []*tun.Device +} + +type TurnSender struct { + turnConns map[string]*TurnConn + addresses []string + devices []*tun.Device +} + +func runTurnWriting(tcpConn net.Conn, testData []byte, testDataLen int, wg *sync.WaitGroup) { + defer wg.Done() + defer tcpConn.Close() + + log.Infof("start to sending test data: %s", tcpConn.RemoteAddr()) + + si := NewStartInidication(time.Now(), testDataLen) + _, err := tcpConn.Write(si) + if err != nil { + log.Errorf("failed to write to tcp: %s", err) + return + } + + pieceSize := 1024 + for j := 0; j < testDataLen; j += pieceSize { + end := j + pieceSize + if end > testDataLen { + end = testDataLen + } + _, writeErr := tcpConn.Write(testData[j:end]) + if writeErr != nil { + log.Errorf("failed to write to tcp conn: %s", writeErr) + return + } + } + + // grant time to flush out packages + time.Sleep(3 * time.Second) +} + +func createSenderDevices(sender *TurnSender, clientAddresses *ClientPeerAddr) { + var i int + devices := make([]*tun.Device, 0, len(clientAddresses.Address)) + for k, v := range clientAddresses.Address { + tc, ok := sender.turnConns[k] + if !ok { + log.Fatalf("failed to find turn conn: %s", k) + } + + addr, err := net.ResolveUDPAddr("udp", v) + if err != nil { + log.Fatalf("failed to resolve udp address: %s", err) + } + device := &tun.Device{ + Name: fmt.Sprintf("mtun-sender-%d", i), + IP: fmt.Sprintf("10.0.%d.1", i), + PConn: tc.relayConn, + DstAddr: addr, + } + + err = device.Up() + if err != nil { + log.Fatalf("failed to bring up device: %s", err) + } + + devices = append(devices, device) + i++ + } + sender.devices = devices +} + +func createTurnConns(p int, sender *TurnSender) { + turnConns := make(map[string]*TurnConn) + addresses := make([]string, 0, len(pairs)) + for i := 0; i < p; i++ { + tc := AllocateTurnClient(turnSrvAddress) + log.Infof("allocated turn client: %s", tc.Address().String()) + turnConns[tc.Address().String()] = tc + addresses = append(addresses, tc.Address().String()) + } + + sender.turnConns = turnConns + sender.addresses = addresses +} + +func runTurnReading(d *tun.Device, durations chan time.Duration) { + tcpListener, err := net.Listen("tcp", d.IP+":9999") + if err != nil { + log.Fatalf("failed to listen on tcp: %s", err) + } + log := log.WithField("device", tcpListener.Addr()) + + tcpConn, err := tcpListener.Accept() + if err != nil { + _ = tcpListener.Close() + log.Fatalf("failed to accept connection: %s", err) + } + log.Infof("remote peer connected") + + buf := make([]byte, 103) + n, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + + si := DecodeStartIndication(buf[:n]) + log.Infof("received start indication: %v, %d", si, n) + + buf = make([]byte, 8192) + i, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + now := time.Now() + for i < si.TransferSize { + n, err := tcpConn.Read(buf) + if err != nil { + _ = tcpListener.Close() + log.Fatalf(errMsgFailedReadTCP, err) + } + i += n + } + durations <- time.Since(now) +} + +func createDevices(addresses []string, receiver *TurnReceiver) error { + receiver.conns = make([]*net.UDPConn, 0, len(addresses)) + receiver.clientAddresses = make(map[string]string, len(addresses)) + receiver.devices = make([]*tun.Device, 0, len(addresses)) + for i, addr := range addresses { + localAddr, err := net.ResolveUDPAddr("udp", udpListener) + if err != nil { + return fmt.Errorf("failed to resolve UDP address: %s", err) + } + + conn, err := net.ListenUDP("udp", localAddr) + if err != nil { + return fmt.Errorf("failed to create UDP connection: %s", err) + } + + receiver.conns = append(receiver.conns, conn) + receiver.clientAddresses[addr] = conn.LocalAddr().String() + + dstAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return fmt.Errorf("failed to resolve address: %s", err) + } + + device := &tun.Device{ + Name: fmt.Sprintf("mtun-%d", i), + IP: fmt.Sprintf("10.0.%d.2", i), + PConn: conn, + DstAddr: dstAddr, + } + + if err = device.Up(); err != nil { + return fmt.Errorf("failed to bring up device: %s, %s", device.Name, err) + } + receiver.devices = append(receiver.devices, device) + } + return nil +} diff --git a/relay/testec2/turn_allocator.go b/relay/testec2/turn_allocator.go new file mode 100644 index 000000000..fd86208df --- /dev/null +++ b/relay/testec2/turn_allocator.go @@ -0,0 +1,83 @@ +//go:build linux || darwin + +package main + +import ( + "fmt" + "net" + + "github.com/pion/logging" + "github.com/pion/turn/v3" + log "github.com/sirupsen/logrus" +) + +type TurnConn struct { + conn net.Conn + turnClient *turn.Client + relayConn net.PacketConn +} + +func (tc *TurnConn) Address() net.Addr { + return tc.relayConn.LocalAddr() +} + +func (tc *TurnConn) Close() { + _ = tc.relayConn.Close() + tc.turnClient.Close() + _ = tc.conn.Close() +} + +func AllocateTurnClient(serverAddr string) *TurnConn { + conn, err := net.Dial("tcp", serverAddr) + if err != nil { + log.Fatal(err) + } + + turnClient, err := getTurnClient(serverAddr, conn) + if err != nil { + log.Fatal(err) + } + + relayConn, err := turnClient.Allocate() + if err != nil { + log.Fatal(err) + } + + return &TurnConn{ + conn: conn, + turnClient: turnClient, + relayConn: relayConn, + } +} + +func getTurnClient(address string, conn net.Conn) (*turn.Client, error) { + // Dial TURN Server + addrStr := fmt.Sprintf("%s:%d", address, 443) + + fac := logging.NewDefaultLoggerFactory() + //fac.DefaultLogLevel = logging.LogLevelTrace + + // Start a new TURN Client and wrap our net.Conn in a STUNConn + // This allows us to simulate datagram based communication over a net.Conn + cfg := &turn.ClientConfig{ + TURNServerAddr: address, + Conn: turn.NewSTUNConn(conn), + Username: "test", + Password: "test", + LoggerFactory: fac, + } + + client, err := turn.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create TURN client for server %s: %s", addrStr, err) + } + + // Start listening on the conn provided. + err = client.Listen() + if err != nil { + client.Close() + return nil, fmt.Errorf("failed to listen on TURN client for server %s: %s", addrStr, err) + } + + return client, nil +} From 3d81a0a07d2b66460bd77057bcfe3d811336a042 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 22 Aug 2024 14:40:25 +0200 Subject: [PATCH 197/199] Fix log with tags --- relay/server/peer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay/server/peer.go b/relay/server/peer.go index e3595b2e7..3ca0bda2a 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -97,12 +97,12 @@ func (p *Peer) CloseGracefully(ctx context.Context) { p.connMu.Lock() _, err := p.writeWithTimeout(ctx, messages.MarshalCloseMsg()) if err != nil { - log.Errorf("failed to send close message to peer: %s", p.String()) + p.log.Errorf("failed to send close message to peer: %s", p.String()) } err = p.conn.Close() if err != nil { - log.Errorf("failed to close connection to peer: %s", err) + p.log.Errorf("failed to close connection to peer: %s", err) } defer p.connMu.Unlock() From f28b4555b5298cd63fe1013fd41f81e621859584 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Thu, 22 Aug 2024 23:04:38 +0200 Subject: [PATCH 198/199] Replace error message to ctx.Err() --- relay/server/peer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/relay/server/peer.go b/relay/server/peer.go index 3ca0bda2a..c09dc8c9f 100644 --- a/relay/server/peer.go +++ b/relay/server/peer.go @@ -2,7 +2,6 @@ package server import ( "context" - "fmt" "io" "net" "sync" @@ -130,7 +129,7 @@ func (p *Peer) writeWithTimeout(ctx context.Context, buf []byte) (int, error) { select { case <-ctx.Done(): - return 0, fmt.Errorf("write operation timed out") + return 0, ctx.Err() case <-writeDone: return n, err } From e4604bb2d9c5619e8950fc12e11a11f38d9ac01d Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Fri, 23 Aug 2024 11:25:03 +0200 Subject: [PATCH 199/199] Fix StunTurn address update (#2471) Pass the atomic store by address --- client/internal/engine.go | 2 +- client/internal/peer/worker_ice.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/internal/engine.go b/client/internal/engine.go index 0f66f5a4e..3a868657b 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -952,7 +952,7 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e RosenpassPubKey: e.getRosenpassPubKey(), RosenpassAddr: e.getRosenpassAddr(), ICEConfig: peer.ICEConfig{ - StunTurn: e.StunTurn, + StunTurn: &e.StunTurn, InterfaceBlackList: e.config.IFaceBlackList, DisableIPv6Discovery: e.config.DisableIPv6Discovery, UDPMux: e.udpMux.UDPMuxDefault, diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index 26f3d04ab..0edb124cf 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -38,7 +38,7 @@ var ( type ICEConfig struct { // StunTurn is a list of STUN and TURN URLs - StunTurn atomic.Value // []*stun.URI + StunTurn *atomic.Value // []*stun.URI // InterfaceBlackList is a list of machine interfaces that should be filtered out by ICE Candidate gathering // (e.g. if eth0 is in the list, host candidate of this interface won't be used)