From b0364da67c027fb1038a34555a4c017f91fbc766 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Wed, 26 Jul 2023 14:00:47 +0200 Subject: [PATCH] Wg ebpf proxy (#911) EBPF proxy between TURN (relay) and WireGuard to reduce number of used ports used by the NetBird agent. - Separate the wg configuration from the proxy logic - In case if eBPF type proxy has only one single proxy instance - In case if the eBPF is not supported fallback to the original proxy Implementation Between the signature of eBPF type proxy and original proxy has differences so this is why the factory structure exists --- client/internal/engine.go | 25 +- client/internal/engine_test.go | 4 +- client/internal/peer/conn.go | 163 +++++++------ client/internal/peer/conn_test.go | 40 +++- client/internal/proxy/dummy.go | 72 ------ client/internal/proxy/noproxy.go | 42 ---- client/internal/proxy/proxy.go | 35 --- client/internal/proxy/wireguard.go | 128 ---------- client/internal/wgproxy/bpf/portreplace.c | 90 +++++++ client/internal/wgproxy/bpf_bpfeb.go | 120 ++++++++++ client/internal/wgproxy/bpf_bpfeb.o | Bin 0 -> 6224 bytes client/internal/wgproxy/bpf_bpfel.go | 120 ++++++++++ client/internal/wgproxy/bpf_bpfel.o | Bin 0 -> 6224 bytes client/internal/wgproxy/factory.go | 20 ++ client/internal/wgproxy/factory_linux.go | 19 ++ client/internal/wgproxy/factory_nonlinux.go | 7 + client/internal/wgproxy/loader.go | 80 +++++++ client/internal/wgproxy/loader_test.go | 18 ++ client/internal/wgproxy/portlookup.go | 32 +++ client/internal/wgproxy/portlookup_test.go | 42 ++++ client/internal/wgproxy/proxy.go | 12 + client/internal/wgproxy/proxy_ebpf.go | 250 ++++++++++++++++++++ client/internal/wgproxy/proxy_ebpf_test.go | 56 +++++ client/internal/wgproxy/proxy_userspace.go | 105 ++++++++ go.mod | 2 +- go.sum | 10 +- iface/bind/udp_mux_universal.go | 2 +- iface/iface.go | 2 +- 28 files changed, 1117 insertions(+), 379 deletions(-) delete mode 100644 client/internal/proxy/dummy.go delete mode 100644 client/internal/proxy/noproxy.go delete mode 100644 client/internal/proxy/proxy.go delete mode 100644 client/internal/proxy/wireguard.go create mode 100644 client/internal/wgproxy/bpf/portreplace.c create mode 100644 client/internal/wgproxy/bpf_bpfeb.go create mode 100644 client/internal/wgproxy/bpf_bpfeb.o create mode 100644 client/internal/wgproxy/bpf_bpfel.go create mode 100644 client/internal/wgproxy/bpf_bpfel.o create mode 100644 client/internal/wgproxy/factory.go create mode 100644 client/internal/wgproxy/factory_linux.go create mode 100644 client/internal/wgproxy/factory_nonlinux.go create mode 100644 client/internal/wgproxy/loader.go create mode 100644 client/internal/wgproxy/loader_test.go create mode 100644 client/internal/wgproxy/portlookup.go create mode 100644 client/internal/wgproxy/portlookup_test.go create mode 100644 client/internal/wgproxy/proxy.go create mode 100644 client/internal/wgproxy/proxy_ebpf.go create mode 100644 client/internal/wgproxy/proxy_ebpf_test.go create mode 100644 client/internal/wgproxy/proxy_userspace.go diff --git a/client/internal/engine.go b/client/internal/engine.go index ad9b79d7a..038f39e5c 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -20,8 +20,8 @@ import ( "github.com/netbirdio/netbird/client/internal/acl" "github.com/netbirdio/netbird/client/internal/dns" "github.com/netbirdio/netbird/client/internal/peer" - "github.com/netbirdio/netbird/client/internal/proxy" "github.com/netbirdio/netbird/client/internal/routemanager" + "github.com/netbirdio/netbird/client/internal/wgproxy" nbssh "github.com/netbirdio/netbird/client/ssh" nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/iface" @@ -101,7 +101,8 @@ type Engine struct { ctx context.Context - wgInterface *iface.WGIface + wgInterface *iface.WGIface + wgProxyFactory *wgproxy.Factory udpMux *bind.UniversalUDPMuxDefault udpMuxConn io.Closer @@ -132,6 +133,7 @@ func NewEngine( signalClient signal.Client, mgmClient mgm.Client, config *EngineConfig, mobileDep MobileDependency, statusRecorder *peer.Status, ) *Engine { + return &Engine{ ctx: ctx, cancel: cancel, @@ -146,6 +148,7 @@ func NewEngine( networkSerial: 0, sshServerFunc: nbssh.DefaultSSHServer, statusRecorder: statusRecorder, + wgProxyFactory: wgproxy.NewFactory(config.WgPort), } } @@ -282,7 +285,7 @@ func (e *Engine) modifyPeers(peersUpdate []*mgmProto.RemotePeerConfig) error { for _, p := range peersUpdate { peerPubKey := p.GetWgPubKey() if peerConn, ok := e.peerConns[peerPubKey]; ok { - if peerConn.GetConf().ProxyConfig.AllowedIps != strings.Join(p.AllowedIps, ",") { + if peerConn.WgConfig().AllowedIps != strings.Join(p.AllowedIps, ",") { modified = append(modified, p) continue } @@ -795,9 +798,7 @@ func (e *Engine) connWorker(conn *peer.Conn, peerKey string) { // we might have received new STUN and TURN servers meanwhile, so update them e.syncMsgMux.Lock() - conf := conn.GetConf() - conf.StunTurn = append(e.STUNs, e.TURNs...) - conn.UpdateConf(conf) + conn.UpdateStunTurn(append(e.STUNs, e.TURNs...)) e.syncMsgMux.Unlock() err := conn.Open() @@ -826,9 +827,9 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e stunTurn = append(stunTurn, e.STUNs...) stunTurn = append(stunTurn, e.TURNs...) - proxyConfig := proxy.Config{ + wgConfig := peer.WgConfig{ RemoteKey: pubKey, - WgListenAddr: fmt.Sprintf("127.0.0.1:%d", e.config.WgPort), + WgListenPort: e.config.WgPort, WgInterface: e.wgInterface, AllowedIps: allowedIPs, PreSharedKey: e.config.PreSharedKey, @@ -845,13 +846,13 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs string) (*peer.Conn, e Timeout: timeout, UDPMux: e.udpMux.UDPMuxDefault, UDPMuxSrflx: e.udpMux, - ProxyConfig: proxyConfig, + WgConfig: wgConfig, LocalWgPort: e.config.WgPort, NATExternalIPs: e.parseNATExternalIPMappings(), UserspaceBind: e.wgInterface.IsUserspaceBind(), } - peerConn, err := peer.NewConn(config, e.statusRecorder, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover) + peerConn, err := peer.NewConn(config, e.statusRecorder, e.wgProxyFactory, e.mobileDep.TunAdapter, e.mobileDep.IFaceDiscover) if err != nil { return nil, err } @@ -1008,6 +1009,10 @@ func (e *Engine) parseNATExternalIPMappings() []string { } func (e *Engine) close() { + if err := e.wgProxyFactory.Free(); err != nil { + log.Errorf("failed closing ebpf proxy: %s", err) + } + log.Debugf("removing Netbird interface %s", e.config.WgIfaceName) if e.wgInterface != nil { if err := e.wgInterface.Close(); err != nil { diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 4d67968f6..60c07c0c9 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -367,9 +367,9 @@ func TestEngine_UpdateNetworkMap(t *testing.T) { t.Errorf("expecting Engine.peerConns to contain peer %s", p) } expectedAllowedIPs := strings.Join(p.AllowedIps, ",") - if conn.GetConf().ProxyConfig.AllowedIps != expectedAllowedIPs { + if conn.WgConfig().AllowedIps != expectedAllowedIPs { t.Errorf("expecting peer %s to have AllowedIPs= %s, got %s", p.GetWgPubKey(), - expectedAllowedIPs, conn.GetConf().ProxyConfig.AllowedIps) + expectedAllowedIPs, conn.WgConfig().AllowedIps) } } }) diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index 9a3ef106b..9247ed3c5 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -10,9 +10,10 @@ import ( "github.com/pion/ice/v2" log "github.com/sirupsen/logrus" + "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - "github.com/netbirdio/netbird/client/internal/proxy" "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" "github.com/netbirdio/netbird/iface/bind" signal "github.com/netbirdio/netbird/signal/client" @@ -23,8 +24,18 @@ import ( const ( iceKeepAliveDefault = 4 * time.Second iceDisconnectedTimeoutDefault = 6 * time.Second + + defaultWgKeepAlive = 25 * time.Second ) +type WgConfig struct { + WgListenPort int + RemoteKey string + WgInterface *iface.WGIface + AllowedIps string + PreSharedKey *wgtypes.Key +} + // ConnConfig is a peer Connection configuration type ConnConfig struct { @@ -43,7 +54,7 @@ type ConnConfig struct { Timeout time.Duration - ProxyConfig proxy.Config + WgConfig WgConfig UDPMux ice.UDPMux UDPMuxSrflx ice.UniversalUDPMux @@ -98,7 +109,9 @@ type Conn struct { statusRecorder *Status - proxy proxy.Proxy + wgProxyFactory *wgproxy.Factory + wgProxy wgproxy.Proxy + remoteModeCh chan ModeMessage meta meta @@ -122,14 +135,19 @@ func (conn *Conn) GetConf() ConnConfig { return conn.config } -// UpdateConf updates the connection config -func (conn *Conn) UpdateConf(conf ConnConfig) { - conn.config = conf +// WgConfig returns the WireGuard config +func (conn *Conn) WgConfig() WgConfig { + return conn.config.WgConfig +} + +// UpdateStunTurn update the turn and stun addresses +func (conn *Conn) UpdateStunTurn(turnStun []*ice.URL) { + conn.config.StunTurn = turnStun } // NewConn creates a new not opened Conn to the remote peer. // To establish a connection run Conn.Open -func NewConn(config ConnConfig, statusRecorder *Status, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover) (*Conn, error) { +func NewConn(config ConnConfig, statusRecorder *Status, wgProxyFactory *wgproxy.Factory, adapter iface.TunAdapter, iFaceDiscover stdnet.ExternalIFaceDiscover) (*Conn, error) { return &Conn{ config: config, mu: sync.Mutex{}, @@ -139,6 +157,7 @@ func NewConn(config ConnConfig, statusRecorder *Status, adapter iface.TunAdapter remoteAnswerCh: make(chan OfferAnswer), statusRecorder: statusRecorder, remoteModeCh: make(chan ModeMessage, 1), + wgProxyFactory: wgProxyFactory, adapter: adapter, iFaceDiscover: iFaceDiscover, }, nil @@ -215,12 +234,12 @@ func (conn *Conn) candidateTypes() []ice.CandidateType { func (conn *Conn) Open() error { log.Debugf("trying to connect to peer %s", conn.config.Key) - peerState := State{PubKey: conn.config.Key} - - peerState.IP = strings.Split(conn.config.ProxyConfig.AllowedIps, "/")[0] - peerState.ConnStatusUpdate = time.Now() - peerState.ConnStatus = conn.status - + peerState := State{ + PubKey: conn.config.Key, + IP: strings.Split(conn.config.WgConfig.AllowedIps, "/")[0], + ConnStatusUpdate: time.Now(), + ConnStatus: conn.status, + } err := conn.statusRecorder.UpdatePeerState(peerState) if err != nil { log.Warnf("erro while updating the state of peer %s,err: %v", conn.config.Key, err) @@ -275,10 +294,11 @@ func (conn *Conn) Open() error { defer conn.notifyDisconnected() conn.mu.Unlock() - peerState = State{PubKey: conn.config.Key} - - peerState.ConnStatus = conn.status - peerState.ConnStatusUpdate = time.Now() + peerState = State{ + PubKey: conn.config.Key, + ConnStatus: conn.status, + ConnStatusUpdate: time.Now(), + } err = conn.statusRecorder.UpdatePeerState(peerState) if err != nil { log.Warnf("erro while updating the state of peer %s,err: %v", conn.config.Key, err) @@ -309,19 +329,12 @@ func (conn *Conn) Open() error { remoteWgPort = remoteOfferAnswer.WgListenPort } // the ice connection has been established successfully so we are ready to start the proxy - err = conn.startProxy(remoteConn, remoteWgPort) + remoteAddr, err := conn.configureConnection(remoteConn, remoteWgPort) if err != nil { return err } - if conn.proxy.Type() == proxy.TypeDirectNoProxy { - host, _, _ := net.SplitHostPort(remoteConn.LocalAddr().String()) - rhost, _, _ := net.SplitHostPort(remoteConn.RemoteAddr().String()) - // direct Wireguard connection - log.Infof("directly connected to peer %s [laddr <-> raddr] [%s:%d <-> %s:%d]", conn.config.Key, host, conn.config.LocalWgPort, rhost, remoteWgPort) - } else { - log.Infof("connected to peer %s [laddr <-> raddr] [%s <-> %s]", conn.config.Key, remoteConn.LocalAddr().String(), remoteConn.RemoteAddr().String()) - } + log.Infof("connected to peer %s, endpoint address: %s", conn.config.Key, remoteAddr.String()) // wait until connection disconnected or has been closed externally (upper layer, e.g. engine) select { @@ -338,54 +351,60 @@ func isRelayCandidate(candidate ice.Candidate) bool { return candidate.Type() == ice.CandidateTypeRelay } -// startProxy starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected -func (conn *Conn) startProxy(remoteConn net.Conn, remoteWgPort int) error { +// configureConnection starts proxying traffic from/to local Wireguard and sets connection status to StatusConnected +func (conn *Conn) configureConnection(remoteConn net.Conn, remoteWgPort int) (net.Addr, error) { conn.mu.Lock() defer conn.mu.Unlock() - var pair *ice.CandidatePair pair, err := conn.agent.GetSelectedCandidatePair() if err != nil { - return err + return nil, err } - peerState := State{PubKey: conn.config.Key} - p := conn.getProxy(pair, remoteWgPort) - conn.proxy = p - err = p.Start(remoteConn) + var endpoint net.Addr + if isRelayCandidate(pair.Local) { + log.Debugf("setup relay connection") + conn.wgProxy = conn.wgProxyFactory.GetProxy() + endpoint, err = conn.wgProxy.AddTurnConn(remoteConn) + if err != nil { + return nil, err + } + } else { + // To support old version's with direct mode we attempt to punch an additional role with the remote wireguard port + go conn.punchRemoteWGPort(pair, remoteWgPort) + endpoint = remoteConn.RemoteAddr() + } + + endpointUdpAddr, _ := net.ResolveUDPAddr(endpoint.Network(), endpoint.String()) + + err = conn.config.WgConfig.WgInterface.UpdatePeer(conn.config.WgConfig.RemoteKey, conn.config.WgConfig.AllowedIps, defaultWgKeepAlive, endpointUdpAddr, conn.config.WgConfig.PreSharedKey) if err != nil { - return err + if conn.wgProxy != nil { + _ = conn.wgProxy.CloseConn() + } + return nil, err } conn.status = StatusConnected - peerState.ConnStatus = conn.status - peerState.ConnStatusUpdate = time.Now() - peerState.LocalIceCandidateType = pair.Local.Type().String() - peerState.RemoteIceCandidateType = pair.Remote.Type().String() + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.status, + ConnStatusUpdate: time.Now(), + LocalIceCandidateType: pair.Local.Type().String(), + RemoteIceCandidateType: pair.Remote.Type().String(), + Direct: !isRelayCandidate(pair.Local), + } if pair.Local.Type() == ice.CandidateTypeRelay || pair.Remote.Type() == ice.CandidateTypeRelay { peerState.Relayed = true } - peerState.Direct = p.Type() == proxy.TypeDirectNoProxy || p.Type() == proxy.TypeNoProxy err = conn.statusRecorder.UpdatePeerState(peerState) if err != nil { log.Warnf("unable to save peer's state, got error: %v", err) } - return nil -} - -// todo rename this method and the proxy package to something more appropriate -func (conn *Conn) getProxy(pair *ice.CandidatePair, remoteWgPort int) proxy.Proxy { - if isRelayCandidate(pair.Local) { - return proxy.NewWireGuardProxy(conn.config.ProxyConfig) - } - - // To support old version's with direct mode we attempt to punch an additional role with the remote wireguard port - go conn.punchRemoteWGPort(pair, remoteWgPort) - - return proxy.NewNoProxy(conn.config.ProxyConfig) + return endpoint, nil } func (conn *Conn) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) { @@ -414,22 +433,22 @@ func (conn *Conn) cleanup() error { conn.mu.Lock() defer conn.mu.Unlock() + var err1, err2, err3 error if conn.agent != nil { - err := conn.agent.Close() - if err != nil { - return err + err1 = conn.agent.Close() + if err1 == nil { + conn.agent = nil } - conn.agent = nil } - if conn.proxy != nil { - err := conn.proxy.Close() - if err != nil { - return err - } - conn.proxy = nil + if conn.wgProxy != nil { + err2 = conn.wgProxy.CloseConn() + conn.wgProxy = nil } + // todo: is it problem if we try to remove a peer what is never existed? + err3 = conn.config.WgConfig.WgInterface.RemovePeer(conn.config.WgConfig.RemoteKey) + if conn.notifyDisconnected != nil { conn.notifyDisconnected() conn.notifyDisconnected = nil @@ -437,10 +456,11 @@ func (conn *Conn) cleanup() error { conn.status = StatusDisconnected - peerState := State{PubKey: conn.config.Key} - peerState.ConnStatus = conn.status - peerState.ConnStatusUpdate = time.Now() - + peerState := State{ + PubKey: conn.config.Key, + ConnStatus: conn.status, + ConnStatusUpdate: time.Now(), + } err := conn.statusRecorder.UpdatePeerState(peerState) if err != nil { // pretty common error because by that time Engine can already remove the peer and status won't be available. @@ -449,8 +469,13 @@ func (conn *Conn) cleanup() error { } log.Debugf("cleaned up connection to peer %s", conn.config.Key) - - return nil + if err1 != nil { + return err1 + } + if err2 != nil { + return err2 + } + return err3 } // SetSignalOffer sets a handler function to be triggered by Conn when a new connection offer has to be signalled to the remote peer diff --git a/client/internal/peer/conn_test.go b/client/internal/peer/conn_test.go index dedc381ac..ac2fa5c00 100644 --- a/client/internal/peer/conn_test.go +++ b/client/internal/peer/conn_test.go @@ -5,12 +5,11 @@ import ( "testing" "time" - "github.com/netbirdio/netbird/client/internal/stdnet" - "github.com/magiconair/properties/assert" "github.com/pion/ice/v2" - "github.com/netbirdio/netbird/client/internal/proxy" + "github.com/netbirdio/netbird/client/internal/stdnet" + "github.com/netbirdio/netbird/client/internal/wgproxy" "github.com/netbirdio/netbird/iface" ) @@ -20,7 +19,6 @@ var connConf = ConnConfig{ StunTurn: []*ice.URL{}, InterfaceBlackList: nil, Timeout: time.Second, - ProxyConfig: proxy.Config{}, LocalWgPort: 51820, } @@ -37,7 +35,11 @@ func TestNewConn_interfaceFilter(t *testing.T) { } func TestConn_GetKey(t *testing.T) { - conn, err := NewConn(connConf, nil, nil, nil) + wgProxyFactory := wgproxy.NewFactory(connConf.LocalWgPort) + defer func() { + _ = wgProxyFactory.Free() + }() + conn, err := NewConn(connConf, nil, wgProxyFactory, nil, nil) if err != nil { return } @@ -48,8 +50,11 @@ func TestConn_GetKey(t *testing.T) { } func TestConn_OnRemoteOffer(t *testing.T) { - - conn, err := NewConn(connConf, NewRecorder("https://mgm"), nil, nil) + wgProxyFactory := wgproxy.NewFactory(connConf.LocalWgPort) + defer func() { + _ = wgProxyFactory.Free() + }() + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) if err != nil { return } @@ -82,8 +87,11 @@ func TestConn_OnRemoteOffer(t *testing.T) { } func TestConn_OnRemoteAnswer(t *testing.T) { - - conn, err := NewConn(connConf, NewRecorder("https://mgm"), nil, nil) + wgProxyFactory := wgproxy.NewFactory(connConf.LocalWgPort) + defer func() { + _ = wgProxyFactory.Free() + }() + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) if err != nil { return } @@ -115,8 +123,11 @@ func TestConn_OnRemoteAnswer(t *testing.T) { wg.Wait() } func TestConn_Status(t *testing.T) { - - conn, err := NewConn(connConf, NewRecorder("https://mgm"), nil, nil) + wgProxyFactory := wgproxy.NewFactory(connConf.LocalWgPort) + defer func() { + _ = wgProxyFactory.Free() + }() + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) if err != nil { return } @@ -142,8 +153,11 @@ func TestConn_Status(t *testing.T) { } func TestConn_Close(t *testing.T) { - - conn, err := NewConn(connConf, NewRecorder("https://mgm"), nil, nil) + wgProxyFactory := wgproxy.NewFactory(connConf.LocalWgPort) + defer func() { + _ = wgProxyFactory.Free() + }() + conn, err := NewConn(connConf, NewRecorder("https://mgm"), wgProxyFactory, nil, nil) if err != nil { return } diff --git a/client/internal/proxy/dummy.go b/client/internal/proxy/dummy.go deleted file mode 100644 index ebd7cd68f..000000000 --- a/client/internal/proxy/dummy.go +++ /dev/null @@ -1,72 +0,0 @@ -package proxy - -import ( - "context" - log "github.com/sirupsen/logrus" - "net" - "time" -) - -// DummyProxy just sends pings to the RemoteKey peer and reads responses -type DummyProxy struct { - conn net.Conn - remote string - ctx context.Context - cancel context.CancelFunc -} - -func NewDummyProxy(remote string) *DummyProxy { - p := &DummyProxy{remote: remote} - p.ctx, p.cancel = context.WithCancel(context.Background()) - return p -} - -func (p *DummyProxy) Close() error { - p.cancel() - return nil -} - -func (p *DummyProxy) Start(remoteConn net.Conn) error { - p.conn = remoteConn - go func() { - buf := make([]byte, 1500) - for { - select { - case <-p.ctx.Done(): - return - default: - _, err := p.conn.Read(buf) - if err != nil { - log.Errorf("error while reading RemoteKey %s proxy %v", p.remote, err) - return - } - //log.Debugf("received %s from %s", string(buf[:n]), p.remote) - } - - } - }() - - go func() { - for { - select { - case <-p.ctx.Done(): - return - default: - _, err := p.conn.Write([]byte("hello")) - //log.Debugf("sent ping to %s", p.remote) - if err != nil { - log.Errorf("error while writing to RemoteKey %s proxy %v", p.remote, err) - return - } - time.Sleep(5 * time.Second) - } - } - - }() - - return nil -} - -func (p *DummyProxy) Type() Type { - return TypeDummy -} diff --git a/client/internal/proxy/noproxy.go b/client/internal/proxy/noproxy.go deleted file mode 100644 index dcfe182fd..000000000 --- a/client/internal/proxy/noproxy.go +++ /dev/null @@ -1,42 +0,0 @@ -package proxy - -import ( - log "github.com/sirupsen/logrus" - "net" -) - -// NoProxy is used just to configure WireGuard without any local proxy in between. -// Used when the WireGuard interface is userspace and uses bind.ICEBind -type NoProxy struct { - config Config -} - -// NewNoProxy creates a new NoProxy with a provided config -func NewNoProxy(config Config) *NoProxy { - return &NoProxy{config: config} -} - -// Close removes peer from the WireGuard interface -func (p *NoProxy) Close() error { - err := p.config.WgInterface.RemovePeer(p.config.RemoteKey) - if err != nil { - return err - } - return nil -} - -// Start just updates WireGuard peer with the remote address -func (p *NoProxy) Start(remoteConn net.Conn) error { - - log.Debugf("using NoProxy to connect to peer %s at %s", p.config.RemoteKey, remoteConn.RemoteAddr().String()) - addr, err := net.ResolveUDPAddr("udp", remoteConn.RemoteAddr().String()) - if err != nil { - return err - } - return p.config.WgInterface.UpdatePeer(p.config.RemoteKey, p.config.AllowedIps, DefaultWgKeepAlive, - addr, p.config.PreSharedKey) -} - -func (p *NoProxy) Type() Type { - return TypeNoProxy -} diff --git a/client/internal/proxy/proxy.go b/client/internal/proxy/proxy.go deleted file mode 100644 index a0b9e98a1..000000000 --- a/client/internal/proxy/proxy.go +++ /dev/null @@ -1,35 +0,0 @@ -package proxy - -import ( - "github.com/netbirdio/netbird/iface" - "golang.zx2c4.com/wireguard/wgctrl/wgtypes" - "io" - "net" - "time" -) - -const DefaultWgKeepAlive = 25 * time.Second - -type Type string - -const ( - TypeDirectNoProxy Type = "DirectNoProxy" - TypeWireGuard Type = "WireGuard" - TypeDummy Type = "Dummy" - TypeNoProxy Type = "NoProxy" -) - -type Config struct { - WgListenAddr string - RemoteKey string - WgInterface *iface.WGIface - AllowedIps string - PreSharedKey *wgtypes.Key -} - -type Proxy interface { - io.Closer - // Start creates a local remoteConn and starts proxying data from/to remoteConn - Start(remoteConn net.Conn) error - Type() Type -} diff --git a/client/internal/proxy/wireguard.go b/client/internal/proxy/wireguard.go deleted file mode 100644 index ec3c6a730..000000000 --- a/client/internal/proxy/wireguard.go +++ /dev/null @@ -1,128 +0,0 @@ -package proxy - -import ( - "context" - log "github.com/sirupsen/logrus" - "net" -) - -// WireGuardProxy proxies -type WireGuardProxy struct { - ctx context.Context - cancel context.CancelFunc - - config Config - - remoteConn net.Conn - localConn net.Conn -} - -func NewWireGuardProxy(config Config) *WireGuardProxy { - p := &WireGuardProxy{config: config} - p.ctx, p.cancel = context.WithCancel(context.Background()) - return p -} - -func (p *WireGuardProxy) updateEndpoint() error { - udpAddr, err := net.ResolveUDPAddr(p.localConn.LocalAddr().Network(), p.localConn.LocalAddr().String()) - if err != nil { - return err - } - // add local proxy connection as a Wireguard peer - err = p.config.WgInterface.UpdatePeer(p.config.RemoteKey, p.config.AllowedIps, DefaultWgKeepAlive, - udpAddr, p.config.PreSharedKey) - if err != nil { - return err - } - - return nil -} - -func (p *WireGuardProxy) Start(remoteConn net.Conn) error { - p.remoteConn = remoteConn - - var err error - p.localConn, err = net.Dial("udp", p.config.WgListenAddr) - if err != nil { - log.Errorf("failed dialing to local Wireguard port %s", err) - return err - } - - err = p.updateEndpoint() - if err != nil { - log.Errorf("error while updating Wireguard peer endpoint [%s] %v", p.config.RemoteKey, err) - return err - } - - go p.proxyToRemote() - go p.proxyToLocal() - - return nil -} - -func (p *WireGuardProxy) Close() error { - p.cancel() - if c := p.localConn; c != nil { - err := p.localConn.Close() - if err != nil { - return err - } - } - err := p.config.WgInterface.RemovePeer(p.config.RemoteKey) - if err != nil { - return err - } - return nil -} - -// proxyToRemote proxies everything from Wireguard to the RemoteKey peer -// blocks -func (p *WireGuardProxy) proxyToRemote() { - - buf := make([]byte, 1500) - for { - select { - case <-p.ctx.Done(): - log.Debugf("stopped proxying to remote peer %s due to closed connection", p.config.RemoteKey) - return - default: - n, err := p.localConn.Read(buf) - if err != nil { - continue - } - - _, err = p.remoteConn.Write(buf[:n]) - if err != nil { - continue - } - } - } -} - -// proxyToLocal proxies everything from the RemoteKey peer to local Wireguard -// blocks -func (p *WireGuardProxy) proxyToLocal() { - - buf := make([]byte, 1500) - for { - select { - case <-p.ctx.Done(): - log.Debugf("stopped proxying from remote peer %s due to closed connection", p.config.RemoteKey) - return - default: - n, err := p.remoteConn.Read(buf) - if err != nil { - continue - } - - _, err = p.localConn.Write(buf[:n]) - if err != nil { - continue - } - } - } -} - -func (p *WireGuardProxy) Type() Type { - return TypeWireGuard -} diff --git a/client/internal/wgproxy/bpf/portreplace.c b/client/internal/wgproxy/bpf/portreplace.c new file mode 100644 index 000000000..25994116b --- /dev/null +++ b/client/internal/wgproxy/bpf/portreplace.c @@ -0,0 +1,90 @@ +#include +#include // ETH_P_IP +#include +#include +#include +#include +#include + +#define bpf_printk(fmt, ...) \ + ({ \ + char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ + }) + +const __u32 map_key_proxy_port = 0; +const __u32 map_key_wg_port = 1; + +struct bpf_map_def SEC("maps") xdp_port_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u16), + .max_entries = 10, +}; + +__u16 proxy_port = 0; +__u16 wg_port = 0; + +bool read_port_settings() { + __u16 *value; + value = bpf_map_lookup_elem(&xdp_port_map, &map_key_proxy_port); + if(!value) { + return false; + } + + proxy_port = *value; + + value = bpf_map_lookup_elem(&xdp_port_map, &map_key_wg_port); + if(!value) { + return false; + } + wg_port = *value; + + return true; +} + +SEC("xdp") +int xdp_prog_func(struct xdp_md *ctx) { + if(proxy_port == 0 || wg_port == 0) { + if(!read_port_settings()){ + return XDP_PASS; + } + bpf_printk("proxy port: %d, wg port: %d", proxy_port, wg_port); + } + + void *data = (void *)(long)ctx->data; + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr *eth = data; + struct iphdr *ip = (data + sizeof(struct ethhdr)); + struct udphdr *udp = (data + sizeof(struct ethhdr) + sizeof(struct iphdr)); + + // return early if not enough data + if (data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) > data_end){ + return XDP_PASS; + } + + // skip non IPv4 packages + if (eth->h_proto != htons(ETH_P_IP)) { + return XDP_PASS; + } + + if (ip->protocol != IPPROTO_UDP) { + return XDP_PASS; + } + + // 2130706433 = 127.0.0.1 + if (ip->daddr != htonl(2130706433)) { + return XDP_PASS; + } + + if (udp->source != htons(wg_port)){ + return XDP_PASS; + } + + __be16 new_src_port = udp->dest; + __be16 new_dst_port = htons(proxy_port); + udp->dest = new_dst_port; + udp->source = new_src_port; + return XDP_PASS; +} +char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/client/internal/wgproxy/bpf_bpfeb.go b/client/internal/wgproxy/bpf_bpfeb.go new file mode 100644 index 000000000..37a5cb22c --- /dev/null +++ b/client/internal/wgproxy/bpf_bpfeb.go @@ -0,0 +1,120 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 +// +build arm64be armbe mips mips64 mips64p32 ppc64 s390 s390x sparc sparc64 + +package wgproxy + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs +} + +// bpfSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + XdpPortMap *ebpf.MapSpec `ebpf:"xdp_port_map"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + XdpPortMap *ebpf.Map `ebpf:"xdp_port_map"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.XdpPortMap, + ) +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.XdpProgFunc, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed bpf_bpfeb.o +var _BpfBytes []byte diff --git a/client/internal/wgproxy/bpf_bpfeb.o b/client/internal/wgproxy/bpf_bpfeb.o new file mode 100644 index 0000000000000000000000000000000000000000..c3aa757d6c3bf49effa51939e90b9896b7b96596 GIT binary patch literal 6224 zcmb_gU1*!v89tw^HE!1|PMoY_E9E$CgPhuqWp`<|%tj4!=L|zc=72leeO9E;)`w(C zmgGcUGU|&B%+`yrA25T`bn8VJV+6)XD1&&t^2T7k=tV9D3xh#mH@$114DWf)k92%$ zk1+Pa_nr57f6sZ(&v&G^re5tysUXY-+zgBiXL%xZ@=XXn8k zsqnLA+)w0B=sM=VsO|1$jQyx;PY#a>rCA;j2Ps+3h^N^uYJKm8F(M%@jS9|8FYKOC zSgQ4OI6oksGx#7D?_2SJI8L0>{$v%!_mNco zedQxsen&x(uQ_>H0S=(gZqa>Lk}IwgN&;@~JyaeseE=cFs=gTy}cD^Bsr4;&k{ zFQ8*Qvng+9Ib|G z-7Cf5SrsH+NDrt$gYudZbS)72q*7_i|0UhkMK_m0PTcPc_H(HyiuqPvy~3!fQMeRA zgfcXuW;2Wz;%YH!i$*(KUWrzsP{U}qHVjwB$HYpt8823&qKvC8QE0Uhq^0s;Xp7;( zO0_^lsZ>?6(sZ-cSSe`XQc<3St1Swu_xTacHQ~$q8U<$%)~F)EkIjP9Ilp zowH@vL34aac7sx5YxU%nT8#Xx3Cn?crL|-*U8z+U2hp+NNvl089CK#h-`vi)Ow(Du zB%AT~qS}HTchoACijBd~$m9B+;+3M7PfFXBLA`?F1(`;y;mH!0TdO7qCS<8qt2Wb9 zv(sTNJf9o1)uL%!AD&cpg<6HC=X1G>7iKSnm%fqn&5HSAv0=ki(qozN(N{)KzdSy^ zDHJlU+RGzyWmZKiE?O&`PXa+EnOtI^f4*ld|;G`;LjirFKMakEjd#wtxQYPNJ) z-K-NTu(fscG{^k7Y0rw!?#+(K^jg?0UW^t*YqcH;?q$xNP`gsIOZhgIOREt_O%-6F zlE)e19s~j#By+mEL7F8TD*9WfMuqE#ZVRP+1IDjjy!h(t;kVC!XDXb1JvSAGBCN!P zsM?Ig$mJ%^r$&u)FRTzFX$t%8l;(VU0WF@y`*^mDjAt}&DfW!SaUH<3gZVk)Sx*~& zBS{huzAHtz;GqenvuOr1LV5`+@-hHvtN6VTTU&I<82V$LrKG&-MJ=DWK(QEW|5zKxWSQ`8eN&!BRHuyskyn!(%&KUd;lm_b}k=0my zDuQAFeqX|ZHrKyhCLaJL{%K%WKMhP9YtQ&@{xx9N ze*@UHzXj~(=UR5+tF`N3wN{;Y3D$ww)FZ+1)867Vc=iWNiIl)usq$Go1I+8LwJ(Ba z{|pvG;5jynzYdC+xQX8mUySEQP#$;+IAD4z%mngBZcjhgSYF+SMi1n#+;uD7q(R&YjDIUtu#Ej%zfg zn@q-Dr5GeLlEF~_FZ2^c;TqV3`iG71@1D}0Zxr))zQz9cAWH+#rh0c@yrcTPp8fK-jmJ&2%w^9x>rdG)@pKY_ zcc>?4fVH_2>URHoos)F;1>MMYfLQxAEBgV-$3$$NKZX zCYP)|+v|8N_VceBCqKWnr~0>!zcIH>&)b?iz%jby#``rC`^;aTzkc{_CE0p${&<&n z`$N53o2;EQnHYD7WqI4~mPt2B^HFOxCQOT2Qwema31ns0Hi1)G_@})}c>l^)_HsydG<*#K2y^a1SO?kOeYEZuJl}j7_dr}@&DM4L-rP`zLySKl; ze-G>61Vqf6Q=eW^15Brt-Gg4$9z7z&S+GN=)kLhtj!DH@Z&bD`(072ldRjs;f22p^ zwU{c@eA{uXL->!qA`Rx-qT(G)br?G(aUA_^3da?C40$)$x5v~3xb+QHcMvM+Btg2j%hh{ZQK$6#)G=I(_cH3di#k^&3ll?G@k^2Li3B@2Q|M5 zenRtiz<(P&sDm+6ikG{?qZ9;Dv=i;yU^wjbL7PYbjK5=?Pfah$aj6`;z5pS+^g8%7 zFmh_=5p)i`*)r!f??FDP`6T!m%`bwV)BGm*?|}z(b>W9`@(Oow{j4cI!WwgaH?=K! z+Wy4BKhyTqAs;p*x!O2Ru4bBSX*)I^o{tpn>I2VCzXwK4c_`|mPp%ogJtY6JWutb7 z@~w7=;y-~6{7FCh2$AvsY{}KhD}+4C{mZ3NK~#ct*5@Mht3j<67FVibxE3^OV))!r zdHEbQm7rWmXM*93@Icqog?ivkdeHi->9SuamGbqn9~6Ss#F<96teinJ=$)ZHAFTUw z-Al#5gme-vCr(O71Kx%dv@GEItW+t>|0Udw6+4zjmfs%;j&n6Dvguk{{`f&rA%8W1 z2xX`Q)v6yZhsA8r5S50%RuAfduV8Rc>-+WbF;Op8!#!pM!cIhacIcwyh&hV$%)~Flq>L`Pn?q7 znrF*_gXYwr*K$gYsa5CoQi%Ah3Cn?cC6r_!Q79Ex1~9OpDWjbbmN}z8-`vc&mtbe_ zb*~zJKPWAmyn|XUm#qvuBM-}mitAY=pYqI52Gj})m%Rjh4Nc{^+-fE76KLK+SR$M{`@ynu30vn%~p)NLSk%meB`Sm7tW85 zA8?3)4NX-`^-3l%w$+5Gt{$WBF9o9)ykc;}&sJ-C8uch!lcM*CWn8UfjIj)p4XQO& zRy*pXbZl%bJ`rogKhPfk zK#1EGjskCM{4cZ-0FKoIPxA2G02Y5xc@e)4+_9r{2FHPCpd&VUlr=VaAT_Bsd2Cvk zd0?ngZ}QmD*yORRvB~41g_*~v8vhC7>x7c`1`jer3mO}JPU9ZT+nGJwnKVukKNg~) zaUOW`De7q9PT<=b{{{Lk+={j>+zEVN{&Pt%onzYmeHVc&F6HKys_rxp$=ubApGe)O*}jga2e(?2foXZYr@a}DX@3ZLtZ9vPs1+O1 z-rUezaUqt(bUW~LD=3J0J|r#7-U}9{ea^zX*WR`;?{Kss(I5LK2tdTt&k%r2eA_?c zZ&C>|{kIfs+TXP><74YdGXD3-fDGn3HJIaR*O+xB?zS+;$8}4>GL`i>Z(+upwd{G{ z&RUqc58@%wKgaM50uV8MrU*brzbH`)mmR$6U{i0?|Bj>IcJPjaA3FGng*m>b4(`zP zG~PaAC3 zi@`kSNCwY1c)`Ir2R9si%fVX?zU$!o4&HU}BL~0S+ZT)D?hzF9_Yt|vb8iguW9bDh zQ}XH}uPTRqa={tD4h~$Kh6{!3t3EHE__Ao-few5k$FGq5q7@nBD ze0kLW3Y!ssT%j@5jE=gae;^(IiN zq_)`6t!a3c*hGK!`D}D?{qeh~1>I;l#ilV|=~*z$r@Uq8fW6rw^Nw0POG1!GadZZ-1+U+kXN&#&G)=w7|7@ z$N3O_8Gj6qO{V_|TL_-EB-7s1yc_@4bK;wR(g}WkYxy!Vw(six6Xu^sTIuHVA>@b6 zUtCYnqV7k?_neszGk=_BcTDWE%a;8oN|BoAg6|y6<`z17+ lc$$>Z`;Ys}CU>mAgwh$JG~d?$Y|7~lY*