diff --git a/client/internal/dns/service_listener.go b/client/internal/dns/service_listener.go index 687ca2459..232f6ebc2 100644 --- a/client/internal/dns/service_listener.go +++ b/client/internal/dns/service_listener.go @@ -11,6 +11,9 @@ import ( "github.com/miekg/dns" log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/ebpf" + ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" ) const ( @@ -24,10 +27,11 @@ type serviceViaListener struct { dnsMux *dns.ServeMux customAddr *netip.AddrPort server *dns.Server - runtimeIP string - runtimePort int + listenIP string + listenPort int listenerIsRunning bool listenerFlagLock sync.Mutex + ebpfService ebpfMgr.Manager } func newServiceViaListener(wgIface WGIface, customAddr *netip.AddrPort) *serviceViaListener { @@ -43,6 +47,7 @@ func newServiceViaListener(wgIface WGIface, customAddr *netip.AddrPort) *service UDPSize: 65535, }, } + return s } @@ -55,13 +60,21 @@ func (s *serviceViaListener) Listen() error { } var err error - s.runtimeIP, s.runtimePort, err = s.evalRuntimeAddress() + s.listenIP, s.listenPort, err = s.evalListenAddress() if err != nil { log.Errorf("failed to eval runtime address: %s", err) return err } - s.server.Addr = fmt.Sprintf("%s:%d", s.runtimeIP, s.runtimePort) + s.server.Addr = fmt.Sprintf("%s:%d", s.listenIP, s.listenPort) + if s.shouldApplyPortFwd() { + s.ebpfService = ebpf.GetEbpfManagerInstance() + err = s.ebpfService.LoadDNSFwd(s.listenIP, s.listenPort) + if err != nil { + log.Warnf("failed to load DNS port forwarder, custom port may not work well on some Linux operating systems: %s", err) + s.ebpfService = nil + } + } log.Debugf("starting dns on %s", s.server.Addr) go func() { s.setListenerStatus(true) @@ -69,9 +82,10 @@ func (s *serviceViaListener) Listen() error { err := s.server.ListenAndServe() if err != nil { - log.Errorf("dns server running with %d port returned an error: %v. Will not retry", s.runtimePort, err) + log.Errorf("dns server running with %d port returned an error: %v. Will not retry", s.listenPort, err) } }() + return nil } @@ -90,6 +104,13 @@ func (s *serviceViaListener) Stop() { if err != nil { log.Errorf("stopping dns server listener returned an error: %v", err) } + + if s.ebpfService != nil { + err = s.ebpfService.FreeDNSFwd() + if err != nil { + log.Errorf("stopping traffic forwarder returned an error: %v", err) + } + } } func (s *serviceViaListener) RegisterMux(pattern string, handler dns.Handler) { @@ -101,11 +122,18 @@ func (s *serviceViaListener) DeregisterMux(pattern string) { } func (s *serviceViaListener) RuntimePort() int { - return s.runtimePort + s.listenerFlagLock.Lock() + defer s.listenerFlagLock.Unlock() + + if s.ebpfService != nil { + return defaultPort + } else { + return s.listenPort + } } func (s *serviceViaListener) RuntimeIP() string { - return s.runtimeIP + return s.listenIP } func (s *serviceViaListener) setListenerStatus(running bool) { @@ -136,10 +164,30 @@ func (s *serviceViaListener) getFirstListenerAvailable() (string, int, error) { return "", 0, fmt.Errorf("unable to find an unused ip and port combination. IPs tested: %v and ports %v", ips, ports) } -func (s *serviceViaListener) evalRuntimeAddress() (string, int, error) { +func (s *serviceViaListener) evalListenAddress() (string, int, error) { if s.customAddr != nil { return s.customAddr.Addr().String(), int(s.customAddr.Port()), nil } return s.getFirstListenerAvailable() } + +// shouldApplyPortFwd decides whether to apply eBPF program to capture DNS traffic on port 53. +// This is needed because on some operating systems if we start a DNS server not on a default port 53, the domain name +// resolution won't work. +// So, in case we are running on Linux and picked a non-default port (53) we should fall back to the eBPF solution that will capture +// traffic on port 53 and forward it to a local DNS server running on 5053. +func (s *serviceViaListener) shouldApplyPortFwd() bool { + if runtime.GOOS != "linux" { + return false + } + + if s.customAddr != nil { + return false + } + + if s.listenPort == defaultPort { + return false + } + return true +} diff --git a/client/internal/wgproxy/ebpf/bpf_bpfeb.go b/client/internal/ebpf/ebpf/bpf_bpfeb.go similarity index 84% rename from client/internal/wgproxy/ebpf/bpf_bpfeb.go rename to client/internal/ebpf/ebpf/bpf_bpfeb.go index c4875c3ae..5d2765862 100644 --- a/client/internal/wgproxy/ebpf/bpf_bpfeb.go +++ b/client/internal/ebpf/ebpf/bpf_bpfeb.go @@ -54,13 +54,16 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - NbWgProxy *ebpf.ProgramSpec `ebpf:"nb_wg_proxy"` + NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"` } // bpfMapSpecs contains maps before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type bpfMapSpecs struct { + NbFeatures *ebpf.MapSpec `ebpf:"nb_features"` + NbMapDnsIp *ebpf.MapSpec `ebpf:"nb_map_dns_ip"` + NbMapDnsPort *ebpf.MapSpec `ebpf:"nb_map_dns_port"` NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"` } @@ -83,11 +86,17 @@ func (o *bpfObjects) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfMaps struct { + NbFeatures *ebpf.Map `ebpf:"nb_features"` + NbMapDnsIp *ebpf.Map `ebpf:"nb_map_dns_ip"` + NbMapDnsPort *ebpf.Map `ebpf:"nb_map_dns_port"` NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"` } func (m *bpfMaps) Close() error { return _BpfClose( + m.NbFeatures, + m.NbMapDnsIp, + m.NbMapDnsPort, m.NbWgProxySettingsMap, ) } @@ -96,12 +105,12 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - NbWgProxy *ebpf.Program `ebpf:"nb_wg_proxy"` + NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"` } func (p *bpfPrograms) Close() error { return _BpfClose( - p.NbWgProxy, + p.NbXdpProg, ) } diff --git a/client/internal/ebpf/ebpf/bpf_bpfeb.o b/client/internal/ebpf/ebpf/bpf_bpfeb.o new file mode 100644 index 000000000..0559da486 Binary files /dev/null and b/client/internal/ebpf/ebpf/bpf_bpfeb.o differ diff --git a/client/internal/wgproxy/ebpf/bpf_bpfel.go b/client/internal/ebpf/ebpf/bpf_bpfel.go similarity index 84% rename from client/internal/wgproxy/ebpf/bpf_bpfel.go rename to client/internal/ebpf/ebpf/bpf_bpfel.go index 5a2ce3bb8..b94b79c54 100644 --- a/client/internal/wgproxy/ebpf/bpf_bpfel.go +++ b/client/internal/ebpf/ebpf/bpf_bpfel.go @@ -54,13 +54,16 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - NbWgProxy *ebpf.ProgramSpec `ebpf:"nb_wg_proxy"` + NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"` } // bpfMapSpecs contains maps before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type bpfMapSpecs struct { + NbFeatures *ebpf.MapSpec `ebpf:"nb_features"` + NbMapDnsIp *ebpf.MapSpec `ebpf:"nb_map_dns_ip"` + NbMapDnsPort *ebpf.MapSpec `ebpf:"nb_map_dns_port"` NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"` } @@ -83,11 +86,17 @@ func (o *bpfObjects) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfMaps struct { + NbFeatures *ebpf.Map `ebpf:"nb_features"` + NbMapDnsIp *ebpf.Map `ebpf:"nb_map_dns_ip"` + NbMapDnsPort *ebpf.Map `ebpf:"nb_map_dns_port"` NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"` } func (m *bpfMaps) Close() error { return _BpfClose( + m.NbFeatures, + m.NbMapDnsIp, + m.NbMapDnsPort, m.NbWgProxySettingsMap, ) } @@ -96,12 +105,12 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - NbWgProxy *ebpf.Program `ebpf:"nb_wg_proxy"` + NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"` } func (p *bpfPrograms) Close() error { return _BpfClose( - p.NbWgProxy, + p.NbXdpProg, ) } diff --git a/client/internal/ebpf/ebpf/bpf_bpfel.o b/client/internal/ebpf/ebpf/bpf_bpfel.o new file mode 100644 index 000000000..05fa42ecd Binary files /dev/null and b/client/internal/ebpf/ebpf/bpf_bpfel.o differ diff --git a/client/internal/ebpf/ebpf/dns_fwd_linux.go b/client/internal/ebpf/ebpf/dns_fwd_linux.go new file mode 100644 index 000000000..1b6493692 --- /dev/null +++ b/client/internal/ebpf/ebpf/dns_fwd_linux.go @@ -0,0 +1,51 @@ +package ebpf + +import ( + "encoding/binary" + "net" + + log "github.com/sirupsen/logrus" +) + +const ( + mapKeyDNSIP uint32 = 0 + mapKeyDNSPort uint32 = 1 +) + +func (tf *GeneralManager) LoadDNSFwd(ip string, dnsPort int) error { + log.Debugf("load ebpf DNS forwarder: address: %s:%d", ip, dnsPort) + tf.lock.Lock() + defer tf.lock.Unlock() + + err := tf.loadXdp() + if err != nil { + return err + } + + err = tf.bpfObjs.NbMapDnsIp.Put(mapKeyDNSIP, ip2int(ip)) + if err != nil { + return err + } + + err = tf.bpfObjs.NbMapDnsPort.Put(mapKeyDNSPort, uint16(dnsPort)) + if err != nil { + return err + } + + tf.setFeatureFlag(featureFlagDnsForwarder) + err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags) + if err != nil { + return err + } + return nil +} + +func (tf *GeneralManager) FreeDNSFwd() error { + log.Debugf("free ebpf DNS forwarder") + return tf.unsetFeatureFlag(featureFlagDnsForwarder) +} + +func ip2int(ipString string) uint32 { + ip := net.ParseIP(ipString) + return binary.BigEndian.Uint32(ip.To4()) +} diff --git a/client/internal/ebpf/ebpf/manager_linux.go b/client/internal/ebpf/ebpf/manager_linux.go new file mode 100644 index 000000000..9dfdc0ad1 --- /dev/null +++ b/client/internal/ebpf/ebpf/manager_linux.go @@ -0,0 +1,116 @@ +package ebpf + +import ( + _ "embed" + "net" + "sync" + + "github.com/cilium/ebpf/link" + "github.com/cilium/ebpf/rlimit" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/ebpf/manager" +) + +const ( + mapKeyFeatures uint32 = 0 + + featureFlagWGProxy = 0b00000001 + featureFlagDnsForwarder = 0b00000010 +) + +var ( + singleton manager.Manager + singletonLock = &sync.Mutex{} +) + +// required packages libbpf-dev, libc6-dev-i386-amd64-cross + +// GeneralManager is used to load multiple eBPF programs with a custom check (if then) done in prog.c +// The manager simply adds a feature (byte) of each program to a map that is shared between the userspace and kernel. +// When packet arrives, the C code checks for each feature (if it is set) and executes each enabled program (e.g., dns_fwd.c and wg_proxy.c). +// +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang-14 bpf src/prog.c -- -I /usr/x86_64-linux-gnu/include +type GeneralManager struct { + lock sync.Mutex + link link.Link + featureFlags uint16 + bpfObjs bpfObjects +} + +// GetEbpfManagerInstance return a static eBpf Manager instance +func GetEbpfManagerInstance() manager.Manager { + singletonLock.Lock() + defer singletonLock.Unlock() + if singleton != nil { + return singleton + } + singleton = &GeneralManager{} + return singleton +} + +func (tf *GeneralManager) setFeatureFlag(feature uint16) { + tf.featureFlags = tf.featureFlags | feature +} + +func (tf *GeneralManager) loadXdp() error { + if tf.link != nil { + return nil + } + // it required for Docker + err := rlimit.RemoveMemlock() + if err != nil { + return err + } + + iFace, err := net.InterfaceByName("lo") + if err != nil { + return err + } + + // load pre-compiled programs into the kernel. + err = loadBpfObjects(&tf.bpfObjs, nil) + if err != nil { + return err + } + + tf.link, err = link.AttachXDP(link.XDPOptions{ + Program: tf.bpfObjs.NbXdpProg, + Interface: iFace.Index, + }) + + if err != nil { + _ = tf.bpfObjs.Close() + tf.link = nil + return err + } + return nil +} + +func (tf *GeneralManager) unsetFeatureFlag(feature uint16) error { + tf.lock.Lock() + defer tf.lock.Unlock() + tf.featureFlags &^= feature + + if tf.link == nil { + return nil + } + + if tf.featureFlags == 0 { + return tf.close() + } + + return tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags) +} + +func (tf *GeneralManager) close() error { + log.Debugf("detach ebpf program ") + err := tf.bpfObjs.Close() + if err != nil { + log.Warnf("failed to close eBpf objects: %s", err) + } + + err = tf.link.Close() + tf.link = nil + return err +} diff --git a/client/internal/ebpf/ebpf/manager_linux_test.go b/client/internal/ebpf/ebpf/manager_linux_test.go new file mode 100644 index 000000000..956499e5b --- /dev/null +++ b/client/internal/ebpf/ebpf/manager_linux_test.go @@ -0,0 +1,40 @@ +package ebpf + +import ( + "testing" +) + +func TestManager_setFeatureFlag(t *testing.T) { + mgr := GeneralManager{} + mgr.setFeatureFlag(featureFlagWGProxy) + if mgr.featureFlags != 1 { + t.Errorf("invalid faeture state") + } + + mgr.setFeatureFlag(featureFlagDnsForwarder) + if mgr.featureFlags != 3 { + t.Errorf("invalid faeture state") + } +} + +func TestManager_unsetFeatureFlag(t *testing.T) { + mgr := GeneralManager{} + mgr.setFeatureFlag(featureFlagWGProxy) + mgr.setFeatureFlag(featureFlagDnsForwarder) + + err := mgr.unsetFeatureFlag(featureFlagWGProxy) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if mgr.featureFlags != 2 { + t.Errorf("invalid faeture state, expected: %d, got: %d", 2, mgr.featureFlags) + } + + err = mgr.unsetFeatureFlag(featureFlagDnsForwarder) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if mgr.featureFlags != 0 { + t.Errorf("invalid faeture state, expected: %d, got: %d", 0, mgr.featureFlags) + } +} diff --git a/client/internal/ebpf/ebpf/src/dns_fwd.c b/client/internal/ebpf/ebpf/src/dns_fwd.c new file mode 100644 index 000000000..5228c7e75 --- /dev/null +++ b/client/internal/ebpf/ebpf/src/dns_fwd.c @@ -0,0 +1,64 @@ +const __u32 map_key_dns_ip = 0; +const __u32 map_key_dns_port = 1; + +struct bpf_map_def SEC("maps") nb_map_dns_ip = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u32), + .max_entries = 10, +}; + +struct bpf_map_def SEC("maps") nb_map_dns_port = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u16), + .max_entries = 10, +}; + +__be32 dns_ip = 0; +__be16 dns_port = 0; + +// 13568 is 53 in big endian +__be16 GENERAL_DNS_PORT = 13568; + +bool read_settings() { + __u16 *port_value; + __u32 *ip_value; + + // read dns ip + ip_value = bpf_map_lookup_elem(&nb_map_dns_ip, &map_key_dns_ip); + if(!ip_value) { + return false; + } + dns_ip = htonl(*ip_value); + + // read dns port + port_value = bpf_map_lookup_elem(&nb_map_dns_port, &map_key_dns_port); + if (!port_value) { + return false; + } + dns_port = htons(*port_value); + return true; +} + +int xdp_dns_fwd(struct iphdr *ip, struct udphdr *udp) { + if (dns_port == 0) { + if(!read_settings()){ + return XDP_PASS; + } + bpf_printk("dns port: %d", ntohs(dns_port)); + bpf_printk("dns ip: %d", ntohl(dns_ip)); + } + + if (udp->dest == GENERAL_DNS_PORT && ip->daddr == dns_ip) { + udp->dest = dns_port; + return XDP_PASS; + } + + if (udp->source == dns_port && ip->saddr == dns_ip) { + udp->source = GENERAL_DNS_PORT; + return XDP_PASS; + } + + return XDP_PASS; +} \ No newline at end of file diff --git a/client/internal/ebpf/ebpf/src/prog.c b/client/internal/ebpf/ebpf/src/prog.c new file mode 100644 index 000000000..09b649370 --- /dev/null +++ b/client/internal/ebpf/ebpf/src/prog.c @@ -0,0 +1,66 @@ +#include +#include // ETH_P_IP +#include +#include +#include +#include +#include +#include "dns_fwd.c" +#include "wg_proxy.c" + +#define bpf_printk(fmt, ...) \ + ({ \ + char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ + }) + +const __u16 flag_feature_wg_proxy = 0b01; +const __u16 flag_feature_dns_fwd = 0b10; + +const __u32 map_key_features = 0; +struct bpf_map_def SEC("maps") nb_features = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u16), + .max_entries = 10, +}; + +SEC("xdp") +int nb_xdp_prog(struct xdp_md *ctx) { + __u16 *features; + features = bpf_map_lookup_elem(&nb_features, &map_key_features); + if (!features) { + return XDP_PASS; + } + + void *data = (void *)(long)ctx->data; + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr *eth = data; + struct iphdr *ip = (data + sizeof(struct ethhdr)); + struct udphdr *udp = (data + sizeof(struct ethhdr) + sizeof(struct iphdr)); + + // return early if not enough data + if (data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) > data_end){ + return XDP_PASS; + } + + // skip non IPv4 packages + if (eth->h_proto != htons(ETH_P_IP)) { + return XDP_PASS; + } + + // skip non UPD packages + if (ip->protocol != IPPROTO_UDP) { + return XDP_PASS; + } + + if (*features & flag_feature_dns_fwd) { + xdp_dns_fwd(ip, udp); + } + + if (*features & flag_feature_wg_proxy) { + xdp_wg_proxy(ip, udp); + } + return XDP_PASS; +} +char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/client/internal/ebpf/ebpf/src/wg_proxy.c b/client/internal/ebpf/ebpf/src/wg_proxy.c new file mode 100644 index 000000000..ecfedc6b3 --- /dev/null +++ b/client/internal/ebpf/ebpf/src/wg_proxy.c @@ -0,0 +1,54 @@ +const __u32 map_key_proxy_port = 0; +const __u32 map_key_wg_port = 1; + +struct bpf_map_def SEC("maps") nb_wg_proxy_settings_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(__u16), + .max_entries = 10, +}; + +__u16 proxy_port = 0; +__u16 wg_port = 0; + +bool read_port_settings() { + __u16 *value; + value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_proxy_port); + if (!value) { + return false; + } + + proxy_port = *value; + + value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_wg_port); + if (!value) { + return false; + } + wg_port = htons(*value); + + return true; +} + +int xdp_wg_proxy(struct iphdr *ip, struct udphdr *udp) { + if (proxy_port == 0 || wg_port == 0) { + if (!read_port_settings()){ + return XDP_PASS; + } + bpf_printk("proxy port: %d, wg port: %d", proxy_port, wg_port); + } + + // 2130706433 = 127.0.0.1 + if (ip->daddr != htonl(2130706433)) { + return XDP_PASS; + } + + if (udp->source != wg_port){ + return XDP_PASS; + } + + __be16 new_src_port = udp->dest; + __be16 new_dst_port = htons(proxy_port); + udp->dest = new_dst_port; + udp->source = new_src_port; + return XDP_PASS; +} \ No newline at end of file diff --git a/client/internal/ebpf/ebpf/wg_proxy_linux.go b/client/internal/ebpf/ebpf/wg_proxy_linux.go new file mode 100644 index 000000000..4e0df7329 --- /dev/null +++ b/client/internal/ebpf/ebpf/wg_proxy_linux.go @@ -0,0 +1,41 @@ +package ebpf + +import log "github.com/sirupsen/logrus" + +const ( + mapKeyProxyPort uint32 = 0 + mapKeyWgPort uint32 = 1 +) + +func (tf *GeneralManager) LoadWgProxy(proxyPort, wgPort int) error { + log.Debugf("load ebpf WG proxy") + tf.lock.Lock() + defer tf.lock.Unlock() + + err := tf.loadXdp() + if err != nil { + return err + } + + err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyProxyPort, uint16(proxyPort)) + if err != nil { + return err + } + + err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyWgPort, uint16(wgPort)) + if err != nil { + return err + } + + tf.setFeatureFlag(featureFlagWGProxy) + err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags) + if err != nil { + return err + } + return nil +} + +func (tf *GeneralManager) FreeWGProxy() error { + log.Debugf("free ebpf WG proxy") + return tf.unsetFeatureFlag(featureFlagWGProxy) +} diff --git a/client/internal/ebpf/instantiater_linux.go b/client/internal/ebpf/instantiater_linux.go new file mode 100644 index 000000000..20d8145b4 --- /dev/null +++ b/client/internal/ebpf/instantiater_linux.go @@ -0,0 +1,15 @@ +//go:build !android + +package ebpf + +import ( + "github.com/netbirdio/netbird/client/internal/ebpf/ebpf" + "github.com/netbirdio/netbird/client/internal/ebpf/manager" +) + +// GetEbpfManagerInstance is a wrapper function. This encapsulation is required because if the code import the internal +// ebpf package the Go compiler will include the object files. But it is not supported on Android. It can cause instant +// panic on older Android version. +func GetEbpfManagerInstance() manager.Manager { + return ebpf.GetEbpfManagerInstance() +} diff --git a/client/internal/ebpf/instantiater_nonlinux.go b/client/internal/ebpf/instantiater_nonlinux.go new file mode 100644 index 000000000..b7c38733a --- /dev/null +++ b/client/internal/ebpf/instantiater_nonlinux.go @@ -0,0 +1,10 @@ +//go:build !linux || android + +package ebpf + +import "github.com/netbirdio/netbird/client/internal/ebpf/manager" + +// GetEbpfManagerInstance return error because ebpf is not supported on all os +func GetEbpfManagerInstance() manager.Manager { + panic("unsupported os") +} diff --git a/client/internal/ebpf/manager/manager.go b/client/internal/ebpf/manager/manager.go new file mode 100644 index 000000000..af10142d5 --- /dev/null +++ b/client/internal/ebpf/manager/manager.go @@ -0,0 +1,9 @@ +package manager + +// Manager is used to load multiple eBPF programs. E.g., current DNS programs and WireGuard proxy +type Manager interface { + LoadDNSFwd(ip string, dnsPort int) error + FreeDNSFwd() error + LoadWgProxy(proxyPort, wgPort int) error + FreeWGProxy() error +} diff --git a/client/internal/wgproxy/ebpf/bpf_bpfeb.o b/client/internal/wgproxy/ebpf/bpf_bpfeb.o deleted file mode 100644 index 82d7fc35b..000000000 Binary files a/client/internal/wgproxy/ebpf/bpf_bpfeb.o and /dev/null differ diff --git a/client/internal/wgproxy/ebpf/bpf_bpfel.o b/client/internal/wgproxy/ebpf/bpf_bpfel.o deleted file mode 100644 index 2fc0ff8fc..000000000 Binary files a/client/internal/wgproxy/ebpf/bpf_bpfel.o and /dev/null differ diff --git a/client/internal/wgproxy/ebpf/loader.go b/client/internal/wgproxy/ebpf/loader.go deleted file mode 100644 index e154c0d9e..000000000 --- a/client/internal/wgproxy/ebpf/loader.go +++ /dev/null @@ -1,84 +0,0 @@ -//go:build linux && !android - -package ebpf - -import ( - _ "embed" - "net" - - "github.com/cilium/ebpf/link" - "github.com/cilium/ebpf/rlimit" -) - -const ( - mapKeyProxyPort uint32 = 0 - mapKeyWgPort uint32 = 1 -) - -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang-14 bpf src/portreplace.c -- - -// EBPF is a wrapper for eBPF program -type EBPF struct { - link link.Link -} - -// NewEBPF create new EBPF instance -func NewEBPF() *EBPF { - return &EBPF{} -} - -// Load load ebpf program -func (l *EBPF) Load(proxyPort, wgPort int) error { - // it required for Docker - err := rlimit.RemoveMemlock() - if err != nil { - return err - } - - ifce, err := net.InterfaceByName("lo") - if err != nil { - return err - } - - // Load pre-compiled programs into the kernel. - objs := bpfObjects{} - err = loadBpfObjects(&objs, nil) - if err != nil { - return err - } - defer func() { - _ = objs.Close() - }() - - err = objs.NbWgProxySettingsMap.Put(mapKeyProxyPort, uint16(proxyPort)) - if err != nil { - return err - } - - err = objs.NbWgProxySettingsMap.Put(mapKeyWgPort, uint16(wgPort)) - if err != nil { - return err - } - - defer func() { - _ = objs.NbWgProxySettingsMap.Close() - }() - - l.link, err = link.AttachXDP(link.XDPOptions{ - Program: objs.NbWgProxy, - Interface: ifce.Index, - }) - if err != nil { - return err - } - - return err -} - -// Free ebpf program -func (l *EBPF) Free() error { - if l.link != nil { - return l.link.Close() - } - return nil -} diff --git a/client/internal/wgproxy/ebpf/loader_test.go b/client/internal/wgproxy/ebpf/loader_test.go deleted file mode 100644 index 6ce323e70..000000000 --- a/client/internal/wgproxy/ebpf/loader_test.go +++ /dev/null @@ -1,18 +0,0 @@ -//go:build linux - -package ebpf - -import ( - "testing" -) - -func Test_newEBPF(t *testing.T) { - ebpf := NewEBPF() - err := ebpf.Load(1234, 51892) - defer func() { - _ = ebpf.Free() - }() - if err != nil { - t.Errorf("%s", err) - } -} diff --git a/client/internal/wgproxy/ebpf/src/portreplace.c b/client/internal/wgproxy/ebpf/src/portreplace.c deleted file mode 100644 index dc95ee53f..000000000 --- a/client/internal/wgproxy/ebpf/src/portreplace.c +++ /dev/null @@ -1,90 +0,0 @@ -#include -#include // ETH_P_IP -#include -#include -#include -#include -#include - -#define bpf_printk(fmt, ...) \ - ({ \ - char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ - }) - -const __u32 map_key_proxy_port = 0; -const __u32 map_key_wg_port = 1; - -struct bpf_map_def SEC("maps") nb_wg_proxy_settings_map = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(__u16), - .max_entries = 10, -}; - -__u16 proxy_port = 0; -__u16 wg_port = 0; - -bool read_port_settings() { - __u16 *value; - value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_proxy_port); - if(!value) { - return false; - } - - proxy_port = *value; - - value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_wg_port); - if(!value) { - return false; - } - wg_port = *value; - - return true; -} - -SEC("xdp") -int nb_wg_proxy(struct xdp_md *ctx) { - if(proxy_port == 0 || wg_port == 0) { - if(!read_port_settings()){ - return XDP_PASS; - } - bpf_printk("proxy port: %d, wg port: %d", proxy_port, wg_port); - } - - void *data = (void *)(long)ctx->data; - void *data_end = (void *)(long)ctx->data_end; - struct ethhdr *eth = data; - struct iphdr *ip = (data + sizeof(struct ethhdr)); - struct udphdr *udp = (data + sizeof(struct ethhdr) + sizeof(struct iphdr)); - - // return early if not enough data - if (data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) > data_end){ - return XDP_PASS; - } - - // skip non IPv4 packages - if (eth->h_proto != htons(ETH_P_IP)) { - return XDP_PASS; - } - - if (ip->protocol != IPPROTO_UDP) { - return XDP_PASS; - } - - // 2130706433 = 127.0.0.1 - if (ip->daddr != htonl(2130706433)) { - return XDP_PASS; - } - - if (udp->source != htons(wg_port)){ - return XDP_PASS; - } - - __be16 new_src_port = udp->dest; - __be16 new_dst_port = htons(proxy_port); - udp->dest = new_dst_port; - udp->source = new_src_port; - return XDP_PASS; -} -char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/client/internal/wgproxy/proxy_ebpf.go b/client/internal/wgproxy/proxy_ebpf.go index 8be6b0c19..ff8ff665b 100644 --- a/client/internal/wgproxy/proxy_ebpf.go +++ b/client/internal/wgproxy/proxy_ebpf.go @@ -12,15 +12,15 @@ import ( "github.com/google/gopacket" "github.com/google/gopacket/layers" - log "github.com/sirupsen/logrus" - ebpf2 "github.com/netbirdio/netbird/client/internal/wgproxy/ebpf" + "github.com/netbirdio/netbird/client/internal/ebpf" + ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager" ) // WGEBPFProxy definition for proxy with EBPF support type WGEBPFProxy struct { - ebpf *ebpf2.EBPF + ebpfManager ebpfMgr.Manager lastUsedPort uint16 localWGListenPort int @@ -36,7 +36,7 @@ func NewWGEBPFProxy(wgPort int) *WGEBPFProxy { log.Debugf("instantiate ebpf proxy") wgProxy := &WGEBPFProxy{ localWGListenPort: wgPort, - ebpf: ebpf2.NewEBPF(), + ebpfManager: ebpf.GetEbpfManagerInstance(), lastUsedPort: 0, turnConnStore: make(map[uint16]net.Conn), } @@ -56,7 +56,7 @@ func (p *WGEBPFProxy) Listen() error { return err } - err = p.ebpf.Load(wgPorxyPort, p.localWGListenPort) + err = p.ebpfManager.LoadWgProxy(wgPorxyPort, p.localWGListenPort) if err != nil { return err } @@ -110,7 +110,7 @@ func (p *WGEBPFProxy) Free() error { err1 = p.conn.Close() } - err2 = p.ebpf.Free() + err2 = p.ebpfManager.FreeWGProxy() if p.rawConn != nil { err3 = p.rawConn.Close() } diff --git a/iface/iface.go b/iface/iface.go index 58167d1e8..55891d047 100644 --- a/iface/iface.go +++ b/iface/iface.go @@ -112,7 +112,7 @@ func (w *WGIface) Close() error { return w.tun.Close() } -// SetFilter sets packet filters for the userspace impelemntation +// SetFilter sets packet filters for the userspace implementation func (w *WGIface) SetFilter(filter PacketFilter) error { w.mu.Lock() defer w.mu.Unlock()